[CI] Build wasm blob with srtool and include prop hashes and blobs in release notes (#1506)

* initial build-wasms commit

* fix wasm builds

* add caching and artifact upload

* test for git dir

* Revert "test for git dir"

This reverts commit 295e4655484691aed9162f479a87f09c1b1c9317.

* fix git version incompability

* attempt to fix caching

* Revert "fix git version incompability"

This reverts commit 1d22ffd26b7eff635c5abcc9bd04cb3c88629a1d.

* add kusama job

* use a build matrix for building runtimes

* combine build-wasms and publish draft release

* attempt to name runtimes according to version

* Revert "attempt to name runtimes according to version"

This reverts commit 82f7b4dc0aea995f2599293bba9f11b879f459e5.
Failed experiment

* name runtimes according to version #2

* fix asset path and name

* build wasms first, include prop hash

polkadot/.github/workflows/publish-draft-release.yml

@@ -6,6 +6,51 @@ on:
       - v**.**.**
 
 jobs:
+  build-runtimes:
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        runtime: ['polkadot', 'kusama']
+    container:
+      image: chevdor/srtool:nightly-2020-07-20
+      volumes:
+        - ${{ github.workspace }}:/build
+      env:
+        PACKAGE: ${{ matrix.runtime }}-runtime
+        RUSTC_VERSION: nightly-2020-07-20
+    steps:
+      - uses: actions/checkout@v2
+      - name: Cache target dir
+        uses: actions/cache@v2
+        with:
+          path: '${{ github.workspace }}/target'
+          key: srtool-target-${{ matrix.runtime }}-${{ github.sha }}
+          restore-keys: |
+            srtool-target-${{ matrix.runtime }}-
+            srtool-target-
+      - name: Build ${{ matrix.runtime }} runtime
+        id: build-runtime
+        shell: bash
+        run: |
+          cd /build
+          pwd
+          ls -la
+          build --json | tee srtool_output.json
+          cat srtool_output.json
+          while IFS= read -r line; do
+            echo "::set-output name=$line::$(jq -r ".$line" < srtool_output.json)"
+          done <<< "$(jq -r 'keys[]' < srtool_output.json)"
+      - name: Upload ${{ matrix.runtime }} srtool json
+        uses: actions/upload-artifact@v2
+        with:
+          name: ${{ matrix.runtime }}-srtool-json
+          path: srtool_output.json
+      - name: Upload ${{ matrix.runtime }} runtime
+        uses: actions/upload-artifact@v2
+        with:
+          name: ${{ matrix.runtime }}-runtime
+          path: "${{ steps.build-runtime.outputs.wasm }}"
+
   get-rust-versions:
     runs-on: ubuntu-latest
     container:
@@ -21,9 +66,10 @@ jobs:
 
   publish-draft-release:
     runs-on: ubuntu-latest
-    needs: get-rust-versions
+    needs: ['get-rust-versions', 'build-runtimes']
     outputs:
       release_url: ${{ steps.create-release.outputs.html_url }}
+      asset_upload_url: ${{ steps.create-release.outputs.upload_url }}
     steps:
     - uses: actions/checkout@v2
       with:
@@ -33,17 +79,15 @@ jobs:
       uses: actions/setup-ruby@v1
       with:
         ruby-version: 2.7
+    - name: Download srtool json output
+      uses: actions/download-artifact@v2
     - name: Generate release text
       env:
         RUSTC_STABLE: ${{ needs.get-rust-versions.outputs.rustc-stable }}
         RUSTC_NIGHTLY: ${{ needs.get-rust-versions.outputs.rustc-nightly }}
       run: |
         gem install changelogerator git toml
-        ruby $GITHUB_WORKSPACE/polkadot/scripts/github/generate_release_text.rb > release_text.md
-    - uses: actions/upload-artifact@v2
-      with:
-        name: release_text
-        path: release_text.md
+        ruby $GITHUB_WORKSPACE/polkadot/scripts/github/generate_release_text.rb | tee release_text.md
     - name: Create draft release
       id: create-release
       uses: actions/create-release@v1
@@ -66,3 +110,33 @@ jobs:
           access_token: ${{ secrets.MATRIX_ACCESS_TOKEN }}
           message: "**New version of polkadot tagged**: ${{ github.ref }}<br/>Gav: Draft release created: ${{ needs.publish-draft-release.outputs.release_url }}"
           server: "matrix.parity.io"
+
+  publish-runtimes:
+    runs-on: ubuntu-latest
+    needs: ['publish-draft-release']
+    strategy:
+      matrix:
+        runtime: ['polkadot', 'kusama']
+    steps:
+      - uses: actions/checkout@v2
+      - uses: actions/download-artifact@v2
+        with:
+          name: ${{ matrix.runtime }}-runtime
+      - name: Set up Ruby 2.7
+        uses: actions/setup-ruby@v1
+        with:
+          ruby-version: 2.7
+      - name: Get runtime version
+        id: get-runtime-ver
+        run: |
+          runtime_ver="$(ruby -e 'require "./scripts/github/lib.rb"; puts get_runtime("${{ matrix.runtime }}")')"
+          echo "::set-output name=runtime_ver::$runtime_ver"
+      - name: Upload ${{ matrix.runtime }} wasm
+        uses: actions/upload-release-asset@v1
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        with:
+          upload_url: ${{ needs.publish-draft-release.outputs.asset_upload_url }}
+          asset_path: ./${{ matrix.runtime }}_runtime.compact.wasm
+          asset_name: ${{ matrix.runtime }}_runtime-v${{ steps.get-runtime-ver.outputs.runtime_ver }}.compact.wasm
+          asset_content_type: application/wasm

polkadot/scripts/github/generate_release_text.rb

@@ -4,6 +4,8 @@ require 'changelogerator'
 require 'git'
 require 'erb'
 require 'toml'
+require 'json'
+require_relative './lib.rb'
 
 version = ENV['GITHUB_REF']
 token = ENV['GITHUB_TOKEN']
@@ -61,14 +63,23 @@ release_priority = Changelog.highest_priority_for_changes(all_changes)
 rustc_stable = ENV['RUSTC_STABLE']
 rustc_nightly = ENV['RUSTC_NIGHTLY']
 
-polkadot_runtime = File.open(polkadot_path + '/runtime/polkadot/src/lib.rs') do |f|
-  f.find { |l| l =~ /spec_version/ }.match(/[0-9]+/)[0]
-end
-kusama_runtime = File.open(polkadot_path + '/runtime/kusama/src/lib.rs') do |f|
-  f.find { |l| l =~ /spec_version/ }.match(/[0-9]+/)[0]
-end
-westend_runtime = File.open(polkadot_path + '/runtime/westend/src/lib.rs') do |f|
-  f.find { |l| l =~ /spec_version/ }.match(/[0-9]+/)[0]
-end
+polkadot_runtime = get_runtime('polkadot', polkadot_path)
+kusama_runtime = get_runtime('kusama', polkadot_path)
+westend_runtime = get_runtime('westend', polkadot_path)
+
+# These json files should have been downloaded as part of the build-runtimes
+# github action
+
+polkadot_json = JSON.parse(
+  File.read(
+    ENV['GITHUB_WORKSPACE'] + '/polkadot-srtool-json/srtool_output.json'
+  )
+)
+
+kusama_json = JSON.parse(
+  File.read(
+    ENV['GITHUB_WORKSPACE'] + '/kusama-srtool-json/srtool_output.json'
+  )
+)
 
 puts renderer.result

polkadot/scripts/github/lib.rb

@@ -0,0 +1,12 @@
+# frozen_string_literal: true
+
+# A collection of helper functions that might be useful for various scripts
+
+# Gets the runtime version for a given runtime.
+# Optionally accepts a path that is the root of the project which defaults to
+# the current working directory
+def get_runtime(runtime, path = '.')
+  File.open(path + "/runtime/#{runtime}/src/lib.rs") do |f|
+    f.find { |l| l =~ /spec_version/ }.match(/[0-9]+/)[0]
+  end
+end

polkadot/scripts/github/polkadot_release.erb

@@ -11,6 +11,12 @@ This release was tested against the following versions of `rustc`. Other version
 - <%= rustc_stable %>
 - <%= rustc_nightly %>
 
+WASM runtimes built with [srtool](https://gitlab.com/chevdor/srtool) using `<%= polkadot_json['rustc'] %>`.
+
+Proposal hashes:
+* `polkadot_runtime-v<%= polkadot_runtime %>.compact.wasm - <%= polkadot_json['prop'] %>`
+* `kusama_runtime-v<%= kusama_runtime %>.compact.wasm - <%= kusama_json['prop'] %>`
+
 <% unless misc_changes.empty? %>
 ## Changes