Fix #851 (#1110)

* Fix doc

* Formatting fixes and bump prio if host functions are detected

* Add canvas

* Bump canvas-kusama's runtime version to 800

* Add db migrations

* Prepare for client releases

* Split the upload of the context artifact

* Rename of the srtool digests to match script default expectations

* Fix runtime name for canvas

* Fix jobs dependencies when skipping jobs/steps for client builds

* Fix CI for the client builds

* Fix issue when building in client mode

* Bump checkout action to v3 and gate some tests

* Fix notification

ref #851

.github/workflows/release-02_create-draft.yml

@@ -5,16 +5,23 @@ on:
     inputs:
       ref1:
         description: The 'from' tag to use for the diff
-        default: statemine-v5.0.0
+        default: parachains-v7.0.0
         required: true
       ref2:
         description: The 'to' tag to use for the diff
-        default: release-statemine-v6
+        default: release-parachains-v8.0.0
         required: true
+      release_type:
+        description: Pass "client" for client releases, leave empty otherwise
+        required: false
       pre_release:
         description: For pre-releases
         default: "true"
         required: true
+      notification:
+        description: Whether or not to notify over Matrix
+        default: "true"
+        required: true
 
 jobs:
   get-rust-versions:
@@ -30,6 +37,8 @@ jobs:
           echo "::set-output name=stable::$(rustc +stable --version)"
           echo "::set-output name=nightly::$(rustc +nightly --version)"
 
+  # We do not skip the entire job for client builds (although we don't need it)
+  # because it is a dep of the next job. However we skip the time consuming steps.
   build-runtimes:
     runs-on: ubuntu-latest
     strategy:
@@ -37,11 +46,12 @@ jobs:
         runtime: ["shell", "statemine", "statemint", "westmint", "rococo-parachain", "canvas-kusama"]
     steps:
       - name: Checkout sources
-        uses: actions/checkout@v2
+        uses: actions/checkout@v3
         with:
           ref: ${{ github.event.inputs.ref2 }}
 
       - name: Cache target dir
+        if: ${{ github.event.inputs.release_type != 'client' }}
         uses: actions/cache@v2
         with:
           path: "${{ github.workspace }}/runtime/${{ matrix.runtime }}/target"
@@ -51,6 +61,7 @@ jobs:
             srtool-target-
 
       - name: Build ${{ matrix.runtime }} runtime
+        if: ${{ github.event.inputs.release_type != 'client' }}
         id: srtool_build
         uses: chevdor/srtool-actions@v0.3.0
         with:
@@ -59,18 +70,21 @@ jobs:
           runtime_dir: polkadot-parachains/${{ matrix.runtime }}
 
       - name: Store srtool digest to disk
+        if: ${{ github.event.inputs.release_type != 'client' }}
         run: |
           echo '${{ steps.srtool_build.outputs.json }}' | \
-            jq > ${{ matrix.runtime }}_srtool_output.json
+            jq > ${{ matrix.runtime }}-srtool-digest.json
 
       - name: Upload ${{ matrix.runtime }} srtool json
-        uses: actions/upload-artifact@v2
+        if: ${{ github.event.inputs.release_type != 'client' }}
+        uses: actions/upload-artifact@v3
         with:
           name: ${{ matrix.runtime }}-srtool-json
-          path: ${{ matrix.runtime }}_srtool_output.json
+          path: ${{ matrix.runtime }}-srtool-digest.json
 
       - name: Upload ${{ matrix.runtime }} runtime
-        uses: actions/upload-artifact@v2
+        if: ${{ github.event.inputs.release_type != 'client' }}
+        uses: actions/upload-artifact@v3
         with:
           name: ${{ matrix.runtime }}-runtime
           path: |
@@ -84,7 +98,7 @@ jobs:
       asset_upload_url: ${{ steps.create-release.outputs.upload_url }}
     steps:
       - name: Checkout sources
-        uses: actions/checkout@v2
+        uses: actions/checkout@v3
         with:
           fetch-depth: 0
           path: cumulus
@@ -116,39 +130,47 @@ jobs:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
           NO_CACHE: 1
           DEBUG: 1
-          SHELL_DIGEST: ${{ github.workspace}}/shell-srtool-json/shell_srtool_output.json
+          SHELL_DIGEST: ${{ github.workspace}}/shell-srtool-json/shell-srtool-digest.json
-          WESTMINT_DIGEST: ${{ github.workspace}}/westmint-srtool-json/westmint_srtool_output.json
+          WESTMINT_DIGEST: ${{ github.workspace}}/westmint-srtool-json/westmint-srtool-digest.json
-          STATEMINE_DIGEST: ${{ github.workspace}}/statemine-srtool-json/statemine_srtool_output.json
+          STATEMINE_DIGEST: ${{ github.workspace}}/statemine-srtool-json/statemine-srtool-digest.json
-          STATEMINT_DIGEST: ${{ github.workspace}}/statemint-srtool-json/statemint_srtool_output.json
+          STATEMINT_DIGEST: ${{ github.workspace}}/statemint-srtool-json/statemint-srtool-digest.json
-          ROCOCO_PARA_DIGEST: ${{ github.workspace}}/rococo-parachain-srtool-json/rococo-parachain_srtool_output.json
+          ROCOCO_PARA_DIGEST: ${{ github.workspace}}/rococo-parachain-srtool-json/rococo-parachain-srtool-digest.json
-          CANVAS_KUSAMA_DIGEST: ${{ github.workspace}}/canvas-kusama-srtool-json/canvas-kusama_srtool_output.json
+          CANVAS_KUSAMA_DIGEST: ${{ github.workspace}}/canvas-kusama-srtool-json/canvas-kusama-srtool-digest.json
           REF1: ${{ github.event.inputs.ref1 }}
           REF2: ${{ github.event.inputs.ref2 }}
           PRE_RELEASE: ${{ github.event.inputs.pre_release }}
-          HIDE_SRTOOL_ROCOCO: false
+          RELEASE_TYPE: ${{ github.event.inputs.release_type }}
-          HIDE_SRTOOL_SHELL: false
         run: |
-          find ${{env.GITHUB_WORKSPACE}} -type f -name "*_srtool_output.json"
+          find ${{env.GITHUB_WORKSPACE}} -type f -name "*-srtool-digest.json"
-          ls -al $SHELL_DIGEST
+
-          ls -al $WESTMINT_DIGEST
+          if [ "$RELEASE_TYPE" == "client" ]; then
-          ls -al $STATEMINE_DIGEST
+            ls -al $SHELL_DIGEST || true
-          ls -al $STATEMINT_DIGEST
+            ls -al $WESTMINT_DIGEST || true
-          ls -al $ROCOCO_PARA_DIGEST
+            ls -al $STATEMINE_DIGEST || true
-          ls -al $CANVAS_KUSAMA_DIGEST
+            ls -al $STATEMINT_DIGEST || true
+            ls -al $ROCOCO_PARA_DIGEST || true
+            ls -al $CANVAS_KUSAMA_DIGEST || true
+          fi
 
           echo "The diff will be computed from $REF1 to $REF2"
           cd cumulus/scripts/ci/changelog
           ./bin/changelog $REF1 $REF2 release-notes.md
-          ls -al release-notes.md
+          ls -al {release-notes.md,context.json} || true
-          ls -al context.json
 
-      - name: Archive artifact context.json
+      - name: Archive srtool json
-        uses: actions/upload-artifact@v2
+        if: ${{ github.event.inputs.release_type != 'client' }}
+        uses: actions/upload-artifact@v3
+        with:
+          name: srtool-json
+          path: |
+            **/*-srtool-digest.json
+
+      - name: Archive context artifact
+        uses: actions/upload-artifact@v3
         with:
           name: release-notes-context
           path: |
             context.json
-            **/*_srtool_output.json
 
       - name: Create draft release
         id: create-release
@@ -156,12 +178,13 @@ jobs:
         env:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
         with:
-          tag_name: parachains-${{ github.ref }}
-          release_name: Parachains ${{ github.ref }}
           body_path: ./cumulus/scripts/ci/changelog/release-notes.md
+          tag_name: ${{ github.event.inputs.ref2 }}
+          release_name: ${{ github.event.inputs.ref2 }}
           draft: true
 
   publish-runtimes:
+    if: ${{ github.event.inputs.release_type != 'client' }}
     runs-on: ubuntu-latest
     needs: ["publish-draft-release"]
     env:
@@ -171,7 +194,7 @@ jobs:
         runtime: ["shell", "statemine", "statemint", "westmint", "rococo-parachain", "canvas-kusama"]
     steps:
       - name: Checkout sources
-        uses: actions/checkout@v2
+        uses: actions/checkout@v3
         with:
           ref: ${{ github.event.inputs.ref2 }}
 
@@ -194,11 +217,12 @@ jobs:
           echo "Found version: >$runtime_ver<"
           echo "::set-output name=runtime_ver::$runtime_ver"
 
-      - name: Fix parachain runtime name
+      - name: Fix runtime name
         id: fix-runtime-path
         run: |
           cd "${{ matrix.runtime }}-runtime/"
           mv "$(sed 's/-parachain/_parachain/' <<< ${{ matrix.runtime }})_runtime.compact.compressed.wasm" "${{ matrix.runtime }}_runtime.compact.compressed.wasm" || true
+          mv "$(sed 's/-kusama/_kusama/' <<< ${{ matrix.runtime }})_runtime.compact.compressed.wasm" "${{ matrix.runtime }}_runtime.compact.compressed.wasm" || true
 
       - name: Upload compressed ${{ matrix.runtime }} wasm
         uses: actions/upload-release-asset@v1
@@ -211,6 +235,7 @@ jobs:
           asset_content_type: application/wasm
 
   post_to_matrix:
+    if: ${{ github.event.inputs.notification == 'true' }}
     runs-on: ubuntu-latest
     needs: publish-draft-release
     steps:
@@ -220,7 +245,7 @@ jobs:
           room_id: ${{ secrets.INTERNAL_CUMULUS_MATRIX_ROOM_ID }}
           access_token: ${{ secrets.MATRIX_ACCESS_TOKEN }}
           message: |
-            **New draft for ${{ github.repository }}**: ${{ github.ref }}<br/>
+            **New draft for ${{ github.repository }}**: ${{ github.event.inputs.ref2 }}<br/>
 
             Draft release created: [draft](${{ needs.publish-draft-release.outputs.release_url }})
 

.gitignore

@@ -7,3 +7,4 @@ polkadot_argument_parsing
 **/node_modules
 **/chains/
 *.iml
+.env

polkadot-parachains/canvas-kusama/src/lib.rs

@@ -111,7 +111,7 @@ pub const VERSION: RuntimeVersion = RuntimeVersion {
 	spec_name: create_runtime_str!("canvas-kusama"),
 	impl_name: create_runtime_str!("canvas-kusama"),
 	authoring_version: 1,
-	spec_version: 16,
+	spec_version: 800,
 	impl_version: 0,
 	apis: RUNTIME_API_VERSIONS,
 	transaction_version: 1,

scripts/ci/changelog/README.md

@@ -16,7 +16,7 @@ Run:
 
 For instance:
 ```
-./bin/changelog statemine-v5.0.0
+./bin/changelog parachains-v7.0.0-rc8
 ```
 
 A file called `release-notes.md` will be generated and can be used for the release.
@@ -36,6 +36,13 @@ REF2=HEAD
 DEBUG=1
 NO_CACHE=1
 ```
+
+By default, the template will include all the information, including the runtime data.
+For clients releases, we don't need those and they can be skipped by setting the following env:
+```
+RELEASE_TYPE=client
+```
+
 ## Considered labels
 
 The following list will likely evolve over time and it will be hard to keep it in sync.

scripts/ci/changelog/bin/changelog

@@ -81,34 +81,56 @@ WESTMINT_DIGEST = ENV['WESTMINT_DIGEST'] || 'digests/westmint-srtool-digest.json
 STATEMINE_DIGEST = ENV['STATEMINE_DIGEST'] || 'digests/statemine-srtool-digest.json'
 STATEMINT_DIGEST = ENV['STATEMINT_DIGEST'] || 'digests/statemint-srtool-digest.json'
 ROCOCO_PARA_DIGEST = ENV['ROCOCO_PARA_DIGEST'] || 'digests/rococo-parachain-srtool-digest.json'
+CANVAS_KUSAMA_DIGEST = ENV['CANVAS_KUSAMA_DIGEST'] || 'digests/canvas-kusama-srtool-digest.json'
 
-# Here we compose all the pieces together into one
+logger.debug("Release type: #{ENV['RELEASE_TYPE']}")
-# single big json file.
+
-cmd = format('jq \
+if ENV['RELEASE_TYPE'] && ENV['RELEASE_TYPE'] == 'client'
+  logger.debug("Building changelog without runtimes")
+  cmd = format('jq \
     --slurpfile cumulus %s \
     --slurpfile substrate %s \
     --slurpfile polkadot %s \
-    --slurpfile srtool_shell %s \
-    --slurpfile srtool_westmint %s \
-    --slurpfile srtool_statemine %s \
-    --slurpfile srtool_statemint %s \
-    --slurpfile srtool_rococo_parachain %s \
     -n \'{
             cumulus: $cumulus[0],
             substrate: $substrate[0],
             polkadot: $polkadot[0],
-        srtool: [
+          }\' > context.json', cumulus_data, substrate_data, polkadot_data,
-        { name: "rococo", data: $srtool_rococo_parachain[0] },
+  )
-        { name: "shell", data: $srtool_shell[0] },
+else
-        { name: "westmint", data: $srtool_westmint[0] },
+  logger.debug("Building changelog with runtimes")
-        { name: "statemint", data: $srtool_statemint[0] },
+
-        { name: "statemine", data: $srtool_statemine[0] }
+  # Here we compose all the pieces together into one
-    ] }\' > context.json', cumulus_data, substrate_data, polkadot_data,
+  # single big json file.
-             SHELL_DIGEST,
+  cmd = format('jq \
-             WESTMINT_DIGEST,
+      --slurpfile cumulus %s \
-             STATEMINE_DIGEST,
+      --slurpfile substrate %s \
-             STATEMINT_DIGEST,
+      --slurpfile polkadot %s \
-             ROCOCO_PARA_DIGEST)
+      --slurpfile srtool_shell %s \
+      --slurpfile srtool_westmint %s \
+      --slurpfile srtool_statemine %s \
+      --slurpfile srtool_statemint %s \
+      --slurpfile srtool_rococo_parachain %s \
+      --slurpfile srtool_canvas_kusama %s \
+      -n \'{
+              cumulus: $cumulus[0],
+              substrate: $substrate[0],
+              polkadot: $polkadot[0],
+          srtool: [
+          { name: "rococo", data: $srtool_rococo_parachain[0] },
+          { name: "shell", data: $srtool_shell[0] },
+          { name: "westmint", data: $srtool_westmint[0] },
+          { name: "statemint", data: $srtool_statemint[0] },
+          { name: "statemine", data: $srtool_statemine[0] },
+          { name: "canvas", data: $srtool_canvas_kusama[0] }
+      ] }\' > context.json', cumulus_data, substrate_data, polkadot_data,
+              SHELL_DIGEST,
+              WESTMINT_DIGEST,
+              STATEMINE_DIGEST,
+              STATEMINT_DIGEST,
+              ROCOCO_PARA_DIGEST,
+              CANVAS_KUSAMA_DIGEST)
+end
 system(cmd)
 
 cmd = format('tera --env --env-key env --include-path templates \

scripts/ci/changelog/templates/global_priority.md.tera

@@ -1,4 +1,4 @@
-{% import "high_priority.md.tera" as m_p -%}
+{%- import "high_priority.md.tera" as m_p -%}
 ## Global Priority
 
 {%- set cumulus_prio = 0 -%}
@@ -23,10 +23,7 @@
 {%- endif -%}
 {%- if substrate_prio > global_prio -%}
     {%- set global_prio = substrate_prio -%}
-{%- endif -%}
+{%- endif %}
-
-{# We show the result #}
-{{ m_p::high_priority(p=global_prio, changes=changes) }}
 
 <!--
 - Cumulus: {{ cumulus_prio }}
@@ -34,4 +31,5 @@
 - Substrate: {{ substrate_prio }}
 -->
 
-{# todo: show high prio list here #}
+{# We show the result #}
+{{ m_p::high_priority(p=global_prio, changes=changes) }}

scripts/ci/changelog/templates/high_priority.md.tera

@@ -3,7 +3,23 @@
 {# This macro convert a priority level into readable output #}
 {%- macro high_priority(p, changes) -%}
 
-{%- if p >= 7 -%}
+{# real globals don't work so we count the number of host functions here as well #}
+{# unfortunately, the next snippet is duplicated in the host_functions.md.tera template #}
+{# as well #}
+{%- set_global host_fn_count = 0 -%}
+
+{# We loop first to count the number of host functions but we do not display anything yet #}
+{%- for pr in changes -%}
+{%- if pr.meta.B and pr.meta.B.value == 0 -%}
+{#- We skip silent ones -#}
+{%- else -%}
+    {%- if pr.meta.E and pr.meta.E.value == 4 -%}
+        {%- set_global host_fn_count = host_fn_count + 1 -%}
+    {%- endif -%}
+{%- endif -%}
+{%- endfor -%}
+
+{%- if p >= 7 or host_fn_count > 0 -%}
     {%- set prio = "‼️ HIGH" -%}
     {%- set text = "This is a **high priority** release and you must upgrade as as soon as possible." -%}
 {%- elif p >= 5 -%}
@@ -12,16 +28,17 @@
 {%- elif p >= 3 -%}
     {%- set prio = "Low" -%}
     {%- set text = "This is a low priority release and you may upgrade at your convenience." -%}
-{%- endif %}
+{%- endif -%}
 
-{%- if prio %}
+{%- if prio -%}
 {{prio}}: {{text}}
 {%- else -%}
 <!-- No relevant Priority label as been detected -->
-{% endif %}
+{%- endif %}
 
+{# We only show details if Medium or High #}
+{%- if p >= 5 -%}
 The changes motivating this priority level are:
-
 {% for pr in changes | sort(attribute="merged_at") -%}
     {%- if pr.meta.C -%}
         {%- if pr.meta.C.value == p %}
@@ -29,9 +46,10 @@ The changes motivating this priority level are:
 {%- if pr.meta.B and pr.meta.B.value == 7 %}
 (RUNTIME)
 {% endif %}
+
         {%- endif -%}
     {%- endif -%}
 {%- endfor %}
-
+{%- endif %}
 
 {%- endmacro priority -%}

scripts/ci/changelog/templates/host_functions.md.tera

@@ -1,30 +1,37 @@
 {%- import "change.md.tera" as m_c -%}
 {%- set_global host_fn_count = 0 -%}
 
+{# We loop first to count the number of host functions but we do not display anything yet #}
+{%- for pr in changes -%}
+{%- if pr.meta.B and pr.meta.B.value == 0 -%}
+{#- We skip silent ones -#}
+{%- else -%}
+    {%- if pr.meta.E and pr.meta.E.value == 4 -%}
+        {%- set_global host_fn_count = host_fn_count + 1 -%}
+    {% endif -%}
+{%- endif -%}
+{%- endfor -%}
+
+<!-- {{ host_fn_count }} host functions were detected -->
+
+{% if host_fn_count == 0 -%}
+<!-- ℹ️ This release does not contain any new host functions. -->
+{%- else -%}
+## Host functions
+
+⚠️ The runtimes in this release contain {{ host_fn_count }} new **host function{{ host_fn_count | pluralize }}**.
+
+⚠️ It is critical that you update your client before the chain switches to the new runtimes.
+
 {% for pr in changes | sort(attribute="merged_at") -%}
 
 {%- if pr.meta.B and pr.meta.B.value == 0 -%}
 {#- We skip silent ones -#}
 {%- else -%}
     {%- if pr.meta.E and pr.meta.E.value == 4 -%}
-        {%- set_global host_fn_count = host_fn_count + 1 -%}
             - {{ m_c::change(c=pr) }}
         {% endif -%}
     {% endif -%}
 {%- endfor -%}
 
-<!-- {{ host_fn_count }} host functions were detected -->
-
-{% if host_fn_count == 0 -%}
-<!-- ℹ️ This release does not contain any new host functions. -->
-{% elif host_fn_count == 1 -%}
-## Host functions
-
-⚠️ The runtimes in this release contain one new **host function**.
-
-⚠️ It is critical that you update your client before the chain switches to the new runtimes.
-{%- else -%}
-⚠️ The runtimes in this release contain {{ host_fn_count }} new **host function{{ host_fn_count | pluralize }}**.
-
-⚠️ It is critical that you update your client before the chain switches to the new runtimes.
 {%- endif %}

scripts/ci/changelog/templates/migrations-db.md.tera

@@ -0,0 +1,26 @@
+{%- import "change.md.tera" as m_c %}
+{%- set_global db_migration_count = 0 -%}
+
+## Database Migrations
+
+{% for pr in changes | sort(attribute="merged_at") -%}
+
+{%- if pr.meta.B and pr.meta.B.value == 0 %}
+{#- We skip silent ones -#}
+{%- else -%}
+{%- if pr.meta.E and pr.meta.E.value == 2 -%}
+{%- set_global db_migration_count = db_migration_count + 1 -%}
+- {{ m_c::change(c=pr) }}
+{% endif -%}
+{% endif -%}
+{% endfor -%}
+
+{%- if db_migration_count == 0 -%}
+No Database migration detected in this release.
+{% else  %}
+
+There is {{ db_migration_count }} database migration(s) in this release.
+
+Database migrations are operations bringing your database to the latest stand.
+Some migrations may break compatibility and making a backup of your database is highly recommended.
+{%- endif %}

scripts/ci/changelog/templates/migrations-runtime.md.tera

@@ -1,6 +1,6 @@
 {%- import "change.md.tera" as m_c %}
 
-## Migrations
+## Runtime Migrations
 
 {% for pr in changes | sort(attribute="merged_at") -%}
 

scripts/ci/changelog/templates/template.md.tera

@@ -1,4 +1,4 @@
-{# This is the entry point of the template -#}
+{# This is the entry point of the template for the parachains-* releases-#}
 <!-- repository: {{ cumulus.repository.name }} -->
 {% include "pre_release.md.tera" -%}
 
@@ -16,16 +16,23 @@ This release contains the changes from `{{ env.REF1 }}` to `{{ env.REF2 }}`.
 {%- set DOT = "[P]" -%}
 {%- set SUB = "[S]" -%}
 
+{# We check for host function first because no matter what the priority is, #}
+{# we will force it to HIGH if at least one host function was detected. #}
+{% include "host_functions.md.tera" -%}
 
 {% include "global_priority.md.tera" -%}
 
-{% include "host_functions.md.tera" -%}
-
 {% include "compiler.md.tera" -%}
 
-{% include "migrations.md.tera" -%}
+{% include "migrations-db.md.tera" -%}
 
+{% include "migrations-runtime.md.tera" -%}
+
+{% if env.RELEASE_TYPE and env.RELEASE_TYPE == "client" %}
+<!-- skipping runtime data for RELEASE_TYPE = {{ env.RELEASE_TYPE }} -->
+{% else %}
 {% include "runtimes.md.tera" -%}
+{% endif %}
 
 {% include "changes.md.tera" -%}