Add check runtimes GH Workflow (#2252)

This PR introduces:
- a new script 
- a new GH Workflow 
- runtime reference spec files for `rococo` and `westend`

It brings a mechanism to check that part(s) of the runtimes' metadata
that should not change over time, actually did not change over time.

Ideally, the GHW should trigger when a release is edited but GH seem to
[have an issue](https://github.com/orgs/community/discussions/47794)
that prevents the trigger from working. This is why the check has been
implemented as `workflow_dispatch` for a start.

The `workflow_dispatch` requires a `release_id` that can be found using:
```
curl -s -H "Authorization: Bearer ${GITHUB_TOKEN}" \
   https://api.github.com/repos/paritytech/polkadot-sdk/releases | \
   jq '.[] | { name: .name, id: .id }'
```
as documented in the workflow.

A sample run can be seen
[here](https://github.com/chevdor/polkadot-sdk/actions/runs/6811176342).

.github/runtime_specs/rococo.json

@@ -0,0 +1,17 @@
+{
+    "pallets": {
+      "1": {
+        "constants": {
+          "EpochDuration": {
+            "value": [ 88, 2, 0, 0, 0, 0, 0, 0 ]}
+        }
+      },
+
+      "2": {
+        "constants": {
+          "MinimumPeriod": {
+            "value": [ 184, 11, 0, 0, 0, 0, 0, 0 ]}
+        }
+      }
+    }
+  }

.github/runtime_specs/westend.json

@@ -0,0 +1,17 @@
+{
+    "pallets": {
+      "1": {
+        "constants": {
+          "EpochDuration": {
+            "value": [ 88, 2, 0, 0, 0, 0, 0, 0 ]}
+        }
+      },
+
+      "2": {
+        "constants": {
+          "MinimumPeriod": {
+            "value": [ 184, 11, 0, 0, 0, 0, 0, 0 ]}
+        }
+      }
+    }
+  }

.github/scripts/check-runtime.py

@@ -0,0 +1,124 @@
+#!/usr/bin/env python3
+
+import json
+import sys
+import logging
+import os
+
+
+def check_constant(spec_pallet_id, spec_pallet_value, meta_constant):
+    """
+    Check a single constant
+
+    :param spec_pallet_id:
+    :param spec_pallet_value:
+    :param meta_constant:
+    :return:
+    """
+    if meta_constant['name'] == list(spec_pallet_value.keys())[0]:
+        constant = meta_constant['name']
+        res = list(spec_pallet_value.values())[0]["value"] == meta_constant["value"]
+
+        logging.debug(f"  Checking pallet:{spec_pallet_id}/constants/{constant}")
+        logging.debug(f"    spec_pallet_value: {spec_pallet_value}")
+        logging.debug(f"    meta_constant: {meta_constant}")
+        logging.info(f"pallet:{spec_pallet_id}/constants/{constant} -> {res}")
+        return res
+    else:
+        # logging.warning(f"  Skipping pallet:{spec_pallet_id}/constants/{meta_constant['name']}")
+        pass
+
+
+def check_pallet(metadata, spec_pallet):
+    """
+    Check one pallet
+
+    :param metadata:
+    :param spec_pallet_id:
+    :param spec_pallet_value:
+    :return:
+    """
+
+    spec_pallet_id, spec_pallet_value = spec_pallet
+    logging.debug(f"Pallet: {spec_pallet_id}")
+
+    metadata_pallets = metadata["pallets"]
+    metadata_pallet = metadata_pallets[spec_pallet_id]
+
+    res = map(lambda meta_constant_value: check_constant(
+        spec_pallet_id, spec_pallet_value["constants"], meta_constant_value),
+              metadata_pallet["constants"].values())
+    res = list(filter(lambda item: item is not None, res))
+    return all(res)
+
+
+def check_pallets(metadata, specs):
+    """
+    CHeck all pallets
+
+    :param metadata:
+    :param specs:
+    :return:
+    """
+
+    res = list(map(lambda spec_pallet: check_pallet(metadata, spec_pallet),
+                   specs['pallets'].items()))
+    res = list(filter(lambda item: item is not None, res))
+    return all(res)
+
+
+def check_metadata(metadata, specs):
+    """
+    Check metadata (json) against a list of expectations
+
+    :param metadata: Metadata in JSON format
+    :param expectation: Expectations
+    :return: Bool
+    """
+
+    res = check_pallets(metadata, specs)
+    return res
+
+
+def help():
+    """ Show some simple help """
+
+    print(f"You must pass 2 args, you passed {len(sys.argv) - 1}")
+    print("Sample call:")
+    print("check-runtime.py <metadata.json> <specs.json>")
+
+
+def load_json(file):
+    """ Load json from a file """
+
+    f = open(file)
+    return json.load(f)
+
+
+def main():
+    LOGLEVEL = os.environ.get('LOGLEVEL', 'INFO').upper()
+    logging.basicConfig(level=LOGLEVEL)
+
+    if len(sys.argv) != 3:
+        help()
+        exit(1)
+
+    metadata_file = sys.argv[1]
+    specs_file = sys.argv[2]
+    print(f"Checking metadata from: {metadata_file} with specs from: {specs_file}")
+
+    metadata = load_json(metadata_file)
+    specs = load_json(specs_file)
+
+    res = check_metadata(metadata, specs)
+
+    if res:
+        logging.info(f"OK")
+        exit(0)
+    else:
+        print("")
+        logging.info(f"Some errors were found, run again with LOGLEVEL=debug")
+        exit(1)
+
+if __name__ == "__main__":
+    main()

.github/scripts/common/lib.sh

@@ -202,21 +202,26 @@ fetch_release_artifacts() {
   echo "Release ID : $RELEASE_ID"
   echo "Repo       : $REPO"
   echo "Binary     : $BINARY"
+  OUTPUT_DIR=${OUTPUT_DIR:-"./release-artifacts/${BINARY}"}
+  echo "OUTPUT_DIR : $OUTPUT_DIR"
 
+  echo "Fetching release info..."
   curl -L -s \
     -H "Accept: application/vnd.github+json" \
     -H "Authorization: Bearer ${GITHUB_TOKEN}" \
     -H "X-GitHub-Api-Version: 2022-11-28" \
     https://api.github.com/repos/${REPO}/releases/${RELEASE_ID} > release.json
 
-  # Get Asset ids
+  echo "Extract asset ids..."
   ids=($(jq -r '.assets[].id' < release.json ))
+  echo "Extract asset count..."
   count=$(jq '.assets|length' < release.json )
 
   # Fetch artifacts
-  mkdir -p "./release-artifacts/${BINARY}"
-  pushd "./release-artifacts/${BINARY}" > /dev/null
+  mkdir -p "$OUTPUT_DIR"
+  pushd "$OUTPUT_DIR" > /dev/null
 
+  echo "Fetching assets..."
   iter=1
   for id in "${ids[@]}"
   do

.github/workflows/build-and-attach-release-runtimes.yml

@@ -31,7 +31,7 @@ jobs:
 
     steps:
     - name: Checkout code
-      uses: actions/checkout@v4
+      uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1
 
     - name: Build ${{ matrix.runtime.name }} ${{ matrix.build_config.type }}
       id: srtool_build

.github/workflows/check-publish.yml

@@ -12,7 +12,7 @@ jobs:
   check-publish:
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@8ade135a41bc03ea155e62e844d188df1ea18608 # v4.1.0
+      - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1
 
       - name: Rust Cache
         uses: Swatinem/rust-cache@3cf7f8cc28d1b4e7d01e3783be10a97d55d483c8 # v2.7.1

.github/workflows/check-runtimes.yml

@@ -0,0 +1,94 @@
+name: Check Runtimes Specs
+# This GH Workflow fetches the runtimes available in a release.
+# It then compares their metadata with reference specs located under
+#  .github/runtime_specs.
+
+on:
+  workflow_dispatch:
+    inputs:
+      release_id:
+        description: |
+          Release ID.
+          You can find it using the command:
+          curl -s \
+            -H "Authorization: Bearer ${GITHUB_TOKEN}" https://api.github.com/repos/paritytech/polkadot-sdk/releases | \
+            jq '.[] | { name: .name, id: .id }'
+        required: true
+        type: string
+
+  # This trigger unfortunately does not work as expected.
+  # https://github.com/orgs/community/discussions/47794
+  # release:
+  #   types: [edited]
+
+env:
+  RUNTIME_SPECS_DIR: .github/runtime_specs
+  DATA_DIR: runtimes
+  RELEASE_ID: ${{ inputs.release_id }}
+  REPO: ${{ github.repository }}
+
+jobs:
+  find-specs:
+    name: Fetch runtime specs
+    outputs:
+      specs: ${{ steps.get-list.outputs.specs }}
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout the repo
+        uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1
+
+      - name: Get list
+        id: get-list
+        run: |
+          lst=$(ls $RUNTIME_SPECS_DIR/*.json | xargs -I{} basename "{}" .json | jq -R .| jq -sc .)
+          echo "Found: $lst"
+          echo "specs=$lst" >> $GITHUB_OUTPUT
+
+  check-runtimes:
+    name: Check runtime specs
+    runs-on: ubuntu-latest
+    needs:
+      - find-specs
+
+    strategy:
+      matrix:
+        specs: ${{ fromJSON(needs.find-specs.outputs.specs) }}
+
+    steps:
+      - name: Checkout the repo
+        uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1
+
+      - name: Fetch release artifacts based on release id
+        env:
+          OUTPUT_DIR: ${{ env.DATA_DIR }}
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        run: |
+          . ./.github/scripts/common/lib.sh
+          fetch_release_artifacts
+
+      - name: Install tooling
+        env:
+          SUBWASM_VERSION: v0.20.0
+          DL_BASE_URL: https://github.com/chevdor/subwasm/releases/download
+        run: |
+          wget $DL_BASE_URL/$SUBWASM_VERSION/subwasm_linux_amd64_$SUBWASM_VERSION.deb \
+            -O subwasm.deb
+          sudo dpkg -i subwasm.deb
+          subwasm --version
+
+      - name: Extract metadata JSON for ${{ matrix.specs }}
+        env:
+          RUNTIME: ${{ matrix.specs }}
+        run: |
+          WASM=$(ls ${DATA_DIR}/${RUNTIME}*.wasm)
+          echo "WASM=$WASM"
+          subwasm show --json "$WASM" > "${DATA_DIR}/${RUNTIME}.json"
+
+      - name: Check specs for ${{ matrix.specs }}
+        id: build
+        env:
+          RUNTIME: ${{ matrix.specs }}
+          LOGLEVEL: info
+        run: |
+          python --version
+          .github/scripts/check-runtime.py "${DATA_DIR}/${RUNTIME}.json" "${RUNTIME_SPECS_DIR}/${RUNTIME}.json"