Weekly Cronjob fetching artifacts and generating polkadot.rs file. (#1352)

* github CI action cronjob

* add commit message

* fix the CI yml files

* binary crate for CI script with substrate-runner

* update the CI script

* correct the artifacts script

* remove bash script

.editorconfig

@@ -11,3 +11,6 @@ trim_trailing_whitespace = true
 insert_final_newline = true
 indent_style = space
 indent_size = 4
+
+[*.yml]
+indent_size = 2

.github/workflows/nightly.yml

@@ -5,7 +5,6 @@ on:
     # Run at 8am every day, well after the new binary is built
     - cron: "0 8 * * *"
 
-
 env:
   CARGO_TERM_COLOR: always
 

.github/workflows/update-artifacts.yml

@@ -0,0 +1,56 @@
+name: Update Artifacts
+
+on:
+  workflow_dispatch: # Allows manual triggering
+  schedule:
+    - cron: "0 0 * * 1" # weekly on Monday at 00:00 UTC
+
+concurrency:
+  group: ${{ github.ref }}-${{ github.workflow }}
+  cancel-in-progress: true
+
+env:
+  CARGO_TERM_COLOR: always
+
+jobs:
+  check:
+    name: Renew Artifacts
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout sources
+        uses: actions/checkout@v4
+
+      # We run this (up-to-date) node locally to fetch metadata from it for the artifacts
+      - name: Use substrate-node binary
+        uses: ./.github/workflows/actions/use-substrate
+
+      - name: Install Rust stable toolchain
+        uses: actions-rs/toolchain@v1
+        with:
+          profile: minimal
+          toolchain: stable
+          override: true
+
+      - name: Rust Cache
+        uses: Swatinem/rust-cache@3cf7f8cc28d1b4e7d01e3783be10a97d55d483c8 # v2.7.1
+
+      # This starts a substrate node and runs a few subxt cli child processes to fetch metadata from it and generate code.
+      # In particular it generates:
+      # - 4 metadata (*.scale) files in the `artifacts` directory
+      # - a polkadot.rs file from the full metadata that is checked in integration tests
+      # - a polkadot.json in the `artifacts/demo_chain_specs` directory
+      - name: Fetch Artifacts
+        run: cargo build --bin artifacts
+
+      - name: Create Pull Request
+        uses: peter-evans/create-pull-request@v5
+        with:
+          base: master
+          branch: update-artifacts
+          commit-message: Update Artifacts (auto-generated)
+          branch-suffix: timestamp
+          title: Update Artifacts (auto-generated)
+          body: |
+            This PR updates the artifacts by fetching fresh metadata from a substrate node.
+            It also recreates the polkadot.rs file used in the integration tests.
+            It was created automatically by a Weekly GitHub Action Cronjob.

Cargo.lock

@@ -331,6 +331,13 @@ version = "0.7.4"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "96d30a06541fbafbc7f82ed10c06164cfbd2c401138f6addd8404629c4b16711"
 
+[[package]]
+name = "artifacts"
+version = "0.33.0"
+dependencies = [
+ "substrate-runner",
+]
+
 [[package]]
 name = "assert_matches"
 version = "1.5.0"

Cargo.toml

@@ -12,6 +12,7 @@ members = [
     "metadata",
     "signer",
     "subxt",
+    "scripts/artifacts"
 ]
 
 # We exclude any crates that would depend on non mutually

scripts/artifacts.sh

@@ -1,26 +0,0 @@
-# # Generates the 3 metadata files in `/artifacts` and the polkadot.rs file for integration tests
-#
-# This script is to be run from the root of the repository: `scripts/artifacts.sh`
-#
-# It expects a local polkadot node to be running a JSON-RPC HTTP server at 127.0.0.1:9933
-# A local polkadot node can be run via:
-# ```
-# git clone https://github.com/paritytech/polkadot.git
-# cd polkadot 
-# cargo build --release
-# ./target/release/polkadot --dev --tmp
-# ```
-
-# get the full metadata
-cargo run --bin subxt metadata --version 15 > artifacts/polkadot_metadata_full.scale
-# use it to generate polkadot.rs
-cargo run --bin subxt codegen --file artifacts/polkadot_metadata_full.scale | rustfmt > testing/integration-tests/src/full_client/codegen/polkadot.rs
-# generate a metadata file that only contains a few pallets that we need for our examples.
-cargo run --bin subxt metadata --file artifacts/polkadot_metadata_full.scale --pallets "Balances,Staking,System,Multisig,Timestamp,ParaInherent" > artifacts/polkadot_metadata_small.scale
-# generate a metadata file that contains no pallets
-cargo run --bin subxt metadata --file artifacts/polkadot_metadata_full.scale --pallets "" > artifacts/polkadot_metadata_tiny.scale
-# generate a metadata file that only contains some custom metadata
-cargo run --bin generate-custom-metadata > artifacts/metadata_with_custom_values.scale 
-
-# Generate the polkadot chain spec.
-cargo run --features chain-spec-pruning --bin subxt chain-spec --url wss://rpc.polkadot.io:443 --output-file artifacts/demo_chain_specs/polkadot.json --state-root-hash --remove-substitutes

scripts/artifacts/.gitignore

@@ -0,0 +1 @@
+/target

scripts/artifacts/Cargo.lock

@@ -0,0 +1,14 @@
+# This file is automatically @generated by Cargo.
+# It is not intended for manual editing.
+version = 3
+
+[[package]]
+name = "artifacts"
+version = "0.1.0"
+dependencies = [
+ "substrate-runner",
+]
+
+[[package]]
+name = "substrate-runner"
+version = "0.33.0"

scripts/artifacts/Cargo.toml

@@ -0,0 +1,14 @@
+[package]
+name = "artifacts"
+authors.workspace = true
+edition.workspace = true
+version.workspace = true
+rust-version.workspace = true
+license.workspace = true
+repository.workspace = true
+documentation.workspace = true
+homepage.workspace = true
+description = "Internal tool to regenerate artifacts"
+
+[dependencies]
+substrate-runner = { workspace = true }

scripts/artifacts/src/main.rs

@@ -0,0 +1,89 @@
+use std::{
+    fs::File,
+    process::{Command, Stdio},
+};
+
+use substrate_runner::SubstrateNode;
+
+/// A Script to generate artifacts that are used in the integration tests.
+///
+/// Run with `cargo run --bin artifacts` from the root of the repository.
+fn main() {
+    let mut node_builder = SubstrateNode::builder();
+    node_builder.binary_paths(["substrate-node", "substrate"]);
+
+    // Spawn the node and retrieve a ws URL to it:
+    let proc = node_builder
+        .spawn()
+        .map_err(|e| e.to_string())
+        .expect("Could not spawn node");
+    let node_url = format!("ws://127.0.0.1:{}", proc.ws_port());
+
+    // Get the full metadata from the spawned substrate node
+    Command::make(&format!(
+        "cargo run --bin subxt metadata --version 15 --url {node_url}"
+    ))
+    .out("artifacts/polkadot_metadata_full.scale");
+
+    // Use it to generate polkadot.rs
+    Command::make("cargo run --bin subxt codegen --file artifacts/polkadot_metadata_full.scale")
+        .pipe("rustfmt")
+        .out("testing/integration-tests/src/full_client/codegen/polkadot.rs");
+
+    // Generate a metadata file that only contains a few pallets that we need for our examples.
+    Command::make(r#"cargo run --bin subxt metadata --file artifacts/polkadot_metadata_full.scale --pallets "Balances,Staking,System,Multisig,Timestamp,ParaInherent""#)
+    .out("artifacts/polkadot_metadata_small.scale");
+
+    // Generate a metadata file that contains no pallets
+    Command::make(r#"cargo run --bin subxt metadata --file artifacts/polkadot_metadata_full.scale --pallets """#)
+    .out("artifacts/polkadot_metadata_tiny.scale");
+
+    // Generate a metadata file that only contains some custom metadata
+    Command::make("cargo run --bin generate-custom-metadata")
+        .out("artifacts/metadata_with_custom_values.scale");
+
+    // Generate the polkadot chain spec.
+    Command::make("cargo run --features chain-spec-pruning --bin subxt chain-spec --url wss://rpc.polkadot.io:443 --output-file artifacts/demo_chain_specs/polkadot.json --state-root-hash --remove-substitutes").spawn().unwrap().wait().unwrap();
+}
+
+trait CommandT {
+    /// Creates a new command, parsing the arg_string provided.
+    fn make(arg_string: &str) -> Self;
+
+    /// Pipes the output of the current command to the next command.
+    fn pipe(self, arg_string: &str) -> Self;
+
+    /// Writes bytes from stdout to a new file at path.
+    fn out(self, path: &str);
+}
+
+impl CommandT for Command {
+    fn make(arg_string: &str) -> Self {
+        // Note: simple space splitting, no fancy parsing of e.g. quotes surrounding whitespace.
+        let mut parts = arg_string.split(' ');
+        let program = parts.next().expect("no program in command string");
+        let mut command = Command::new(program);
+        for e in parts {
+            command.arg(e);
+        }
+        command
+    }
+
+    fn pipe(mut self, arg_string: &str) -> Self {
+        // execute self
+        let old_cmd = self.stdout(Stdio::piped()).spawn().unwrap();
+        let mut next_cmd = Self::make(arg_string);
+        next_cmd.stdin(Stdio::from(old_cmd.stdout.unwrap()));
+        next_cmd
+    }
+
+    fn out(mut self, path: &str) {
+        dbg!(path);
+        let file = File::create(path).unwrap();
+        self.stdout(Stdio::from(file))
+            .spawn()
+            .unwrap()
+            .wait()
+            .unwrap();
+    }
+}