Ensure metadata is in sync with running node during tests (#333)

* add test-runtime crate to dynamically obtain metadata/node_runtime for tests

* cargo fmt

* Download substrate prior to cargo calls (hopefully)

* add README explaining test-runtime

* Fix CI, fmt and clippy

* more clippy

* tweak test-node readme

* fmt the clippied

* A little tidy up in build.rs

* use ureq and raw string

* Don't export unnecessary metadata

* async_std/RpcClient/bytes instead of ureq/Value/hex

* newline

* document try_from_url error unwrap

.github/workflows/rust.yml

@@ -23,6 +23,13 @@ jobs:
       - name: Checkout sources
         uses: actions/checkout@v2
 
+      - name: Download Substrate
+        run: |
+            curl "https://releases.parity.io/substrate/x86_64-debian:stretch/latest/substrate/substrate" --output substrate --location
+            chmod +x substrate
+            mkdir -p ~/.local/bin
+            mv substrate ~/.local/bin
+
       - name: Install Rust stable toolchain
         uses: actions-rs/toolchain@v1
         with:
@@ -70,6 +77,13 @@ jobs:
       - name: Checkout sources
         uses: actions/checkout@v2
 
+      - name: Download Substrate
+        run: |
+            curl "https://releases.parity.io/substrate/x86_64-debian:stretch/latest/substrate/substrate" --output substrate --location
+            chmod +x substrate
+            mkdir -p ~/.local/bin
+            mv substrate ~/.local/bin
+
       - name: Install Rust stable toolchain
         uses: actions-rs/toolchain@v1
         with:
@@ -93,7 +107,7 @@ jobs:
       - name: Download Substrate
         run: |
             curl "https://releases.parity.io/substrate/x86_64-debian:stretch/latest/substrate/substrate" --output substrate --location
-            chmod +x ./substrate
+            chmod +x substrate
             mkdir -p ~/.local/bin
             mv substrate ~/.local/bin
 
@@ -120,6 +134,13 @@ jobs:
       - name: Checkout sources
         uses: actions/checkout@v2
 
+      - name: Download Substrate
+        run: |
+            curl "https://releases.parity.io/substrate/x86_64-debian:stretch/latest/substrate/substrate" --output substrate --location
+            chmod +x substrate
+            mkdir -p ~/.local/bin
+            mv substrate ~/.local/bin
+
       - name: Install Rust stable toolchain
         uses: actions-rs/toolchain@v1
         with:

Cargo.toml

@@ -1,5 +1,5 @@
 [workspace]
-members = [".", "cli", "codegen", "macro"]
+members = [".", "cli", "codegen", "macro", "test-runtime"]
 
 [package]
 name = "subxt"
@@ -48,6 +48,7 @@ env_logger = "0.8.3"
 tempdir = "0.3.7"
 wabt = "0.10.0"
 which = "4.0.2"
+test-runtime = { path = "test-runtime" }
 
 sp-keyring = { package = "sp-keyring", git = "https://github.com/paritytech/substrate/", branch = "master" }
 

macro/src/lib.rs

@@ -39,7 +39,6 @@ struct GeneratedTypeDerives(Punctuated<syn::Path, syn::Token![,]>);
 pub fn subxt(args: TokenStream, input: TokenStream) -> TokenStream {
     let attr_args = parse_macro_input!(args as syn::AttributeArgs);
     let item_mod = parse_macro_input!(input as syn::ItemMod);
-
     let args = match RuntimeMetadataArgs::from_list(&attr_args) {
         Ok(v) => v,
         Err(e) => return TokenStream::from(e.write_errors()),

test-runtime/Cargo.toml

@@ -0,0 +1,14 @@
+[package]
+name = "test-runtime"
+version = "0.1.0"
+edition = "2021"
+
+[dependencies]
+subxt = { path = ".." }
+sp-runtime = { package = "sp-runtime", git = "https://github.com/paritytech/substrate/", branch = "master" }
+codec = { package = "parity-scale-codec", version = "2", default-features = false, features = ["derive", "full", "bit-vec"] }
+
+[build-dependencies]
+subxt = { path = ".." }
+async-std = { version = "1.9.0", features = ["attributes"] }
+sp-core = { package = "sp-core", git = "https://github.com/paritytech/substrate/", branch = "master" }

test-runtime/README.md

@@ -0,0 +1,10 @@
+# test-runtime
+
+The logic for this crate exists mainly in the `build.rs` file.
+
+At compile time, this crate will:
+- Spin up a local `substrate` binary (set the `SUBSTRATE_NODE_PATH` env var to point to a custom binary, otehrwise it'll look for `substrate` on your PATH).
+- Obtain metadata from this node.
+- Export the metadata and a `node_runtime` module which has been annotated using the `subxt` proc macro and is based off the above metadata.
+
+The reason for doing this is that our integration tests (which also spin up a Substrate node) can then use the generated `subxt` types from the exact node being tested against, so that we don't have to worry about metadata getting out of sync with the binary under test.

test-runtime/build.rs

@@ -0,0 +1,144 @@
+// Copyright 2019-2021 Parity Technologies (UK) Ltd.
+// This file is part of subxt.
+//
+// subxt is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// subxt is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with subxt.  If not, see <http://www.gnu.org/licenses/>.
+
+use std::{
+    env,
+    fs,
+    net::TcpListener,
+    path::Path,
+    process::Command,
+    sync::atomic::{
+        AtomicU16,
+        Ordering,
+    },
+    thread,
+    time,
+};
+
+static SUBSTRATE_BIN_ENV_VAR: &str = "SUBSTRATE_NODE_PATH";
+
+#[async_std::main]
+async fn main() {
+    // Select substrate binary to run based on env var.
+    let substrate_bin =
+        env::var(SUBSTRATE_BIN_ENV_VAR).unwrap_or_else(|_| "substrate".to_owned());
+
+    // Run binary.
+    let port = next_open_port()
+        .expect("Cannot spawn substrate: no available ports in the given port range");
+    let cmd = Command::new(&substrate_bin)
+        .arg("--dev")
+        .arg("--tmp")
+        .arg(format!("--rpc-port={}", port))
+        .spawn();
+    let mut cmd = match cmd {
+        Ok(cmd) => cmd,
+        Err(e) => {
+            panic!("Cannot spawn substrate command '{}': {}", substrate_bin, e)
+        }
+    };
+
+    // Download metadata from binary; retry until successful, or a limit is hit.
+    let metadata_bytes: sp_core::Bytes = {
+        const MAX_RETRIES: usize = 20;
+        let mut retries = 0;
+        let mut wait_secs = 1;
+        loop {
+            if retries >= MAX_RETRIES {
+                panic!("Cannot connect to substrate node after {} retries", retries);
+            }
+            let res =
+                subxt::RpcClient::try_from_url(&format!("http://localhost:{}", port))
+                    .await
+                    .expect("should only error if malformed URL for an HTTP connection")
+                    .request("state_getMetadata", &[])
+                    .await;
+            match res {
+                Ok(res) => {
+                    let _ = cmd.kill();
+                    break res
+                }
+                _ => {
+                    thread::sleep(time::Duration::from_secs(wait_secs));
+                    retries += 1;
+                    wait_secs += 1;
+                }
+            };
+        }
+    };
+
+    // Save metadata to a file:
+    let out_dir = env::var_os("OUT_DIR").unwrap();
+    let metadata_path = Path::new(&out_dir).join("metadata.scale");
+    fs::write(&metadata_path, &metadata_bytes.0).expect("Couldn't write metadata output");
+
+    // Write out our expression to generate the runtime API to a file. Ideally, we'd just write this code
+    // in lib.rs, but we must pass a string literal (and not `concat!(..)`) as an arg to runtime_metadata_path,
+    // and so we need to spit it out here and include it verbatim instead.
+    let runtime_api_contents = format!(
+        r#"
+        #[subxt::subxt(
+            runtime_metadata_path = "{}",
+            generated_type_derives = "Debug, Eq, PartialEq"
+        )]
+        pub mod node_runtime {{
+            #[subxt(substitute_type = "sp_arithmetic::per_things::Perbill")]
+            use sp_runtime::Perbill;
+        }}
+    "#,
+        metadata_path
+            .to_str()
+            .expect("Path to metadata should be stringifiable")
+    );
+    let runtime_path = Path::new(&out_dir).join("runtime.rs");
+    fs::write(&runtime_path, runtime_api_contents)
+        .expect("Couldn't write runtime rust output");
+
+    // Re-build if we point to a different substrate binary:
+    println!("cargo:rerun-if-env-changed={}", SUBSTRATE_BIN_ENV_VAR);
+    // Re-build if this file changes:
+    println!("cargo:rerun-if-changed=build.rs");
+}
+
+/// Returns the next open port, or None if no port found in range.
+fn next_open_port() -> Option<u16> {
+    /// The start of the port range to scan.
+    const START_PORT: u16 = 9900;
+    /// The end of the port range to scan.
+    const END_PORT: u16 = 10000;
+    /// The maximum number of ports to scan before giving up.
+    const MAX_PORTS: u16 = 1000;
+
+    let next_port: AtomicU16 = AtomicU16::new(START_PORT);
+    let mut ports_scanned = 0u16;
+    loop {
+        // Loop back from the beginning if needed
+        let _ = next_port.compare_exchange(
+            END_PORT,
+            START_PORT,
+            Ordering::SeqCst,
+            Ordering::SeqCst,
+        );
+        let next = next_port.fetch_add(1, Ordering::SeqCst);
+        if TcpListener::bind(("0.0.0.0", next)).is_ok() {
+            return Some(next)
+        }
+        ports_scanned += 1;
+        if ports_scanned == MAX_PORTS {
+            return None
+        }
+    }
+}

test-runtime/src/lib.rs

@@ -16,11 +16,7 @@
 
 #![allow(clippy::too_many_arguments)]
 
-#[subxt::subxt(
+/// The SCALE encoded metadata obtained from a local run of a substrate node.
-    runtime_metadata_path = "tests/integration/node_runtime.scale",
+pub static METADATA: &[u8] = include_bytes!(concat!(env!("OUT_DIR"), "/metadata.scale"));
-    generated_type_derives = "Debug, Eq, PartialEq"
+
-)]
+include!(concat!(env!("OUT_DIR"), "/runtime.rs"));
-pub mod node_runtime {
-    #[subxt(substitute_type = "sp_arithmetic::per_things::Perbill")]
-    use sp_runtime::Perbill;
-}

tests/integration/client.rs

@@ -15,9 +15,9 @@
 // along with subxt.  If not, see <http://www.gnu.org/licenses/>.
 
 use crate::{
-    runtime::node_runtime::system,
     test_node_process,
     test_node_process_with,
+    utils::node_runtime::system,
 };
 
 use sp_core::storage::{

tests/integration/main.rs

@@ -15,7 +15,6 @@
 // along with subxt.  If not, see <http://www.gnu.org/licenses/>.
 
 mod codegen;
-mod runtime;
 mod utils;
 
 #[cfg(test)]
@@ -23,5 +22,5 @@ mod client;
 #[cfg(test)]
 mod frame;
 
-pub use runtime::node_runtime;
+pub use test_runtime::node_runtime;
 pub use utils::*;

tests/integration/utils/node_proc.rs

@@ -64,13 +64,9 @@ where
 
     /// Attempt to kill the running substrate process.
     pub fn kill(&mut self) -> Result<(), String> {
-        log::info!("Killing contracts node process {}", self.proc.id());
+        log::info!("Killing node process {}", self.proc.id());
         if let Err(err) = self.proc.kill() {
-            let err = format!(
+            let err = format!("Error killing node process {}: {}", self.proc.id(), err);
-                "Error killing contracts node process {}: {}",
-                self.proc.id(),
-                err
-            );
             log::error!("{}", err);
             return Err(err)
         }