diff --git a/.github/workflows/rust.yml b/.github/workflows/rust.yml index dd587f9a1b..865afb6468 100644 --- a/.github/workflows/rust.yml +++ b/.github/workflows/rust.yml @@ -23,6 +23,13 @@ jobs: - name: Checkout sources uses: actions/checkout@v2 + - name: Download Substrate + run: | + curl "https://releases.parity.io/substrate/x86_64-debian:stretch/latest/substrate/substrate" --output substrate --location + chmod +x substrate + mkdir -p ~/.local/bin + mv substrate ~/.local/bin + - name: Install Rust stable toolchain uses: actions-rs/toolchain@v1 with: @@ -70,6 +77,13 @@ jobs: - name: Checkout sources uses: actions/checkout@v2 + - name: Download Substrate + run: | + curl "https://releases.parity.io/substrate/x86_64-debian:stretch/latest/substrate/substrate" --output substrate --location + chmod +x substrate + mkdir -p ~/.local/bin + mv substrate ~/.local/bin + - name: Install Rust stable toolchain uses: actions-rs/toolchain@v1 with: @@ -93,7 +107,7 @@ jobs: - name: Download Substrate run: | curl "https://releases.parity.io/substrate/x86_64-debian:stretch/latest/substrate/substrate" --output substrate --location - chmod +x ./substrate + chmod +x substrate mkdir -p ~/.local/bin mv substrate ~/.local/bin @@ -120,6 +134,13 @@ jobs: - name: Checkout sources uses: actions/checkout@v2 + - name: Download Substrate + run: | + curl "https://releases.parity.io/substrate/x86_64-debian:stretch/latest/substrate/substrate" --output substrate --location + chmod +x substrate + mkdir -p ~/.local/bin + mv substrate ~/.local/bin + - name: Install Rust stable toolchain uses: actions-rs/toolchain@v1 with: diff --git a/Cargo.toml b/Cargo.toml index e826c58c14..e7141690f1 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,5 +1,5 @@ [workspace] -members = [".", "cli", "codegen", "macro"] +members = [".", "cli", "codegen", "macro", "test-runtime"] [package] name = "subxt" @@ -48,6 +48,7 @@ env_logger = "0.8.3" tempdir = "0.3.7" wabt = "0.10.0" which = "4.0.2" +test-runtime = { path = "test-runtime" } sp-keyring = { package = "sp-keyring", git = "https://github.com/paritytech/substrate/", branch = "master" } diff --git a/macro/src/lib.rs b/macro/src/lib.rs index 41706634cf..17fcf49c48 100644 --- a/macro/src/lib.rs +++ b/macro/src/lib.rs @@ -39,7 +39,6 @@ struct GeneratedTypeDerives(Punctuated); pub fn subxt(args: TokenStream, input: TokenStream) -> TokenStream { let attr_args = parse_macro_input!(args as syn::AttributeArgs); let item_mod = parse_macro_input!(input as syn::ItemMod); - let args = match RuntimeMetadataArgs::from_list(&attr_args) { Ok(v) => v, Err(e) => return TokenStream::from(e.write_errors()), diff --git a/test-runtime/Cargo.toml b/test-runtime/Cargo.toml new file mode 100644 index 0000000000..14707745ae --- /dev/null +++ b/test-runtime/Cargo.toml @@ -0,0 +1,14 @@ +[package] +name = "test-runtime" +version = "0.1.0" +edition = "2021" + +[dependencies] +subxt = { path = ".." } +sp-runtime = { package = "sp-runtime", git = "https://github.com/paritytech/substrate/", branch = "master" } +codec = { package = "parity-scale-codec", version = "2", default-features = false, features = ["derive", "full", "bit-vec"] } + +[build-dependencies] +subxt = { path = ".." } +async-std = { version = "1.9.0", features = ["attributes"] } +sp-core = { package = "sp-core", git = "https://github.com/paritytech/substrate/", branch = "master" } diff --git a/test-runtime/README.md b/test-runtime/README.md new file mode 100644 index 0000000000..9763412df2 --- /dev/null +++ b/test-runtime/README.md @@ -0,0 +1,10 @@ +# test-runtime + +The logic for this crate exists mainly in the `build.rs` file. + +At compile time, this crate will: +- Spin up a local `substrate` binary (set the `SUBSTRATE_NODE_PATH` env var to point to a custom binary, otehrwise it'll look for `substrate` on your PATH). +- Obtain metadata from this node. +- Export the metadata and a `node_runtime` module which has been annotated using the `subxt` proc macro and is based off the above metadata. + +The reason for doing this is that our integration tests (which also spin up a Substrate node) can then use the generated `subxt` types from the exact node being tested against, so that we don't have to worry about metadata getting out of sync with the binary under test. \ No newline at end of file diff --git a/test-runtime/build.rs b/test-runtime/build.rs new file mode 100644 index 0000000000..0251219bc6 --- /dev/null +++ b/test-runtime/build.rs @@ -0,0 +1,144 @@ +// Copyright 2019-2021 Parity Technologies (UK) Ltd. +// This file is part of subxt. +// +// subxt is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// subxt is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU General Public License +// along with subxt. If not, see . + +use std::{ + env, + fs, + net::TcpListener, + path::Path, + process::Command, + sync::atomic::{ + AtomicU16, + Ordering, + }, + thread, + time, +}; + +static SUBSTRATE_BIN_ENV_VAR: &str = "SUBSTRATE_NODE_PATH"; + +#[async_std::main] +async fn main() { + // Select substrate binary to run based on env var. + let substrate_bin = + env::var(SUBSTRATE_BIN_ENV_VAR).unwrap_or_else(|_| "substrate".to_owned()); + + // Run binary. + let port = next_open_port() + .expect("Cannot spawn substrate: no available ports in the given port range"); + let cmd = Command::new(&substrate_bin) + .arg("--dev") + .arg("--tmp") + .arg(format!("--rpc-port={}", port)) + .spawn(); + let mut cmd = match cmd { + Ok(cmd) => cmd, + Err(e) => { + panic!("Cannot spawn substrate command '{}': {}", substrate_bin, e) + } + }; + + // Download metadata from binary; retry until successful, or a limit is hit. + let metadata_bytes: sp_core::Bytes = { + const MAX_RETRIES: usize = 20; + let mut retries = 0; + let mut wait_secs = 1; + loop { + if retries >= MAX_RETRIES { + panic!("Cannot connect to substrate node after {} retries", retries); + } + let res = + subxt::RpcClient::try_from_url(&format!("http://localhost:{}", port)) + .await + .expect("should only error if malformed URL for an HTTP connection") + .request("state_getMetadata", &[]) + .await; + match res { + Ok(res) => { + let _ = cmd.kill(); + break res + } + _ => { + thread::sleep(time::Duration::from_secs(wait_secs)); + retries += 1; + wait_secs += 1; + } + }; + } + }; + + // Save metadata to a file: + let out_dir = env::var_os("OUT_DIR").unwrap(); + let metadata_path = Path::new(&out_dir).join("metadata.scale"); + fs::write(&metadata_path, &metadata_bytes.0).expect("Couldn't write metadata output"); + + // Write out our expression to generate the runtime API to a file. Ideally, we'd just write this code + // in lib.rs, but we must pass a string literal (and not `concat!(..)`) as an arg to runtime_metadata_path, + // and so we need to spit it out here and include it verbatim instead. + let runtime_api_contents = format!( + r#" + #[subxt::subxt( + runtime_metadata_path = "{}", + generated_type_derives = "Debug, Eq, PartialEq" + )] + pub mod node_runtime {{ + #[subxt(substitute_type = "sp_arithmetic::per_things::Perbill")] + use sp_runtime::Perbill; + }} + "#, + metadata_path + .to_str() + .expect("Path to metadata should be stringifiable") + ); + let runtime_path = Path::new(&out_dir).join("runtime.rs"); + fs::write(&runtime_path, runtime_api_contents) + .expect("Couldn't write runtime rust output"); + + // Re-build if we point to a different substrate binary: + println!("cargo:rerun-if-env-changed={}", SUBSTRATE_BIN_ENV_VAR); + // Re-build if this file changes: + println!("cargo:rerun-if-changed=build.rs"); +} + +/// Returns the next open port, or None if no port found in range. +fn next_open_port() -> Option { + /// The start of the port range to scan. + const START_PORT: u16 = 9900; + /// The end of the port range to scan. + const END_PORT: u16 = 10000; + /// The maximum number of ports to scan before giving up. + const MAX_PORTS: u16 = 1000; + + let next_port: AtomicU16 = AtomicU16::new(START_PORT); + let mut ports_scanned = 0u16; + loop { + // Loop back from the beginning if needed + let _ = next_port.compare_exchange( + END_PORT, + START_PORT, + Ordering::SeqCst, + Ordering::SeqCst, + ); + let next = next_port.fetch_add(1, Ordering::SeqCst); + if TcpListener::bind(("0.0.0.0", next)).is_ok() { + return Some(next) + } + ports_scanned += 1; + if ports_scanned == MAX_PORTS { + return None + } + } +} diff --git a/tests/integration/runtime.rs b/test-runtime/src/lib.rs similarity index 74% rename from tests/integration/runtime.rs rename to test-runtime/src/lib.rs index 8cb0651e06..68b389dc2e 100644 --- a/tests/integration/runtime.rs +++ b/test-runtime/src/lib.rs @@ -16,11 +16,7 @@ #![allow(clippy::too_many_arguments)] -#[subxt::subxt( - runtime_metadata_path = "tests/integration/node_runtime.scale", - generated_type_derives = "Debug, Eq, PartialEq" -)] -pub mod node_runtime { - #[subxt(substitute_type = "sp_arithmetic::per_things::Perbill")] - use sp_runtime::Perbill; -} +/// The SCALE encoded metadata obtained from a local run of a substrate node. +pub static METADATA: &[u8] = include_bytes!(concat!(env!("OUT_DIR"), "/metadata.scale")); + +include!(concat!(env!("OUT_DIR"), "/runtime.rs")); diff --git a/tests/integration/client.rs b/tests/integration/client.rs index c6aa506d6c..a583d3586b 100644 --- a/tests/integration/client.rs +++ b/tests/integration/client.rs @@ -15,9 +15,9 @@ // along with subxt. If not, see . use crate::{ - runtime::node_runtime::system, test_node_process, test_node_process_with, + utils::node_runtime::system, }; use sp_core::storage::{ diff --git a/tests/integration/main.rs b/tests/integration/main.rs index 40ee6f31b7..afe6e603f2 100644 --- a/tests/integration/main.rs +++ b/tests/integration/main.rs @@ -15,7 +15,6 @@ // along with subxt. If not, see . mod codegen; -mod runtime; mod utils; #[cfg(test)] @@ -23,5 +22,5 @@ mod client; #[cfg(test)] mod frame; -pub use runtime::node_runtime; +pub use test_runtime::node_runtime; pub use utils::*; diff --git a/tests/integration/node_runtime.scale b/tests/integration/node_runtime.scale deleted file mode 100644 index 2ab10bd2a8..0000000000 Binary files a/tests/integration/node_runtime.scale and /dev/null differ diff --git a/tests/integration/utils/node_proc.rs b/tests/integration/utils/node_proc.rs index a63f023055..caeb74e9da 100644 --- a/tests/integration/utils/node_proc.rs +++ b/tests/integration/utils/node_proc.rs @@ -64,13 +64,9 @@ where /// Attempt to kill the running substrate process. pub fn kill(&mut self) -> Result<(), String> { - log::info!("Killing contracts node process {}", self.proc.id()); + log::info!("Killing node process {}", self.proc.id()); if let Err(err) = self.proc.kill() { - let err = format!( - "Error killing contracts node process {}: {}", - self.proc.id(), - err - ); + let err = format!("Error killing node process {}: {}", self.proc.id(), err); log::error!("{}", err); return Err(err) }