Factor substrate node runner into separate crate (#913)

* factor out node starting thing to separate crate to use in test-runtime and integration-tests

* remove subxt dep on test-runtime build, and clippy tidyup

* remove now-unneeded dep

* Update testing/substrate-runner/Cargo.toml

Co-authored-by: Alexandru Vasile <60601340+lexnv@users.noreply.github.com>

* Update testing/integration-tests/Cargo.toml

Co-authored-by: Niklas Adolfsson <niklasadolfsson1@gmail.com>

* remove unneeded port things for backward compat

---------

Co-authored-by: Alexandru Vasile <60601340+lexnv@users.noreply.github.com>
Co-authored-by: Niklas Adolfsson <niklasadolfsson1@gmail.com>

testing/test-runtime/Cargo.toml

@@ -6,11 +6,11 @@ publish = false
 
 [dependencies]
 subxt = { path = "../../subxt" }
-sp-runtime = "23.0.0"
-codec = { package = "parity-scale-codec", version = "3.0.0", default-features = false, features = ["derive", "full", "bit-vec"] }
 
 [build-dependencies]
-subxt = { path = "../../subxt" }
+substrate-runner = { path = "../substrate-runner" }
+impl-serde = "0.4.0"
+serde = "1.0.160"
 tokio = { version = "1.27", features = ["macros", "rt-multi-thread"] }
 which = "4.4.0"
 jsonrpsee = { version = "0.16.0", features = ["async-client", "client-ws-transport"] }

testing/test-runtime/build.rs

@@ -2,14 +2,8 @@
 // This file is dual-licensed as Apache-2.0 or GPL-3.0.
 // see LICENSE for license details.
 
-use std::{
-    env, fs,
-    net::TcpListener,
-    ops::{Deref, DerefMut},
-    path::Path,
-    process::Command,
-    thread, time,
-};
+use std::{env, fs, path::Path};
+use substrate_runner::{Error as SubstrateNodeError, SubstrateNode};
 
 static SUBSTRATE_BIN_ENV_VAR: &str = "SUBSTRATE_NODE_PATH";
 
@@ -22,19 +16,15 @@ async fn run() {
     // Select substrate binary to run based on env var.
     let substrate_bin = env::var(SUBSTRATE_BIN_ENV_VAR).unwrap_or_else(|_| "substrate".to_owned());
 
-    // Run binary.
-    let port = next_open_port().expect("Cannot spawn substrate: no available ports");
-    let cmd = Command::new(&substrate_bin)
-        .arg("--dev")
-        .arg("--tmp")
-        .arg(format!("--ws-port={port}"))
-        .spawn();
-    let mut cmd = match cmd {
-        Ok(cmd) => KillOnDrop(cmd),
-        Err(ref e) if e.kind() == std::io::ErrorKind::NotFound => {
+    let mut node_builder = SubstrateNode::builder();
+    node_builder.binary_path(substrate_bin.clone());
+
+    let node = match node_builder.spawn() {
+        Ok(node) => node,
+        Err(SubstrateNodeError::Io(e)) if e.kind() == std::io::ErrorKind::NotFound => {
             panic!(
                 "A substrate binary should be installed on your path for testing purposes. \
-            See https://github.com/paritytech/subxt/tree/master#integration-testing"
+                 See https://github.com/paritytech/subxt/tree/master#integration-testing"
             )
         }
         Err(e) => {
@@ -42,35 +32,20 @@ async fn run() {
         }
     };
 
-    // Download metadata from binary; retry until successful, or a limit is hit.
-    let metadata_bytes: subxt::rpc::types::Bytes = {
-        const MAX_RETRIES: usize = 6;
-        let mut retries = 0;
+    let port = node.ws_port();
 
-        loop {
-            if retries >= MAX_RETRIES {
-                panic!("Cannot connect to substrate node after {retries} retries");
-            }
-
-            // It might take a while for substrate node that spin up the RPC server.
-            // Thus, the connection might get rejected a few times.
-            use client::ClientT;
-            let res = match client::build(&format!("ws://localhost:{port}")).await {
-                Ok(c) => c.request("state_getMetadata", client::rpc_params![]).await,
-                Err(e) => Err(e),
-            };
-
-            match res {
-                Ok(res) => {
-                    let _ = cmd.kill();
-                    break res;
-                }
-                _ => {
-                    thread::sleep(time::Duration::from_secs(1 << retries));
-                    retries += 1;
-                }
-            };
-        }
+    // Download metadata from binary. Avoid Subxt dep on `subxt::rpc::types::Bytes`and just impl here.
+    // This may at least prevent this script from running so often (ie whenever we change Subxt).
+    #[derive(serde::Deserialize)]
+    pub struct Bytes(#[serde(with = "impl_serde::serialize")] pub Vec<u8>);
+    let metadata_bytes: Bytes = {
+        use client::ClientT;
+        client::build(&format!("ws://localhost:{port}"))
+            .await
+            .unwrap_or_else(|e| panic!("Failed to connect to node: {e}"))
+            .request("state_getMetadata", client::rpc_params![])
+            .await
+            .unwrap_or_else(|e| panic!("Failed to obtain metadata from node: {e}"))
     };
 
     // Save metadata to a file:
@@ -110,43 +85,6 @@ async fn run() {
     println!("cargo:rerun-if-changed=build.rs");
 }
 
-/// Returns the next open port, or None if no port found.
-fn next_open_port() -> Option<u16> {
-    match TcpListener::bind(("127.0.0.1", 0)) {
-        Ok(listener) => {
-            if let Ok(address) = listener.local_addr() {
-                Some(address.port())
-            } else {
-                None
-            }
-        }
-        Err(_) => None,
-    }
-}
-
-/// If the substrate process isn't explicitly killed on drop,
-/// it seems that panics that occur while the command is running
-/// will leave it running and block the build step from ever finishing.
-/// Wrapping it in this prevents this from happening.
-struct KillOnDrop(std::process::Child);
-
-impl Deref for KillOnDrop {
-    type Target = std::process::Child;
-    fn deref(&self) -> &Self::Target {
-        &self.0
-    }
-}
-impl DerefMut for KillOnDrop {
-    fn deref_mut(&mut self) -> &mut Self::Target {
-        &mut self.0
-    }
-}
-impl Drop for KillOnDrop {
-    fn drop(&mut self) {
-        let _ = self.0.kill();
-    }
-}
-
 // Use jsonrpsee to obtain metadata from the node.
 mod client {
     pub use jsonrpsee::{