Put integration tests behind feature flag (#515)

* subxt: Add integration-tests feature flag

Signed-off-by: Alexandru Vasile <alexandru.vasile@parity.io>

* subxt: Guard integration tests under feature flag

Signed-off-by: Alexandru Vasile <alexandru.vasile@parity.io>

* test-runtime: Place build.rs under feature flag

Signed-off-by: Alexandru Vasile <alexandru.vasile@parity.io>

* subxt: Pass `integration-tests` feature to `test-runtime`

Signed-off-by: Alexandru Vasile <alexandru.vasile@parity.io>

* CI: Use `integration-tests` feature to run all tests

Signed-off-by: Alexandru Vasile <alexandru.vasile@parity.io>

* subxt: Rely on `#[cfg(feature = "integration-tests")]` for integration

Signed-off-by: Alexandru Vasile <alexandru.vasile@parity.io>

* subxt/metadata: Manually construct test metadata

Signed-off-by: Alexandru Vasile <alexandru.vasile@parity.io>

* artifacts: Move scale binary blob to dedicated folder

Signed-off-by: Alexandru Vasile <alexandru.vasile@parity.io>

* examples: Update path to metadata blob

Signed-off-by: Alexandru Vasile <alexandru.vasile@parity.io>

* metadata: Rely on artifact metadata blob for benches

Signed-off-by: Alexandru Vasile <alexandru.vasile@parity.io>

* metadata: Remove `test-runtime` dependency

Signed-off-by: Alexandru Vasile <alexandru.vasile@parity.io>

* examples: Modify runtime path for `custom_type_derives`

Signed-off-by: Alexandru Vasile <alexandru.vasile@parity.io>

* subxt: Remove tests folder

Signed-off-by: Alexandru Vasile <alexandru.vasile@parity.io>

* test-runtime: Remove `integration-tests` feature flag

Signed-off-by: Alexandru Vasile <alexandru.vasile@parity.io>

* integration-tests: Add an integration test crate for subxt

Signed-off-by: Alexandru Vasile <alexandru.vasile@parity.io>

* subxt: Remove `test-runtime` dependency

Signed-off-by: Alexandru Vasile <alexandru.vasile@parity.io>

* subxt: Add comment for feature flags

Signed-off-by: Alexandru Vasile <alexandru.vasile@parity.io>

* integration-tests: Trim dependencies

Signed-off-by: Alexandru Vasile <alexandru.vasile@parity.io>

* integration-tests: Move dependencies under dev

Signed-off-by: Alexandru Vasile <alexandru.vasile@parity.io>

* Revert "CI: Use `integration-tests` feature to run all tests"

This reverts commit 8e5f38ba8c633ac40420fadf58700ac402f762d4.

* integration-tests: Remove integration folder

Signed-off-by: Alexandru Vasile <alexandru.vasile@parity.io>

* subxt: Fix feature flag and test comment

Signed-off-by: Alexandru Vasile <alexandru.vasile@parity.io>

* subxt: Extra feature flag comment

Signed-off-by: Alexandru Vasile <alexandru.vasile@parity.io>

* integration-tests: Move tests content under src

Signed-off-by: Alexandru Vasile <alexandru.vasile@parity.io>

Cargo.toml

@@ -4,6 +4,7 @@ members = [
     "cli",
     "codegen",
     "examples",
+    "integration-tests",
     "macro",
     "metadata",
     "subxt",

examples/examples/balance_transfer.rs

@@ -30,7 +30,7 @@ use subxt::{
     PolkadotExtrinsicParams,
 };
 
-#[subxt::subxt(runtime_metadata_path = "examples/polkadot_metadata.scale")]
+#[subxt::subxt(runtime_metadata_path = "../artifacts/polkadot_metadata.scale")]
 pub mod polkadot {}
 
 #[tokio::main]

examples/examples/balance_transfer_with_params.rs

@@ -35,7 +35,7 @@ use subxt::{
     PolkadotExtrinsicParamsBuilder as Params,
 };
 
-#[subxt::subxt(runtime_metadata_path = "examples/polkadot_metadata.scale")]
+#[subxt::subxt(runtime_metadata_path = "../artifacts/polkadot_metadata.scale")]
 pub mod polkadot {}
 
 #[tokio::main]

examples/examples/custom_config.rs

@@ -31,7 +31,7 @@ use subxt::{
     PolkadotExtrinsicParams,
 };
 
-#[subxt::subxt(runtime_metadata_path = "examples/polkadot_metadata.scale")]
+#[subxt::subxt(runtime_metadata_path = "../artifacts/polkadot_metadata.scale")]
 pub mod polkadot {}
 
 /// Custom [`Config`] impl where the default types for the target chain differ from the

examples/examples/custom_type_derives.rs

@@ -19,8 +19,7 @@
 #![allow(clippy::redundant_clone)]
 
 #[subxt::subxt(
-    runtime_metadata_path = "examples/polkadot_metadata.scale",
-
+    runtime_metadata_path = "../artifacts/polkadot_metadata.scale",
     // We can add (certain) custom derives to the generated types by providing
     // a comma separated list to the below attribute. Most useful for adding `Clone`.
     // The derives that we can add ultimately is limited to the traits that the base

examples/examples/fetch_all_accounts.rs

@@ -28,7 +28,7 @@ use subxt::{
     PolkadotExtrinsicParams,
 };
 
-#[subxt::subxt(runtime_metadata_path = "examples/polkadot_metadata.scale")]
+#[subxt::subxt(runtime_metadata_path = "../artifacts/polkadot_metadata.scale")]
 pub mod polkadot {}
 
 #[tokio::main]

examples/examples/fetch_staking_details.rs

@@ -34,7 +34,7 @@ use subxt::{
     PolkadotExtrinsicParams,
 };
 
-#[subxt::subxt(runtime_metadata_path = "examples/polkadot_metadata.scale")]
+#[subxt::subxt(runtime_metadata_path = "../artifacts/polkadot_metadata.scale")]
 pub mod polkadot {}
 
 #[tokio::main]

examples/examples/metadata_compatibility.rs

@@ -28,7 +28,7 @@ use subxt::{
     PolkadotExtrinsicParams,
 };
 
-#[subxt::subxt(runtime_metadata_path = "examples/polkadot_metadata.scale")]
+#[subxt::subxt(runtime_metadata_path = "../artifacts/polkadot_metadata.scale")]
 pub mod polkadot {}
 
 #[tokio::main]

examples/examples/rpc_call.rs

@@ -28,7 +28,7 @@ use subxt::{
     PolkadotExtrinsicParams,
 };
 
-#[subxt::subxt(runtime_metadata_path = "examples/polkadot_metadata.scale")]
+#[subxt::subxt(runtime_metadata_path = "../artifacts/polkadot_metadata.scale")]
 pub mod polkadot {}
 
 #[tokio::main]

examples/examples/submit_and_watch.rs

@@ -31,7 +31,7 @@ use subxt::{
     PolkadotExtrinsicParams,
 };
 
-#[subxt::subxt(runtime_metadata_path = "examples/polkadot_metadata.scale")]
+#[subxt::subxt(runtime_metadata_path = "../artifacts/polkadot_metadata.scale")]
 pub mod polkadot {}
 
 #[tokio::main]

examples/examples/subscribe_all_events.rs

@@ -32,7 +32,7 @@ use subxt::{
     PolkadotExtrinsicParams,
 };
 
-#[subxt::subxt(runtime_metadata_path = "examples/polkadot_metadata.scale")]
+#[subxt::subxt(runtime_metadata_path = "../artifacts/polkadot_metadata.scale")]
 pub mod polkadot {}
 
 /// Subscribe to all events, and then manually look through them and

examples/examples/subscribe_one_event.rs

@@ -32,7 +32,7 @@ use subxt::{
     PolkadotExtrinsicParams,
 };
 
-#[subxt::subxt(runtime_metadata_path = "examples/polkadot_metadata.scale")]
+#[subxt::subxt(runtime_metadata_path = "../artifacts/polkadot_metadata.scale")]
 pub mod polkadot {}
 
 /// Subscribe to all events, and then manually look through them and

examples/examples/subscribe_some_events.rs

@@ -32,7 +32,7 @@ use subxt::{
     PolkadotExtrinsicParams,
 };
 
-#[subxt::subxt(runtime_metadata_path = "examples/polkadot_metadata.scale")]
+#[subxt::subxt(runtime_metadata_path = "../artifacts/polkadot_metadata.scale")]
 pub mod polkadot {}
 
 /// Subscribe to all events, and then manually look through them and

integration-tests/Cargo.toml

@@ -0,0 +1,34 @@
+[package]
+name = "subxt-integration"
+version = "0.20.0"
+authors = ["Parity Technologies <admin@parity.io>"]
+edition = "2021"
+
+license = "GPL-3.0"
+readme = "../README.md"
+repository = "https://github.com/paritytech/subxt"
+documentation = "https://docs.rs/subxt"
+homepage = "https://www.parity.io/"
+description = "Subxt integration tests that rely on the Substrate binary"
+
+[features]
+default = ["subxt/integration-tests"]
+
+[dev-dependencies]
+assert_matches = "1.5.0"
+async-trait = "0.1.49"
+codec = { package = "parity-scale-codec", version = "3.0.0", default-features = false, features = ["derive", "full", "bit-vec"] }
+env_logger = "0.9.0"
+frame-metadata = "15.0.0"
+futures = "0.3.13"
+hex = "0.4.3"
+log = "0.4.14"
+scale-info = { version = "2.0.0", features = ["bit-vec"] }
+sp-core = { version = "6.0.0", default-features = false  }
+sp-keyring = "6.0.0"
+sp-runtime = "6.0.0"
+subxt = { version = "0.20.0", path = "../subxt" }
+test-runtime = { path = "../test-runtime" }
+tokio = { version = "1.8", features = ["macros", "time"] }
+wabt = "0.10.0"
+which = "4.0.2"

integration-tests/src/lib.rs

@@ -14,7 +14,9 @@
 // You should have received a copy of the GNU General Public License
 // along with subxt.  If not, see <http://www.gnu.org/licenses/>.
 
+#[cfg(test)]
 mod codegen;
+#[cfg(test)]
 mod utils;
 
 #[cfg(test)]
@@ -24,10 +26,11 @@ mod events;
 #[cfg(test)]
 mod frame;
 #[cfg(test)]
-#[cfg(integration_tests)]
 mod metadata;
 #[cfg(test)]
 mod storage;
 
+#[cfg(test)]
 use test_runtime::node_runtime;
+#[cfg(test)]
 use utils::*;

integration-tests/src/utils/node_proc.rs

@@ -37,7 +37,6 @@ use subxt::{
 pub struct TestNodeProcess<R: Config> {
     proc: process::Child,
     client: Client<R>,
-    #[cfg(integration_tests)]
     ws_url: String,
 }
 
@@ -79,7 +78,6 @@ where
     }
 
     /// Returns the address to which the client is connected.
-    #[cfg(integration_tests)]
     pub fn ws_url(&self) -> &str {
         &self.ws_url
     }
@@ -149,7 +147,6 @@ impl TestNodeProcessBuilder {
                 Ok(TestNodeProcess {
                     proc,
                     client,
-                    #[cfg(integration_tests)]
                     ws_url,
                 })
             }

metadata/Cargo.toml

@@ -21,7 +21,6 @@ sp-core = { version = "6.0.0"  }
 bitvec = { version = "1.0.0", default-features = false, features = ["alloc"] }
 criterion = "0.3"
 scale-info = { version = "2.0.0", features = ["bit-vec"] }
-test-runtime = { path = "../test-runtime" }
 
 [lib]
 # Without this, libtest cli opts interfere with criteron benches:

metadata/benches/bench.rs

@@ -26,6 +26,10 @@ use scale_info::{
     TypeDef,
     TypeDefVariant,
 };
+use std::{
+    fs,
+    path::Path,
+};
 use subxt_metadata::{
     get_call_hash,
     get_constant_hash,
@@ -35,7 +39,8 @@ use subxt_metadata::{
 };
 
 fn load_metadata() -> RuntimeMetadataV14 {
-    let bytes = test_runtime::METADATA;
+    let bytes = fs::read(Path::new("../artifacts/polkadot_metadata.scale"))
+        .expect("Cannot read metadata blob");
     let meta: RuntimeMetadataPrefixed =
         Decode::decode(&mut &*bytes).expect("Cannot decode scale metadata");
 

subxt/Cargo.toml

@@ -12,6 +12,12 @@ homepage = "https://www.parity.io/"
 description = "Submit extrinsics (transactions) to a substrate node via RPC"
 keywords = ["parity", "substrate", "blockchain"]
 
+[features]
+# Activate this to expose functionality only used for integration testing.
+# The exposed functionality is subject to breaking changes at any point,
+# and should not be relied upon.
+integration-tests = []
+
 [dependencies]
 bitvec = { version = "1.0.0", default-features = false, features = ["alloc"] }
 codec = { package = "parity-scale-codec", version = "3.0.0", default-features = false, features = ["derive", "full", "bit-vec"] }
@@ -42,5 +48,4 @@ env_logger = "0.9.0"
 tempdir = "0.3.7"
 wabt = "0.10.0"
 which = "4.0.2"
-test-runtime = { path = "../test-runtime" }
 sp-keyring = "6.0.0"

subxt/src/client.rs

@@ -88,7 +88,7 @@ impl ClientBuilder {
     /// Set the metadata.
     ///
     /// *Note:* Metadata will no longer be downloaded from the runtime node.
-    #[cfg(integration_tests)]
+    #[cfg(feature = "integration-tests")]
     pub fn set_metadata(mut self, metadata: Metadata) -> Self {
         self.metadata = Some(metadata);
         self

subxt/src/metadata/metadata_type.rs

@@ -478,13 +478,64 @@ impl TryFrom<RuntimeMetadataPrefixed> for Metadata {
 mod tests {
     use super::*;
     use crate::StorageEntryKey;
+    use frame_metadata::{
+        ExtrinsicMetadata,
+        PalletStorageMetadata,
+        StorageEntryModifier,
+        StorageEntryType,
+    };
+    use scale_info::{
+        meta_type,
+        TypeInfo,
+    };
 
     fn load_metadata() -> Metadata {
-        let bytes = test_runtime::METADATA;
-        let meta: RuntimeMetadataPrefixed =
-            codec::Decode::decode(&mut &*bytes).expect("Cannot decode scale metadata");
+        #[allow(dead_code)]
+        #[allow(non_camel_case_types)]
+        #[derive(TypeInfo)]
+        enum Call {
+            fill_block { param: u128 },
+        }
+        let storage = PalletStorageMetadata {
+            prefix: "System",
+            entries: vec![StorageEntryMetadata {
+                name: "Account",
+                modifier: StorageEntryModifier::Optional,
+                ty: StorageEntryType::Plain(meta_type::<u32>()),
+                default: vec![0],
+                docs: vec![],
+            }],
+        };
+        let constant = PalletConstantMetadata {
+            name: "BlockWeights",
+            ty: meta_type::<u32>(),
+            value: vec![1, 2, 3],
+            docs: vec![],
+        };
+        let pallet = frame_metadata::PalletMetadata {
+            index: 0,
+            name: "System",
+            calls: Some(frame_metadata::PalletCallMetadata {
+                ty: meta_type::<Call>(),
+            }),
+            storage: Some(storage),
+            constants: vec![constant],
+            event: None,
+            error: None,
+        };
 
-        Metadata::try_from(meta)
+        let metadata = RuntimeMetadataV14::new(
+            vec![pallet],
+            ExtrinsicMetadata {
+                ty: meta_type::<()>(),
+                version: 0,
+                signed_extensions: vec![],
+            },
+            meta_type::<()>(),
+        );
+        let prefixed = RuntimeMetadataPrefixed::from(metadata);
+
+        Metadata::try_from(prefixed)
             .expect("Cannot translate runtime metadata to internal Metadata")
     }
 
@@ -501,10 +552,10 @@ mod tests {
             hash
         );
 
-        // Currently the caching does not take into account different pallets
-        // as the intended behavior is to use this method only once.
-        // Enforce this behavior into testing.
-        let hash_old = metadata.metadata_hash(&["Balances"]);
+        // The cache `metadata.inner.cached_metadata_hash` is already populated from
+        // the previous call. Therefore, changing the pallets argument must not
+        // change the methods behavior.
+        let hash_old = metadata.metadata_hash(&["no-pallet"]);
         assert_eq!(hash_old, hash);
     }