Introduce Metadata type (#974)

* WIP new Metadata type

* Finish basic Metadata impl inc hashing and validation

* remove caching from metadata; can add that higher up

* remove caches

* update retain to use Metadata

* clippy fixes

* update codegen to use Metadata

* clippy

* WIP fixing subxt lib

* WIP fixing tests, rebuild artifacts, fix OrderedMap::retain

* get --all-targets compiling

* move DispatchError type lookup back to being optional

* cargo clippy

* fix docs

* re-use VariantIndex to get variants

* add docs and enforce docs on metadata crate

* fix docs

* add test and fix docs

* cargo fmt

* address review comments

* update lockfiles

* ExactSizeIter so we can ask for len() of things (and hopefully soon is_empty()

testing/integration-tests/src/blocks/mod.rs

@@ -3,11 +3,11 @@
 // see LICENSE for license details.
 
 use crate::{pair_signer, test_context, utils::node_runtime};
-use codec::Compact;
-use frame_metadata::RuntimeMetadataPrefixed;
+use codec::{Compact, Encode};
 use futures::StreamExt;
 use sp_keyring::AccountKeyring;
 use subxt::blocks::BlocksClient;
+use subxt_metadata::Metadata;
 
 // Check that we can subscribe to non-finalized blocks.
 #[tokio::test]
@@ -103,21 +103,17 @@ async fn runtime_api_call() -> Result<(), subxt::Error> {
     let block = sub.next().await.unwrap()?;
     let rt = block.runtime_api().await?;
 
-    let (_, meta) = rt
-        .call_raw::<(Compact<u32>, RuntimeMetadataPrefixed)>("Metadata_metadata", None)
+    // get metadata via state_call.
+    let (_, meta1) = rt
+        .call_raw::<(Compact<u32>, Metadata)>("Metadata_metadata", None)
         .await?;
-    let metadata_call = match meta.1 {
-        frame_metadata::RuntimeMetadata::V14(metadata) => {
-            subxt_metadata::metadata_v14_to_latest(metadata)
-        }
-        frame_metadata::RuntimeMetadata::V15(metadata) => metadata,
-        _ => panic!("Metadata V14 or V15 unavailable"),
-    };
 
-    // Compare the runtime API call against the `state_getMetadata`.
-    let metadata = api.rpc().metadata_legacy(None).await?;
-    let metadata = metadata.runtime_metadata();
-    assert_eq!(&metadata_call, metadata);
+    // get metadata via `state_getMetadata`.
+    let meta2 = api.rpc().metadata_legacy(None).await?;
+
+    // They should be the same.
+    assert_eq!(meta1.encode(), meta2.encode());
+
     Ok(())
 }
 

testing/integration-tests/src/client/mod.rs

@@ -8,7 +8,6 @@ use crate::{
 };
 use assert_matches::assert_matches;
 use codec::{Compact, Decode, Encode};
-use frame_metadata::RuntimeMetadataPrefixed;
 use sp_core::storage::well_known_keys;
 use sp_core::{sr25519::Pair as Sr25519Pair, Pair};
 use sp_keyring::AccountKeyring;
@@ -21,6 +20,7 @@ use subxt::{
     tx::Signer,
     utils::AccountId32,
 };
+use subxt_metadata::Metadata;
 
 #[tokio::test]
 async fn insert_key() {
@@ -420,27 +420,23 @@ async fn unsigned_extrinsic_is_same_shape_as_polkadotjs() {
 }
 
 #[tokio::test]
-async fn rpc_state_call() {
+async fn rpc_state_call() -> Result<(), subxt::Error> {
     let ctx = test_context().await;
     let api = ctx.client();
 
-    // Call into the runtime of the chain to get the Metadata.
-    let (_, meta) = api
+    // get metadata via state_call.
+    let (_, meta1) = api
         .rpc()
-        .state_call::<(Compact<u32>, RuntimeMetadataPrefixed)>("Metadata_metadata", None, None)
-        .await
-        .unwrap();
-    let metadata_call = match meta.1 {
-        frame_metadata::RuntimeMetadata::V14(metadata) => {
-            subxt_metadata::metadata_v14_to_latest(metadata)
-        }
-        frame_metadata::RuntimeMetadata::V15(metadata) => metadata,
-        _ => panic!("Metadata V14 or V15 unavailable"),
-    };
-    // Compare the runtime API call against the `state_getMetadata`.
-    let metadata = api.rpc().metadata_legacy(None).await.unwrap();
-    let metadata = metadata.runtime_metadata();
-    assert_eq!(&metadata_call, metadata);
+        .state_call::<(Compact<u32>, Metadata)>("Metadata_metadata", None, None)
+        .await?;
+
+    // get metadata via `state_getMetadata`.
+    let meta2 = api.rpc().metadata_legacy(None).await?;
+
+    // They should be the same.
+    assert_eq!(meta1.encode(), meta2.encode());
+
+    Ok(())
 }
 
 #[tokio::test]

testing/integration-tests/src/codegen/codegen_tests.rs

@@ -11,6 +11,9 @@ use subxt_codegen::{CratePath, DerivesRegistry, RuntimeGenerator, TypeSubstitute
 
 fn generate_runtime_interface_from_metadata(metadata: RuntimeMetadataPrefixed) -> String {
     // Generate a runtime interface from the provided metadata.
+    let metadata = metadata
+        .try_into()
+        .expect("frame_metadata should be convertible into Metadata");
     let generator = RuntimeGenerator::new(metadata);
     let item_mod = syn::parse_quote!(
         pub mod api {}

testing/integration-tests/src/codegen/documentation.rs

@@ -2,49 +2,46 @@
 // This file is dual-licensed as Apache-2.0 or GPL-3.0.
 // see LICENSE for license details.
 
+use codec::Decode;
 use regex::Regex;
 use subxt_codegen::{CratePath, DerivesRegistry, RuntimeGenerator, TypeSubstitutes};
+use subxt_metadata::Metadata;
 
-fn load_test_metadata() -> frame_metadata::RuntimeMetadataPrefixed {
+fn load_test_metadata() -> Metadata {
     let bytes = test_runtime::METADATA;
-    codec::Decode::decode(&mut &*bytes).expect("Cannot decode scale metadata")
+    Metadata::decode(&mut &*bytes).expect("Cannot decode scale metadata")
 }
 
 fn metadata_docs() -> Vec<String> {
     // Load the runtime metadata downloaded from a node via `test-runtime`.
-    let meta = load_test_metadata();
-    let metadata = match meta.1 {
-        frame_metadata::RuntimeMetadata::V14(v14) => subxt_metadata::metadata_v14_to_latest(v14),
-        frame_metadata::RuntimeMetadata::V15(v15) => v15,
-        _ => panic!("Unsupported metadata version {:?}", meta.1),
-    };
+    let metadata = load_test_metadata();
 
     // Inspect the metadata types and collect the documentation.
     let mut docs = Vec::new();
-    for ty in &metadata.types.types {
+    for ty in &metadata.types().types {
         docs.extend_from_slice(&ty.ty.docs);
     }
 
-    for pallet in metadata.pallets {
-        if let Some(storage) = pallet.storage {
-            for entry in storage.entries {
-                docs.extend(entry.docs);
+    for pallet in metadata.pallets() {
+        if let Some(storage) = pallet.storage() {
+            for entry in storage.entries() {
+                docs.extend_from_slice(entry.docs());
             }
         }
         // Note: Calls, Events and Errors are deduced directly to
         // PortableTypes which are handled above.
-        for constant in pallet.constants {
-            docs.extend(constant.docs);
+        for constant in pallet.constants() {
+            docs.extend_from_slice(constant.docs());
         }
     }
     // Note: Extrinsics do not have associated documentation, but is implied by
     // associated Type.
 
     // Inspect the runtime API types and collect the documentation.
-    for api in metadata.apis {
-        docs.extend(api.docs);
-        for method in api.methods {
-            docs.extend(method.docs);
+    for api in metadata.runtime_api_traits() {
+        docs.extend_from_slice(api.docs());
+        for method in api.methods() {
+            docs.extend_from_slice(method.docs());
         }
     }
 

testing/integration-tests/src/frame/balances.rs

@@ -383,9 +383,11 @@ async fn constant_existential_deposit() {
 
     // get and decode constant manually via metadata:
     let metadata = api.metadata();
-    let balances_metadata = metadata.pallet("Balances").unwrap();
-    let constant_metadata = balances_metadata.constant("ExistentialDeposit").unwrap();
-    let existential_deposit = u128::decode(&mut &constant_metadata.value[..]).unwrap();
+    let balances_metadata = metadata.pallet_by_name("Balances").unwrap();
+    let constant_metadata = balances_metadata
+        .constant_by_name("ExistentialDeposit")
+        .unwrap();
+    let existential_deposit = u128::decode(&mut constant_metadata.value()).unwrap();
     assert_eq!(existential_deposit, 100_000_000_000_000);
 
     // constant address for API access:

testing/integration-tests/src/frame/staking.rs

@@ -69,8 +69,8 @@ async fn validate_not_possible_for_stash_account() -> Result<(), Error> {
         .await;
     assert_matches!(announce_validator, Err(Error::Runtime(DispatchError::Module(err))) => {
         let details = err.details().unwrap();
-        assert_eq!(details.pallet(), "Staking");
-        assert_eq!(details.error(), "NotController");
+        assert_eq!(details.pallet.name(), "Staking");
+        assert_eq!(&details.variant.name, "NotController");
     });
     Ok(())
 }
@@ -118,8 +118,8 @@ async fn nominate_not_possible_for_stash_account() -> Result<(), Error> {
 
     assert_matches!(nomination, Err(Error::Runtime(DispatchError::Module(err))) => {
         let details = err.details().unwrap();
-        assert_eq!(details.pallet(), "Staking");
-        assert_eq!(details.error(), "NotController");
+        assert_eq!(details.pallet.name(), "Staking");
+        assert_eq!(&details.variant.name, "NotController");
     });
     Ok(())
 }
@@ -164,8 +164,8 @@ async fn chill_works_for_controller_only() -> Result<(), Error> {
 
     assert_matches!(chill, Err(Error::Runtime(DispatchError::Module(err))) => {
         let details = err.details().unwrap();
-        assert_eq!(details.pallet(), "Staking");
-        assert_eq!(details.error(), "NotController");
+        assert_eq!(details.pallet.name(), "Staking");
+        assert_eq!(&details.variant.name, "NotController");
     });
 
     let is_chilled = api
@@ -211,8 +211,8 @@ async fn tx_bond() -> Result<(), Error> {
 
     assert_matches!(bond_again, Err(Error::Runtime(DispatchError::Module(err))) => {
         let details = err.details().unwrap();
-        assert_eq!(details.pallet(), "Staking");
-        assert_eq!(details.error(), "AlreadyBonded");
+        assert_eq!(details.pallet.name(), "Staking");
+        assert_eq!(&details.variant.name, "AlreadyBonded");
     });
     Ok(())
 }

testing/integration-tests/src/metadata/validation.rs

@@ -3,12 +3,9 @@
 // see LICENSE for license details.
 
 use crate::{node_runtime, test_context, TestContext};
-use frame_metadata::{
-    v15::{
-        ExtrinsicMetadata, PalletCallMetadata, PalletMetadata, PalletStorageMetadata,
-        RuntimeMetadataV15, StorageEntryMetadata, StorageEntryModifier, StorageEntryType,
-    },
-    RuntimeMetadataPrefixed,
+use frame_metadata::v15::{
+    ExtrinsicMetadata, PalletCallMetadata, PalletMetadata, PalletStorageMetadata,
+    RuntimeMetadataV15, StorageEntryMetadata, StorageEntryModifier, StorageEntryType,
 };
 use scale_info::{
     build::{Fields, Variants},
@@ -16,13 +13,7 @@ use scale_info::{
 };
 use subxt::{Metadata, OfflineClient, SubstrateConfig};
 
-async fn metadata_to_api(
-    metadata: RuntimeMetadataV15,
-    ctx: &TestContext,
-) -> OfflineClient<SubstrateConfig> {
-    let prefixed = RuntimeMetadataPrefixed::from(metadata);
-    let metadata = Metadata::try_from(prefixed).unwrap();
-
+async fn metadata_to_api(metadata: Metadata, ctx: &TestContext) -> OfflineClient<SubstrateConfig> {
     OfflineClient::new(
         ctx.client().genesis_hash(),
         ctx.client().runtime_version(),
@@ -30,56 +21,18 @@ async fn metadata_to_api(
     )
 }
 
-#[tokio::test]
-async fn full_metadata_check() {
-    let ctx = test_context().await;
-    let api = ctx.client();
-
-    // Runtime metadata is identical to the metadata used during API generation.
-    assert!(node_runtime::validate_codegen(&api).is_ok());
-
-    // Modify the metadata.
-    let mut metadata = api.metadata().runtime_metadata().clone();
-    metadata.pallets[0].name = "NewPallet".to_string();
-
-    let api = metadata_to_api(metadata, &ctx).await;
-    assert_eq!(
-        node_runtime::validate_codegen(&api)
-            .expect_err("Validation should fail for incompatible metadata"),
-        ::subxt::error::MetadataError::IncompatibleMetadata
-    );
+fn v15_to_metadata(v15: RuntimeMetadataV15) -> Metadata {
+    let subxt_md: subxt_metadata::Metadata = v15.try_into().unwrap();
+    subxt_md.into()
 }
 
-#[tokio::test]
-async fn constant_values_are_not_validated() {
-    let ctx = test_context().await;
-    let api = ctx.client();
-
-    let deposit_addr = node_runtime::constants().balances().existential_deposit();
-
-    // Retrieve existential deposit to validate it and confirm that it's OK.
-    assert!(api.constants().at(&deposit_addr).is_ok());
-
-    // Modify the metadata.
-    let mut metadata = api.metadata().runtime_metadata().clone();
-
-    let mut existential = metadata
-        .pallets
-        .iter_mut()
-        .find(|pallet| pallet.name == "Balances")
-        .expect("Metadata must contain Balances pallet")
-        .constants
-        .iter_mut()
-        .find(|constant| constant.name == "ExistentialDeposit")
-        .expect("ExistentialDeposit constant must be present");
-
-    // Modifying a constant value should not lead to an error:
-    existential.value = vec![0u8; 32];
-
-    let api = metadata_to_api(metadata, &ctx).await;
-
-    assert!(node_runtime::validate_codegen(&api).is_ok());
-    assert!(api.constants().at(&deposit_addr).is_ok());
+fn modified_metadata<F>(metadata: Metadata, f: F) -> Metadata
+where
+    F: FnOnce(&mut RuntimeMetadataV15),
+{
+    let mut metadata = RuntimeMetadataV15::from((*metadata).clone());
+    f(&mut metadata);
+    v15_to_metadata(metadata)
 }
 
 fn default_pallet() -> PalletMetadata {
@@ -95,7 +48,7 @@ fn default_pallet() -> PalletMetadata {
     }
 }
 
-fn pallets_to_metadata(pallets: Vec<PalletMetadata>) -> RuntimeMetadataV15 {
+fn pallets_to_metadata(pallets: Vec<PalletMetadata>) -> Metadata {
     // Extrinsic needs to contain at least the generic type parameter "Call"
     // for the metadata to be valid.
     // The "Call" type from the metadata is used to decode extrinsics.
@@ -120,7 +73,7 @@ fn pallets_to_metadata(pallets: Vec<PalletMetadata>) -> RuntimeMetadataV15 {
         SomeCall,
     }
 
-    RuntimeMetadataV15::new(
+    v15_to_metadata(RuntimeMetadataV15::new(
         pallets,
         ExtrinsicMetadata {
             ty: meta_type::<ExtrinsicType<RuntimeCall>>(),
@@ -129,7 +82,60 @@ fn pallets_to_metadata(pallets: Vec<PalletMetadata>) -> RuntimeMetadataV15 {
         },
         meta_type::<()>(),
         vec![],
-    )
+    ))
+}
+
+#[tokio::test]
+async fn full_metadata_check() {
+    let ctx = test_context().await;
+    let api = ctx.client();
+
+    // Runtime metadata is identical to the metadata used during API generation.
+    assert!(node_runtime::validate_codegen(&api).is_ok());
+
+    // Modify the metadata.
+    let metadata = modified_metadata(api.metadata(), |md| {
+        md.pallets[0].name = "NewPallet".to_string();
+    });
+
+    let api = metadata_to_api(metadata, &ctx).await;
+    assert_eq!(
+        node_runtime::validate_codegen(&api)
+            .expect_err("Validation should fail for incompatible metadata"),
+        ::subxt::error::MetadataError::IncompatibleCodegen
+    );
+}
+
+#[tokio::test]
+async fn constant_values_are_not_validated() {
+    let ctx = test_context().await;
+    let api = ctx.client();
+
+    let deposit_addr = node_runtime::constants().balances().existential_deposit();
+
+    // Retrieve existential deposit to validate it and confirm that it's OK.
+    assert!(api.constants().at(&deposit_addr).is_ok());
+
+    // Modify the metadata.
+    let metadata = modified_metadata(api.metadata(), |md| {
+        let mut existential = md
+            .pallets
+            .iter_mut()
+            .find(|pallet| pallet.name == "Balances")
+            .expect("Metadata must contain Balances pallet")
+            .constants
+            .iter_mut()
+            .find(|constant| constant.name == "ExistentialDeposit")
+            .expect("ExistentialDeposit constant must be present");
+
+        // Modifying a constant value should not lead to an error:
+        existential.value = vec![0u8; 32];
+    });
+
+    let api = metadata_to_api(metadata, &ctx).await;
+
+    assert!(node_runtime::validate_codegen(&api).is_ok());
+    assert!(api.constants().at(&deposit_addr).is_ok());
 }
 
 #[tokio::test]