Introduce Metadata type (#974)

* WIP new Metadata type

* Finish basic Metadata impl inc hashing and validation

* remove caching from metadata; can add that higher up

* remove caches

* update retain to use Metadata

* clippy fixes

* update codegen to use Metadata

* clippy

* WIP fixing subxt lib

* WIP fixing tests, rebuild artifacts, fix OrderedMap::retain

* get --all-targets compiling

* move DispatchError type lookup back to being optional

* cargo clippy

* fix docs

* re-use VariantIndex to get variants

* add docs and enforce docs on metadata crate

* fix docs

* add test and fix docs

* cargo fmt

* address review comments

* update lockfiles

* ExactSizeIter so we can ask for len() of things (and hopefully soon is_empty()

cli/src/commands/compatibility.rs

@@ -4,15 +4,12 @@
 
 use clap::Parser as ClapParser;
 use codec::Decode;
-use color_eyre::eyre::{self, WrapErr};
-use frame_metadata::{
-    v15::RuntimeMetadataV15, RuntimeMetadata, RuntimeMetadataPrefixed, META_RESERVED,
-};
+use color_eyre::eyre::WrapErr;
 use jsonrpsee::client_transport::ws::Uri;
 use serde::{Deserialize, Serialize};
 use std::collections::HashMap;
 use subxt_codegen::utils::MetadataVersion;
-use subxt_metadata::{get_pallet_hash, metadata_v14_to_latest, MetadataHasher};
+use subxt_metadata::Metadata;
 
 /// Verify metadata compatibility between substrate nodes.
 #[derive(Debug, ClapParser)]
@@ -66,9 +63,9 @@ async fn handle_pallet_metadata(
     for node in nodes.iter() {
         let metadata = fetch_runtime_metadata(node, version).await?;
 
-        match metadata.pallets.iter().find(|pallet| pallet.name == name) {
+        match metadata.pallet_by_name(name) {
             Some(pallet_metadata) => {
-                let hash = get_pallet_hash(&metadata.types, pallet_metadata);
+                let hash = pallet_metadata.hash();
                 let hex_hash = hex::encode(hash);
                 println!("Node {node:?} has pallet metadata hash {hex_hash:?}");
 
@@ -97,7 +94,7 @@ async fn handle_full_metadata(nodes: &[Uri], version: MetadataVersion) -> color_
     let mut compatibility_map: HashMap<String, Vec<String>> = HashMap::new();
     for node in nodes.iter() {
         let metadata = fetch_runtime_metadata(node, version).await?;
-        let hash = MetadataHasher::new().hash(&metadata);
+        let hash = metadata.hasher().hash();
         let hex_hash = hex::encode(hash);
         println!("Node {node:?} has metadata hash {hex_hash:?}",);
 
@@ -119,26 +116,8 @@ async fn handle_full_metadata(nodes: &[Uri], version: MetadataVersion) -> color_
 async fn fetch_runtime_metadata(
     url: &Uri,
     version: MetadataVersion,
-) -> color_eyre::Result<RuntimeMetadataV15> {
+) -> color_eyre::Result<Metadata> {
     let bytes = subxt_codegen::utils::fetch_metadata_bytes(url, version).await?;
-
-    let metadata = <RuntimeMetadataPrefixed as Decode>::decode(&mut &bytes[..])?;
-    if metadata.0 != META_RESERVED {
-        return Err(eyre::eyre!(
-            "Node {:?} has invalid metadata prefix: {:?} expected prefix: {:?}",
-            url,
-            metadata.0,
-            META_RESERVED
-        ));
-    }
-
-    match metadata.1 {
-        RuntimeMetadata::V14(v14) => Ok(metadata_v14_to_latest(v14)),
-        RuntimeMetadata::V15(v15) => Ok(v15),
-        _ => Err(eyre::eyre!(
-            "Node {:?} with unsupported metadata version: {:?}",
-            url,
-            metadata.1
-        )),
-    }
+    let metadata = Metadata::decode(&mut &bytes[..])?;
+    Ok(metadata)
 }

cli/src/commands/explore/calls.rs

@@ -1,22 +1,23 @@
-use crate::utils::type_description::print_type_description;
-use crate::utils::type_example::print_type_examples;
-use crate::utils::with_indent;
 use clap::Args;
-
+use color_eyre::eyre::eyre;
+use scale_info::form::PortableForm;
+use scale_info::{PortableRegistry, Type, TypeDef, TypeDefVariant};
+use scale_value::{Composite, ValueDef};
 use std::fmt::Write;
 use std::str::FromStr;
 use std::write;
 
-use color_eyre::eyre::eyre;
-use frame_metadata::v15::PalletMetadata;
-
-use scale_info::form::PortableForm;
-use scale_info::{PortableRegistry, Type, TypeDef, TypeDefVariant};
-use scale_value::{Composite, ValueDef};
-
 use subxt::tx;
 use subxt::utils::H256;
-use subxt::{config::SubstrateConfig, Metadata, OfflineClient};
+use subxt::{
+    config::SubstrateConfig,
+    metadata::{types::PalletMetadata, Metadata},
+    OfflineClient,
+};
+
+use crate::utils::type_description::print_type_description;
+use crate::utils::type_example::print_type_examples;
+use crate::utils::with_indent;
 
 #[derive(Debug, Clone, Args)]
 pub struct CallsSubcommand {
@@ -28,13 +29,13 @@ pub struct CallsSubcommand {
 pub(crate) fn explore_calls(
     command: CallsSubcommand,
     metadata: &Metadata,
-    pallet_metadata: &PalletMetadata<PortableForm>,
+    pallet_metadata: PalletMetadata,
 ) -> color_eyre::Result<()> {
-    let pallet_name = pallet_metadata.name.as_str();
+    let pallet_name = pallet_metadata.name();
 
     // get the enum that stores the possible calls:
     let (calls_enum_type_def, _calls_enum_type) =
-        get_calls_enum_type(pallet_metadata, &metadata.runtime_metadata().types)?;
+        get_calls_enum_type(pallet_metadata, metadata.types())?;
 
     // if no call specified, show user the calls to choose from:
     let Some(call_name) = command.call else {
@@ -55,14 +56,9 @@ pub(crate) fn explore_calls(
 
     // if no trailing arguments specified show user the expected type of arguments with examples:
     if trailing_args.is_empty() {
-        let mut type_description =
-            print_type_description(&call.fields, &metadata.runtime_metadata().types)?;
+        let mut type_description = print_type_description(&call.fields, metadata.types())?;
         type_description = with_indent(type_description, 4);
-        let mut type_examples = print_type_examples(
-            &call.fields,
-            &metadata.runtime_metadata().types,
-            "SCALE_VALUE",
-        )?;
+        let mut type_examples = print_type_examples(&call.fields, metadata.types(), "SCALE_VALUE")?;
         type_examples = with_indent(type_examples, 4);
         let mut output = String::new();
         write!(output, "Usage:\n    subxt explore {pallet_name} calls {call_name} <SCALE_VALUE>\n        construct the call by providing a valid argument\n\n")?;
@@ -102,22 +98,19 @@ fn print_available_calls(pallet_calls: &TypeDefVariant<PortableForm>, pallet_nam
 }
 
 fn get_calls_enum_type<'a>(
-    pallet: &'a frame_metadata::v15::PalletMetadata<PortableForm>,
+    pallet: PalletMetadata,
     registry: &'a PortableRegistry,
 ) -> color_eyre::Result<(&'a TypeDefVariant<PortableForm>, &'a Type<PortableForm>)> {
-    let calls = pallet
-        .calls
-        .as_ref()
-        .ok_or(eyre!("The \"{}\" pallet has no calls.", pallet.name))?;
+    let call_ty = pallet
+        .call_ty_id()
+        .ok_or(eyre!("The \"{}\" pallet has no calls.", pallet.name()))?;
     let calls_enum_type = registry
-        .resolve(calls.ty.id)
-        .ok_or(eyre!("calls type with id {} not found.", calls.ty.id))?;
+        .resolve(call_ty)
+        .ok_or(eyre!("calls type with id {} not found.", call_ty))?;
+
     // should always be a variant type, where each variant corresponds to one call.
-    let calls_enum_type_def = match &calls_enum_type.type_def {
-        TypeDef::Variant(variant) => variant,
-        _ => {
-            return Err(eyre!("calls type is not a variant"));
-        }
+    let TypeDef::Variant(calls_enum_type_def) = &calls_enum_type.type_def else {
+        return Err(eyre!("calls type is not a variant"));
     };
     Ok((calls_enum_type_def, calls_enum_type))
 }

cli/src/commands/explore/constants.rs

@@ -1,16 +1,11 @@
-use crate::utils::type_description::print_type_description;
-use crate::utils::{print_docs_with_indent, with_indent};
 use clap::Args;
-
+use color_eyre::eyre::eyre;
 use std::fmt::Write;
 use std::write;
+use subxt::metadata::{types::PalletMetadata, Metadata};
 
-use color_eyre::eyre::eyre;
-use frame_metadata::v15::PalletMetadata;
-
-use scale_info::form::PortableForm;
-
-use subxt::Metadata;
+use crate::utils::type_description::print_type_description;
+use crate::utils::{print_docs_with_indent, with_indent};
 
 #[derive(Debug, Clone, Args)]
 pub struct ConstantsSubcommand {
@@ -20,9 +15,9 @@ pub struct ConstantsSubcommand {
 pub(crate) fn explore_constants(
     command: ConstantsSubcommand,
     metadata: &Metadata,
-    pallet_metadata: &PalletMetadata<PortableForm>,
+    pallet_metadata: PalletMetadata,
 ) -> color_eyre::Result<()> {
-    let pallet_name = pallet_metadata.name.as_str();
+    let pallet_name = pallet_metadata.name();
     let Some(constant_name) = command.constant else {
         let available_constants = print_available_constants(pallet_metadata, pallet_name);
         println!("Usage:\n    subxt explore {pallet_name} constants <CONSTANT>\n        explore a specific call within this pallet\n\n{available_constants}", );
@@ -30,7 +25,7 @@ pub(crate) fn explore_constants(
     };
 
     // if specified constant is wrong, show user the constants to choose from (but this time as an error):
-    let Some(constant) = pallet_metadata.constants.iter().find(|constant| constant.name.to_lowercase() == constant_name.to_lowercase())   else {
+    let Some(constant) = pallet_metadata.constants().find(|constant| constant.name().to_lowercase() == constant_name.to_lowercase()) else {
         let available_constants = print_available_constants(pallet_metadata, pallet_name);
         let description = format!("Usage:\n    subxt explore {pallet_name} constants <CONSTANT>\n        explore a specific call within this pallet\n\n{available_constants}", );
         let err = eyre!("constant \"{constant_name}\" not found in \"{pallet_name}\" pallet!\n\n{description}");
@@ -39,13 +34,13 @@ pub(crate) fn explore_constants(
 
     // docs
     let mut output = String::new();
-    let doc_string = print_docs_with_indent(&constant.docs, 4);
+    let doc_string = print_docs_with_indent(constant.docs(), 4);
     if !doc_string.is_empty() {
         write!(output, "Description:\n{doc_string}")?;
     }
 
     // shape
-    let mut type_description = print_type_description(&constant.ty.id, metadata.types())?;
+    let mut type_description = print_type_description(&constant.ty(), metadata.types())?;
     type_description = with_indent(type_description, 4);
     write!(
         output,
@@ -53,11 +48,8 @@ pub(crate) fn explore_constants(
     )?;
 
     // value
-    let scale_val = scale_value::scale::decode_as_type(
-        &mut &constant.value[..],
-        constant.ty.id,
-        metadata.types(),
-    )?;
+    let scale_val =
+        scale_value::scale::decode_as_type(&mut constant.value(), constant.ty(), metadata.types())?;
     write!(
         output,
         "\n\nThe value of the constant is:\n    {}",
@@ -68,15 +60,12 @@ pub(crate) fn explore_constants(
     Ok(())
 }
 
-fn print_available_constants(
-    pallet_metadata: &PalletMetadata<PortableForm>,
-    pallet_name: &str,
-) -> String {
-    if pallet_metadata.constants.is_empty() {
+fn print_available_constants(pallet_metadata: PalletMetadata, pallet_name: &str) -> String {
+    if pallet_metadata.constants().len() == 0 {
         return format!("No <CONSTANT>'s available in the \"{pallet_name}\" pallet.");
     }
     let mut output = format!("Available <CONSTANT>'s in the \"{pallet_name}\" pallet:");
-    let mut strings: Vec<_> = pallet_metadata.constants.iter().map(|c| &c.name).collect();
+    let mut strings: Vec<_> = pallet_metadata.constants().map(|c| c.name()).collect();
     strings.sort();
     for constant in strings {
         output.push_str("\n    ");

cli/src/commands/explore/mod.rs

@@ -7,10 +7,6 @@ use std::write;
 
 use codec::Decode;
 use color_eyre::eyre::eyre;
-use frame_metadata::v15::RuntimeMetadataV15;
-use frame_metadata::RuntimeMetadataPrefixed;
-
-use syn::__private::str;
 
 use crate::commands::explore::calls::{explore_calls, CallsSubcommand};
 use crate::commands::explore::constants::{explore_constants, ConstantsSubcommand};
@@ -89,24 +85,23 @@ pub enum PalletSubcommand {
 pub async fn run(opts: Opts) -> color_eyre::Result<()> {
     // get the metadata
     let bytes = opts.file_or_url.fetch().await?;
-    let metadata_prefixed = <RuntimeMetadataPrefixed as Decode>::decode(&mut &bytes[..])?;
-    let metadata = Metadata::try_from(metadata_prefixed)?;
+    let metadata = Metadata::decode(&mut &bytes[..])?;
 
     // if no pallet specified, show user the pallets to choose from:
     let Some(pallet_name) = opts.pallet else {
-        let available_pallets = print_available_pallets(metadata.runtime_metadata());
+        let available_pallets = print_available_pallets(&metadata);
         println!("Usage:\n    subxt explore <PALLET>\n        explore a specific pallet\n\n{available_pallets}", );
         return Ok(());
     };
 
     // if specified pallet is wrong, show user the pallets to choose from (but this time as an error):
-    let Some(pallet_metadata) = metadata.runtime_metadata().pallets.iter().find(|pallet| pallet.name.to_lowercase() == pallet_name.to_lowercase())else {
-        return Err(eyre!("pallet \"{}\" not found in metadata!\n{}", pallet_name, print_available_pallets(metadata.runtime_metadata())));
+    let Some(pallet_metadata) = metadata.pallets().find(|pallet| pallet.name().to_lowercase() == pallet_name.to_lowercase()) else {
+        return Err(eyre!("pallet \"{}\" not found in metadata!\n{}", pallet_name, print_available_pallets(&metadata)));
     };
 
     // if correct pallet was specified but no subcommand, instruct the user how to proceed:
     let Some(pallet_subcomand) = opts.pallet_subcommand else {
-        let docs_string = print_docs_with_indent(&pallet_metadata.docs, 4);
+        let docs_string = print_docs_with_indent(pallet_metadata.docs(), 4);
         let mut output = String::new();
         if !docs_string.is_empty() {
             write!(output, "Description:\n{docs_string}")?;
@@ -132,12 +127,12 @@ pub async fn run(opts: Opts) -> color_eyre::Result<()> {
     }
 }
 
-fn print_available_pallets(metadata_v15: &RuntimeMetadataV15) -> String {
-    if metadata_v15.pallets.is_empty() {
+fn print_available_pallets(metadata: &Metadata) -> String {
+    if metadata.pallets().len() == 0 {
         "There are no <PALLET> values available.".to_string()
     } else {
         let mut output = "Available <PALLET> values are:".to_string();
-        let mut strings: Vec<_> = metadata_v15.pallets.iter().map(|p| &p.name).collect();
+        let mut strings: Vec<_> = metadata.pallets().map(|p| p.name()).collect();
         strings.sort();
         for pallet in strings {
             write!(output, "\n    {}", pallet).unwrap();

cli/src/commands/explore/storage.rs

@@ -1,18 +1,20 @@
-use crate::utils::type_description::print_type_description;
-use crate::utils::type_example::print_type_examples;
-use crate::utils::{print_docs_with_indent, with_indent};
 use clap::Args;
-
+use color_eyre::eyre::eyre;
 use std::fmt::Write;
 use std::write;
 
-use color_eyre::eyre::eyre;
-use frame_metadata::v15::{PalletMetadata, PalletStorageMetadata, StorageEntryType};
-
-use scale_info::form::PortableForm;
-
 use subxt::OnlineClient;
-use subxt::{config::SubstrateConfig, Metadata};
+use subxt::{
+    config::SubstrateConfig,
+    metadata::{
+        types::{PalletMetadata, StorageEntryType, StorageMetadata},
+        Metadata,
+    },
+};
+
+use crate::utils::type_description::print_type_description;
+use crate::utils::type_example::print_type_examples;
+use crate::utils::{print_docs_with_indent, with_indent};
 
 #[derive(Debug, Clone, Args)]
 pub struct StorageSubcommand {
@@ -24,14 +26,14 @@ pub struct StorageSubcommand {
 pub(crate) async fn explore_storage(
     command: StorageSubcommand,
     metadata: &Metadata,
-    pallet_metadata: &PalletMetadata<PortableForm>,
+    pallet_metadata: PalletMetadata<'_>,
     custom_online_client_url: Option<String>,
 ) -> color_eyre::Result<()> {
-    let pallet_name = pallet_metadata.name.as_str();
+    let pallet_name = pallet_metadata.name();
     let trailing_args = command.trailing_args.join(" ");
     let trailing_args = trailing_args.trim();
 
-    let Some(storage_metadata) = &pallet_metadata.storage else {
+    let Some(storage_metadata) = pallet_metadata.storage() else {
         println!("The \"{pallet_name}\" pallet has no storage entries.");
         return Ok(());
     };
@@ -44,15 +46,17 @@ pub(crate) async fn explore_storage(
     };
 
     // if specified call storage entry wrong, show user the storage entries to choose from (but this time as an error):
-    let Some(storage) = storage_metadata.entries.iter().find(|entry| entry.name.to_lowercase() == entry_name.to_lowercase())   else {
+    let Some(storage) = storage_metadata.entries().find(|entry| entry.name().to_lowercase() == entry_name.to_lowercase()) else {
         let storage_entries = print_available_storage_entries(storage_metadata, pallet_name);
         let description = format!("Usage:\n    subxt explore {pallet_name} storage <STORAGE_ENTRY>\n        view details for a specific storage entry\n\n{storage_entries}");
         return Err(eyre!("Storage entry \"{entry_name}\" not found in \"{pallet_name}\" pallet!\n\n{description}"));
     };
 
-    let (return_ty_id, key_ty_id) = match storage.ty {
-        StorageEntryType::Plain(value) => (value.id, None),
-        StorageEntryType::Map { value, key, .. } => (value.id, Some(key.id)),
+    let (return_ty_id, key_ty_id) = match storage.entry_type() {
+        StorageEntryType::Plain(value) => (*value, None),
+        StorageEntryType::Map {
+            value_ty, key_ty, ..
+        } => (*value_ty, Some(*key_ty)),
     };
 
     // get the type and type description for the return and key type:
@@ -66,7 +70,7 @@ pub(crate) async fn explore_storage(
         )?;
     }
 
-    let docs_string = print_docs_with_indent(&storage.docs, 4);
+    let docs_string = print_docs_with_indent(storage.docs(), 4);
     if !docs_string.is_empty() {
         write!(output, "Description:\n{docs_string}")?;
     }
@@ -156,17 +160,17 @@ pub(crate) async fn explore_storage(
 }
 
 fn print_available_storage_entries(
-    storage_metadata: &PalletStorageMetadata<PortableForm>,
+    storage_metadata: &StorageMetadata,
     pallet_name: &str,
 ) -> String {
-    if storage_metadata.entries.is_empty() {
+    if storage_metadata.entries().len() == 0 {
         format!("No <STORAGE_ENTRY>'s available in the \"{pallet_name}\" pallet.")
     } else {
         let mut output = format!(
             "Available <STORAGE_ENTRY>'s in the \"{}\" pallet:",
             pallet_name
         );
-        let mut strings: Vec<_> = storage_metadata.entries.iter().map(|s| &s.name).collect();
+        let mut strings: Vec<_> = storage_metadata.entries().map(|s| s.name()).collect();
         strings.sort();
         for entry in strings {
             write!(output, "\n    {}", entry).unwrap();

cli/src/commands/metadata.rs

@@ -5,10 +5,10 @@
 use crate::utils::FileOrUrl;
 use clap::Parser as ClapParser;
 use codec::{Decode, Encode};
-use color_eyre::eyre;
-use frame_metadata::{RuntimeMetadata, RuntimeMetadataPrefixed};
+use color_eyre::eyre::{self, bail};
+use frame_metadata::{v15::RuntimeMetadataV15, RuntimeMetadata, RuntimeMetadataPrefixed};
 use std::io::{self, Write};
-use subxt_metadata::{metadata_v14_to_latest, retain_metadata};
+use subxt_metadata::Metadata;
 
 /// Download metadata from a substrate node, for use with `subxt` codegen.
 #[derive(Debug, ClapParser)]
@@ -36,19 +36,19 @@ pub struct Opts {
 
 pub async fn run(opts: Opts) -> color_eyre::Result<()> {
     let bytes = opts.file_or_url.fetch().await?;
-    let mut metadata = <RuntimeMetadataPrefixed as Decode>::decode(&mut &bytes[..])?;
+    let mut metadata = RuntimeMetadataPrefixed::decode(&mut &bytes[..])?;
+
+    let version = match &metadata.1 {
+        RuntimeMetadata::V14(_) => Version::V14,
+        RuntimeMetadata::V15(_) => Version::V15,
+        _ => Version::Unknown,
+    };
 
     if opts.pallets.is_some() || opts.runtime_apis.is_some() {
-        let mut metadata_v15 = match metadata.1 {
-            RuntimeMetadata::V14(metadata_v14) => metadata_v14_to_latest(metadata_v14),
-            RuntimeMetadata::V15(metadata_v15) => metadata_v15,
-            _ => {
-                return Err(eyre::eyre!(
-                    "Unsupported metadata version {:?}, expected V14.",
-                    metadata.1
-                ));
-            }
-        };
+        // convert to internal type:
+        let mut md = Metadata::try_from(metadata)?;
+
+        // retain pallets and/or runtime APIs given:
         let retain_pallets_fn: Box<dyn Fn(&str) -> bool> = match opts.pallets.as_ref() {
             Some(pallets) => Box::new(|name| pallets.iter().any(|p| &**p == name)),
             None => Box::new(|_| true),
@@ -57,8 +57,16 @@ pub async fn run(opts: Opts) -> color_eyre::Result<()> {
             Some(apis) => Box::new(|name| apis.iter().any(|p| &**p == name)),
             None => Box::new(|_| true),
         };
-        retain_metadata(&mut metadata_v15, retain_pallets_fn, retain_runtime_apis_fn);
-        metadata = metadata_v15.into();
+        md.retain(retain_pallets_fn, retain_runtime_apis_fn);
+
+        // Convert back to wire format, preserving version:
+        metadata = match version {
+            Version::V14 => RuntimeMetadataV15::from(md).into(),
+            Version::V15 => RuntimeMetadataV15::from(md).into(),
+            Version::Unknown => {
+                bail!("Unsupported metadata version; V14 or V15 metadata is expected.")
+            }
+        }
     }
 
     match opts.format.as_str() {
@@ -82,3 +90,9 @@ pub async fn run(opts: Opts) -> color_eyre::Result<()> {
         )),
     }
 }
+
+enum Version {
+    V14,
+    V15,
+    Unknown,
+}