Merge v0.50.x to master (#2127)

* v0.50.0: Integrate frame-decode, redo storage APIs and break up Error. (#2100)

* WIP integrating new frame-decode and working out new storage APIS

* WIP: first pass adding new storage things to subxt-core

* Second pass over Address type and start impl in Subxt

* WIP new storage APIs

* WIP New storage APIs roughly completed, lots of errors still

* Remove PlainorMap enum; plain and map values now use same struct to simplify usage

* Begin 'fixing' errors

* WIP splitting errors and tidying payload/address traits

* Get subxt-core compiling

* Small fixes in subxt-core and remove metadata mod

* subxt-core: cargo check --all-targets passes

* Fix test

* WIP starting to update subxt from subxt-core changes

* WIP splitting up subxt errors into smaller variants

* WIP errors: add DispatchError errors

* Port new Storage APIs to subxt-core

* cargo check -p subxt passes

* Quick-fix errors in subxt-cli (explore subcommand)

* fmt

* Finish fixing codegen up and start fixing examples

* get Subxt examples compiling and bytes_at for constants

* Add some arcs to limit lifetimes in subxt/subxt-core storage APIs

* A little Arcing to allow more method chaining in Storage APIs, aligning with Subxt

* Update codegen test

* cargo check --all-targets passing

* cargo check --features 'unstable-light-client' passing

* clippy

* Remove unused dep in subxt

* use published frame-decode

* fix wasm-example

* Add new tx extension to fix daily tests

* Remove unused subxt_core::dynamic::DecodedValue type

* Update book to match changes

* Update docs to fix more broken bits

* Add missing docs

* fmt

* allow larger result errs for now

* Add missing alloc imports in subxt-core

* Fix doc tests and fix bug getting constant info

* Fix V14 -> Metadata transform for storage & constants

* Fix parachain example

* Fix FFI example

* BlockLength decodes t ostruct, not u128

* use fetch/iter shorthands rather than entry in most storage tests

* Fix some integration tests

* Fix Runtime codegen tests

* Expose the dynamic custom_value selecter and use in a UI test

* Update codegen metadata

* Tidy CLI storage query and support (str,str) as a storage address

* Add (str,str) as valid constant address too

* Show string tuple in constants example

* Via the magic of traits, avoid needing any clones of queries/addresses and accept references to them

* clippy

* [v0.50] update scale-info-legacy and frame-decode to latest (#2119)

* bump scale-info-legacy and frame-decode to latest

* Remove something we don't need in this PR

* Fully remove unused for now dep

* [v0.50] Convert historic metadata to subxt::Metadata (#2120)

* First pass converting historic metadatas to our subxt::Metadata type

* use published frame-decode

* fmt and rename legacy metadata macro

* Enable legacy feature where needed in subxt_metadata so it compiles on its own

* Use cargo hack more in CI and fix subxt-metadata features

* Add tests for metadata conversion (need to optimise; some too expensive right now

* Address performance and equality issues in metadata conversion testing

* fmt

* fmt all

* clippy

* Fix a doc link

* Test codegen and fixes to make it work

* Remove local frame-decode patch

* bump frame-decode to latest

* [v0.50.0] Allow visiting extrinsic fields in subxt_historic (#2124)

* Allow visiting extrinsic fields

* fmt

* Don't use local scale-decode dep

* Clippy and tidy

* Extend 'subxt codegen' CLI to work with legacy metadatas

* Simplify historic extrinsics example now that AccountId32s have paths/names

* clippy

* clippy

* clippy..

* Allow visiting storage values, too, and clean up extrinsic visiting a little by narrowing lifetime

* Try to fix flaky test

* Add custom value decode to extrinsics example

* Remove useless else branch ra thought I needed

* Simplify examples

* Prep to release v0.0.5 (#2126)

cli/Cargo.toml

@@ -30,16 +30,18 @@ subxt-codegen = { workspace = true }
 scale-typegen = { workspace = true }
 subxt-utils-fetchmetadata = { workspace = true, features = ["url"] }
 subxt-utils-stripmetadata = { workspace = true }
-subxt-metadata = { workspace = true }
+subxt-metadata = { workspace = true, features = ["legacy"] }
 subxt = { workspace = true, features = ["default"] }
 clap = { workspace = true }
 serde = { workspace = true, features = ["derive"] }
 color-eyre = { workspace = true }
 serde_json = { workspace = true }
 hex = { workspace = true }
+frame-decode = { workspace = true, features = ["legacy-types"] }
 frame-metadata = { workspace = true }
 codec = { package = "parity-scale-codec", workspace = true }
 scale-info = { workspace = true }
+scale-info-legacy = { workspace = true }
 scale-value = { workspace = true }
 syn = { workspace = true }
 quote = { workspace = true }

cli/src/commands/codegen.rs

@@ -4,12 +4,12 @@
 
 use crate::utils::{FileOrUrl, validate_url_security};
 use clap::Parser as ClapParser;
-use codec::Decode;
 use color_eyre::eyre::eyre;
 use scale_typegen_description::scale_typegen::typegen::{
     settings::substitutes::path_segments,
     validation::{registry_contains_type_path, similar_type_paths_in_registry},
 };
+use std::path::PathBuf;
 use subxt_codegen::CodegenBuilder;
 use subxt_metadata::Metadata;
 
@@ -28,6 +28,12 @@ pub struct Opts {
     /// Additional attributes
     #[clap(long = "attribute")]
     attributes: Vec<String>,
+    /// Path to legacy type definitions (required for metadatas pre-V14)
+    #[clap(long)]
+    legacy_types: Option<PathBuf>,
+    /// The spec version of the legacy metadata (required for metadatas pre-V14)
+    #[clap(long)]
+    legacy_spec_version: Option<u64>,
     /// Additional derives for a given type.
     ///
     /// Example 1: `--derive-for-type my_module::my_type=serde::Serialize`.
@@ -145,9 +151,20 @@ pub async fn run(opts: Opts, output: &mut impl std::io::Write) -> color_eyre::Re
     validate_url_security(opts.file_or_url.url.as_ref(), opts.allow_insecure)?;
 
     let bytes = opts.file_or_url.fetch().await?;
+    let legacy_types = opts
+        .legacy_types
+        .map(|path| {
+            let bytes = std::fs::read(path).map_err(|e| eyre!("Cannot read legacy_types: {e}"))?;
+            let types = frame_decode::legacy_types::from_bytes(&bytes)
+                .map_err(|e| eyre!("Cannot deserialize legacy_types: {e}"))?;
+            Ok::<_, color_eyre::eyre::Error>(types)
+        })
+        .transpose()?;
 
     codegen(
         &bytes,
+        legacy_types,
+        opts.legacy_spec_version,
         opts.derives,
         opts.attributes,
         opts.derives_for_type,
@@ -175,6 +192,8 @@ impl syn::parse::Parse for OuterAttribute {
 #[allow(clippy::too_many_arguments)]
 fn codegen(
     metadata_bytes: &[u8],
+    legacy_types: Option<scale_info_legacy::ChainTypeRegistry>,
+    legacy_spec_version: Option<u64>,
     raw_derives: Vec<String>,
     raw_attributes: Vec<String>,
     derives_for_type: Vec<DeriveForType>,
@@ -211,8 +230,79 @@ fn codegen(
     }
 
     let metadata = {
-        let mut metadata = subxt_metadata::Metadata::decode(&mut &*metadata_bytes)
-            .map_err(|e| eyre!("Cannot decode the provided metadata: {e}"))?;
+        let runtime_metadata = subxt_metadata::decode_runtime_metadata(metadata_bytes)?;
+        let mut metadata = match runtime_metadata {
+            // Too old to work with:
+            frame_metadata::RuntimeMetadata::V0(_)
+            | frame_metadata::RuntimeMetadata::V1(_)
+            | frame_metadata::RuntimeMetadata::V2(_)
+            | frame_metadata::RuntimeMetadata::V3(_)
+            | frame_metadata::RuntimeMetadata::V4(_)
+            | frame_metadata::RuntimeMetadata::V5(_)
+            | frame_metadata::RuntimeMetadata::V6(_)
+            | frame_metadata::RuntimeMetadata::V7(_) => {
+                Err(eyre!("Metadata V1-V7 cannot be decoded from"))
+            }
+            // Converting legacy metadatas:
+            frame_metadata::RuntimeMetadata::V8(md) => {
+                let legacy_types = legacy_types
+                    .ok_or_else(|| eyre!("--legacy-types needed to load V8 metadata"))?;
+                let legacy_spec = legacy_spec_version
+                    .ok_or_else(|| eyre!("--legacy-spec-version needed to load V8 metadata"))?;
+                Metadata::from_v8(&md, &legacy_types.for_spec_version(legacy_spec))
+                    .map_err(|e| eyre!("Cannot load V8 metadata: {e}"))
+            }
+            frame_metadata::RuntimeMetadata::V9(md) => {
+                let legacy_types = legacy_types
+                    .ok_or_else(|| eyre!("--legacy-types needed to load V9 metadata"))?;
+                let legacy_spec = legacy_spec_version
+                    .ok_or_else(|| eyre!("--legacy-spec-version needed to load V9 metadata"))?;
+                Metadata::from_v9(&md, &legacy_types.for_spec_version(legacy_spec))
+                    .map_err(|e| eyre!("Cannot load V9 metadata: {e}"))
+            }
+            frame_metadata::RuntimeMetadata::V10(md) => {
+                let legacy_types = legacy_types
+                    .ok_or_else(|| eyre!("--legacy-types needed to load V10 metadata"))?;
+                let legacy_spec = legacy_spec_version
+                    .ok_or_else(|| eyre!("--legacy-spec-version needed to load V10 metadata"))?;
+                Metadata::from_v10(&md, &legacy_types.for_spec_version(legacy_spec))
+                    .map_err(|e| eyre!("Cannot load V10 metadata: {e}"))
+            }
+            frame_metadata::RuntimeMetadata::V11(md) => {
+                let legacy_types = legacy_types
+                    .ok_or_else(|| eyre!("--legacy-types needed to load V11 metadata"))?;
+                let legacy_spec = legacy_spec_version
+                    .ok_or_else(|| eyre!("--legacy-spec-version needed to load V11 metadata"))?;
+                Metadata::from_v11(&md, &legacy_types.for_spec_version(legacy_spec))
+                    .map_err(|e| eyre!("Cannot load V11 metadata: {e}"))
+            }
+            frame_metadata::RuntimeMetadata::V12(md) => {
+                let legacy_types = legacy_types
+                    .ok_or_else(|| eyre!("--legacy-types needed to load V12 metadata"))?;
+                let legacy_spec = legacy_spec_version
+                    .ok_or_else(|| eyre!("--legacy-spec-version needed to load V12 metadata"))?;
+                Metadata::from_v12(&md, &legacy_types.for_spec_version(legacy_spec))
+                    .map_err(|e| eyre!("Cannot load V12 metadata: {e}"))
+            }
+            frame_metadata::RuntimeMetadata::V13(md) => {
+                let legacy_types = legacy_types
+                    .ok_or_else(|| eyre!("--legacy-types needed to load V13 metadata"))?;
+                let legacy_spec = legacy_spec_version
+                    .ok_or_else(|| eyre!("--legacy-spec-version needed to load V13 metadata"))?;
+                Metadata::from_v13(&md, &legacy_types.for_spec_version(legacy_spec))
+                    .map_err(|e| eyre!("Cannot load V13 metadata: {e}"))
+            }
+            // Converting modern metadatas:
+            frame_metadata::RuntimeMetadata::V14(md) => {
+                Metadata::from_v14(md).map_err(|e| eyre!("Cannot load V14 metadata: {e}"))
+            }
+            frame_metadata::RuntimeMetadata::V15(md) => {
+                Metadata::from_v15(md).map_err(|e| eyre!("Cannot load V15 metadata: {e}"))
+            }
+            frame_metadata::RuntimeMetadata::V16(md) => {
+                Metadata::from_v16(md).map_err(|e| eyre!("Cannot load V16 metadata: {e}"))
+            }
+        }?;
 
         // Run this first to ensure type paths are unique (which may result in 1,2,3 suffixes being added
         // to type paths), so that when we validate derives/substitutions below, they are allowed for such

cli/src/commands/diff.rs

@@ -215,7 +215,6 @@ struct StorageEntryDiff {
     key_different: bool,
     value_different: bool,
     default_different: bool,
-    modifier_different: bool,
 }
 
 impl StorageEntryDiff {
@@ -225,41 +224,32 @@ impl StorageEntryDiff {
         metadata_1: &Metadata,
         metadata_2: &Metadata,
     ) -> Self {
-        let value_1_ty_id = storage_entry_1.entry_type().value_ty();
+        let value_1_ty_id = storage_entry_1.value_ty();
         let value_1_hash = metadata_1
             .type_hash(value_1_ty_id)
             .expect("type is in metadata; qed");
-        let value_2_ty_id = storage_entry_2.entry_type().value_ty();
+        let value_2_ty_id = storage_entry_2.value_ty();
         let value_2_hash = metadata_2
             .type_hash(value_2_ty_id)
             .expect("type is in metadata; qed");
         let value_different = value_1_hash != value_2_hash;
 
-        let key_1_hash = storage_entry_1
-            .entry_type()
-            .key_ty()
-            .map(|key_ty| {
-                metadata_1
-                    .type_hash(key_ty)
-                    .expect("type is in metadata; qed")
-            })
-            .unwrap_or_default();
-        let key_2_hash = storage_entry_2
-            .entry_type()
-            .key_ty()
-            .map(|key_ty| {
-                metadata_2
-                    .type_hash(key_ty)
-                    .expect("type is in metadata; qed")
-            })
-            .unwrap_or_default();
-        let key_different = key_1_hash != key_2_hash;
+        let key_parts_same = storage_entry_1.keys().len() == storage_entry_2.keys().len()
+            && storage_entry_1
+                .keys()
+                .zip(storage_entry_2.keys())
+                .all(|(a, b)| {
+                    let a_hash = metadata_1.type_hash(a.key_id).expect("type is in metadata");
+                    let b_hash = metadata_2.type_hash(b.key_id).expect("type is in metadata");
+                    a.hasher == b.hasher && a_hash == b_hash
+                });
+
+        let key_different = !key_parts_same;
 
         StorageEntryDiff {
             key_different,
             value_different,
-            default_different: storage_entry_1.default_bytes() != storage_entry_2.default_bytes(),
-            modifier_different: storage_entry_1.modifier() != storage_entry_2.modifier(),
+            default_different: storage_entry_1.default_value() != storage_entry_2.default_value(),
         }
     }
 
@@ -271,9 +261,6 @@ impl StorageEntryDiff {
         if self.value_different {
             strings.push("value type");
         }
-        if self.modifier_different {
-            strings.push("modifier");
-        }
         if self.default_different {
             strings.push("default value");
         }

cli/src/commands/explore/pallets/calls.rs

@@ -12,7 +12,7 @@ use subxt::utils::H256;
 use subxt::{
     OfflineClient,
     config::SubstrateConfig,
-    metadata::{Metadata, types::PalletMetadata},
+    metadata::{Metadata, PalletMetadata},
 };
 
 use crate::utils::{

cli/src/commands/explore/pallets/constants.rs

@@ -2,7 +2,7 @@ use clap::Args;
 use color_eyre::eyre::eyre;
 use indoc::{formatdoc, writedoc};
 use scale_typegen_description::type_description;
-use subxt::metadata::{Metadata, types::PalletMetadata};
+use subxt::metadata::{Metadata, PalletMetadata};
 
 use crate::utils::{Indent, SyntaxHighlight, first_paragraph_of_docs, format_scale_value};
 

cli/src/commands/explore/pallets/events.rs

@@ -2,7 +2,7 @@ use clap::Args;
 use color_eyre::eyre::eyre;
 use indoc::{formatdoc, writedoc};
 use scale_info::{Variant, form::PortableForm};
-use subxt::metadata::{Metadata, types::PalletMetadata};
+use subxt::metadata::{Metadata, PalletMetadata};
 
 use crate::utils::{Indent, fields_description, first_paragraph_of_docs};
 

cli/src/commands/explore/pallets/storage.rs

@@ -1,17 +1,11 @@
 use clap::Args;
-use color_eyre::{
-    eyre::{bail, eyre},
-    owo_colors::OwoColorize,
-};
+use color_eyre::{eyre::bail, owo_colors::OwoColorize};
 use indoc::{formatdoc, writedoc};
 use scale_typegen_description::type_description;
 use scale_value::Value;
 use std::fmt::Write;
 use std::write;
-use subxt::metadata::{
-    Metadata,
-    types::{PalletMetadata, StorageEntryType, StorageMetadata},
-};
+use subxt::metadata::{Metadata, PalletMetadata, StorageMetadata};
 
 use crate::utils::{
     FileOrUrl, Indent, SyntaxHighlight, create_client, first_paragraph_of_docs,
@@ -75,12 +69,7 @@ pub async fn explore_storage(
         );
     };
 
-    let (return_ty_id, key_ty_id) = match storage.entry_type() {
-        StorageEntryType::Plain(value) => (*value, None),
-        StorageEntryType::Map {
-            value_ty, key_ty, ..
-        } => (*value_ty, Some(*key_ty)),
-    };
+    let return_ty_id = storage.value_ty();
 
     let key_value_placeholder = "<KEY_VALUE>".blue();
 
@@ -114,15 +103,30 @@ pub async fn explore_storage(
     "}?;
 
     // inform user about shape of the key if it can be provided:
-    if let Some(key_ty_id) = key_ty_id {
-        let key_ty_description = type_description(key_ty_id, metadata.types(), true)
-            .expect("No type Description")
-            .indent(4)
-            .highlight();
+    let storage_keys = storage.keys().collect::<Vec<_>>();
+    if !storage_keys.is_empty() {
+        let key_ty_description = format!(
+            "({})",
+            storage_keys
+                .iter()
+                .map(|key| type_description(key.key_id, metadata.types(), true)
+                    .expect("No type Description"))
+                .collect::<Vec<_>>()
+                .join(", ")
+        )
+        .indent(4)
+        .highlight();
 
-        let key_ty_example = type_example(key_ty_id, metadata.types())
-            .indent(4)
-            .highlight();
+        let key_ty_example = format!(
+            "({})",
+            storage_keys
+                .iter()
+                .map(|key| type_example(key.key_id, metadata.types()).to_string())
+                .collect::<Vec<_>>()
+                .join(", ")
+        )
+        .indent(4)
+        .highlight();
 
         writedoc! {output, "
 
@@ -144,7 +148,8 @@ pub async fn explore_storage(
         return Ok(());
     }
 
-    let storage_entry_keys: Vec<Value> = match (!trailing_args.is_empty(), key_ty_id.is_some()) {
+    let storage_entry_keys: Vec<Value> = match (!trailing_args.is_empty(), !storage_keys.is_empty())
+    {
         // keys provided, keys not needed.
         (true, false) => {
             let trailing_args_str = trailing_args.join(" ");
@@ -190,18 +195,17 @@ pub async fn explore_storage(
     // construct the client:
     let client = create_client(&file_or_url).await?;
 
-    let storage_query = subxt::dynamic::storage(pallet_name, storage.name(), storage_entry_keys);
-    let decoded_value_thunk_or_none = client
+    // Fetch the value:
+    let storage_value = client
         .storage()
         .at_latest()
         .await?
-        .fetch(&storage_query)
-        .await?;
+        .fetch((pallet_name, storage.name()), storage_entry_keys)
+        .await?
+        .decode()?;
 
-    let decoded_value_thunk =
-        decoded_value_thunk_or_none.ok_or(eyre!("Value not found in storage."))?;
+    let value = storage_value.to_string().highlight();
 
-    let value = decoded_value_thunk.to_value()?.to_string().highlight();
     writedoc! {output, "
 
     The value of the storage entry is:

cli/src/commands/explore/runtime_apis/mod.rs

@@ -101,15 +101,13 @@ pub async fn run<'a>(
             return format!("The method does not require an {input_value_placeholder}");
         }
 
-        let fields: Vec<(Option<&str>, u32)> = method
-            .inputs()
-            .map(|f| (Some(f.name.as_str()), f.ty))
-            .collect();
+        let fields: Vec<(Option<&str>, u32)> =
+            method.inputs().map(|f| (Some(&*f.name), f.id)).collect();
         let fields_description =
             fields_description(&fields, method.name(), metadata.types()).indent(4);
 
         let fields_example =
-            fields_composite_example(method.inputs().map(|e| e.ty), metadata.types())
+            fields_composite_example(method.inputs().map(|e| e.id), metadata.types())
                 .indent(4)
                 .highlight();
 
@@ -164,13 +162,14 @@ pub async fn run<'a>(
             {value_str}
             "}?;
             // encode, then decode. This ensures that the scale value is of the correct shape for the param:
-            let bytes = value.encode_as_type(ty.ty, metadata.types())?;
-            let value = Value::decode_as_type(&mut &bytes[..], ty.ty, metadata.types())?;
+            let bytes = value.encode_as_type(ty.id, metadata.types())?;
+            let value = Value::decode_as_type(&mut &bytes[..], ty.id, metadata.types())?;
             Ok(value)
         })
         .collect::<color_eyre::Result<Vec<Value>>>()?;
 
-    let method_call = subxt::dynamic::runtime_api_call(api_name, method.name(), args_data);
+    let method_call =
+        subxt::dynamic::runtime_api_call::<_, Value>(api_name, method.name(), args_data);
     let client = create_client(&file_or_url).await?;
     let output_value = client
         .runtime_api()
@@ -179,7 +178,7 @@ pub async fn run<'a>(
         .call(method_call)
         .await?;
 
-    let output_value = output_value.to_value()?.to_string().highlight();
+    let output_value = output_value.to_string().highlight();
     writedoc! {output, "
 
     Returned value: