From 3bbba1b0053965875993a7a8ebb5c6a2d4f5e372 Mon Sep 17 00:00:00 2001 From: James Wilson Date: Fri, 21 Nov 2025 15:32:32 +0000 Subject: [PATCH] [v0.50.0] Allow visiting extrinsic fields in subxt_historic (#2124) * Allow visiting extrinsic fields * fmt * Don't use local scale-decode dep * Clippy and tidy * Extend 'subxt codegen' CLI to work with legacy metadatas * Simplify historic extrinsics example now that AccountId32s have paths/names * clippy * clippy * clippy.. * Allow visiting storage values, too, and clean up extrinsic visiting a little by narrowing lifetime * Try to fix flaky test * Add custom value decode to extrinsics example * Remove useless else branch ra thought I needed * Simplify examples --- Cargo.lock | 18 +- Cargo.toml | 6 +- cli/Cargo.toml | 4 +- cli/src/commands/codegen.rs | 96 ++++- historic/Cargo.toml | 4 +- historic/examples/extrinsics.rs | 377 +++++++++++++++++- historic/examples/storage.rs | 74 +++- historic/src/extrinsics/extrinsic_call.rs | 38 +- historic/src/extrinsics/extrinsics_type.rs | 3 - historic/src/lib.rs | 5 + historic/src/storage/storage_value.rs | 31 +- historic/src/utils.rs | 2 + historic/src/utils/any_resolver.rs | 188 +++++++++ historic/src/utils/either.rs | 1 + metadata/src/lib.rs | 59 ++- .../src/full_client/blocks.rs | 2 +- 16 files changed, 871 insertions(+), 37 deletions(-) create mode 100644 historic/src/utils/any_resolver.rs diff --git a/Cargo.lock b/Cargo.lock index 91c0c41824..db8208349c 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1953,9 +1953,9 @@ dependencies = [ [[package]] name = "frame-decode" -version = "0.13.0" +version = "0.15.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "73d29c7f2987ea24ab2eaea315aadb9ba598188823181cdf0476049b625a5844" +checksum = "0fb3bfa2988ef40247e0e0eecfb171a01ad6f4e399485503aad4c413a5f236f3" dependencies = [ "frame-metadata 23.0.0", "parity-scale-codec", @@ -4383,9 +4383,9 @@ dependencies = [ [[package]] name = "scale-decode" -version = "0.16.0" +version = "0.16.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4d78196772d25b90a98046794ce0fe2588b39ebdfbdc1e45b4c6c85dd43bebad" +checksum = "8d6ed61699ad4d54101ab5a817169259b5b0efc08152f8632e61482d8a27ca3d" dependencies = [ "parity-scale-codec", "primitive-types", @@ -4398,9 +4398,9 @@ dependencies = [ [[package]] name = "scale-decode-derive" -version = "0.16.0" +version = "0.16.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2f4b54a1211260718b92832b661025d1f1a4b6930fbadd6908e00edd265fa5f7" +checksum = "65cb245f7fdb489e7ba43a616cbd34427fe3ba6fe0edc1d0d250085e6c84f3ec" dependencies = [ "darling", "proc-macro2", @@ -4464,9 +4464,9 @@ dependencies = [ [[package]] name = "scale-info-legacy" -version = "0.3.2" +version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "06423f0d7ea951547143aff4695c4c3e821e66c9b80729a3ff55fa93d23e93e6" +checksum = "2500adfb429a0ffda37919df92c05d0c1359c10e0444c17253c84b84dfce542f" dependencies = [ "hashbrown 0.15.3", "scale-type-resolver", @@ -5658,6 +5658,7 @@ version = "0.44.0" dependencies = [ "clap", "color-eyre", + "frame-decode", "frame-metadata 23.0.0", "heck", "hex", @@ -5667,6 +5668,7 @@ dependencies = [ "pretty_assertions", "quote", "scale-info", + "scale-info-legacy", "scale-typegen 0.12.0", "scale-typegen-description", "scale-value", diff --git a/Cargo.toml b/Cargo.toml index 4acf44b7dc..efb0b03d3b 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -81,7 +81,7 @@ darling = "0.20.10" derive-where = "1.2.7" either = { version = "1.13.0", default-features = false } finito = { version = "0.1.0", default-features = false } -frame-decode = { version = "0.14.0", default-features = false } +frame-decode = { version = "0.15.0", default-features = false } frame-metadata = { version = "23.0.0", default-features = false } futures = { version = "0.3.31", default-features = false, features = ["std"] } getrandom = { version = "0.2", default-features = false } @@ -100,10 +100,10 @@ regex = { version = "1.11.0", default-features = false } scale-info = { version = "2.11.4", default-features = false } scale-value = { version = "0.18.1", default-features = false } scale-bits = { version = "0.7.0", default-features = false } -scale-decode = { version = "0.16.0", default-features = false } +scale-decode = { version = "0.16.2", default-features = false } scale-encode = { version = "0.10.0", default-features = false } scale-type-resolver = { version = "0.2.0" } -scale-info-legacy = { version = "0.3.2", default-features = false } +scale-info-legacy = { version = "0.4.0", default-features = false } scale-typegen = "0.12.0" scale-typegen-description = "0.11.0" serde = { version = "1.0.210", default-features = false, features = ["derive"] } diff --git a/cli/Cargo.toml b/cli/Cargo.toml index 47e8ddccee..63ae941585 100644 --- a/cli/Cargo.toml +++ b/cli/Cargo.toml @@ -30,16 +30,18 @@ subxt-codegen = { workspace = true } scale-typegen = { workspace = true } subxt-utils-fetchmetadata = { workspace = true, features = ["url"] } subxt-utils-stripmetadata = { workspace = true } -subxt-metadata = { workspace = true } +subxt-metadata = { workspace = true, features = ["legacy"] } subxt = { workspace = true, features = ["default"] } clap = { workspace = true } serde = { workspace = true, features = ["derive"] } color-eyre = { workspace = true } serde_json = { workspace = true } hex = { workspace = true } +frame-decode = { workspace = true, features = ["legacy-types"] } frame-metadata = { workspace = true } codec = { package = "parity-scale-codec", workspace = true } scale-info = { workspace = true } +scale-info-legacy = { workspace = true } scale-value = { workspace = true } syn = { workspace = true } quote = { workspace = true } diff --git a/cli/src/commands/codegen.rs b/cli/src/commands/codegen.rs index 10744aed76..95213077ed 100644 --- a/cli/src/commands/codegen.rs +++ b/cli/src/commands/codegen.rs @@ -4,12 +4,12 @@ use crate::utils::{FileOrUrl, validate_url_security}; use clap::Parser as ClapParser; -use codec::Decode; use color_eyre::eyre::eyre; use scale_typegen_description::scale_typegen::typegen::{ settings::substitutes::path_segments, validation::{registry_contains_type_path, similar_type_paths_in_registry}, }; +use std::path::PathBuf; use subxt_codegen::CodegenBuilder; use subxt_metadata::Metadata; @@ -28,6 +28,12 @@ pub struct Opts { /// Additional attributes #[clap(long = "attribute")] attributes: Vec, + /// Path to legacy type definitions (required for metadatas pre-V14) + #[clap(long)] + legacy_types: Option, + /// The spec version of the legacy metadata (required for metadatas pre-V14) + #[clap(long)] + legacy_spec_version: Option, /// Additional derives for a given type. /// /// Example 1: `--derive-for-type my_module::my_type=serde::Serialize`. @@ -145,9 +151,20 @@ pub async fn run(opts: Opts, output: &mut impl std::io::Write) -> color_eyre::Re validate_url_security(opts.file_or_url.url.as_ref(), opts.allow_insecure)?; let bytes = opts.file_or_url.fetch().await?; + let legacy_types = opts + .legacy_types + .map(|path| { + let bytes = std::fs::read(path).map_err(|e| eyre!("Cannot read legacy_types: {e}"))?; + let types = frame_decode::legacy_types::from_bytes(&bytes) + .map_err(|e| eyre!("Cannot deserialize legacy_types: {e}"))?; + Ok::<_, color_eyre::eyre::Error>(types) + }) + .transpose()?; codegen( &bytes, + legacy_types, + opts.legacy_spec_version, opts.derives, opts.attributes, opts.derives_for_type, @@ -175,6 +192,8 @@ impl syn::parse::Parse for OuterAttribute { #[allow(clippy::too_many_arguments)] fn codegen( metadata_bytes: &[u8], + legacy_types: Option, + legacy_spec_version: Option, raw_derives: Vec, raw_attributes: Vec, derives_for_type: Vec, @@ -211,8 +230,79 @@ fn codegen( } let metadata = { - let mut metadata = subxt_metadata::Metadata::decode(&mut &*metadata_bytes) - .map_err(|e| eyre!("Cannot decode the provided metadata: {e}"))?; + let runtime_metadata = subxt_metadata::decode_runtime_metadata(metadata_bytes)?; + let mut metadata = match runtime_metadata { + // Too old to work with: + frame_metadata::RuntimeMetadata::V0(_) + | frame_metadata::RuntimeMetadata::V1(_) + | frame_metadata::RuntimeMetadata::V2(_) + | frame_metadata::RuntimeMetadata::V3(_) + | frame_metadata::RuntimeMetadata::V4(_) + | frame_metadata::RuntimeMetadata::V5(_) + | frame_metadata::RuntimeMetadata::V6(_) + | frame_metadata::RuntimeMetadata::V7(_) => { + Err(eyre!("Metadata V1-V7 cannot be decoded from")) + } + // Converting legacy metadatas: + frame_metadata::RuntimeMetadata::V8(md) => { + let legacy_types = legacy_types + .ok_or_else(|| eyre!("--legacy-types needed to load V8 metadata"))?; + let legacy_spec = legacy_spec_version + .ok_or_else(|| eyre!("--legacy-spec-version needed to load V8 metadata"))?; + Metadata::from_v8(&md, &legacy_types.for_spec_version(legacy_spec)) + .map_err(|e| eyre!("Cannot load V8 metadata: {e}")) + } + frame_metadata::RuntimeMetadata::V9(md) => { + let legacy_types = legacy_types + .ok_or_else(|| eyre!("--legacy-types needed to load V9 metadata"))?; + let legacy_spec = legacy_spec_version + .ok_or_else(|| eyre!("--legacy-spec-version needed to load V9 metadata"))?; + Metadata::from_v9(&md, &legacy_types.for_spec_version(legacy_spec)) + .map_err(|e| eyre!("Cannot load V9 metadata: {e}")) + } + frame_metadata::RuntimeMetadata::V10(md) => { + let legacy_types = legacy_types + .ok_or_else(|| eyre!("--legacy-types needed to load V10 metadata"))?; + let legacy_spec = legacy_spec_version + .ok_or_else(|| eyre!("--legacy-spec-version needed to load V10 metadata"))?; + Metadata::from_v10(&md, &legacy_types.for_spec_version(legacy_spec)) + .map_err(|e| eyre!("Cannot load V10 metadata: {e}")) + } + frame_metadata::RuntimeMetadata::V11(md) => { + let legacy_types = legacy_types + .ok_or_else(|| eyre!("--legacy-types needed to load V11 metadata"))?; + let legacy_spec = legacy_spec_version + .ok_or_else(|| eyre!("--legacy-spec-version needed to load V11 metadata"))?; + Metadata::from_v11(&md, &legacy_types.for_spec_version(legacy_spec)) + .map_err(|e| eyre!("Cannot load V11 metadata: {e}")) + } + frame_metadata::RuntimeMetadata::V12(md) => { + let legacy_types = legacy_types + .ok_or_else(|| eyre!("--legacy-types needed to load V12 metadata"))?; + let legacy_spec = legacy_spec_version + .ok_or_else(|| eyre!("--legacy-spec-version needed to load V12 metadata"))?; + Metadata::from_v12(&md, &legacy_types.for_spec_version(legacy_spec)) + .map_err(|e| eyre!("Cannot load V12 metadata: {e}")) + } + frame_metadata::RuntimeMetadata::V13(md) => { + let legacy_types = legacy_types + .ok_or_else(|| eyre!("--legacy-types needed to load V13 metadata"))?; + let legacy_spec = legacy_spec_version + .ok_or_else(|| eyre!("--legacy-spec-version needed to load V13 metadata"))?; + Metadata::from_v13(&md, &legacy_types.for_spec_version(legacy_spec)) + .map_err(|e| eyre!("Cannot load V13 metadata: {e}")) + } + // Converting modern metadatas: + frame_metadata::RuntimeMetadata::V14(md) => { + Metadata::from_v14(md).map_err(|e| eyre!("Cannot load V14 metadata: {e}")) + } + frame_metadata::RuntimeMetadata::V15(md) => { + Metadata::from_v15(md).map_err(|e| eyre!("Cannot load V15 metadata: {e}")) + } + frame_metadata::RuntimeMetadata::V16(md) => { + Metadata::from_v16(md).map_err(|e| eyre!("Cannot load V16 metadata: {e}")) + } + }?; // Run this first to ensure type paths are unique (which may result in 1,2,3 suffixes being added // to type paths), so that when we validate derives/substitutions below, they are allowed for such diff --git a/historic/Cargo.toml b/historic/Cargo.toml index b458e3cae3..9e0b91b7a8 100644 --- a/historic/Cargo.toml +++ b/historic/Cargo.toml @@ -45,7 +45,7 @@ jsonrpsee = [ subxt-rpcs = { workspace = true } frame-decode = { workspace = true, features = ["legacy", "legacy-types"] } frame-metadata = { workspace = true, features = ["std", "legacy"] } -scale-type-resolver = { workspace = true } +scale-type-resolver = { workspace = true, features = ["scale-info"] } codec = { workspace = true } primitive-types = { workspace = true } scale-info = { workspace = true } @@ -60,4 +60,4 @@ futures = { workspace = true } tokio = { workspace = true, features = ["full"] } scale-value = { workspace = true } scale-decode = { workspace = true, features = ["derive"] } -hex = { workspace = true } \ No newline at end of file +hex = { workspace = true } diff --git a/historic/examples/extrinsics.rs b/historic/examples/extrinsics.rs index 82ba674b89..13dc2631aa 100644 --- a/historic/examples/extrinsics.rs +++ b/historic/examples/extrinsics.rs @@ -1,8 +1,8 @@ #![allow(missing_docs)] -use subxt_historic::{Error, OnlineClient, PolkadotConfig}; +use subxt_historic::{OnlineClient, PolkadotConfig}; #[tokio::main] -async fn main() -> Result<(), Error> { +async fn main() -> Result<(), Box> { // Configuration for the Polkadot relay chain. let config = PolkadotConfig::new(); @@ -10,7 +10,7 @@ async fn main() -> Result<(), Error> { let client = OnlineClient::from_url(config, "wss://rpc.polkadot.io").await?; // Iterate through some randomly selected old blocks to show how to fetch and decode extrinsics. - for block_number in 123456.. { + for block_number in 1234567.. { println!("=== Block {block_number} ==="); // Point the client at a specific block number. By default this will download and cache @@ -40,10 +40,26 @@ async fn main() -> Result<(), Error> { // scale_value::Value type, which can represent any SCALE encoded data, but if you // have an idea of the type then you can try to decode into that type instead): for field in extrinsic.call().fields().iter() { + // We can visit fields, which gives us the ability to inspect and decode information + // from them selectively, returning whatever we like from it. Here we demo our + // type name visitor which is defined below: + let tn = field + .visit(type_name::GetTypeName::new())? + .unwrap_or_default(); + + // When visiting fields we can also decode into a custom shape like so: + let _custom_value = field.visit(value::GetValue::new())?; + + // We can also obtain and decode things without the complexity of the above: println!( - " {}: {}", + " {}: {} {}", field.name(), - field.decode_as::().unwrap() + field.decode_as::().unwrap(), + if tn.is_empty() { + String::new() + } else { + format!("(type name: {tn})") + }, ); } @@ -81,3 +97,354 @@ async fn main() -> Result<(), Error> { Ok(()) } + +/// This module defines an example visitor which retrieves the name of a type. +/// This is a more advanced use case and can typically be avoided. +mod type_name { + use scale_decode::{ + Visitor, + visitor::types::{Composite, Sequence, Variant}, + visitor::{TypeIdFor, Unexpected}, + }; + use scale_type_resolver::TypeResolver; + + /// This is a visitor which obtains type names. + pub struct GetTypeName { + marker: core::marker::PhantomData, + } + + impl GetTypeName { + /// Construct our TypeName visitor. + pub fn new() -> Self { + GetTypeName { + marker: core::marker::PhantomData, + } + } + } + + impl Visitor for GetTypeName { + type Value<'scale, 'resolver> = Option<&'resolver str>; + type Error = scale_decode::Error; + type TypeResolver = R; + + // Look at the path of types that have paths and return the ident from that. + fn visit_composite<'scale, 'resolver>( + self, + value: &mut Composite<'scale, 'resolver, Self::TypeResolver>, + _type_id: TypeIdFor, + ) -> Result, Self::Error> { + Ok(value.path().last()) + } + fn visit_variant<'scale, 'resolver>( + self, + value: &mut Variant<'scale, 'resolver, Self::TypeResolver>, + _type_id: TypeIdFor, + ) -> Result, Self::Error> { + Ok(value.path().last()) + } + fn visit_sequence<'scale, 'resolver>( + self, + value: &mut Sequence<'scale, 'resolver, Self::TypeResolver>, + _type_id: TypeIdFor, + ) -> Result, Self::Error> { + Ok(value.path().last()) + } + + // Else, we return nothing as we can't find a name for the type. + fn visit_unexpected<'scale, 'resolver>( + self, + _unexpected: Unexpected, + ) -> Result, Self::Error> { + Ok(None) + } + } +} + +/// This visitor demonstrates how to decode and return a custom Value shape +mod value { + use scale_decode::{ + Visitor, + visitor::TypeIdFor, + visitor::types::{Array, BitSequence, Composite, Sequence, Str, Tuple, Variant}, + }; + use scale_type_resolver::TypeResolver; + use std::collections::HashMap; + + /// A value type we're decoding into. + #[allow(dead_code)] + pub enum Value { + Number(f64), + BigNumber(String), + Bool(bool), + Char(char), + Array(Vec), + String(String), + Address(Vec), + I256([u8; 32]), + U256([u8; 32]), + Struct(HashMap), + Variant(String, VariantFields), + } + + pub enum VariantFields { + Unnamed(Vec), + Named(HashMap), + } + + /// An error we can encounter trying to decode things into a [`Value`] + #[derive(Debug, thiserror::Error)] + pub enum ValueError { + #[error("Decode error: {0}")] + Decode(#[from] scale_decode::visitor::DecodeError), + #[error("Cannot decode bit sequence: {0}")] + CannotDecodeBitSequence(codec::Error), + } + + /// This is a visitor which obtains type names. + pub struct GetValue { + marker: core::marker::PhantomData, + } + + impl GetValue { + /// Construct our TypeName visitor. + pub fn new() -> Self { + GetValue { + marker: core::marker::PhantomData, + } + } + } + + impl Visitor for GetValue { + type Value<'scale, 'resolver> = Value; + type Error = ValueError; + type TypeResolver = R; + + fn visit_i256<'resolver>( + self, + value: &[u8; 32], + _type_id: TypeIdFor, + ) -> Result, Self::Error> { + Ok(Value::I256(*value)) + } + + fn visit_u256<'resolver>( + self, + value: &[u8; 32], + _type_id: TypeIdFor, + ) -> Result, Self::Error> { + Ok(Value::U256(*value)) + } + + fn visit_i128<'scale, 'resolver>( + self, + value: i128, + _type_id: TypeIdFor, + ) -> Result, Self::Error> { + let attempt = value as f64; + if attempt as i128 == value { + Ok(Value::Number(attempt)) + } else { + Ok(Value::BigNumber(value.to_string())) + } + } + + fn visit_i64<'scale, 'resolver>( + self, + value: i64, + type_id: TypeIdFor, + ) -> Result, Self::Error> { + self.visit_i128(value.into(), type_id) + } + + fn visit_i32<'scale, 'resolver>( + self, + value: i32, + type_id: TypeIdFor, + ) -> Result, Self::Error> { + self.visit_i128(value.into(), type_id) + } + + fn visit_i16<'scale, 'resolver>( + self, + value: i16, + type_id: TypeIdFor, + ) -> Result, Self::Error> { + self.visit_i128(value.into(), type_id) + } + + fn visit_i8<'scale, 'resolver>( + self, + value: i8, + type_id: TypeIdFor, + ) -> Result, Self::Error> { + self.visit_i128(value.into(), type_id) + } + + fn visit_u128<'scale, 'resolver>( + self, + value: u128, + _type_id: TypeIdFor, + ) -> Result, Self::Error> { + let attempt = value as f64; + if attempt as u128 == value { + Ok(Value::Number(attempt)) + } else { + Ok(Value::BigNumber(value.to_string())) + } + } + + fn visit_u64<'scale, 'resolver>( + self, + value: u64, + type_id: TypeIdFor, + ) -> Result, Self::Error> { + self.visit_u128(value.into(), type_id) + } + + fn visit_u32<'scale, 'resolver>( + self, + value: u32, + type_id: TypeIdFor, + ) -> Result, Self::Error> { + self.visit_u128(value.into(), type_id) + } + + fn visit_u16<'scale, 'resolver>( + self, + value: u16, + type_id: TypeIdFor, + ) -> Result, Self::Error> { + self.visit_u128(value.into(), type_id) + } + + fn visit_u8<'scale, 'resolver>( + self, + value: u8, + type_id: TypeIdFor, + ) -> Result, Self::Error> { + self.visit_u128(value.into(), type_id) + } + + fn visit_bool<'scale, 'resolver>( + self, + value: bool, + _type_id: TypeIdFor, + ) -> Result, Self::Error> { + Ok(Value::Bool(value)) + } + + fn visit_char<'scale, 'resolver>( + self, + value: char, + _type_id: TypeIdFor, + ) -> Result, Self::Error> { + Ok(Value::Char(value)) + } + + fn visit_array<'scale, 'resolver>( + self, + values: &mut Array<'scale, 'resolver, Self::TypeResolver>, + _type_id: TypeIdFor, + ) -> Result, Self::Error> { + Ok(Value::Array(to_array(values.remaining(), values)?)) + } + + fn visit_sequence<'scale, 'resolver>( + self, + values: &mut Sequence<'scale, 'resolver, Self::TypeResolver>, + _type_id: TypeIdFor, + ) -> Result, Self::Error> { + Ok(Value::Array(to_array(values.remaining(), values)?)) + } + + fn visit_str<'scale, 'resolver>( + self, + value: &mut Str<'scale>, + _type_id: TypeIdFor, + ) -> Result, Self::Error> { + Ok(Value::String(value.as_str()?.to_owned())) + } + + fn visit_tuple<'scale, 'resolver>( + self, + values: &mut Tuple<'scale, 'resolver, Self::TypeResolver>, + _type_id: TypeIdFor, + ) -> Result, Self::Error> { + Ok(Value::Array(to_array(values.remaining(), values)?)) + } + + fn visit_bitsequence<'scale, 'resolver>( + self, + value: &mut BitSequence<'scale>, + _type_id: TypeIdFor, + ) -> Result, Self::Error> { + let bits = value.decode()?; + let mut out = Vec::with_capacity(bits.len()); + for b in bits { + let b = b.map_err(ValueError::CannotDecodeBitSequence)?; + out.push(Value::Bool(b)); + } + Ok(Value::Array(out)) + } + + fn visit_composite<'scale, 'resolver>( + self, + value: &mut Composite<'scale, 'resolver, Self::TypeResolver>, + _type_id: TypeIdFor, + ) -> Result, Self::Error> { + // Special case for ss58 addresses: + if let Some(n) = value.name() + && n == "AccountId32" + && value.bytes_from_start().len() == 32 + { + return Ok(Value::Address(value.bytes_from_start().to_vec())); + } + + // Reuse logic for decoding variant fields: + match to_variant_fieldish(value)? { + VariantFields::Named(s) => Ok(Value::Struct(s)), + VariantFields::Unnamed(a) => Ok(Value::Array(a)), + } + } + + fn visit_variant<'scale, 'resolver>( + self, + value: &mut Variant<'scale, 'resolver, Self::TypeResolver>, + _type_id: TypeIdFor, + ) -> Result, Self::Error> { + let name = value.name().to_owned(); + let fields = to_variant_fieldish(value.fields())?; + Ok(Value::Variant(name, fields)) + } + } + + fn to_variant_fieldish<'scale, 'resolver, R: TypeResolver>( + value: &mut Composite<'scale, 'resolver, R>, + ) -> Result { + // If fields are unnamed, treat as array: + if value.fields().iter().all(|f| f.name.is_none()) { + return Ok(VariantFields::Unnamed(to_array(value.remaining(), value)?)); + } + + // Otherwise object: + let mut out = HashMap::new(); + for field in value { + let field = field?; + let name = field.name().unwrap().to_string(); + let value = field.decode_with_visitor(GetValue::new())?; + out.insert(name, value); + } + Ok(VariantFields::Named(out)) + } + + fn to_array<'scale, 'resolver, R: TypeResolver>( + len: usize, + mut values: impl scale_decode::visitor::DecodeItemIterator<'scale, 'resolver, R>, + ) -> Result, ValueError> { + let mut out = Vec::with_capacity(len); + while let Some(value) = values.decode_item(GetValue::new()) { + out.push(value?); + } + Ok(out) + } +} diff --git a/historic/examples/storage.rs b/historic/examples/storage.rs index 8dacf4c169..f61ed9ac02 100644 --- a/historic/examples/storage.rs +++ b/historic/examples/storage.rs @@ -1,8 +1,8 @@ #![allow(missing_docs)] -use subxt_historic::{Error, OnlineClient, PolkadotConfig, ext::StreamExt}; +use subxt_historic::{OnlineClient, PolkadotConfig, ext::StreamExt}; #[tokio::main] -async fn main() -> Result<(), Error> { +async fn main() -> Result<(), Box> { // Configuration for the Polkadot relay chain. let config = PolkadotConfig::new(); @@ -36,6 +36,12 @@ async fn main() -> Result<(), Error> { // represent any SCALE-encoded value, like so: let _balance_info = entry.decode_as::()?; + // We can visit the value, which is a more advanced use case and allows us to extract more + // data from the type, here the name of it, if it exists: + let tn = entry + .visit(type_name::GetTypeName::new())? + .unwrap_or(""); + // Or, if we know what shape to expect, we can decode the parts of the value that we care // about directly into a static type, which is more efficient and allows easy type-safe // access, like so: @@ -53,7 +59,7 @@ async fn main() -> Result<(), Error> { let balance_info = entry.decode_as::()?; println!( - " Single balance info from {account_id_hex} => free: {} reserved: {} misc_frozen: {} fee_frozen: {}", + " Single balance info from {account_id_hex} => free: {} reserved: {} misc_frozen: {} fee_frozen: {} (type name: {tn})", balance_info.data.free, balance_info.data.reserved, balance_info.data.misc_frozen, @@ -105,3 +111,65 @@ async fn main() -> Result<(), Error> { Ok(()) } + +/// This module defines an example visitor which retrieves the name of a type. +/// This is a more advanced use case and can typically be avoided. +mod type_name { + use scale_decode::{ + Visitor, + visitor::types::{Composite, Sequence, Variant}, + visitor::{TypeIdFor, Unexpected}, + }; + use scale_type_resolver::TypeResolver; + + /// This is a visitor which obtains type names. + pub struct GetTypeName { + marker: core::marker::PhantomData, + } + + impl GetTypeName { + /// Construct our TypeName visitor. + pub fn new() -> Self { + GetTypeName { + marker: core::marker::PhantomData, + } + } + } + + impl Visitor for GetTypeName { + type Value<'scale, 'resolver> = Option<&'resolver str>; + type Error = scale_decode::Error; + type TypeResolver = R; + + // Look at the path of types that have paths and return the ident from that. + fn visit_composite<'scale, 'resolver>( + self, + value: &mut Composite<'scale, 'resolver, Self::TypeResolver>, + _type_id: TypeIdFor, + ) -> Result, Self::Error> { + Ok(value.path().last()) + } + fn visit_variant<'scale, 'resolver>( + self, + value: &mut Variant<'scale, 'resolver, Self::TypeResolver>, + _type_id: TypeIdFor, + ) -> Result, Self::Error> { + Ok(value.path().last()) + } + fn visit_sequence<'scale, 'resolver>( + self, + value: &mut Sequence<'scale, 'resolver, Self::TypeResolver>, + _type_id: TypeIdFor, + ) -> Result, Self::Error> { + Ok(value.path().last()) + } + + // Else, we return nothing as we can't find a name for the type. + fn visit_unexpected<'scale, 'resolver>( + self, + _unexpected: Unexpected, + ) -> Result, Self::Error> { + Ok(None) + } + } +} diff --git a/historic/src/extrinsics/extrinsic_call.rs b/historic/src/extrinsics/extrinsic_call.rs index f42339cd73..3f767b58a2 100644 --- a/historic/src/extrinsics/extrinsic_call.rs +++ b/historic/src/extrinsics/extrinsic_call.rs @@ -1,6 +1,7 @@ use super::extrinsic_info::{AnyExtrinsicInfo, with_info}; use crate::error::ExtrinsicCallError; use crate::utils::Either; +use crate::utils::{AnyResolver, AnyTypeId}; use scale_info_legacy::{LookupName, TypeRegistrySet}; /// This represents the call data in the extrinsic. @@ -53,6 +54,7 @@ impl<'extrinsics, 'atblock> ExtrinsicCall<'extrinsics, 'atblock> { pub struct ExtrinsicCallFields<'extrinsics, 'atblock> { all_bytes: &'extrinsics [u8], info: &'extrinsics AnyExtrinsicInfo<'atblock>, + resolver: AnyResolver<'atblock>, } impl<'extrinsics, 'atblock> ExtrinsicCallFields<'extrinsics, 'atblock> { @@ -60,7 +62,16 @@ impl<'extrinsics, 'atblock> ExtrinsicCallFields<'extrinsics, 'atblock> { all_bytes: &'extrinsics [u8], info: &'extrinsics AnyExtrinsicInfo<'atblock>, ) -> Self { - Self { all_bytes, info } + let resolver = match info { + AnyExtrinsicInfo::Legacy(info) => AnyResolver::B(info.resolver), + AnyExtrinsicInfo::Current(info) => AnyResolver::A(info.resolver), + }; + + Self { + all_bytes, + info, + resolver, + } } /// Return the bytes representing the fields stored in this extrinsic. @@ -74,11 +85,12 @@ impl<'extrinsics, 'atblock> ExtrinsicCallFields<'extrinsics, 'atblock> { } /// Iterate over each of the fields of the extrinsic call data. - pub fn iter(&self) -> impl Iterator> { + pub fn iter(&self) -> impl Iterator> { match &self.info { AnyExtrinsicInfo::Legacy(info) => { Either::A(info.info.call_data().map(|named_arg| ExtrinsicCallField { field_bytes: &self.all_bytes[named_arg.range()], + resolver: &self.resolver, info: AnyExtrinsicCallFieldInfo::Legacy(ExtrinsicCallFieldInfo { info: named_arg, resolver: info.resolver, @@ -88,6 +100,7 @@ impl<'extrinsics, 'atblock> ExtrinsicCallFields<'extrinsics, 'atblock> { AnyExtrinsicInfo::Current(info) => { Either::B(info.info.call_data().map(|named_arg| ExtrinsicCallField { field_bytes: &self.all_bytes[named_arg.range()], + resolver: &self.resolver, info: AnyExtrinsicCallFieldInfo::Current(ExtrinsicCallFieldInfo { info: named_arg, resolver: info.resolver, @@ -119,9 +132,10 @@ impl<'extrinsics, 'atblock> ExtrinsicCallFields<'extrinsics, 'atblock> { } } -pub struct ExtrinsicCallField<'extrinsics, 'atblock> { +pub struct ExtrinsicCallField<'fields, 'extrinsics, 'atblock> { field_bytes: &'extrinsics [u8], info: AnyExtrinsicCallFieldInfo<'extrinsics, 'atblock>, + resolver: &'fields AnyResolver<'atblock>, } enum AnyExtrinsicCallFieldInfo<'extrinsics, 'atblock> { @@ -144,7 +158,7 @@ macro_rules! with_call_field_info { }; } -impl<'extrinsics, 'atblock> ExtrinsicCallField<'extrinsics, 'atblock> { +impl<'fields, 'extrinsics, 'atblock> ExtrinsicCallField<'fields, 'extrinsics, 'atblock> { /// Get the raw bytes for this field. pub fn bytes(&self) -> &'extrinsics [u8] { self.field_bytes @@ -155,6 +169,22 @@ impl<'extrinsics, 'atblock> ExtrinsicCallField<'extrinsics, 'atblock> { with_call_field_info!(&self.info => info.info.name()) } + /// Visit the given field with a [`scale_decode::visitor::Visitor`]. This is like a lower level + /// version of [`ExtrinsicCallField::decode_as`], as the visitor is able to preserve lifetimes + /// and has access to more type information than is available via [`ExtrinsicCallField::decode_as`]. + pub fn visit>>( + &self, + visitor: V, + ) -> Result, V::Error> { + let type_id = match &self.info { + AnyExtrinsicCallFieldInfo::Current(info) => AnyTypeId::A(*info.info.ty()), + AnyExtrinsicCallFieldInfo::Legacy(info) => AnyTypeId::B(info.info.ty().clone()), + }; + let cursor = &mut self.bytes(); + + scale_decode::visitor::decode_with_visitor(cursor, type_id, self.resolver, visitor) + } + /// Attempt to decode the value of this field into the given type. pub fn decode_as(&self) -> Result { with_call_field_info!(&self.info => { diff --git a/historic/src/extrinsics/extrinsics_type.rs b/historic/src/extrinsics/extrinsics_type.rs index 3c7c27943d..520b314759 100644 --- a/historic/src/extrinsics/extrinsics_type.rs +++ b/historic/src/extrinsics/extrinsics_type.rs @@ -110,6 +110,3 @@ impl<'extrinsics, 'atblock> Extrinsic<'extrinsics, 'atblock> { ExtrinsicTransactionParams::new(self.bytes, self.info) } } - -// TODO add extrinsic.call() with .bytes, and .decode function to make it easy to decode call fields into Value or whatever. -// Then add this to the example. Make sure we can do everything that dot-block-decoder does easily. diff --git a/historic/src/lib.rs b/historic/src/lib.rs index d4243d1ec5..8c15ab2e6f 100644 --- a/historic/src/lib.rs +++ b/historic/src/lib.rs @@ -20,3 +20,8 @@ pub use error::Error; pub mod ext { pub use futures::stream::{Stream, StreamExt}; } + +/// Helper types that could be useful. +pub mod helpers { + pub use crate::utils::{AnyResolver, AnyResolverError, AnyTypeId}; +} diff --git a/historic/src/storage/storage_value.rs b/historic/src/storage/storage_value.rs index 16a502f7b9..c19bebaa67 100644 --- a/historic/src/storage/storage_value.rs +++ b/historic/src/storage/storage_value.rs @@ -1,6 +1,7 @@ use super::storage_info::AnyStorageInfo; use super::storage_info::with_info; use crate::error::StorageValueError; +use crate::utils::{AnyResolver, AnyTypeId}; use scale_decode::DecodeAsType; use std::borrow::Cow; use std::sync::Arc; @@ -9,12 +10,22 @@ use std::sync::Arc; pub struct StorageValue<'atblock> { pub(crate) info: Arc>, bytes: Cow<'atblock, [u8]>, + resolver: AnyResolver<'atblock>, } impl<'atblock> StorageValue<'atblock> { /// Create a new storage value. - pub fn new(info: Arc>, bytes: Cow<'atblock, [u8]>) -> Self { - Self { info, bytes } + pub(crate) fn new(info: Arc>, bytes: Cow<'atblock, [u8]>) -> Self { + let resolver = match &*info { + AnyStorageInfo::Current(info) => AnyResolver::A(info.resolver), + AnyStorageInfo::Legacy(info) => AnyResolver::B(info.resolver), + }; + + Self { + info, + bytes, + resolver, + } } /// Get the raw bytes for this storage value. @@ -27,6 +38,22 @@ impl<'atblock> StorageValue<'atblock> { self.bytes.to_vec() } + /// Visit the given field with a [`scale_decode::visitor::Visitor`]. This is like a lower level + /// version of [`StorageValue::decode_as`], as the visitor is able to preserve lifetimes + /// and has access to more type information than is available via [`StorageValue::decode_as`]. + pub fn visit>>( + &self, + visitor: V, + ) -> Result, V::Error> { + let type_id = match &*self.info { + AnyStorageInfo::Current(info) => AnyTypeId::A(info.info.value_id), + AnyStorageInfo::Legacy(info) => AnyTypeId::B(info.info.value_id.clone()), + }; + let cursor = &mut self.bytes(); + + scale_decode::visitor::decode_with_visitor(cursor, type_id, &self.resolver, visitor) + } + /// Decode this storage value. pub fn decode_as(&self) -> Result { with_info!(info = &*self.info => { diff --git a/historic/src/utils.rs b/historic/src/utils.rs index c715c91ed4..4a4edf859b 100644 --- a/historic/src/utils.rs +++ b/historic/src/utils.rs @@ -1,5 +1,7 @@ +mod any_resolver; mod either; mod range_map; +pub use any_resolver::{AnyResolver, AnyResolverError, AnyTypeId}; pub use either::Either; pub use range_map::RangeMap; diff --git a/historic/src/utils/any_resolver.rs b/historic/src/utils/any_resolver.rs new file mode 100644 index 0000000000..8d92bcf4bd --- /dev/null +++ b/historic/src/utils/any_resolver.rs @@ -0,0 +1,188 @@ +use super::Either; +use scale_info_legacy::LookupName; +use scale_type_resolver::ResolvedTypeVisitor; + +/// A type resolver which could either be for modern or historic resolving. +pub type AnyResolver<'resolver> = Either< + &'resolver scale_info::PortableRegistry, + &'resolver scale_info_legacy::TypeRegistrySet<'resolver>, +>; + +/// A type ID which is either a modern or historic ID. +pub type AnyTypeId = Either; + +impl Default for AnyTypeId { + fn default() -> Self { + // Not a sensible default, but we don't need / can't provide a sensible one. + AnyTypeId::A(u32::MAX) + } +} +impl From for AnyTypeId { + fn from(value: u32) -> Self { + AnyTypeId::A(value) + } +} +impl From for AnyTypeId { + fn from(value: LookupName) -> Self { + AnyTypeId::B(value) + } +} +impl TryFrom for u32 { + type Error = (); + fn try_from(value: AnyTypeId) -> Result { + match value { + AnyTypeId::A(v) => Ok(v), + AnyTypeId::B(_) => Err(()), + } + } +} +impl TryFrom for LookupName { + type Error = (); + fn try_from(value: AnyTypeId) -> Result { + match value { + AnyTypeId::A(_) => Err(()), + AnyTypeId::B(v) => Ok(v), + } + } +} + +/// A resolve error that comes from using [`AnyResolver`] to resolve some [`AnyTypeId`] into a type. +#[derive(Debug, thiserror::Error)] +pub enum AnyResolverError { + #[error("got a {got} type ID but expected a {expected} type ID")] + TypeIdMismatch { + got: &'static str, + expected: &'static str, + }, + #[error("{0}")] + ScaleInfo(scale_type_resolver::portable_registry::Error), + #[error("{0}")] + ScaleInfoLegacy(scale_info_legacy::type_registry::TypeRegistryResolveError), +} + +impl<'resolver> scale_type_resolver::TypeResolver for AnyResolver<'resolver> { + type TypeId = AnyTypeId; + type Error = AnyResolverError; + + fn resolve_type<'this, V: ResolvedTypeVisitor<'this, TypeId = Self::TypeId>>( + &'this self, + type_id: Self::TypeId, + visitor: V, + ) -> Result { + match (self, type_id) { + (Either::A(resolver), Either::A(id)) => resolver + .resolve_type(id, ModernVisitor(visitor)) + .map_err(AnyResolverError::ScaleInfo), + (Either::B(resolver), Either::B(id)) => resolver + .resolve_type(id, LegacyVisitor(visitor)) + .map_err(AnyResolverError::ScaleInfoLegacy), + (Either::A(_), Either::B(_)) => Err(AnyResolverError::TypeIdMismatch { + got: "LookupName", + expected: "u32", + }), + (Either::B(_), Either::A(_)) => Err(AnyResolverError::TypeIdMismatch { + got: "u32", + expected: "LookupName", + }), + } + } +} + +// We need to have a visitor which understands only modern or legacy types, and can wrap the more generic visitor +// that must be provided to AnyResolver::resolve_type. This then allows us to visit historic _or_ modern types +// using the single visitor provided by the user. +struct LegacyVisitor(V); +struct ModernVisitor(V); + +mod impls { + use super::{AnyTypeId, LegacyVisitor, LookupName, ModernVisitor}; + use scale_type_resolver::*; + + // An ugly implementation which maps from modern or legacy types into our AnyTypeId, + // to make LegacyVisitor and ModernVisitor valid visitors when wrapping a generic "any" visitor. + macro_rules! impl_visitor_mapper { + ($struc:ident, $type_id_ty:ident, $variant:ident) => { + impl<'this, V> ResolvedTypeVisitor<'this> for $struc + where + V: ResolvedTypeVisitor<'this, TypeId = AnyTypeId>, + { + type TypeId = $type_id_ty; + type Value = V::Value; + + fn visit_unhandled(self, kind: UnhandledKind) -> Self::Value { + self.0.visit_unhandled(kind) + } + fn visit_array(self, type_id: Self::TypeId, len: usize) -> Self::Value { + self.0.visit_array(AnyTypeId::$variant(type_id), len) + } + fn visit_not_found(self) -> Self::Value { + self.0.visit_not_found() + } + fn visit_composite(self, path: Path, fields: Fields) -> Self::Value + where + Path: PathIter<'this>, + Fields: FieldIter<'this, Self::TypeId>, + { + self.0.visit_composite( + path, + fields.map(|field| Field { + name: field.name, + id: AnyTypeId::$variant(field.id), + }), + ) + } + fn visit_variant(self, path: Path, variants: Var) -> Self::Value + where + Path: PathIter<'this>, + Fields: FieldIter<'this, Self::TypeId>, + Var: VariantIter<'this, Fields>, + { + self.0.visit_variant( + path, + variants.map(|variant| Variant { + index: variant.index, + name: variant.name, + fields: variant.fields.map(|field| Field { + name: field.name, + id: AnyTypeId::$variant(field.id), + }), + }), + ) + } + fn visit_sequence(self, path: Path, type_id: Self::TypeId) -> Self::Value + where + Path: PathIter<'this>, + { + self.0.visit_sequence(path, AnyTypeId::$variant(type_id)) + } + + fn visit_tuple(self, type_ids: TypeIds) -> Self::Value + where + TypeIds: ExactSizeIterator, + { + self.0 + .visit_tuple(type_ids.map(|id| AnyTypeId::$variant(id))) + } + + fn visit_primitive(self, primitive: Primitive) -> Self::Value { + self.0.visit_primitive(primitive) + } + + fn visit_compact(self, type_id: Self::TypeId) -> Self::Value { + self.0.visit_compact(AnyTypeId::$variant(type_id)) + } + + fn visit_bit_sequence( + self, + store_format: BitsStoreFormat, + order_format: BitsOrderFormat, + ) -> Self::Value { + self.0.visit_bit_sequence(store_format, order_format) + } + } + }; + } + + impl_visitor_mapper!(ModernVisitor, u32, A); + impl_visitor_mapper!(LegacyVisitor, LookupName, B); +} diff --git a/historic/src/utils/either.rs b/historic/src/utils/either.rs index a81fe55b11..081b52e92e 100644 --- a/historic/src/utils/either.rs +++ b/historic/src/utils/either.rs @@ -1,5 +1,6 @@ macro_rules! either { ($name:ident( $fst:ident, $($variant:ident),* )) => { + #[derive(Clone, Copy, Debug)] pub enum $name<$fst, $($variant),*> { $fst($fst), $($variant($variant),)* diff --git a/metadata/src/lib.rs b/metadata/src/lib.rs index 53b1402e08..daadc37205 100644 --- a/metadata/src/lib.rs +++ b/metadata/src/lib.rs @@ -374,6 +374,27 @@ impl Metadata { ::decode(&mut bytes) } + /// Convert V16 metadata into [`Metadata`]. + pub fn from_v16( + metadata: frame_metadata::v16::RuntimeMetadataV16, + ) -> Result { + metadata.try_into() + } + + /// Convert V15 metadata into [`Metadata`]. + pub fn from_v15( + metadata: frame_metadata::v15::RuntimeMetadataV15, + ) -> Result { + metadata.try_into() + } + + /// Convert V14 metadata into [`Metadata`]. + pub fn from_v14( + metadata: frame_metadata::v14::RuntimeMetadataV14, + ) -> Result { + metadata.try_into() + } + /// Convert V13 metadata into [`Metadata`], given the necessary extra type information. #[cfg(feature = "legacy")] pub fn from_v13( @@ -1222,6 +1243,38 @@ impl<'a> CustomValueMetadata<'a> { } } +/// Decode SCALE encoded metadata. +/// +/// - The default assumption is that metadata is encoded as [`frame_metadata::RuntimeMetadataPrefixed`]. This is the +/// expected format that metadata is encoded into. +/// - if this fails, we also try to decode as [`frame_metadata::RuntimeMetadata`]. +/// - If this all fails, we also try to decode as [`frame_metadata::OpaqueMetadata`]. +pub fn decode_runtime_metadata( + input: &[u8], +) -> Result { + use codec::Decode; + + let err = match frame_metadata::RuntimeMetadataPrefixed::decode(&mut &*input) { + Ok(md) => return Ok(md.1), + Err(e) => e, + }; + + if let Ok(md) = frame_metadata::RuntimeMetadata::decode(&mut &*input) { + return Ok(md); + } + + // frame_metadata::OpaqueMetadata is a vec of bytes. If we can decode the length, AND + // the length definitely corresponds to the number of remaining bytes, then we try to + // decode the inner bytes. + if let Ok(len) = codec::Compact::::decode(&mut &*input) { + if input.len() == len.0 as usize { + return decode_runtime_metadata(input); + } + } + + Err(err) +} + // Support decoding metadata from the "wire" format directly into this. // Errors may be lost in the case that the metadata content is somehow invalid. impl codec::Decode for Metadata { @@ -1231,9 +1284,11 @@ impl codec::Decode for Metadata { frame_metadata::RuntimeMetadata::V14(md) => md.try_into(), frame_metadata::RuntimeMetadata::V15(md) => md.try_into(), frame_metadata::RuntimeMetadata::V16(md) => md.try_into(), - _ => return Err("Cannot try_into() to Metadata: unsupported metadata version".into()), + _ => { + return Err("Metadata::decode failed: Cannot try_into() to Metadata: unsupported metadata version".into()) + }, }; - metadata.map_err(|_e| "Cannot try_into() to Metadata.".into()) + metadata.map_err(|_| "Metadata::decode failed: Cannot try_into() to Metadata".into()) } } diff --git a/testing/integration-tests/src/full_client/blocks.rs b/testing/integration-tests/src/full_client/blocks.rs index 9a81e2290d..770290a5fa 100644 --- a/testing/integration-tests/src/full_client/blocks.rs +++ b/testing/integration-tests/src/full_client/blocks.rs @@ -401,7 +401,7 @@ async fn decode_block_mortality() { } // Explicit Mortal: - for for_n_blocks in [4, 16, 128] { + for for_n_blocks in [16, 64, 128] { let tx = submit_extrinsic_and_get_it_back( &api, DefaultExtrinsicParamsBuilder::new().mortal(for_n_blocks),