diff --git a/cli/src/commands/diff.rs b/cli/src/commands/diff.rs new file mode 100644 index 0000000000..9200e85209 --- /dev/null +++ b/cli/src/commands/diff.rs @@ -0,0 +1,464 @@ +use clap::Args; +use codec::Decode; + +use frame_metadata::RuntimeMetadataPrefixed; +use std::collections::HashMap; +use std::hash::Hash; + +use crate::utils::FileOrUrl; +use color_eyre::owo_colors::OwoColorize; + +use scale_info::form::PortableForm; +use scale_info::Variant; + +use subxt_metadata::{ + ConstantMetadata, Metadata, PalletMetadata, RuntimeApiMetadata, StorageEntryMetadata, + StorageEntryType, +}; + +/// Explore the differences between two nodes +/// +/// # Example +/// ``` +/// subxt diff ./artifacts/polkadot_metadata_small.scale ./artifacts/polkadot_metadata_tiny.scale +/// subxt diff ./artifacts/polkadot_metadata_small.scale wss://rpc.polkadot.io:443 +/// ``` +#[derive(Debug, Args)] +#[command(author, version, about, long_about = None)] +pub struct Opts { + /// metadata file or node URL + metadata_or_url_1: FileOrUrl, + /// metadata file or node URL + metadata_or_url_2: FileOrUrl, +} + +pub async fn run(opts: Opts, output: &mut impl std::io::Write) -> color_eyre::Result<()> { + let (entry_1_metadata, entry_2_metadata) = get_metadata(&opts).await?; + + let metadata_diff = MetadataDiff::construct(&entry_1_metadata, &entry_2_metadata); + + if metadata_diff.is_empty() { + writeln!(output, "No difference in metadata found.")?; + return Ok(()); + } + if !metadata_diff.pallets.is_empty() { + writeln!(output, "Pallets:")?; + for diff in metadata_diff.pallets { + match diff { + Diff::Added(new) => { + writeln!(output, "{}", format!(" + {}", new.name()).green())? + } + Diff::Removed(old) => { + writeln!(output, "{}", format!(" - {}", old.name()).red())? + } + Diff::Changed { from, to } => { + writeln!(output, "{}", format!(" ~ {}", from.name()).yellow())?; + + let pallet_diff = PalletDiff::construct(&from, &to); + if !pallet_diff.calls.is_empty() { + writeln!(output, " Calls:")?; + for diff in pallet_diff.calls { + match diff { + Diff::Added(new) => writeln!( + output, + "{}", + format!(" + {}", &new.name).green() + )?, + Diff::Removed(old) => writeln!( + output, + "{}", + format!(" - {}", &old.name).red() + )?, + Diff::Changed { from, to: _ } => { + writeln!( + output, + "{}", + format!(" ~ {}", &from.name).yellow() + )?; + } + } + } + } + + if !pallet_diff.constants.is_empty() { + writeln!(output, " Constants:")?; + for diff in pallet_diff.constants { + match diff { + Diff::Added(new) => writeln!( + output, + "{}", + format!(" + {}", new.name()).green() + )?, + Diff::Removed(old) => writeln!( + output, + "{}", + format!(" - {}", old.name()).red() + )?, + Diff::Changed { from, to: _ } => writeln!( + output, + "{}", + format!(" ~ {}", from.name()).yellow() + )?, + } + } + } + + if !pallet_diff.storage_entries.is_empty() { + writeln!(output, " Storage Entries:")?; + for diff in pallet_diff.storage_entries { + match diff { + Diff::Added(new) => writeln!( + output, + "{}", + format!(" + {}", new.name()).green() + )?, + Diff::Removed(old) => writeln!( + output, + "{}", + format!(" - {}", old.name()).red() + )?, + Diff::Changed { from, to } => { + let storage_diff = StorageEntryDiff::construct( + from, + to, + &entry_1_metadata, + &entry_2_metadata, + ); + + writeln!( + output, + "{}", + format!( + " ~ {} (Changed: {})", + from.name(), + storage_diff.to_strings().join(", ") + ) + .yellow() + )?; + } + } + } + } + } + } + } + } + + if !metadata_diff.runtime_apis.is_empty() { + writeln!(output, "Runtime APIs:")?; + for diff in metadata_diff.runtime_apis { + match diff { + Diff::Added(new) => { + writeln!(output, "{}", format!(" + {}", new.name()).green())? + } + Diff::Removed(old) => { + writeln!(output, "{}", format!(" - {}", old.name()).red())? + } + Diff::Changed { from, to: _ } => { + writeln!(output, "{}", format!(" ~ {}", from.name()).yellow())? + } + } + } + } + Ok(()) +} + +struct MetadataDiff<'a> { + pallets: Vec>>, + runtime_apis: Vec>>, +} + +impl<'a> MetadataDiff<'a> { + fn construct(metadata_1: &'a Metadata, metadata_2: &'a Metadata) -> MetadataDiff<'a> { + let pallets = pallet_differences(metadata_1, metadata_2); + let runtime_apis = runtime_api_differences(metadata_1, metadata_2); + MetadataDiff { + pallets, + runtime_apis, + } + } + + fn is_empty(&self) -> bool { + self.pallets.is_empty() && self.runtime_apis.is_empty() + } +} + +#[derive(Default)] +struct PalletDiff<'a> { + calls: Vec>>, + constants: Vec>, + storage_entries: Vec>, +} + +impl<'a> PalletDiff<'a> { + fn construct( + pallet_metadata_1: &'a PalletMetadata<'a>, + pallet_metadata_2: &'a PalletMetadata<'a>, + ) -> PalletDiff<'a> { + let calls = calls_differences(pallet_metadata_1, pallet_metadata_2); + let constants = constants_differences(pallet_metadata_1, pallet_metadata_2); + let storage_entries = storage_differences(pallet_metadata_1, pallet_metadata_2); + PalletDiff { + calls, + constants, + storage_entries, + } + } +} + +struct StorageEntryDiff { + key_different: bool, + value_different: bool, + default_different: bool, + modifier_different: bool, +} + +impl StorageEntryDiff { + fn construct( + storage_entry_1: &StorageEntryMetadata, + storage_entry_2: &StorageEntryMetadata, + metadata_1: &Metadata, + metadata_2: &Metadata, + ) -> Self { + let value_1_ty_id = match storage_entry_1.entry_type() { + StorageEntryType::Plain(value_ty) | StorageEntryType::Map { value_ty, .. } => value_ty, + }; + let value_1_hash = metadata_1 + .type_hash(*value_1_ty_id) + .expect("type should be present"); + let value_2_ty_id = match storage_entry_2.entry_type() { + StorageEntryType::Plain(value_ty) | StorageEntryType::Map { value_ty, .. } => value_ty, + }; + let value_2_hash = metadata_1 + .type_hash(*value_2_ty_id) + .expect("type should be present"); + let value_different = value_1_hash != value_2_hash; + + let key_1_hash = match storage_entry_1.entry_type() { + StorageEntryType::Plain(_) => None, + StorageEntryType::Map { key_ty, .. } => Some(*key_ty), + } + .map(|key_ty| { + metadata_1 + .type_hash(key_ty) + .expect("type should be present") + }) + .unwrap_or_default(); + let key_2_hash = match storage_entry_2.entry_type() { + StorageEntryType::Plain(_) => None, + StorageEntryType::Map { key_ty, .. } => Some(*key_ty), + } + .map(|key_ty| { + metadata_2 + .type_hash(key_ty) + .expect("type should be present") + }) + .unwrap_or_default(); + let key_different = key_1_hash != key_2_hash; + + StorageEntryDiff { + key_different, + value_different, + default_different: storage_entry_1.default_bytes() != storage_entry_2.default_bytes(), + modifier_different: storage_entry_1.modifier() != storage_entry_2.modifier(), + } + } + + fn to_strings(&self) -> Vec<&str> { + let mut strings = Vec::<&str>::new(); + if self.key_different { + strings.push("key type"); + } + if self.value_different { + strings.push("value type"); + } + if self.modifier_different { + strings.push("modifier"); + } + if self.default_different { + strings.push("default value"); + } + strings + } +} + +async fn get_metadata(opts: &Opts) -> color_eyre::Result<(Metadata, Metadata)> { + let bytes = opts.metadata_or_url_1.fetch().await?; + let entry_1_metadata: Metadata = + RuntimeMetadataPrefixed::decode(&mut &bytes[..])?.try_into()?; + + let bytes = opts.metadata_or_url_2.fetch().await?; + let entry_2_metadata: Metadata = + RuntimeMetadataPrefixed::decode(&mut &bytes[..])?.try_into()?; + + Ok((entry_1_metadata, entry_2_metadata)) +} + +fn storage_differences<'a>( + pallet_metadata_1: &'a PalletMetadata<'a>, + pallet_metadata_2: &'a PalletMetadata<'a>, +) -> Vec> { + diff( + pallet_metadata_1 + .storage() + .map(|s| s.entries()) + .unwrap_or_default(), + pallet_metadata_2 + .storage() + .map(|s| s.entries()) + .unwrap_or_default(), + |e| { + pallet_metadata_1 + .storage_hash(e.name()) + .expect("storage entry should be present") + }, + |e| { + pallet_metadata_2 + .storage_hash(e.name()) + .expect("storage entry should be present") + }, + |e| e.name(), + ) +} + +fn calls_differences<'a>( + pallet_metadata_1: &'a PalletMetadata<'a>, + pallet_metadata_2: &'a PalletMetadata<'a>, +) -> Vec>> { + return diff( + pallet_metadata_1.call_variants().unwrap_or_default(), + pallet_metadata_2.call_variants().unwrap_or_default(), + |e| { + pallet_metadata_1 + .call_hash(&e.name) + .expect("call should be present") + }, + |e| { + pallet_metadata_2 + .call_hash(&e.name) + .expect("call should be present") + }, + |e| &e.name, + ); +} + +fn constants_differences<'a>( + pallet_metadata_1: &'a PalletMetadata<'a>, + pallet_metadata_2: &'a PalletMetadata<'a>, +) -> Vec> { + diff( + pallet_metadata_1.constants(), + pallet_metadata_2.constants(), + |e| { + pallet_metadata_1 + .constant_hash(e.name()) + .expect("constant should be present") + }, + |e| { + pallet_metadata_2 + .constant_hash(e.name()) + .expect("constant should be present") + }, + |e| e.name(), + ) +} + +fn runtime_api_differences<'a>( + metadata_1: &'a Metadata, + metadata_2: &'a Metadata, +) -> Vec>> { + diff( + metadata_1.runtime_api_traits(), + metadata_2.runtime_api_traits(), + RuntimeApiMetadata::hash, + RuntimeApiMetadata::hash, + RuntimeApiMetadata::name, + ) +} + +fn pallet_differences<'a>( + metadata_1: &'a Metadata, + metadata_2: &'a Metadata, +) -> Vec>> { + diff( + metadata_1.pallets(), + metadata_2.pallets(), + PalletMetadata::hash, + PalletMetadata::hash, + PalletMetadata::name, + ) +} + +#[derive(Debug, Clone, PartialEq, Eq)] +enum Diff { + Added(T), + Changed { from: T, to: T }, + Removed(T), +} + +fn diff( + items_a: impl IntoIterator, + items_b: impl IntoIterator, + hash_fn_a: impl Fn(&T) -> C, + hash_fn_b: impl Fn(&T) -> C, + key_fn: impl Fn(&T) -> I, +) -> Vec> { + let mut entries: HashMap, Option)> = HashMap::new(); + + for t1 in items_a { + let key = key_fn(&t1); + let (e1, _) = entries.entry(key).or_default(); + *e1 = Some(t1); + } + + for t2 in items_b { + let key = key_fn(&t2); + let (e1, e2) = entries.entry(key).or_default(); + // skip all entries with the same hash: + if let Some(e1_inner) = e1 { + let e1_hash = hash_fn_a(e1_inner); + let e2_hash = hash_fn_b(&t2); + if e1_hash == e2_hash { + entries.remove(&key_fn(&t2)); + continue; + } + } + *e2 = Some(t2); + } + + // sort the values by key before returning + let mut diff_vec_with_keys: Vec<_> = entries.into_iter().collect(); + diff_vec_with_keys.sort_by(|a, b| a.0.cmp(&b.0)); + diff_vec_with_keys + .into_iter() + .map(|(_, tuple)| match tuple { + (None, None) => panic!("At least one value is inserted when the key exists; qed"), + (Some(old), None) => Diff::Removed(old), + (None, Some(new)) => Diff::Added(new), + (Some(old), Some(new)) => Diff::Changed { from: old, to: new }, + }) + .collect() +} + +#[cfg(test)] +mod test { + use crate::commands::diff::{diff, Diff}; + + #[test] + fn test_diff_fn() { + let old_pallets = [("Babe", 7), ("Claims", 9), ("Balances", 23)]; + let new_pallets = [("Claims", 9), ("Balances", 22), ("System", 3), ("NFTs", 5)]; + let hash_fn = |e: &(&str, i32)| e.0.len() as i32 * e.1; + let differences = diff(old_pallets, new_pallets, hash_fn, hash_fn, |e| e.0); + let expected_differences = vec![ + Diff::Removed(("Babe", 7)), + Diff::Changed { + from: ("Balances", 23), + to: ("Balances", 22), + }, + Diff::Added(("NFTs", 5)), + Diff::Added(("System", 3)), + ]; + assert_eq!(differences, expected_differences); + } +} diff --git a/cli/src/commands/explore/mod.rs b/cli/src/commands/explore/mod.rs index 4a3334b37d..dba52364d8 100644 --- a/cli/src/commands/explore/mod.rs +++ b/cli/src/commands/explore/mod.rs @@ -80,7 +80,6 @@ pub enum PalletSubcommand { Storage(StorageSubcommand), } -/// cargo run -- explore --file=../artifacts/polkadot_metadata.scale pub async fn run(opts: Opts, output: &mut impl std::io::Write) -> color_eyre::Result<()> { // get the metadata let bytes = opts.file_or_url.fetch().await?; diff --git a/cli/src/commands/explore/storage.rs b/cli/src/commands/explore/storage.rs index a26778d317..0796386464 100644 --- a/cli/src/commands/explore/storage.rs +++ b/cli/src/commands/explore/storage.rs @@ -47,7 +47,7 @@ pub async fn explore_storage( }; // if specified call storage entry wrong, show user the storage entries to choose from (but this time as an error): - let Some(storage) = storage_metadata.entries().find(|entry| entry.name().to_lowercase() == entry_name.to_lowercase()) else { + let Some(storage) = storage_metadata.entries().iter().find(|entry| entry.name().to_lowercase() == entry_name.to_lowercase()) else { let storage_entries = print_available_storage_entries(storage_metadata, pallet_name); let description = format!("Usage:\n subxt explore {pallet_name} storage \n view details for a specific storage entry\n\n{storage_entries}"); return Err(eyre!("Storage entry \"{entry_name}\" not found in \"{pallet_name}\" pallet!\n\n{description}")); @@ -164,14 +164,18 @@ fn print_available_storage_entries( storage_metadata: &StorageMetadata, pallet_name: &str, ) -> String { - if storage_metadata.entries().len() == 0 { + if storage_metadata.entries().is_empty() { format!("No 's available in the \"{pallet_name}\" pallet.") } else { let mut output = format!( "Available 's in the \"{}\" pallet:", pallet_name ); - let mut strings: Vec<_> = storage_metadata.entries().map(|s| s.name()).collect(); + let mut strings: Vec<_> = storage_metadata + .entries() + .iter() + .map(|s| s.name()) + .collect(); strings.sort(); for entry in strings { write!(output, "\n {}", entry).unwrap(); diff --git a/cli/src/commands/mod.rs b/cli/src/commands/mod.rs index 69542ad40d..569c93bcb0 100644 --- a/cli/src/commands/mod.rs +++ b/cli/src/commands/mod.rs @@ -4,6 +4,7 @@ pub mod codegen; pub mod compatibility; +pub mod diff; pub mod explore; pub mod metadata; pub mod version; diff --git a/cli/src/main.rs b/cli/src/main.rs index d605e54a38..a2c74a74f1 100644 --- a/cli/src/main.rs +++ b/cli/src/main.rs @@ -15,6 +15,7 @@ enum Command { Metadata(commands::metadata::Opts), Codegen(commands::codegen::Opts), Compatibility(commands::compatibility::Opts), + Diff(commands::diff::Opts), Version(commands::version::Opts), Explore(commands::explore::Opts), } @@ -28,6 +29,7 @@ async fn main() -> color_eyre::Result<()> { Command::Metadata(opts) => commands::metadata::run(opts, &mut output).await, Command::Codegen(opts) => commands::codegen::run(opts, &mut output).await, Command::Compatibility(opts) => commands::compatibility::run(opts, &mut output).await, + Command::Diff(opts) => commands::diff::run(opts, &mut output).await, Command::Version(opts) => commands::version::run(opts, &mut output), Command::Explore(opts) => commands::explore::run(opts, &mut output).await, } diff --git a/cli/src/utils.rs b/cli/src/utils.rs index 78c869fc58..2a549b0a56 100644 --- a/cli/src/utils.rs +++ b/cli/src/utils.rs @@ -5,14 +5,16 @@ use clap::Args; use color_eyre::eyre; +use std::str::FromStr; use std::{fs, io::Read, path::PathBuf}; + use subxt_codegen::utils::{MetadataVersion, Uri}; pub mod type_description; pub mod type_example; /// The source of the metadata. -#[derive(Debug, Args)] +#[derive(Debug, Args, Clone)] pub struct FileOrUrl { /// The url of the substrate node to query for metadata for codegen. #[clap(long, value_parser)] @@ -94,3 +96,26 @@ pub fn with_indent(s: String, indent: usize) -> String { .collect::>() .join("\n") } + +impl FromStr for FileOrUrl { + type Err = &'static str; + + fn from_str(s: &str) -> Result { + let path = std::path::Path::new(s); + if path.exists() { + Ok(FileOrUrl { + url: None, + file: Some(PathBuf::from(s)), + version: None, + }) + } else { + Uri::from_str(s) + .map_err(|_| "no path or uri could be crated") + .map(|uri| FileOrUrl { + url: Some(uri), + file: None, + version: None, + }) + } + } +} diff --git a/codegen/src/api/storage.rs b/codegen/src/api/storage.rs index ca432082e8..640234b698 100644 --- a/codegen/src/api/storage.rs +++ b/codegen/src/api/storage.rs @@ -30,11 +30,12 @@ pub fn generate_storage( should_gen_docs: bool, ) -> Result { let Some(storage) = pallet.storage() else { - return Ok(quote!()) + return Ok(quote!()); }; let storage_fns = storage .entries() + .iter() .map(|entry| { generate_storage_entry_fns(type_gen, pallet, entry, crate_path, should_gen_docs) }) @@ -104,7 +105,7 @@ fn generate_storage_entry_fns( let pallet_name = pallet.name(); let storage_name = storage_entry.name(); let Some(storage_hash) = pallet.storage_hash(storage_name) else { - return Err(CodegenError::MissingStorageMetadata(pallet_name.into(), storage_name.into())) + return Err(CodegenError::MissingStorageMetadata(pallet_name.into(), storage_name.into())); }; let fn_name = format_ident!("{}", storage_entry.name().to_snake_case()); @@ -157,7 +158,7 @@ fn generate_storage_entry_fns( // so expose a function to create this entry, too: let root_entry_fn = if is_map_type { let fn_name_root = format_ident!("{}_root", fn_name); - quote! ( + quote!( #docs pub fn #fn_name_root( &self, diff --git a/metadata/src/lib.rs b/metadata/src/lib.rs index b629cbe47e..4d6620d04e 100644 --- a/metadata/src/lib.rs +++ b/metadata/src/lib.rs @@ -20,7 +20,7 @@ mod from_into; mod utils; use scale_info::{form::PortableForm, PortableRegistry, Variant}; -use std::collections::HashMap; +use std::collections::{HashMap, HashSet}; use std::sync::Arc; use utils::ordered_map::OrderedMap; use utils::variant_index::VariantIndex; @@ -138,6 +138,16 @@ impl Metadata { { utils::retain::retain_metadata(self, pallet_filter, api_filter); } + + /// Get type hash for a type in the registry + pub fn type_hash(&self, id: u32) -> Option<[u8; 32]> { + self.types.resolve(id)?; + Some(crate::utils::validation::get_type_hash( + &self.types, + id, + &mut HashSet::::new(), + )) + } } /// Metadata for a specific pallet. @@ -303,8 +313,8 @@ impl StorageMetadata { } /// An iterator over the storage entries. - pub fn entries(&self) -> impl ExactSizeIterator { - self.entries.values().iter() + pub fn entries(&self) -> &[StorageEntryMetadata] { + self.entries.values() } /// Return a specific storage entry given its name. @@ -387,7 +397,7 @@ pub enum StorageHasher { } /// Is the storage entry optional, or does it have a default value. -#[derive(Debug, Clone, Copy)] +#[derive(Debug, Clone, Copy, Eq, PartialEq)] pub enum StorageEntryModifier { /// The storage entry returns an `Option`, with `None` if the key is not present. Optional, @@ -490,7 +500,7 @@ pub struct RuntimeApiMetadata<'a> { impl<'a> RuntimeApiMetadata<'a> { /// Trait name. - pub fn name(&self) -> &str { + pub fn name(&self) -> &'a str { &self.inner.name } /// Trait documentation. @@ -509,6 +519,11 @@ impl<'a> RuntimeApiMetadata<'a> { pub fn method_hash(&self, method_name: &str) -> Option<[u8; 32]> { crate::utils::validation::get_runtime_api_hash(self, method_name) } + + /// Return a hash for the runtime API trait. + pub fn hash(&self) -> [u8; 32] { + crate::utils::validation::get_runtime_trait_hash(*self) + } } #[derive(Debug, Clone)] diff --git a/metadata/src/utils/validation.rs b/metadata/src/utils/validation.rs index 6c58f396a3..a16f33fae9 100644 --- a/metadata/src/utils/validation.rs +++ b/metadata/src/utils/validation.rs @@ -175,7 +175,7 @@ fn get_type_def_hash( } /// Obtain the hash representation of a `scale_info::Type` identified by id. -fn get_type_hash( +pub fn get_type_hash( registry: &PortableRegistry, id: u32, visited_ids: &mut HashSet, @@ -283,7 +283,7 @@ fn get_runtime_method_hash( } /// Obtain the hash of all of a runtime API trait, including all of its methods. -fn get_runtime_trait_hash(trait_metadata: RuntimeApiMetadata) -> [u8; HASH_LEN] { +pub fn get_runtime_trait_hash(trait_metadata: RuntimeApiMetadata) -> [u8; HASH_LEN] { let mut visited_ids = HashSet::new(); let trait_name = &*trait_metadata.inner.name; let method_bytes = trait_metadata @@ -379,14 +379,17 @@ pub fn get_pallet_hash(pallet: PalletMetadata) -> [u8; HASH_LEN] { let storage_bytes = match pallet.storage() { Some(storage) => { let prefix_hash = hash(storage.prefix().as_bytes()); - let entries_hash = storage.entries().fold([0u8; HASH_LEN], |bytes, entry| { - // We don't care what order the storage entries occur in, so XOR them together - // to make the order irrelevant. - xor( - bytes, - get_storage_entry_hash(registry, entry, &mut visited_ids), - ) - }); + let entries_hash = storage + .entries() + .iter() + .fold([0u8; HASH_LEN], |bytes, entry| { + // We don't care what order the storage entries occur in, so XOR them together + // to make the order irrelevant. + xor( + bytes, + get_storage_entry_hash(registry, entry, &mut visited_ids), + ) + }); concat_and_hash2(&prefix_hash, &entries_hash) } None => [0u8; HASH_LEN], @@ -496,6 +499,7 @@ mod tests { struct A { pub b: Box, } + #[allow(dead_code)] #[derive(scale_info::TypeInfo)] struct B { @@ -507,6 +511,7 @@ mod tests { #[derive(scale_info::TypeInfo)] // TypeDef::Composite with TypeDef::Array with Typedef::Primitive. struct AccountId32([u8; HASH_LEN]); + #[allow(dead_code)] #[derive(scale_info::TypeInfo)] // TypeDef::Variant. @@ -525,6 +530,7 @@ mod tests { // TypeDef::BitSequence. BitSeq(BitVec), } + #[allow(dead_code)] #[derive(scale_info::TypeInfo)] // Ensure recursive types and TypeDef variants are captured. @@ -533,6 +539,7 @@ mod tests { composite: AccountId32, type_def: DigestItem, } + #[allow(dead_code)] #[derive(scale_info::TypeInfo)] // Simulate a PalletCallMetadata.