Rework Subxt API to support offline and dynamic transactions (#593)

* WIP API changes * debug impls * Get main crate compiling with first round of changes * Some tidy up * Add WithExtrinsicParams, and have SubstrateConfig + PolkadotConfig, not DefaultConfig * move transaction into extrinsic folder * Add runtime updates back to OnlineClient * rework to be 'client first' to fit better with storage + events * add support for events to Client * tidy dupe trait bound * Wire storage into client, but need to remove static reliance * various tidy up and start stripping codegen to remove bits we dont need now * First pass updating calls and constants codegen * WIP storage client updates * First pass migrated runtime storage over to new format * pass over codegen to generate StorageAddresses and throw other stuff out * don't need a Call trait any more * shuffle things around a bit * Various proc_macro fixes to get 'cargo check' working * organise what's exposed from subxt * Get first example working; balance_transfer_with_params * get balance_transfer example compiling * get concurrent_storage_requests.rs example compiling * get fetch_all_accounts example compiling * get a bunch more of the examples compiling * almost get final example working; type mismatch to look into * wee tweaks * move StorageAddress to separate file * pass Defaultable/Iterable info to StorageAddress in codegen * fix storage validation ne, and partial run through example code * Remove static iteration and strip a generic param from everything * fix doc tests in subxt crate * update test utils and start fixing frame tests * fix frame staking tests * fix the rest of the test compile issues, Borrow on storage values * cargo fmt * remove extra logging during tests * Appease clippy and no more need for into_iter on events * cargo fmt * fix dryRun tests by waiting for blocks * wait for blocks instead of sleeping or other test hacks * cargo fmt * Fix doc links * Traitify StorageAddress * remove out-of-date doc comments * optimise decoding storage a little * cleanup tx stuff, trait for TxPayload, remove Err type param and decode at runtime * clippy fixes * fix doc links * fix doc example * constant address trait for consistency * fix a typo and remove EncodeWithMetadata stuff * Put EventDetails behind a proper interface and allow decoding into top level event, too * fix docs * tweak StorageAddress docs * re-export StorageAddress at root for consistency * fix clippy things * Add support for dynamic values * fix double encoding of storage map key after refactor * clippy fix * Fixes and add a dynamic usage example (needs new scale_value release) * bump scale_value version * cargo fmt * Tweak event bits * cargo fmt * Add a test and bump scale-value to 0.4.0 to support this * remove unnecessary vec from dynamic example * Various typo/grammar fixes Co-authored-by: Alexandru Vasile <60601340+lexnv@users.noreply.github.com> * Address PR nits * Undo accidental rename in changelog * Small PR nits/tidyups * fix tests; codegen change against latest substrate * tweak storage address util names * move error decoding to DecodeError and expose * impl some basic traits on the extrinsic param builder Co-authored-by: Alexandru Vasile <60601340+lexnv@users.noreply.github.com>
2026-06-19 17:01:02 +00:00 · 2022-08-08 11:55:20 +01:00
parent 7a09ac6cd7
commit e48f0e3b1d
84 changed files with 23097 additions and 35863 deletions
@@ -0,0 +1,79 @@
+// Copyright 2019-2022 Parity Technologies (UK) Ltd.
+// This file is dual-licensed as Apache-2.0 or GPL-3.0.
+// see LICENSE for license details.
+
+use super::Metadata;
+use crate::{
+    dynamic::DecodedValue,
+    error::Error,
+};
+use codec::Decode;
+use frame_metadata::StorageEntryType;
+
+/// This trait is implemented for types which can be decoded with the help of metadata.
+pub trait DecodeWithMetadata {
+    /// The type that we'll get back from decoding.
+    type Target;
+    /// Given some metadata and a type ID, attempt to SCALE decode the provided bytes into `Self`.
+    fn decode_with_metadata(
+        bytes: &mut &[u8],
+        type_id: u32,
+        metadata: &Metadata,
+    ) -> Result<Self::Target, Error>;
+
+    /// Decode a storage item using metadata. By default, this uses the metadata to
+    /// work out the type ID to use, but for static items we can short circuit this
+    /// lookup.
+    fn decode_storage_with_metadata(
+        bytes: &mut &[u8],
+        pallet_name: &str,
+        storage_entry: &str,
+        metadata: &Metadata,
+    ) -> Result<Self::Target, Error> {
+        let ty = &metadata.pallet(pallet_name)?.storage(storage_entry)?.ty;
+
+        let id = match ty {
+            StorageEntryType::Plain(ty) => ty.id(),
+            StorageEntryType::Map { value, .. } => value.id(),
+        };
+
+        Self::decode_with_metadata(bytes, id, metadata)
+    }
+}
+
+// Things can be dynamically decoded to our Value type:
+impl DecodeWithMetadata for DecodedValue {
+    type Target = Self;
+    fn decode_with_metadata(
+        bytes: &mut &[u8],
+        type_id: u32,
+        metadata: &Metadata,
+    ) -> Result<Self::Target, Error> {
+        let res = scale_value::scale::decode_as_type(bytes, type_id, metadata.types())?;
+        Ok(res)
+    }
+}
+
+/// Any type implementing [`Decode`] can also be decoded with the help of metadata.
+pub struct DecodeStaticType<T>(std::marker::PhantomData<T>);
+
+impl<T: Decode> DecodeWithMetadata for DecodeStaticType<T> {
+    type Target = T;
+
+    fn decode_with_metadata(
+        bytes: &mut &[u8],
+        _type_id: u32,
+        _metadata: &Metadata,
+    ) -> Result<Self::Target, Error> {
+        T::decode(bytes).map_err(|e| e.into())
+    }
+
+    fn decode_storage_with_metadata(
+        bytes: &mut &[u8],
+        _pallet_name: &str,
+        _storage_entry: &str,
+        _metadata: &Metadata,
+    ) -> Result<Self::Target, Error> {
+        T::decode(bytes).map_err(|e| e.into())
+    }
+}
@@ -0,0 +1,67 @@
+// Copyright 2019-2022 Parity Technologies (UK) Ltd.
+// This file is dual-licensed as Apache-2.0 or GPL-3.0.
+// see LICENSE for license details.
+
+use crate::{
+    dynamic::Value,
+    error::Error,
+    metadata::Metadata,
+};
+use codec::Encode;
+
+/// This trait is implemented for types which can be encoded with the help of metadata.
+pub trait EncodeWithMetadata {
+    /// SCALE encode this type to bytes, possibly with the help of metadata.
+    fn encode_with_metadata(
+        &self,
+        type_id: u32,
+        metadata: &Metadata,
+        bytes: &mut Vec<u8>,
+    ) -> Result<(), Error>;
+}
+
+impl EncodeWithMetadata for Value<()> {
+    fn encode_with_metadata(
+        &self,
+        type_id: u32,
+        metadata: &Metadata,
+        bytes: &mut Vec<u8>,
+    ) -> Result<(), Error> {
+        scale_value::scale::encode_as_type(self, type_id, metadata.types(), bytes)
+            .map_err(|e| e.into())
+    }
+}
+
+/// Any type implementing [`Encode`] can also be encoded with the help of metadata.
+pub struct EncodeStaticType<T>(pub T);
+
+impl<T: Encode> EncodeWithMetadata for EncodeStaticType<T> {
+    fn encode_with_metadata(
+        &self,
+        _type_id: u32,
+        _metadata: &Metadata,
+        bytes: &mut Vec<u8>,
+    ) -> Result<(), Error> {
+        self.0.encode_to(bytes);
+        Ok(())
+    }
+}
+
+// We can transparently Encode anything wrapped in EncodeStaticType, too.
+impl<E: Encode> Encode for EncodeStaticType<E> {
+    fn size_hint(&self) -> usize {
+        self.0.size_hint()
+    }
+    fn encode_to<T: codec::Output + ?Sized>(&self, dest: &mut T) {
+        self.0.encode_to(dest)
+    }
+    fn encode(&self) -> Vec<u8> {
+        self.0.encode()
+    }
+    fn using_encoded<R, F: FnOnce(&[u8]) -> R>(&self, f: F) -> R {
+        self.0.using_encoded(f)
+    }
+    fn encoded_size(&self) -> usize {
+        self.0.encoded_size()
+    }
+}
@@ -0,0 +1,14 @@
+// Copyright 2019-2022 Parity Technologies (UK) Ltd.
+// This file is dual-licensed as Apache-2.0 or GPL-3.0.
+// see LICENSE for license details.
+
+/// Locate an item of a known type in the metadata.
+/// We should already know that the item we're looking
+/// for is a call or event for instance, and then with this,
+/// we can dig up details for that item in the metadata.
+pub trait MetadataLocation {
+    /// The pallet in which the item lives.
+    fn pallet(&self) -> &str;
+    /// The name of the item.
+    fn item(&self) -> &str;
+}
@@ -3,7 +3,6 @@
 // see LICENSE for license details.

 use super::hash_cache::HashCache;
-use crate::Call;
 use codec::Error as CodecError;
 use frame_metadata::{
    PalletConstantMetadata,
@@ -16,8 +15,8 @@ use frame_metadata::{
 use parking_lot::RwLock;
 use scale_info::{
    form::PortableForm,
+    PortableRegistry,
    Type,
-    Variant,
 };
 use std::{
    collections::HashMap,
@@ -75,7 +74,11 @@ struct MetadataInner {
    metadata: RuntimeMetadataV14,
    pallets: HashMap<String, PalletMetadata>,
    events: HashMap<(u8, u8), EventMetadata>,
+    // Errors are hashed by pallet index.
    errors: HashMap<(u8, u8), ErrorMetadata>,
+    // Type of the DispatchError type, which is what comes back if
+    // an extrinsic fails.
+    dispatch_error_ty: Option<u32>,
    // The hashes uniquely identify parts of the metadata; different
    // hashes mean some type difference exists between static and runtime
    // versions. We cache them here to avoid recalculating:
@@ -93,7 +96,7 @@ pub struct Metadata {

 impl Metadata {
    /// Returns a reference to [`PalletMetadata`].
-    pub fn pallet(&self, name: &'static str) -> Result<&PalletMetadata, MetadataError> {
+    pub fn pallet(&self, name: &str) -> Result<&PalletMetadata, MetadataError> {
        self.inner
            .pallets
            .get(name)
@@ -128,6 +131,16 @@ impl Metadata {
        Ok(error)
    }

+    /// Return the DispatchError type ID if it exists.
+    pub fn dispatch_error_ty(&self) -> Option<u32> {
+        self.inner.dispatch_error_ty
+    }
+
+    /// Return the type registry embedded within the metadata.
+    pub fn types(&self) -> &PortableRegistry {
+        &self.inner.metadata.types
+    }
+
    /// Resolve a type definition.
    pub fn resolve_type(&self, id: u32) -> Option<&Type<PortableForm>> {
        self.inner.metadata.types.resolve(id)
@@ -139,23 +152,25 @@ impl Metadata {
    }

    /// Obtain the unique hash for a specific storage entry.
-    pub fn storage_hash<S: crate::StorageEntry>(
+    pub fn storage_hash(
        &self,
+        pallet: &str,
+        storage: &str,
    ) -> Result<[u8; 32], MetadataError> {
        self.inner
            .cached_storage_hashes
-            .get_or_insert(S::PALLET, S::STORAGE, || {
-                subxt_metadata::get_storage_hash(
-                    &self.inner.metadata,
-                    S::PALLET,
-                    S::STORAGE,
-                )
-                .map_err(|e| {
-                    match e {
-                        subxt_metadata::NotFound::Pallet => MetadataError::PalletNotFound,
-                        subxt_metadata::NotFound::Item => MetadataError::StorageNotFound,
-                    }
-                })
+            .get_or_insert(pallet, storage, || {
+                subxt_metadata::get_storage_hash(&self.inner.metadata, pallet, storage)
+                    .map_err(|e| {
+                        match e {
+                            subxt_metadata::NotFound::Pallet => {
+                                MetadataError::PalletNotFound
+                            }
+                            subxt_metadata::NotFound::Item => {
+                                MetadataError::StorageNotFound
+                            }
+                        }
+                    })
            })
    }

@@ -183,21 +198,23 @@ impl Metadata {
    }

    /// Obtain the unique hash for a call.
-    pub fn call_hash<C: crate::Call>(&self) -> Result<[u8; 32], MetadataError> {
+    pub fn call_hash(
+        &self,
+        pallet: &str,
+        function: &str,
+    ) -> Result<[u8; 32], MetadataError> {
        self.inner
            .cached_call_hashes
-            .get_or_insert(C::PALLET, C::FUNCTION, || {
-                subxt_metadata::get_call_hash(
-                    &self.inner.metadata,
-                    C::PALLET,
-                    C::FUNCTION,
-                )
-                .map_err(|e| {
-                    match e {
-                        subxt_metadata::NotFound::Pallet => MetadataError::PalletNotFound,
-                        subxt_metadata::NotFound::Item => MetadataError::CallNotFound,
-                    }
-                })
+            .get_or_insert(pallet, function, || {
+                subxt_metadata::get_call_hash(&self.inner.metadata, pallet, function)
+                    .map_err(|e| {
+                        match e {
+                            subxt_metadata::NotFound::Pallet => {
+                                MetadataError::PalletNotFound
+                            }
+                            subxt_metadata::NotFound::Item => MetadataError::CallNotFound,
+                        }
+                    })
            })
    }

@@ -222,7 +239,8 @@ impl Metadata {
 pub struct PalletMetadata {
    index: u8,
    name: String,
-    calls: HashMap<String, u8>,
+    call_indexes: HashMap<String, u8>,
+    call_ty_id: Option<u32>,
    storage: HashMap<String, StorageEntryMetadata<PortableForm>>,
    constants: HashMap<String, PalletConstantMetadata<PortableForm>>,
 }
@@ -238,15 +256,18 @@ impl PalletMetadata {
        self.index
    }

+    /// If calls exist for this pallet, this returns the type ID of the variant
+    /// representing the different possible calls.
+    pub fn call_ty_id(&self) -> Option<u32> {
+        self.call_ty_id
+    }
+
    /// Attempt to resolve a call into an index in this pallet, failing
    /// if the call is not found in this pallet.
-    pub fn call_index<C>(&self) -> Result<u8, MetadataError>
-    where
-        C: Call,
-    {
+    pub fn call_index(&self, function: &str) -> Result<u8, MetadataError> {
        let fn_index = *self
-            .calls
-            .get(C::FUNCTION)
+            .call_indexes
+            .get(function)
            .ok_or(MetadataError::CallNotFound)?;
        Ok(fn_index)
    }
@@ -273,9 +294,12 @@ impl PalletMetadata {
 /// Metadata for specific events.
 #[derive(Clone, Debug)]
 pub struct EventMetadata {
-    pallet: String,
+    // The pallet name is shared across every event, so put it
+    // behind an Arc to avoid lots of needless clones of it existing.
+    pallet: Arc<str>,
    event: String,
-    variant: Variant<PortableForm>,
+    fields: Vec<(Option<String>, u32)>,
+    docs: Vec<String>,
 }

 impl EventMetadata {
@@ -289,21 +313,25 @@ impl EventMetadata {
        &self.event
    }

-    /// Get the type def variant for the pallet event.
-    pub fn variant(&self) -> &Variant<PortableForm> {
-        &self.variant
+    /// The names and types of each field in the event.
+    pub fn fields(&self) -> &[(Option<String>, u32)] {
+        &self.fields
+    }
+
+    /// Documentation for this event.
+    pub fn docs(&self) -> &[String] {
+        &self.docs
    }
 }

-/// Metadata for specific errors obtained from the pallet's `PalletErrorMetadata`.
-///
-/// This holds in memory information regarding the Pallet's name, Error's name, and the underlying
-/// metadata representation.
+/// Details about a specific runtime error.
 #[derive(Clone, Debug)]
 pub struct ErrorMetadata {
-    pallet: String,
+    // The pallet name is shared across every event, so put it
+    // behind an Arc to avoid lots of needless clones of it existing.
+    pallet: Arc<str>,
    error: String,
-    variant: Variant<PortableForm>,
+    docs: Vec<String>,
 }

 impl ErrorMetadata {
@@ -312,29 +340,31 @@ impl ErrorMetadata {
        &self.pallet
    }

-    /// Get the name of the specific pallet error.
+    /// The name of the error.
    pub fn error(&self) -> &str {
        &self.error
    }

-    /// Get the description of the specific pallet error.
-    pub fn description(&self) -> &[String] {
-        self.variant.docs()
+    /// Documentation for the error.
+    pub fn docs(&self) -> &[String] {
+        &self.docs
    }
 }

 /// Error originated from converting a runtime metadata [RuntimeMetadataPrefixed] to
 /// the internal [Metadata] representation.
-///
-/// The runtime metadata is converted when building the [crate::client::Client].
 #[derive(Debug, thiserror::Error)]
 pub enum InvalidMetadataError {
+    /// Invalid prefix
    #[error("Invalid prefix")]
    InvalidPrefix,
+    /// Invalid version
    #[error("Invalid version")]
    InvalidVersion,
+    /// Type missing from type registry
    #[error("Type {0} missing from type registry")]
    MissingType(u32),
+    /// Type was not a variant/enum type
    #[error("Type {0} was not a variant/enum type")]
    TypeDefNotVariant(u32),
 }
@@ -366,15 +396,18 @@ impl TryFrom<RuntimeMetadataPrefixed> for Metadata {
            .pallets
            .iter()
            .map(|pallet| {
-                let calls = pallet.calls.as_ref().map_or(Ok(HashMap::new()), |call| {
-                    let type_def_variant = get_type_def_variant(call.ty.id())?;
-                    let calls = type_def_variant
-                        .variants()
-                        .iter()
-                        .map(|v| (v.name().clone(), v.index()))
-                        .collect();
-                    Ok(calls)
-                })?;
+                let call_ty_id = pallet.calls.as_ref().map(|c| c.ty.id());
+
+                let call_indexes =
+                    pallet.calls.as_ref().map_or(Ok(HashMap::new()), |call| {
+                        let type_def_variant = get_type_def_variant(call.ty.id())?;
+                        let call_indexes = type_def_variant
+                            .variants()
+                            .iter()
+                            .map(|v| (v.name().clone(), v.index()))
+                            .collect();
+                        Ok(call_indexes)
+                    })?;

                let storage = pallet.storage.as_ref().map_or(HashMap::new(), |storage| {
                    storage
@@ -393,7 +426,8 @@ impl TryFrom<RuntimeMetadataPrefixed> for Metadata {
                let pallet_metadata = PalletMetadata {
                    index: pallet.index,
                    name: pallet.name.to_string(),
-                    calls,
+                    call_indexes,
+                    call_ty_id,
                    storage,
                    constants,
                };
@@ -402,55 +436,54 @@ impl TryFrom<RuntimeMetadataPrefixed> for Metadata {
            })
            .collect::<Result<_, _>>()?;

-        let pallet_events = metadata
-            .pallets
-            .iter()
-            .filter_map(|pallet| {
-                pallet.event.as_ref().map(|event| {
-                    let type_def_variant = get_type_def_variant(event.ty.id())?;
-                    Ok((pallet, type_def_variant))
-                })
-            })
-            .collect::<Result<Vec<_>, _>>()?;
-        let events = pallet_events
-            .iter()
-            .flat_map(|(pallet, type_def_variant)| {
-                type_def_variant.variants().iter().map(move |var| {
-                    let key = (pallet.index, var.index());
-                    let value = EventMetadata {
-                        pallet: pallet.name.clone(),
-                        event: var.name().clone(),
-                        variant: var.clone(),
-                    };
-                    (key, value)
-                })
-            })
-            .collect();
+        let mut events = HashMap::<(u8, u8), EventMetadata>::new();
+        for pallet in &metadata.pallets {
+            if let Some(event) = &pallet.event {
+                let pallet_name: Arc<str> = pallet.name.to_string().into();
+                let event_type_id = event.ty.id();
+                let event_variant = get_type_def_variant(event_type_id)?;
+                for variant in event_variant.variants() {
+                    events.insert(
+                        (pallet.index, variant.index()),
+                        EventMetadata {
+                            pallet: pallet_name.clone(),
+                            event: variant.name().to_owned(),
+                            fields: variant
+                                .fields()
+                                .iter()
+                                .map(|f| (f.name().map(|n| n.to_owned()), f.ty().id()))
+                                .collect(),
+                            docs: variant.docs().to_vec(),
+                        },
+                    );
+                }
+            }
+        }

-        let pallet_errors = metadata
-            .pallets
+        let mut errors = HashMap::<(u8, u8), ErrorMetadata>::new();
+        for pallet in &metadata.pallets {
+            if let Some(error) = &pallet.error {
+                let pallet_name: Arc<str> = pallet.name.to_string().into();
+                let error_variant = get_type_def_variant(error.ty.id())?;
+                for variant in error_variant.variants() {
+                    errors.insert(
+                        (pallet.index, variant.index()),
+                        ErrorMetadata {
+                            pallet: pallet_name.clone(),
+                            error: variant.name().clone(),
+                            docs: variant.docs().to_vec(),
+                        },
+                    );
+                }
+            }
+        }
+
+        let dispatch_error_ty = metadata
+            .types
+            .types()
            .iter()
-            .filter_map(|pallet| {
-                pallet.error.as_ref().map(|error| {
-                    let type_def_variant = get_type_def_variant(error.ty.id())?;
-                    Ok((pallet, type_def_variant))
-                })
-            })
-            .collect::<Result<Vec<_>, _>>()?;
-        let errors = pallet_errors
-            .iter()
-            .flat_map(|(pallet, type_def_variant)| {
-                type_def_variant.variants().iter().map(move |var| {
-                    let key = (pallet.index, var.index());
-                    let value = ErrorMetadata {
-                        pallet: pallet.name.clone(),
-                        error: var.name().clone(),
-                        variant: var.clone(),
-                    };
-                    (key, value)
-                })
-            })
-            .collect();
+            .find(|ty| ty.ty().path().segments() == ["sp_runtime", "DispatchError"])
+            .map(|ty| ty.id());

        Ok(Metadata {
            inner: Arc::new(MetadataInner {
@@ -458,6 +491,7 @@ impl TryFrom<RuntimeMetadataPrefixed> for Metadata {
                pallets,
                events,
                errors,
+                dispatch_error_ty,
                cached_metadata_hash: Default::default(),
                cached_call_hashes: Default::default(),
                cached_constant_hashes: Default::default(),
@@ -470,7 +504,6 @@ impl TryFrom<RuntimeMetadataPrefixed> for Metadata {
 #[cfg(test)]
 mod tests {
    use super::*;
-    use crate::StorageEntryKey;
    use frame_metadata::{
        ExtrinsicMetadata,
        PalletStorageMetadata,
@@ -553,14 +586,7 @@ mod tests {
    fn metadata_call_inner_cache() {
        let metadata = load_metadata();

-        #[derive(codec::Encode)]
-        struct ValidCall;
-        impl crate::Call for ValidCall {
-            const PALLET: &'static str = "System";
-            const FUNCTION: &'static str = "fill_block";
-        }
-
-        let hash = metadata.call_hash::<ValidCall>();
+        let hash = metadata.call_hash("System", "fill_block");

        let mut call_number = 0;
        let hash_cached = metadata.inner.cached_call_hashes.get_or_insert(
@@ -601,20 +627,7 @@ mod tests {
    #[test]
    fn metadata_storage_inner_cache() {
        let metadata = load_metadata();
-
-        #[derive(codec::Encode)]
-        struct ValidStorage;
-        impl crate::StorageEntry for ValidStorage {
-            const PALLET: &'static str = "System";
-            const STORAGE: &'static str = "Account";
-            type Value = ();
-
-            fn key(&self) -> StorageEntryKey {
-                unreachable!("Should not be called");
-            }
-        }
-
-        let hash = metadata.storage_hash::<ValidStorage>();
+        let hash = metadata.storage_hash("System", "Account");

        let mut call_number = 0;
        let hash_cached = metadata.inner.cached_storage_hashes.get_or_insert(
@@ -2,9 +2,16 @@
 // This file is dual-licensed as Apache-2.0 or GPL-3.0.
 // see LICENSE for license details.

+//! Types representing the metadata obtained from a node.
+
+mod decode_with_metadata;
+mod encode_with_metadata;
 mod hash_cache;
+mod metadata_location;
 mod metadata_type;

+pub use metadata_location::MetadataLocation;
+
 pub use metadata_type::{
    ErrorMetadata,
    EventMetadata,
@@ -13,3 +20,13 @@ pub use metadata_type::{
    MetadataError,
    PalletMetadata,
 };
+
+pub use decode_with_metadata::{
+    DecodeStaticType,
+    DecodeWithMetadata,
+};
+
+pub use encode_with_metadata::{
+    EncodeStaticType,
+    EncodeWithMetadata,
+};