V15 Metadata: Support accessing custom types (#1106)

* work in progress

* add custom types access

* nit

* custom values client

* adjust light client

* adjust doc comments

* adjust book for custom values in code gen

* format and check docs

* use ignore in docs in book

metadata/src/from_into/v15.rs

@@ -5,10 +5,10 @@
 use super::TryFromError;
 use crate::utils::variant_index::VariantIndex;
 use crate::{
-    utils::ordered_map::OrderedMap, ArcStr, ConstantMetadata, ExtrinsicMetadata, Metadata,
-    OuterEnumsMetadata, PalletMetadataInner, RuntimeApiMetadataInner, RuntimeApiMethodMetadata,
-    RuntimeApiMethodParamMetadata, SignedExtensionMetadata, StorageEntryMetadata,
-    StorageEntryModifier, StorageEntryType, StorageHasher, StorageMetadata,
+    utils::ordered_map::OrderedMap, ArcStr, ConstantMetadata, CustomMetadata, ExtrinsicMetadata,
+    Metadata, OuterEnumsMetadata, PalletMetadataInner, RuntimeApiMetadataInner,
+    RuntimeApiMethodMetadata, RuntimeApiMethodParamMetadata, SignedExtensionMetadata,
+    StorageEntryMetadata, StorageEntryModifier, StorageEntryType, StorageHasher, StorageMetadata,
 };
 use frame_metadata::v15;
 use scale_info::form::PortableForm;
@@ -93,6 +93,7 @@ mod from_v15 {
                     event_enum_ty: m.outer_enums.event_enum_ty.id,
                     error_enum_ty: m.outer_enums.error_enum_ty.id,
                 },
+                custom: CustomMetadata { map: m.custom.map },
             })
         }
     }

metadata/src/lib.rs

@@ -20,7 +20,7 @@ mod from_into;
 mod utils;
 
 use scale_info::{form::PortableForm, PortableRegistry, Variant};
-use std::collections::HashMap;
+use std::collections::{BTreeMap, HashMap};
 use std::sync::Arc;
 use utils::ordered_map::OrderedMap;
 use utils::variant_index::VariantIndex;
@@ -51,6 +51,8 @@ pub struct Metadata {
     dispatch_error_ty: Option<u32>,
     /// Details about each of the runtime API traits.
     apis: OrderedMap<ArcStr, RuntimeApiMetadataInner>,
+    /// Allows users to add custom types to the metadata. A map that associates a string key to a `CustomValueMetadata`.
+    custom: CustomMetadata,
 }
 
 impl Metadata {
@@ -132,6 +134,11 @@ impl Metadata {
         })
     }
 
+    /// Returns custom user defined types
+    pub fn custom(&self) -> &CustomMetadata {
+        &self.custom
+    }
+
     /// Obtain a unique hash representing this metadata or specific parts of it.
     pub fn hasher(&self) -> MetadataHasher {
         MetadataHasher::new(self)
@@ -631,6 +638,40 @@ pub struct RuntimeApiMethodParamMetadata {
     pub ty: u32,
 }
 
+/// Metadata of custom types with custom values, basically the same as `frame_metadata::v15::CustomMetadata<PortableForm>>`.
+#[derive(Debug, Clone)]
+pub struct CustomMetadata {
+    map: BTreeMap<String, frame_metadata::v15::CustomValueMetadata<PortableForm>>,
+}
+
+impl CustomMetadata {
+    /// Get a certain [CustomMetadataValue] by its name.
+    pub fn get(&self, name: &str) -> Option<CustomMetadataValue<'_>> {
+        self.map.get(name).map(|e| CustomMetadataValue {
+            type_id: e.ty.id,
+            data: &e.value,
+        })
+    }
+}
+
+/// Basically the same as `frame_metadata::v15::CustomValueMetadata<PortableForm>>`, but borrowed.
+pub struct CustomMetadataValue<'a> {
+    type_id: u32,
+    data: &'a [u8],
+}
+
+impl<'a> CustomMetadataValue<'a> {
+    /// the scale encoded value
+    pub fn bytes(&self) -> &'a [u8] {
+        self.data
+    }
+
+    /// the type id in the TypeRegistry
+    pub fn type_id(&self) -> u32 {
+        self.type_id
+    }
+}
+
 // Support decoding metadata from the "wire" format directly into this.
 // Errors may be lost in the case that the metadata content is somehow invalid.
 impl codec::Decode for Metadata {

subxt/src/book/mod.rs

@@ -3,7 +3,7 @@
 // see LICENSE for license details.
 
 // Dev note; I used the following command to normalize and wrap comments:
-// rustfmt +nightly --config wrap_comments=true,comment_width=100,normalize_comments=true subxt/src/book/mod.rs
+// rustfmt +nightly --config wrap_comments=true,comment_width=100,normalize_comments=true subxt/src/book/custom_values
 // It messed up comments in code blocks though, so be prepared to go and fix those.
 
 //! # The Subxt Guide

subxt/src/book/usage/custom_values.rs

@@ -0,0 +1,54 @@
+// Copyright 2019-2023 Parity Technologies (UK) Ltd.
+// This file is dual-licensed as Apache-2.0 or GPL-3.0.
+// see LICENSE for license details.
+
+//! # Custom Values
+//!
+//! Substrate-based chains can expose custom values in their metadata.
+//! Each of these values:
+//!
+//! - can be accessed by a unique __name__.
+//! - refers to a concrete __type__ stored in the metadata.
+//! - contains a scale encoded __value__ of that type.
+//!
+//! ## Getting a custom value
+//!
+//! Custom values can be accessed via a [`CustomValuesClient`](crate::custom_values::CustomValuesClient).
+//! The client exposes an `at` function by which a custom value can be fetched, given an address to this custom value.
+//! An address can be as simple as the aforementioned __name__ as a [str]. This will return a dynamic value, that you can manually decode into the type you want.
+//! Suppose, the custom types contain a value of type `Foo` under the name `"foo"` you can access it like in this example:
+//!
+//! ```rust,ignore
+//! use subxt::{OnlineClient, PolkadotConfig, ext::{codec::Decode, scale_decode::DecodeAsType}};
+//!
+//! #[derive(Decode, DecodeAsType, Debug)]
+//! struct Foo {
+//!     n: u8,
+//!     b: bool,
+//! }
+//!
+//! let api = OnlineClient::<PolkadotConfig>::new().await?;
+//! let custom_value_client = api.custom_values();
+//! let foo_dynamic = custom_value_client.at("foo")?;
+//! let foo: Foo = foo_dynamic.as_type()?;
+//!
+//! ```
+//!
+//! Alternatively we also provide a statically generated api for custom values:
+//!
+//! ```rust,ignore
+//! #[subxt::subxt(runtime_metadata_path = "some_metadata.scale")]
+//! pub mod interface {}
+//!
+//! let static_address = interface::custom().foo();
+//!
+//! let api = OnlineClient::<PolkadotConfig>::new().await?;
+//! let custom_value_client = api.custom_values();
+//!
+//! // Now the `at()` function already decodes the value into the Foo type:
+//! let foo = custom_value_client.at(&static_address)?;
+//! ```         
+//!
+//! Note: Names of custom values are converted to __snake_case__ to produce a valid function name during code generation.
+//! If there are multiple values where the names would be equal when converted to __snake_case__, functions might not be statically generated for some of them, because of naming conflicts.
+//! Make sure names in the custom values of your metadata differ significantly.

subxt/src/book/usage/mod.rs

@@ -11,11 +11,13 @@
 //! - [Blocks](blocks)
 //! - [Runtime APIs](runtime_apis)
 //! - [Unstable Light Client](light_client)
+//! - [Custom Values](custom_values)
 //!
 //! Alternately, [go back](super).
 
 pub mod blocks;
 pub mod constants;
+pub mod custom_values;
 pub mod events;
 pub mod light_client;
 pub mod runtime_apis;

subxt/src/client/lightclient/mod.rs

@@ -12,6 +12,7 @@ use crate::{
     client::{OfflineClientT, OnlineClientT},
     config::Config,
     constants::ConstantsClient,
+    custom_values::CustomValuesClient,
     events::EventsClient,
     runtime_api::RuntimeApiClient,
     storage::StorageClient,
@@ -100,6 +101,11 @@ impl<T: Config> LightClient<T> {
         <Self as OfflineClientT<T>>::constants(self)
     }
 
+    /// Access custom types.
+    pub fn custom_values(&self) -> CustomValuesClient<T, Self> {
+        <Self as OfflineClientT<T>>::custom_values(self)
+    }
+
     /// Work with blocks.
     pub fn blocks(&self) -> BlocksClient<T, Self> {
         <Self as OfflineClientT<T>>::blocks(self)

subxt/src/client/offline_client.rs

@@ -2,12 +2,14 @@
 // This file is dual-licensed as Apache-2.0 or GPL-3.0.
 // see LICENSE for license details.
 
+use crate::custom_values::CustomValuesClient;
 use crate::{
     blocks::BlocksClient, constants::ConstantsClient, events::EventsClient,
     rpc::types::RuntimeVersion, runtime_api::RuntimeApiClient, storage::StorageClient,
     tx::TxClient, Config, Metadata,
 };
 use derivative::Derivative;
+
 use std::sync::Arc;
 
 /// A trait representing a client that can perform
@@ -49,6 +51,11 @@ pub trait OfflineClientT<T: Config>: Clone + Send + Sync + 'static {
     fn runtime_api(&self) -> RuntimeApiClient<T, Self> {
         RuntimeApiClient::new(self.clone())
     }
+
+    /// Work this custom types.
+    fn custom_values(&self) -> CustomValuesClient<T, Self> {
+        CustomValuesClient::new(self.clone())
+    }
 }
 
 /// A client that is capable of performing offline-only operations.
@@ -121,6 +128,11 @@ impl<T: Config> OfflineClient<T> {
     pub fn constants(&self) -> ConstantsClient<T, Self> {
         <Self as OfflineClientT<T>>::constants(self)
     }
+
+    /// Access custom types
+    pub fn custom_values(&self) -> CustomValuesClient<T, Self> {
+        <Self as OfflineClientT<T>>::custom_values(self)
+    }
 }
 
 impl<T: Config> OfflineClientT<T> for OfflineClient<T> {

subxt/src/client/online_client.rs

@@ -3,6 +3,7 @@
 // see LICENSE for license details.
 
 use super::{OfflineClient, OfflineClientT};
+use crate::custom_values::CustomValuesClient;
 use crate::{
     blocks::BlocksClient,
     constants::ConstantsClient,
@@ -18,7 +19,9 @@ use crate::{
     Config, Metadata,
 };
 use derivative::Derivative;
+
 use futures::future;
+
 use std::sync::{Arc, RwLock};
 
 /// A trait representing a client that can perform
@@ -292,6 +295,11 @@ impl<T: Config> OnlineClient<T> {
         <Self as OfflineClientT<T>>::constants(self)
     }
 
+    /// Access custom types.
+    pub fn custom_values(&self) -> CustomValuesClient<T, Self> {
+        <Self as OfflineClientT<T>>::custom_values(self)
+    }
+
     /// Work with blocks.
     pub fn blocks(&self) -> BlocksClient<T, Self> {
         <Self as OfflineClientT<T>>::blocks(self)

subxt/src/custom_values/custom_value_address.rs

@@ -0,0 +1,21 @@
+use crate::dynamic::DecodedValueThunk;
+use crate::metadata::DecodeWithMetadata;
+
+/// This represents the address of a custom value in in the metadata.
+/// Anything, that implements the [CustomValueAddress] trait can be used, to fetch
+/// custom values from the metadata.
+pub trait CustomValueAddress {
+    /// The type of the custom value.
+    type Target: DecodeWithMetadata;
+
+    /// the name (key) by which the custom value can be accessed in the metadata.
+    fn name(&self) -> &str;
+}
+
+impl CustomValueAddress for str {
+    type Target = DecodedValueThunk;
+
+    fn name(&self) -> &str {
+        self
+    }
+}

subxt/src/custom_values/custom_values_client.rs

@@ -0,0 +1,135 @@
+use crate::client::OfflineClientT;
+use crate::custom_values::custom_value_address::CustomValueAddress;
+use crate::error::MetadataError;
+use crate::metadata::DecodeWithMetadata;
+use crate::{Config, Error};
+use derivative::Derivative;
+
+/// A client for accessing custom values stored in the metadata.
+#[derive(Derivative)]
+#[derivative(Clone(bound = "Client: Clone"))]
+pub struct CustomValuesClient<T, Client> {
+    client: Client,
+    _marker: std::marker::PhantomData<T>,
+}
+
+impl<T, Client> CustomValuesClient<T, Client> {
+    /// Create a new [`CustomValuesClient`].
+    pub fn new(client: Client) -> Self {
+        Self {
+            client,
+            _marker: std::marker::PhantomData,
+        }
+    }
+}
+
+impl<T: Config, Client: OfflineClientT<T>> CustomValuesClient<T, Client> {
+    /// Access a custom value by the address it is registered under. This can be just a [str] to get back a dynamic value,
+    /// or a static address from the generated static interface to get a value of a static type returned.
+    pub fn at<Address: CustomValueAddress + ?Sized>(
+        &self,
+        address: &Address,
+    ) -> Result<Address::Target, Error> {
+        let metadata = self.client.metadata();
+        let custom_value = metadata
+            .custom()
+            .get(address.name())
+            .ok_or_else(|| MetadataError::CustomValueNameNotFound(address.name().to_string()))?;
+
+        let value = <Address::Target as DecodeWithMetadata>::decode_with_metadata(
+            &mut custom_value.bytes(),
+            custom_value.type_id(),
+            &metadata,
+        )?;
+        Ok(value)
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use std::collections::BTreeMap;
+
+    use crate::custom_values::CustomValuesClient;
+    use crate::rpc::types::RuntimeVersion;
+    use crate::{Metadata, OfflineClient, SubstrateConfig};
+    use scale_decode::DecodeAsType;
+    use scale_info::form::PortableForm;
+    use scale_info::TypeInfo;
+    use sp_core::Encode;
+
+    #[derive(Debug, Clone, PartialEq, Eq, Encode, TypeInfo, DecodeAsType)]
+    pub struct Person {
+        age: u16,
+        name: String,
+    }
+
+    fn mock_metadata() -> Metadata {
+        let person_ty = scale_info::MetaType::new::<Person>();
+        let unit = scale_info::MetaType::new::<()>();
+        let mut types = scale_info::Registry::new();
+        let person_ty_id = types.register_type(&person_ty);
+        let unit_id = types.register_type(&unit);
+        let types: scale_info::PortableRegistry = types.into();
+
+        let person = Person {
+            age: 42,
+            name: "Neo".into(),
+        };
+
+        let person_value_metadata: frame_metadata::v15::CustomValueMetadata<PortableForm> =
+            frame_metadata::v15::CustomValueMetadata {
+                ty: person_ty_id,
+                value: person.encode(),
+            };
+
+        let frame_metadata = frame_metadata::v15::RuntimeMetadataV15 {
+            types,
+            pallets: vec![],
+            extrinsic: frame_metadata::v15::ExtrinsicMetadata {
+                version: 0,
+                address_ty: unit_id,
+                call_ty: unit_id,
+                signature_ty: unit_id,
+                extra_ty: unit_id,
+                signed_extensions: vec![],
+            },
+            ty: unit_id,
+            apis: vec![],
+            outer_enums: frame_metadata::v15::OuterEnums {
+                call_enum_ty: unit_id,
+                event_enum_ty: unit_id,
+                error_enum_ty: unit_id,
+            },
+            custom: frame_metadata::v15::CustomMetadata {
+                map: BTreeMap::from_iter([("Person".to_string(), person_value_metadata)]),
+            },
+        };
+
+        let metadata: subxt_metadata::Metadata = frame_metadata.try_into().unwrap();
+        Metadata::new(metadata)
+    }
+
+    #[test]
+    fn test_decoding() {
+        let client = OfflineClient::<SubstrateConfig>::new(
+            Default::default(),
+            RuntimeVersion {
+                spec_version: 0,
+                transaction_version: 0,
+                other: Default::default(),
+            },
+            mock_metadata(),
+        );
+        let custom_value_client = CustomValuesClient::new(client);
+        assert!(custom_value_client.at("No one").is_err());
+        let person_decoded_value_thunk = custom_value_client.at("Person").unwrap();
+        let person: Person = person_decoded_value_thunk.as_type().unwrap();
+        assert_eq!(
+            person,
+            Person {
+                age: 42,
+                name: "Neo".into()
+            }
+        )
+    }
+}

subxt/src/custom_values/mod.rs

@@ -0,0 +1,11 @@
+// Copyright 2019-2023 Parity Technologies (UK) Ltd.
+// This file is dual-licensed as Apache-2.0 or GPL-3.0.
+// see LICENSE for license details.
+
+//! Types associated with accessing custom types
+
+mod custom_value_address;
+mod custom_values_client;
+
+pub use custom_value_address::CustomValueAddress;
+pub use custom_values_client::CustomValuesClient;

subxt/src/dynamic.rs

@@ -75,4 +75,12 @@ impl DecodedValueThunk {
         )?;
         Ok(val)
     }
+    /// decode the `DecodedValueThunk` into a concrete type.
+    pub fn as_type<T: DecodeAsType>(&self) -> Result<T, scale_decode::Error> {
+        T::decode_as_type(
+            &mut &self.scale_bytes[..],
+            self.type_id,
+            self.metadata.types(),
+        )
+    }
 }

subxt/src/error/mod.rs

@@ -234,4 +234,7 @@ pub enum MetadataError {
     /// The generated interface used is not compatible with the node.
     #[error("The generated code is not compatible with the node")]
     IncompatibleCodegen,
+    /// Custom value not found.
+    #[error("Custom value with name {0} not found")]
+    CustomValueNameNotFound(String),
 }

subxt/src/lib.rs

@@ -67,6 +67,7 @@ pub mod blocks;
 pub mod client;
 pub mod config;
 pub mod constants;
+pub mod custom_values;
 pub mod dynamic;
 pub mod error;
 pub mod events;

subxt/src/metadata/metadata_type.rs

@@ -3,6 +3,7 @@
 // see LICENSE for license details.
 
 use crate::error::MetadataError;
+
 use std::sync::Arc;
 
 /// A cheaply clone-able representation of the runtime metadata received from a node.