Use scale-encode and scale-decode to encode and decode based on metadata (#842)

* WIP EncodeAsType and DecodeAsType

* remove silly cli experiment code

* Get things finally compiling with EncodeAsType and DecodeAsType

* update codegen test and WrapperKeepOpaque proper impl (in case it shows up in codegen)

* fix tests

* accomodate scale-value changes

* starting to migrate to EncodeAsType/DecodeAsType

* static event decoding and tx encoding to use DecodeAsFields/EncodeAsFields

* some tidy up and add decode(skip) attrs where needed

* fix root event decoding

* #[codec(skip)] will do, and combine map_key stuff into storage_address since it's all specific to that

* fmt and clippy

* update Cargo.lock

* remove patched scale-encode

* bump scale-encode to 0.1 and remove unused dep in testing crate

* update deps and use released scale-decode

* update scale-value to latest to remove git branch

* Apply suggestions from code review

Co-authored-by: Alexandru Vasile <60601340+lexnv@users.noreply.github.com>

* remove sorting in derives/attr generation; spit them out in order given

* re-add derive sorting; it's a hashmap

* StaticTxPayload and DynamicTxPayload rolled into single Payload struct

* StaticStorageAddress and DynamicStorageAddress into single Address struct

* Fix storage address byte retrieval

* StaticConstantAddress and DynamicConstantAddress => Address

* Simplify storage codegen to fix test

* Add comments

* Alias to RuntimeEvent rather than making another, and prep for substituting call type

* remove unnecessary clone

* Fix docs and failing UI test

* root_bytes -> to_root_bytes

* document error case in StorageClient::address_bytes()

---------

Co-authored-by: Alexandru Vasile <60601340+lexnv@users.noreply.github.com>

subxt/src/metadata/decode_encode_traits.rs

@@ -0,0 +1,51 @@
+// Copyright 2019-2022 Parity Technologies (UK) Ltd.
+// This file is dual-licensed as Apache-2.0 or GPL-3.0.
+// see LICENSE for license details.
+
+use super::Metadata;
+use crate::error::Error;
+
+/// This trait is implemented for all types that also implement [`scale_decode::DecodeAsType`].
+pub trait DecodeWithMetadata: Sized {
+    /// Given some metadata and a type ID, attempt to SCALE decode the provided bytes into `Self`.
+    fn decode_with_metadata(
+        bytes: &mut &[u8],
+        type_id: u32,
+        metadata: &Metadata,
+    ) -> Result<Self, Error>;
+}
+
+impl<T: scale_decode::DecodeAsType> DecodeWithMetadata for T {
+    fn decode_with_metadata(
+        bytes: &mut &[u8],
+        type_id: u32,
+        metadata: &Metadata,
+    ) -> Result<T, Error> {
+        let val = T::decode_as_type(bytes, type_id, metadata.types())?;
+        Ok(val)
+    }
+}
+
+/// This trait is implemented for all types that also implement [`scale_encode::EncodeAsType`].
+pub trait EncodeWithMetadata {
+    /// SCALE encode this type to bytes, possibly with the help of metadata.
+    fn encode_with_metadata(
+        &self,
+        type_id: u32,
+        metadata: &Metadata,
+        bytes: &mut Vec<u8>,
+    ) -> Result<(), Error>;
+}
+
+impl<T: scale_encode::EncodeAsType> EncodeWithMetadata for T {
+    /// SCALE encode this type to bytes, possibly with the help of metadata.
+    fn encode_with_metadata(
+        &self,
+        type_id: u32,
+        metadata: &Metadata,
+        bytes: &mut Vec<u8>,
+    ) -> Result<(), Error> {
+        self.encode_as_type_to(type_id, metadata.types(), bytes)?;
+        Ok(())
+    }
+}

subxt/src/metadata/decode_with_metadata.rs

@@ -1,79 +0,0 @@
-// Copyright 2019-2022 Parity Technologies (UK) Ltd.
-// This file is dual-licensed as Apache-2.0 or GPL-3.0.
-// see LICENSE for license details.
-
-use super::Metadata;
-use crate::{
-    dynamic::DecodedValue,
-    error::Error,
-};
-use codec::Decode;
-use frame_metadata::StorageEntryType;
-
-/// This trait is implemented for types which can be decoded with the help of metadata.
-pub trait DecodeWithMetadata {
-    /// The type that we'll get back from decoding.
-    type Target;
-    /// Given some metadata and a type ID, attempt to SCALE decode the provided bytes into `Self`.
-    fn decode_with_metadata(
-        bytes: &mut &[u8],
-        type_id: u32,
-        metadata: &Metadata,
-    ) -> Result<Self::Target, Error>;
-
-    /// Decode a storage item using metadata. By default, this uses the metadata to
-    /// work out the type ID to use, but for static items we can short circuit this
-    /// lookup.
-    fn decode_storage_with_metadata(
-        bytes: &mut &[u8],
-        pallet_name: &str,
-        storage_entry: &str,
-        metadata: &Metadata,
-    ) -> Result<Self::Target, Error> {
-        let ty = &metadata.pallet(pallet_name)?.storage(storage_entry)?.ty;
-
-        let id = match ty {
-            StorageEntryType::Plain(ty) => ty.id(),
-            StorageEntryType::Map { value, .. } => value.id(),
-        };
-
-        Self::decode_with_metadata(bytes, id, metadata)
-    }
-}
-
-// Things can be dynamically decoded to our Value type:
-impl DecodeWithMetadata for DecodedValue {
-    type Target = Self;
-    fn decode_with_metadata(
-        bytes: &mut &[u8],
-        type_id: u32,
-        metadata: &Metadata,
-    ) -> Result<Self::Target, Error> {
-        let res = scale_value::scale::decode_as_type(bytes, type_id, metadata.types())?;
-        Ok(res)
-    }
-}
-
-/// Any type implementing [`Decode`] can also be decoded with the help of metadata.
-pub struct DecodeStaticType<T>(std::marker::PhantomData<T>);
-
-impl<T: Decode> DecodeWithMetadata for DecodeStaticType<T> {
-    type Target = T;
-
-    fn decode_with_metadata(
-        bytes: &mut &[u8],
-        _type_id: u32,
-        _metadata: &Metadata,
-    ) -> Result<Self::Target, Error> {
-        T::decode(bytes).map_err(|e| e.into())
-    }
-
-    fn decode_storage_with_metadata(
-        bytes: &mut &[u8],
-        _pallet_name: &str,
-        _storage_entry: &str,
-        _metadata: &Metadata,
-    ) -> Result<Self::Target, Error> {
-        T::decode(bytes).map_err(|e| e.into())
-    }
-}

subxt/src/metadata/encode_with_metadata.rs

@@ -1,67 +0,0 @@
-// Copyright 2019-2022 Parity Technologies (UK) Ltd.
-// This file is dual-licensed as Apache-2.0 or GPL-3.0.
-// see LICENSE for license details.
-
-use crate::{
-    dynamic::Value,
-    error::Error,
-    metadata::Metadata,
-};
-use codec::Encode;
-
-/// This trait is implemented for types which can be encoded with the help of metadata.
-pub trait EncodeWithMetadata {
-    /// SCALE encode this type to bytes, possibly with the help of metadata.
-    fn encode_with_metadata(
-        &self,
-        type_id: u32,
-        metadata: &Metadata,
-        bytes: &mut Vec<u8>,
-    ) -> Result<(), Error>;
-}
-
-impl EncodeWithMetadata for Value<()> {
-    fn encode_with_metadata(
-        &self,
-        type_id: u32,
-        metadata: &Metadata,
-        bytes: &mut Vec<u8>,
-    ) -> Result<(), Error> {
-        scale_value::scale::encode_as_type(self, type_id, metadata.types(), bytes)
-            .map_err(|e| e.into())
-    }
-}
-
-/// Any type implementing [`Encode`] can also be encoded with the help of metadata.
-pub struct EncodeStaticType<T>(pub T);
-
-impl<T: Encode> EncodeWithMetadata for EncodeStaticType<T> {
-    fn encode_with_metadata(
-        &self,
-        _type_id: u32,
-        _metadata: &Metadata,
-        bytes: &mut Vec<u8>,
-    ) -> Result<(), Error> {
-        self.0.encode_to(bytes);
-        Ok(())
-    }
-}
-
-// We can transparently Encode anything wrapped in EncodeStaticType, too.
-impl<E: Encode> Encode for EncodeStaticType<E> {
-    fn size_hint(&self) -> usize {
-        self.0.size_hint()
-    }
-    fn encode_to<T: codec::Output + ?Sized>(&self, dest: &mut T) {
-        self.0.encode_to(dest)
-    }
-    fn encode(&self) -> Vec<u8> {
-        self.0.encode()
-    }
-    fn using_encoded<R, F: FnOnce(&[u8]) -> R>(&self, f: F) -> R {
-        self.0.using_encoded(f)
-    }
-    fn encoded_size(&self) -> usize {
-        self.0.encoded_size()
-    }
-}

subxt/src/metadata/metadata_type.rs

@@ -78,13 +78,19 @@ pub enum MetadataError {
 #[derive(Debug)]
 struct MetadataInner {
     metadata: RuntimeMetadataV14,
-    pallets: HashMap<String, PalletMetadata>,
+
+    // Events are hashed by pallet an error index (decode oriented)
     events: HashMap<(u8, u8), EventMetadata>,
-    // Errors are hashed by pallet index.
+    // Errors are hashed by pallet and error index (decode oriented)
     errors: HashMap<(u8, u8), ErrorMetadata>,
+
+    // Other pallet details are hashed by pallet name.
+    pallets: HashMap<String, PalletMetadata>,
+
     // Type of the DispatchError type, which is what comes back if
     // an extrinsic fails.
     dispatch_error_ty: Option<u32>,
+
     // The hashes uniquely identify parts of the metadata; different
     // hashes mean some type difference exists between static and runtime
     // versions. We cache them here to avoid recalculating:
@@ -245,8 +251,9 @@ impl Metadata {
 pub struct PalletMetadata {
     index: u8,
     name: String,
-    call_indexes: HashMap<String, u8>,
+    call_metadata: HashMap<String, CallMetadata>,
     call_ty_id: Option<u32>,
+    event_ty_id: Option<u32>,
     storage: HashMap<String, StorageEntryMetadata<PortableForm>>,
     constants: HashMap<String, PalletConstantMetadata<PortableForm>>,
 }
@@ -268,11 +275,17 @@ impl PalletMetadata {
         self.call_ty_id
     }
 
+    /// If events exist for this pallet, this returns the type ID of the variant
+    /// representing the different possible events.
+    pub fn event_ty_id(&self) -> Option<u32> {
+        self.event_ty_id
+    }
+
     /// Attempt to resolve a call into an index in this pallet, failing
     /// if the call is not found in this pallet.
-    pub fn call_index(&self, function: &str) -> Result<u8, MetadataError> {
-        let fn_index = *self
-            .call_indexes
+    pub fn call(&self, function: &str) -> Result<&CallMetadata, MetadataError> {
+        let fn_index = self
+            .call_metadata
             .get(function)
             .ok_or(MetadataError::CallNotFound)?;
         Ok(fn_index)
@@ -297,37 +310,21 @@ impl PalletMetadata {
     }
 }
 
-/// Metadata for specific field.
 #[derive(Clone, Debug)]
-pub struct EventFieldMetadata {
-    name: Option<String>,
-    type_name: Option<String>,
-    type_id: u32,
+pub struct CallMetadata {
+    call_index: u8,
+    fields: Vec<scale_info::Field<scale_info::form::PortableForm>>,
 }
 
-impl EventFieldMetadata {
-    /// Construct a new [`EventFieldMetadata`]
-    pub fn new(name: Option<String>, type_name: Option<String>, type_id: u32) -> Self {
-        EventFieldMetadata {
-            name,
-            type_name,
-            type_id,
-        }
+impl CallMetadata {
+    /// Index of this call.
+    pub fn index(&self) -> u8 {
+        self.call_index
     }
 
-    /// Get the name of the field.
-    pub fn name(&self) -> Option<&str> {
-        self.name.as_deref()
-    }
-
-    /// Get the type name of the field as it appears in the code
-    pub fn type_name(&self) -> Option<&str> {
-        self.type_name.as_deref()
-    }
-
-    /// Get the id of a type
-    pub fn type_id(&self) -> u32 {
-        self.type_id
+    /// The names, type names & types of each field in the call data.
+    pub fn fields(&self) -> &[scale_info::Field<scale_info::form::PortableForm>] {
+        &self.fields
     }
 }
 
@@ -338,7 +335,7 @@ pub struct EventMetadata {
     // behind an Arc to avoid lots of needless clones of it existing.
     pallet: Arc<str>,
     event: String,
-    fields: Vec<EventFieldMetadata>,
+    fields: Vec<scale_info::Field<scale_info::form::PortableForm>>,
     docs: Vec<String>,
 }
 
@@ -354,7 +351,7 @@ impl EventMetadata {
     }
 
     /// The names, type names & types of each field in the event.
-    pub fn fields(&self) -> &[EventFieldMetadata] {
+    pub fn fields(&self) -> &[scale_info::Field<scale_info::form::PortableForm>] {
         &self.fields
     }
 
@@ -437,14 +434,23 @@ impl TryFrom<RuntimeMetadataPrefixed> for Metadata {
             .iter()
             .map(|pallet| {
                 let call_ty_id = pallet.calls.as_ref().map(|c| c.ty.id());
+                let event_ty_id = pallet.event.as_ref().map(|e| e.ty.id());
 
-                let call_indexes =
+                let call_metadata =
                     pallet.calls.as_ref().map_or(Ok(HashMap::new()), |call| {
                         let type_def_variant = get_type_def_variant(call.ty.id())?;
                         let call_indexes = type_def_variant
                             .variants()
                             .iter()
-                            .map(|v| (v.name().clone(), v.index()))
+                            .map(|v| {
+                                (
+                                    v.name().clone(),
+                                    CallMetadata {
+                                        call_index: v.index(),
+                                        fields: v.fields().to_vec(),
+                                    },
+                                )
+                            })
                             .collect();
                         Ok(call_indexes)
                     })?;
@@ -466,8 +472,9 @@ impl TryFrom<RuntimeMetadataPrefixed> for Metadata {
                 let pallet_metadata = PalletMetadata {
                     index: pallet.index,
                     name: pallet.name.to_string(),
-                    call_indexes,
+                    call_metadata,
                     call_ty_id,
+                    event_ty_id,
                     storage,
                     constants,
                 };
@@ -488,17 +495,7 @@ impl TryFrom<RuntimeMetadataPrefixed> for Metadata {
                         EventMetadata {
                             pallet: pallet_name.clone(),
                             event: variant.name().to_owned(),
-                            fields: variant
-                                .fields()
-                                .iter()
-                                .map(|f| {
-                                    EventFieldMetadata::new(
-                                        f.name().map(|n| n.to_owned()),
-                                        f.type_name().map(|n| n.to_owned()),
-                                        f.ty().id(),
-                                    )
-                                })
-                                .collect(),
+                            fields: variant.fields().to_vec(),
                             docs: variant.docs().to_vec(),
                         },
                     );

subxt/src/metadata/mod.rs

@@ -4,8 +4,7 @@
 
 //! Types representing the metadata obtained from a node.
 
-mod decode_with_metadata;
-mod encode_with_metadata;
+mod decode_encode_traits;
 mod hash_cache;
 mod metadata_location;
 mod metadata_type;
@@ -14,7 +13,6 @@ pub use metadata_location::MetadataLocation;
 
 pub use metadata_type::{
     ErrorMetadata,
-    EventFieldMetadata,
     EventMetadata,
     InvalidMetadataError,
     Metadata,
@@ -22,12 +20,7 @@ pub use metadata_type::{
     PalletMetadata,
 };
 
-pub use decode_with_metadata::{
-    DecodeStaticType,
+pub use decode_encode_traits::{
     DecodeWithMetadata,
-};
-
-pub use encode_with_metadata::{
-    EncodeStaticType,
     EncodeWithMetadata,
 };