Use scale-encode and scale-decode to encode and decode based on metadata (#842)

* WIP EncodeAsType and DecodeAsType

* remove silly cli experiment code

* Get things finally compiling with EncodeAsType and DecodeAsType

* update codegen test and WrapperKeepOpaque proper impl (in case it shows up in codegen)

* fix tests

* accomodate scale-value changes

* starting to migrate to EncodeAsType/DecodeAsType

* static event decoding and tx encoding to use DecodeAsFields/EncodeAsFields

* some tidy up and add decode(skip) attrs where needed

* fix root event decoding

* #[codec(skip)] will do, and combine map_key stuff into storage_address since it's all specific to that

* fmt and clippy

* update Cargo.lock

* remove patched scale-encode

* bump scale-encode to 0.1 and remove unused dep in testing crate

* update deps and use released scale-decode

* update scale-value to latest to remove git branch

* Apply suggestions from code review

Co-authored-by: Alexandru Vasile <60601340+lexnv@users.noreply.github.com>

* remove sorting in derives/attr generation; spit them out in order given

* re-add derive sorting; it's a hashmap

* StaticTxPayload and DynamicTxPayload rolled into single Payload struct

* StaticStorageAddress and DynamicStorageAddress into single Address struct

* Fix storage address byte retrieval

* StaticConstantAddress and DynamicConstantAddress => Address

* Simplify storage codegen to fix test

* Add comments

* Alias to RuntimeEvent rather than making another, and prep for substituting call type

* remove unnecessary clone

* Fix docs and failing UI test

* root_bytes -> to_root_bytes

* document error case in StorageClient::address_bytes()

---------

Co-authored-by: Alexandru Vasile <60601340+lexnv@users.noreply.github.com>

subxt/src/utils/account_id.rs

@@ -18,7 +18,18 @@ use serde::{
 /// A 32-byte cryptographic identifier. This is a simplified version of Substrate's
 /// `sp_core::crypto::AccountId32`. To obtain more functionality, convert this into
 /// that type.
-#[derive(Clone, Eq, PartialEq, Ord, PartialOrd, Encode, Decode, Debug)]
+#[derive(
+    Clone,
+    Eq,
+    PartialEq,
+    Ord,
+    PartialOrd,
+    Encode,
+    Decode,
+    Debug,
+    scale_encode::EncodeAsType,
+    scale_decode::DecodeAsType,
+)]
 pub struct AccountId32(pub [u8; 32]);
 
 impl AsRef<[u8]> for AccountId32 {

subxt/src/utils/bits.rs

@@ -16,55 +16,53 @@ use scale_bits::{
     },
     Bits,
 };
+use scale_decode::IntoVisitor;
 use std::marker::PhantomData;
 
-macro_rules! store {
-        ($ident: ident; $(($ty: ident, $wrapped: ty)),*) => {
-            /// Associates `bitvec::store::BitStore` trait with corresponding, type-erased `scale_bits::StoreFormat` enum.
-            ///
-            /// Used to decode bit sequences by providing `scale_bits::StoreFormat` using
-            /// `bitvec`-like type type parameters.
-            pub trait $ident {
-                /// Corresponding `scale_bits::StoreFormat` value.
-                const FORMAT: StoreFormat;
-                /// Number of bits that the backing store types holds.
-                const BITS: u32;
-            }
+/// Associates `bitvec::store::BitStore` trait with corresponding, type-erased `scale_bits::StoreFormat` enum.
+///
+/// Used to decode bit sequences by providing `scale_bits::StoreFormat` using
+/// `bitvec`-like type type parameters.
+pub trait BitStore {
+    /// Corresponding `scale_bits::StoreFormat` value.
+    const FORMAT: StoreFormat;
+    /// Number of bits that the backing store types holds.
+    const BITS: u32;
+}
+macro_rules! impl_store {
+    ($ty:ident, $wrapped:ty) => {
+        impl BitStore for $wrapped {
+            const FORMAT: StoreFormat = StoreFormat::$ty;
+            const BITS: u32 = <$wrapped>::BITS;
+        }
+    };
+}
+impl_store!(U8, u8);
+impl_store!(U16, u16);
+impl_store!(U32, u32);
+impl_store!(U64, u64);
 
-            $(
-                impl $ident for $wrapped {
-                    const FORMAT: StoreFormat = StoreFormat::$ty;
-                    const BITS: u32 = <$wrapped>::BITS;
-                }
-            )*
-        };
-    }
-
-macro_rules! order {
-        ($ident: ident; $($ty: ident),*) => {
-            /// Associates `bitvec::order::BitOrder` trait with corresponding, type-erased `scale_bits::OrderFormat` enum.
-            ///
-            /// Used to decode bit sequences in runtime by providing `scale_bits::OrderFormat` using
-            /// `bitvec`-like type type parameters.
-            pub trait $ident {
-                /// Corresponding `scale_bits::OrderFormat` value.
-                const FORMAT: OrderFormat;
-            }
-
-            $(
-                #[doc = concat!("Type-level value that corresponds to `scale_bits::OrderFormat::", stringify!($ty), "` at run-time")]
-                #[doc = concat!(" and `bitvec::order::BitOrder::", stringify!($ty), "` at the type level.")]
-                #[derive(Clone, Debug, PartialEq, Eq)]
-                pub enum $ty {}
-                impl $ident for $ty {
-                    const FORMAT: OrderFormat = OrderFormat::$ty;
-                }
-            )*
-        };
-    }
-
-store!(BitStore; (U8, u8), (U16, u16), (U32, u32), (U64, u64));
-order!(BitOrder; Lsb0, Msb0);
+/// Associates `bitvec::order::BitOrder` trait with corresponding, type-erased `scale_bits::OrderFormat` enum.
+///
+/// Used to decode bit sequences in runtime by providing `scale_bits::OrderFormat` using
+/// `bitvec`-like type type parameters.
+pub trait BitOrder {
+    /// Corresponding `scale_bits::OrderFormat` value.
+    const FORMAT: OrderFormat;
+}
+macro_rules! impl_order {
+    ($ty:ident) => {
+        #[doc = concat!("Type-level value that corresponds to `scale_bits::OrderFormat::", stringify!($ty), "` at run-time")]
+        #[doc = concat!(" and `bitvec::order::BitOrder::", stringify!($ty), "` at the type level.")]
+        #[derive(Clone, Debug, PartialEq, Eq)]
+        pub enum $ty {}
+        impl BitOrder for $ty {
+            const FORMAT: OrderFormat = OrderFormat::$ty;
+        }
+    };
+}
+impl_order!(Lsb0);
+impl_order!(Msb0);
 
 /// Constructs a run-time format parameters based on the corresponding type-level parameters.
 fn bit_format<Store: BitStore, Order: BitOrder>() -> Format {
@@ -77,29 +75,29 @@ fn bit_format<Store: BitStore, Order: BitOrder>() -> Format {
 /// `scale_bits::Bits` generic over the bit store (`u8`/`u16`/`u32`/`u64`) and bit order (LSB, MSB)
 /// used for SCALE encoding/decoding. Uses `scale_bits::Bits`-default `u8` and LSB format underneath.
 #[derive(Debug, Clone, PartialEq, Eq)]
-pub struct DecodedBits<Store: BitStore, Order: BitOrder>(
-    Bits,
-    PhantomData<Store>,
-    PhantomData<Order>,
-);
+pub struct DecodedBits<Store, Order> {
+    bits: Bits,
+    _marker: PhantomData<(Store, Order)>,
+}
 
-impl<Store: BitStore, Order: BitOrder> DecodedBits<Store, Order> {
+impl<Store, Order> DecodedBits<Store, Order> {
     /// Extracts the underlying `scale_bits::Bits` value.
     pub fn into_bits(self) -> Bits {
-        self.0
+        self.bits
     }
 
     /// References the underlying `scale_bits::Bits` value.
     pub fn as_bits(&self) -> &Bits {
-        &self.0
+        &self.bits
     }
 }
 
-impl<Store: BitStore, Order: BitOrder> core::iter::FromIterator<bool>
-    for DecodedBits<Store, Order>
-{
+impl<Store, Order> core::iter::FromIterator<bool> for DecodedBits<Store, Order> {
     fn from_iter<T: IntoIterator<Item = bool>>(iter: T) -> Self {
-        DecodedBits(Bits::from_iter(iter), PhantomData, PhantomData)
+        DecodedBits {
+            bits: Bits::from_iter(iter),
+            _marker: PhantomData,
+        }
     }
 }
 
@@ -132,21 +130,72 @@ impl<Store: BitStore, Order: BitOrder> codec::Decode for DecodedBits<Store, Orde
         let bits = decoder.collect::<Result<Vec<_>, _>>()?;
         let bits = Bits::from_iter(bits);
 
-        Ok(DecodedBits(bits, PhantomData, PhantomData))
+        Ok(DecodedBits {
+            bits,
+            _marker: PhantomData,
+        })
     }
 }
 
 impl<Store: BitStore, Order: BitOrder> codec::Encode for DecodedBits<Store, Order> {
     fn size_hint(&self) -> usize {
-        self.0.size_hint()
+        self.bits.size_hint()
     }
 
     fn encoded_size(&self) -> usize {
-        self.0.encoded_size()
+        self.bits.encoded_size()
     }
 
     fn encode(&self) -> Vec<u8> {
-        scale_bits::encode_using_format(self.0.iter(), bit_format::<Store, Order>())
+        scale_bits::encode_using_format(self.bits.iter(), bit_format::<Store, Order>())
+    }
+}
+
+#[doc(hidden)]
+pub struct DecodedBitsVisitor<S, O>(std::marker::PhantomData<(S, O)>);
+impl<Store, Order> scale_decode::Visitor for DecodedBitsVisitor<Store, Order> {
+    type Value<'scale, 'info> = DecodedBits<Store, Order>;
+    type Error = scale_decode::Error;
+
+    fn unchecked_decode_as_type<'scale, 'info>(
+        self,
+        input: &mut &'scale [u8],
+        type_id: scale_decode::visitor::TypeId,
+        types: &'info scale_info::PortableRegistry,
+    ) -> scale_decode::visitor::DecodeAsTypeResult<
+        Self,
+        Result<Self::Value<'scale, 'info>, Self::Error>,
+    > {
+        let res = scale_decode::visitor::decode_with_visitor(
+            input,
+            type_id.0,
+            types,
+            Bits::into_visitor(),
+        )
+        .map(|bits| {
+            DecodedBits {
+                bits,
+                _marker: PhantomData,
+            }
+        });
+        scale_decode::visitor::DecodeAsTypeResult::Decoded(res)
+    }
+}
+impl<Store, Order> scale_decode::IntoVisitor for DecodedBits<Store, Order> {
+    type Visitor = DecodedBitsVisitor<Store, Order>;
+    fn into_visitor() -> Self::Visitor {
+        DecodedBitsVisitor(PhantomData)
+    }
+}
+
+impl<Store, Order> scale_encode::EncodeAsType for DecodedBits<Store, Order> {
+    fn encode_as_type_to(
+        &self,
+        type_id: u32,
+        types: &scale_info::PortableRegistry,
+        out: &mut Vec<u8>,
+    ) -> Result<(), scale_encode::Error> {
+        self.bits.encode_as_type_to(type_id, types, out)
     }
 }
 

subxt/src/utils/mod.rs

@@ -4,14 +4,14 @@
 
 //! Miscellaneous utility helpers.
 
-pub mod account_id;
+mod account_id;
 pub mod bits;
-pub mod multi_address;
-pub mod multi_signature;
+mod multi_address;
+mod multi_signature;
+mod wrapper_opaque;
 
 use codec::{
     Decode,
-    DecodeAll,
     Encode,
 };
 use derivative::Derivative;
@@ -19,6 +19,7 @@ use derivative::Derivative;
 pub use account_id::AccountId32;
 pub use multi_address::MultiAddress;
 pub use multi_signature::MultiSignature;
+pub use wrapper_opaque::WrapperKeepOpaque;
 
 // Used in codegen
 #[doc(hidden)]
@@ -39,51 +40,6 @@ impl codec::Encode for Encoded {
     }
 }
 
-/// A wrapper for any type `T` which implement encode/decode in a way compatible with `Vec<u8>`.
-///
-/// [`WrapperKeepOpaque`] stores the type only in its opaque format, aka as a `Vec<u8>`. To
-/// access the real type `T` [`Self::try_decode`] needs to be used.
-#[derive(Derivative, Encode, Decode)]
-#[derivative(
-    Debug(bound = ""),
-    Clone(bound = ""),
-    PartialEq(bound = ""),
-    Eq(bound = ""),
-    Default(bound = ""),
-    Hash(bound = "")
-)]
-pub struct WrapperKeepOpaque<T> {
-    data: Vec<u8>,
-    _phantom: PhantomDataSendSync<T>,
-}
-
-impl<T: Decode> WrapperKeepOpaque<T> {
-    /// Try to decode the wrapped type from the inner `data`.
-    ///
-    /// Returns `None` if the decoding failed.
-    pub fn try_decode(&self) -> Option<T> {
-        T::decode_all(&mut &self.data[..]).ok()
-    }
-
-    /// Returns the length of the encoded `T`.
-    pub fn encoded_len(&self) -> usize {
-        self.data.len()
-    }
-
-    /// Returns the encoded data.
-    pub fn encoded(&self) -> &[u8] {
-        &self.data
-    }
-
-    /// Create from the given encoded `data`.
-    pub fn from_encoded(data: Vec<u8>) -> Self {
-        Self {
-            data,
-            _phantom: PhantomDataSendSync::new(),
-        }
-    }
-}
-
 /// A version of [`std::marker::PhantomData`] that is also Send and Sync (which is fine
 /// because regardless of the generic param, it is always possible to Send + Sync this
 /// 0 size type).

subxt/src/utils/multi_address.rs

@@ -14,7 +14,18 @@ use codec::{
 /// A multi-format address wrapper for on-chain accounts. This is a simplified version of Substrate's
 /// `sp_runtime::MultiAddress`. To obtain more functionality, convert this into that type (this conversion
 /// functionality is provided via `From` impls if the `substrate-compat` feature is enabled).
-#[derive(Clone, Eq, PartialEq, Ord, PartialOrd, Encode, Decode, Debug)]
+#[derive(
+    Clone,
+    Eq,
+    PartialEq,
+    Ord,
+    PartialOrd,
+    Encode,
+    Decode,
+    Debug,
+    scale_encode::EncodeAsType,
+    scale_decode::DecodeAsType,
+)]
 pub enum MultiAddress<AccountId, AccountIndex> {
     /// It's an account ID (pubkey).
     Id(AccountId),

subxt/src/utils/wrapper_opaque.rs

@@ -0,0 +1,266 @@
+// Copyright 2019-2022 Parity Technologies (UK) Ltd.
+// This file is dual-licensed as Apache-2.0 or GPL-3.0.
+// see LICENSE for license details.
+
+use super::PhantomDataSendSync;
+use codec::{
+    Compact,
+    Decode,
+    DecodeAll,
+    Encode,
+};
+use derivative::Derivative;
+use scale_decode::{
+    IntoVisitor,
+    Visitor,
+};
+use scale_encode::EncodeAsType;
+
+/// A wrapper for any type `T` which implement encode/decode in a way compatible with `Vec<u8>`.
+/// [`WrapperKeepOpaque`] stores the type only in its opaque format, aka as a `Vec<u8>`. To
+/// access the real type `T` [`Self::try_decode`] needs to be used.
+// Dev notes:
+//
+// - This is adapted from [here](https://github.com/paritytech/substrate/blob/master/frame/support/src/traits/misc.rs).
+// - The encoded bytes will be a compact encoded length followed by that number of bytes.
+// - However, the TypeInfo describes the type as a composite with first a compact encoded length and next the type itself.
+//  [`Encode`] and [`Decode`] impls will "just work" to take this into a `Vec<u8>`, but we need a custom [`EncodeAsType`]
+//  and [`Visitor`] implementation to encode and decode based on TypeInfo.
+#[derive(Derivative, Encode, Decode)]
+#[derivative(
+    Debug(bound = ""),
+    Clone(bound = ""),
+    PartialEq(bound = ""),
+    Eq(bound = ""),
+    Default(bound = ""),
+    Hash(bound = "")
+)]
+pub struct WrapperKeepOpaque<T> {
+    data: Vec<u8>,
+    _phantom: PhantomDataSendSync<T>,
+}
+
+impl<T> WrapperKeepOpaque<T> {
+    /// Try to decode the wrapped type from the inner `data`.
+    ///
+    /// Returns `None` if the decoding failed.
+    pub fn try_decode(&self) -> Option<T>
+    where
+        T: Decode,
+    {
+        T::decode_all(&mut &self.data[..]).ok()
+    }
+
+    /// Returns the length of the encoded `T`.
+    pub fn encoded_len(&self) -> usize {
+        self.data.len()
+    }
+
+    /// Returns the encoded data.
+    pub fn encoded(&self) -> &[u8] {
+        &self.data
+    }
+
+    /// Create from the given encoded `data`.
+    pub fn from_encoded(data: Vec<u8>) -> Self {
+        Self {
+            data,
+            _phantom: PhantomDataSendSync::new(),
+        }
+    }
+
+    /// Create from some raw value by encoding it.
+    pub fn from_value(value: T) -> Self
+    where
+        T: Encode,
+    {
+        Self {
+            data: value.encode(),
+            _phantom: PhantomDataSendSync::new(),
+        }
+    }
+}
+
+impl<T> EncodeAsType for WrapperKeepOpaque<T> {
+    fn encode_as_type_to(
+        &self,
+        type_id: u32,
+        types: &scale_info::PortableRegistry,
+        out: &mut Vec<u8>,
+    ) -> Result<(), scale_encode::Error> {
+        use scale_encode::error::{
+            Error,
+            ErrorKind,
+            Kind,
+        };
+
+        let Some(ty) = types.resolve(type_id) else {
+            return Err(Error::new(ErrorKind::TypeNotFound(type_id)))
+        };
+
+        // Do a basic check that the target shape lines up.
+        let scale_info::TypeDef::Composite(_) = ty.type_def() else {
+            return Err(Error::new(ErrorKind::WrongShape {
+                actual: Kind::Struct,
+                expected: type_id,
+            }))
+        };
+
+        // Check that the name also lines up.
+        if ty.path().ident().as_deref() != Some("WrapperKeepOpaque") {
+            return Err(Error::new(ErrorKind::WrongShape {
+                actual: Kind::Struct,
+                expected: type_id,
+            }))
+        }
+
+        // Just blat the bytes out.
+        self.data.encode_to(out);
+        Ok(())
+    }
+}
+
+pub struct WrapperKeepOpaqueVisitor<T>(std::marker::PhantomData<T>);
+impl<T> Visitor for WrapperKeepOpaqueVisitor<T> {
+    type Value<'scale, 'info> = WrapperKeepOpaque<T>;
+    type Error = scale_decode::Error;
+
+    fn visit_composite<'scale, 'info>(
+        self,
+        value: &mut scale_decode::visitor::types::Composite<'scale, 'info>,
+        _type_id: scale_decode::visitor::TypeId,
+    ) -> Result<Self::Value<'scale, 'info>, Self::Error> {
+        use scale_decode::error::{
+            Error,
+            ErrorKind,
+        };
+
+        if value.path().ident().as_deref() != Some("WrapperKeepOpaque") {
+            return Err(Error::new(ErrorKind::Custom(
+                "Type to decode is not 'WrapperTypeKeepOpaque'".into(),
+            )))
+        }
+        if value.remaining() != 2 {
+            return Err(Error::new(ErrorKind::WrongLength {
+                actual_len: value.remaining(),
+                expected_len: 2,
+            }))
+        }
+
+        // The field to decode is a compact len followed by bytes. Decode the length, then grab the bytes.
+        let Compact(len) = value
+            .decode_item(Compact::<u32>::into_visitor())
+            .expect("length checked")?;
+        let field = value.next().expect("length checked")?;
+
+        // Sanity check that the compact length we decoded lines up with the number of bytes encoded in the next field.
+        if field.bytes().len() != len as usize {
+            return Err(Error::new(ErrorKind::Custom("WrapperTypeKeepOpaque compact encoded length doesn't line up with encoded byte len".into())));
+        }
+
+        Ok(WrapperKeepOpaque {
+            data: field.bytes().to_vec(),
+            _phantom: PhantomDataSendSync::new(),
+        })
+    }
+}
+
+impl<T> IntoVisitor for WrapperKeepOpaque<T> {
+    type Visitor = WrapperKeepOpaqueVisitor<T>;
+    fn into_visitor() -> Self::Visitor {
+        WrapperKeepOpaqueVisitor(std::marker::PhantomData)
+    }
+}
+
+#[cfg(test)]
+mod test {
+    use scale_decode::DecodeAsType;
+
+    use super::*;
+
+    // Copied from https://github.com/paritytech/substrate/blob/master/frame/support/src/traits/misc.rs
+    // and used for tests to check that we can work with the expected TypeInfo without needing to import
+    // the frame_support crate, which has quite a lot of dependencies.
+    impl<T: scale_info::TypeInfo + 'static> scale_info::TypeInfo for WrapperKeepOpaque<T> {
+        type Identity = Self;
+        fn type_info() -> scale_info::Type {
+            use scale_info::{
+                build::Fields,
+                meta_type,
+                Path,
+                Type,
+                TypeParameter,
+            };
+
+            Type::builder()
+                .path(Path::new("WrapperKeepOpaque", module_path!()))
+                .type_params(vec![TypeParameter::new("T", Some(meta_type::<T>()))])
+                .composite(
+                    Fields::unnamed()
+                        .field(|f| f.compact::<u32>())
+                        .field(|f| f.ty::<T>().type_name("T")),
+                )
+        }
+    }
+
+    /// Given a type definition, return type ID and registry representing it.
+    fn make_type<T: scale_info::TypeInfo + 'static>(
+    ) -> (u32, scale_info::PortableRegistry) {
+        let m = scale_info::MetaType::new::<T>();
+        let mut types = scale_info::Registry::new();
+        let id = types.register_type(&m);
+        let portable_registry: scale_info::PortableRegistry = types.into();
+        (id.id(), portable_registry)
+    }
+
+    fn roundtrips_like_scale_codec<T>(t: T)
+    where
+        T: EncodeAsType
+            + DecodeAsType
+            + Encode
+            + Decode
+            + PartialEq
+            + std::fmt::Debug
+            + scale_info::TypeInfo
+            + 'static,
+    {
+        let (type_id, types) = make_type::<T>();
+
+        let scale_codec_encoded = t.encode();
+        let encode_as_type_encoded = t.encode_as_type(type_id, &types).unwrap();
+
+        assert_eq!(
+            scale_codec_encoded, encode_as_type_encoded,
+            "encoded bytes should match"
+        );
+
+        let decode_as_type_bytes = &mut &*scale_codec_encoded;
+        let decoded_as_type = T::decode_as_type(decode_as_type_bytes, type_id, &types)
+            .expect("decode-as-type decodes");
+
+        let decode_scale_codec_bytes = &mut &*scale_codec_encoded;
+        let decoded_scale_codec =
+            T::decode(decode_scale_codec_bytes).expect("scale-codec decodes");
+
+        assert!(
+            decode_as_type_bytes.is_empty(),
+            "no bytes should remain in decode-as-type impl"
+        );
+        assert!(
+            decode_scale_codec_bytes.is_empty(),
+            "no bytes should remain in codec-decode impl"
+        );
+
+        assert_eq!(
+            decoded_as_type, decoded_scale_codec,
+            "decoded values should match"
+        );
+    }
+
+    #[test]
+    fn wrapper_keep_opaque_roundtrips_ok() {
+        roundtrips_like_scale_codec(WrapperKeepOpaque::from_value(123u64));
+        roundtrips_like_scale_codec(WrapperKeepOpaque::from_value(true));
+        roundtrips_like_scale_codec(WrapperKeepOpaque::from_value(vec![1u8, 2, 3, 4]));
+    }
+}