Add bitvec-like generic support to the scale-bits type for use in codegen (#718)

* Add bitvec-like generic support to the scale-bits type for use in codegen * Use nightly 1.66 formatting * Fix reading input while decoding bit sequences * Add tests for our DecodedBits wrapper * Add convenience DecodedBits::(in)to_bits functions * Don't expose DecodedBits::bit_format * Re-export scale_bits as peer dependency * Move subxt::utils into a separate file * Hide DecodedBits internals * Don't re-export types from the `bits` module * Update subxt/src/utils/bits.rs Co-authored-by: Niklas Adolfsson <niklasadolfsson1@gmail.com> * Update subxt/src/utils/bits.rs Co-authored-by: Niklas Adolfsson <niklasadolfsson1@gmail.com> * Address review feedback * Clarify the byte needed calculation in DecodedBits encoding * Remove remaining dbg! invocations Co-authored-by: Niklas Adolfsson <niklasadolfsson1@gmail.com>
2026-06-09 20:11:09 +00:00 · 2022-11-24 15:09:20 +01:00
parent a80d6cfd30
commit f0ce26db7b
9 changed files with 273 additions and 28 deletions
@@ -12,7 +12,7 @@ homepage = "https://www.parity.io/"
 description = "Generate an API for interacting with a substrate node from FRAME metadata"

 [dependencies]
-codec = { package = "parity-scale-codec", version = "3.0.0", default-features = false, features = ["derive", "full", "bit-vec"] }
+codec = { package = "parity-scale-codec", version = "3.0.0", default-features = false, features = ["derive", "full"] }
 darling = "0.14.0"
 frame-metadata = "15.0.0"
 heck = "0.4.0"
@@ -20,7 +20,7 @@ proc-macro2 = "1.0.24"
 proc-macro-error = "1.0.4"
 quote = "1.0.8"
 syn = "1.0.58"
-scale-info = { version = "2.0.0", features = ["bit-vec"] }
+scale-info = "2.0.0"
 subxt-metadata = { version = "0.25.0", path = "../metadata" }
 jsonrpsee = { version = "0.16.0", features = ["async-client", "client-ws-transport", "http-client"] }
 hex = "0.4.3"
@@ -28,4 +28,5 @@ tokio = { version = "1.8", features = ["macros", "rt-multi-thread"] }

 [dev-dependencies]
 bitvec = { version = "1.0.0", default-features = false, features = ["alloc"] }
+scale-info = { version = "2.0.0", features = ["bit-vec"] }
 pretty_assertions = "1.0.0"
@@ -161,11 +161,11 @@ impl RuntimeGenerator {
        let mut type_substitutes = [
            (
                "bitvec::order::Lsb0",
-                parse_quote!(#crate_path::ext::bitvec::order::Lsb0),
+                parse_quote!(#crate_path::utils::bits::Lsb0),
            ),
            (
                "bitvec::order::Msb0",
-                parse_quote!(#crate_path::ext::bitvec::order::Msb0),
+                parse_quote!(#crate_path::utils::bits::Msb0),
            ),
            (
                "sp_core::crypto::AccountId32",
@@ -745,10 +745,22 @@ fn generate_bitvec() {
    registry.register_type(&meta_type::<S>());
    let portable_types: PortableRegistry = registry.into();

+    let substitutes = [
+        (
+            String::from("bitvec::order::Lsb0"),
+            parse_quote!(::subxt_path::utils::bits::Lsb0),
+        ),
+        (
+            String::from("bitvec::order::Msb0"),
+            parse_quote!(::subxt_path::utils::bits::Msb0),
+        ),
+    ]
+    .into();
+
    let type_gen = TypeGenerator::new(
        &portable_types,
        "root",
-        Default::default(),
+        substitutes,
        DerivesRegistry::new(&"::subxt_path".into()),
        "::subxt_path".into(),
    );
@@ -762,8 +774,8 @@ fn generate_bitvec() {
                use super::root;
                #[derive(::subxt_path::ext::codec::Decode, ::subxt_path::ext::codec::Encode, Debug)]
                pub struct S {
-                    pub lsb: ::subxt_path::ext::bitvec::vec::BitVec<::core::primitive::u8, root::bitvec::order::Lsb0>,
-                    pub msb: ::subxt_path::ext::bitvec::vec::BitVec<::core::primitive::u16, root::bitvec::order::Msb0>,
+                    pub lsb: ::subxt_path::utils::bits::DecodedBits<::core::primitive::u8, ::subxt_path::utils::bits::Lsb0>,
+                    pub msb: ::subxt_path::utils::bits::DecodedBits<::core::primitive::u16, ::subxt_path::utils::bits::Msb0>,
                }
            }
        }
@@ -246,7 +246,7 @@ impl TypePathType {
                bit_store_type,
                crate_path,
            } => {
-                let type_path = parse_quote! { #crate_path::ext::bitvec::vec::BitVec<#bit_store_type, #bit_order_type> };
+                let type_path = parse_quote! { #crate_path::utils::bits::DecodedBits<#bit_store_type, #bit_order_type> };
                syn::Type::Path(type_path)
            }
        }
@@ -27,10 +27,10 @@ jsonrpsee-ws = ["jsonrpsee/async-client", "jsonrpsee/client-ws-transport"]
 jsonrpsee-web = ["jsonrpsee/async-wasm-client", "jsonrpsee/client-web-transport"]

 [dependencies]
-bitvec = { version = "1.0.0", default-features = false, features = ["alloc"] }
-codec = { package = "parity-scale-codec", version = "3.0.0", default-features = false, features = ["derive", "full", "bit-vec"] }
-scale-info = { version = "2.0.0", features = ["bit-vec"] }
+codec = { package = "parity-scale-codec", version = "3.0.0", default-features = false, features = ["derive", "full"] }
+scale-info = "2.0.0"
 scale-value = "0.6.0"
+scale-bits = "0.3"
 scale-decode = "0.4.0"
 futures = { version = "0.3.13", default-features = false }
 hex = "0.4.3"
@@ -54,4 +54,7 @@ derivative = "2.2.0"
 getrandom = { version = "0.2", features = ["js"] }

 [dev-dependencies]
+bitvec = "1"
+codec = { package = "parity-scale-codec", version = "3.0.0", default-features = false, features = ["derive", "full", "bit-vec"] }
+scale-info = { version = "2.0.0", features = ["bit-vec"] }
 tokio = { version = "1.8", features = ["macros", "time", "rt-multi-thread"] }
@@ -177,9 +177,9 @@ pub use crate::{

 /// Re-export external crates that are made use of in the subxt API.
 pub mod ext {
-    pub use bitvec;
    pub use codec;
    pub use frame_metadata;
+    pub use scale_bits;
    pub use scale_value;
    pub use sp_core;
    pub use sp_runtime;
@@ -0,0 +1,227 @@
+// Copyright 2019-2022 Parity Technologies (UK) Ltd.
+// This file is dual-licensed as Apache-2.0 or GPL-3.0.
+// see LICENSE for license details.
+
+//! Generic `scale_bits` over `bitvec`-like `BitOrder` and `BitFormat` types.
+
+use codec::{
+    Compact,
+    Input,
+};
+use scale_bits::{
+    scale::format::{
+        Format,
+        OrderFormat,
+        StoreFormat,
+    },
+    Bits,
+};
+use std::marker::PhantomData;
+
+macro_rules! store {
+        ($ident: ident; $(($ty: ident, $wrapped: ty)),*) => {
+            /// Associates `bitvec::store::BitStore` trait with corresponding, type-erased `scale_bits::StoreFormat` enum.
+            ///
+            /// Used to decode bit sequences by providing `scale_bits::StoreFormat` using
+            /// `bitvec`-like type type parameters.
+            pub trait $ident {
+                /// Corresponding `scale_bits::StoreFormat` value.
+                const FORMAT: StoreFormat;
+                /// Number of bits that the backing store types holds.
+                const BITS: u32;
+            }
+
+            $(
+                impl $ident for $wrapped {
+                    const FORMAT: StoreFormat = StoreFormat::$ty;
+                    const BITS: u32 = <$wrapped>::BITS;
+                }
+            )*
+        };
+    }
+
+macro_rules! order {
+        ($ident: ident; $($ty: ident),*) => {
+            /// Associates `bitvec::order::BitOrder` trait with corresponding, type-erased `scale_bits::OrderFormat` enum.
+            ///
+            /// Used to decode bit sequences in runtime by providing `scale_bits::OrderFormat` using
+            /// `bitvec`-like type type parameters.
+            pub trait $ident {
+                /// Corresponding `scale_bits::OrderFormat` value.
+                const FORMAT: OrderFormat;
+            }
+
+            $(
+                #[doc = concat!("Type-level value that corresponds to `scale_bits::OrderFormat::", stringify!($ty), "` at run-time")]
+                #[doc = concat!(" and `bitvec::order::BitOrder::", stringify!($ty), "` at the type level.")]
+                #[derive(Clone, Debug, PartialEq, Eq)]
+                pub enum $ty {}
+                impl $ident for $ty {
+                    const FORMAT: OrderFormat = OrderFormat::$ty;
+                }
+            )*
+        };
+    }
+
+store!(BitStore; (U8, u8), (U16, u16), (U32, u32), (U64, u64));
+order!(BitOrder; Lsb0, Msb0);
+
+/// Constructs a run-time format parameters based on the corresponding type-level parameters.
+fn bit_format<Store: BitStore, Order: BitOrder>() -> Format {
+    Format {
+        order: Order::FORMAT,
+        store: Store::FORMAT,
+    }
+}
+
+/// `scale_bits::Bits` generic over the bit store (`u8`/`u16`/`u32`/`u64`) and bit order (LSB, MSB)
+/// used for SCALE encoding/decoding. Uses `scale_bits::Bits`-default `u8` and LSB format underneath.
+#[derive(Debug, Clone, PartialEq, Eq)]
+pub struct DecodedBits<Store: BitStore, Order: BitOrder>(
+    Bits,
+    PhantomData<Store>,
+    PhantomData<Order>,
+);
+
+impl<Store: BitStore, Order: BitOrder> DecodedBits<Store, Order> {
+    /// Extracts the underlying `scale_bits::Bits` value.
+    pub fn into_bits(self) -> Bits {
+        self.0
+    }
+
+    /// References the underlying `scale_bits::Bits` value.
+    pub fn as_bits(&self) -> &Bits {
+        &self.0
+    }
+}
+
+impl<Store: BitStore, Order: BitOrder> core::iter::FromIterator<bool>
+    for DecodedBits<Store, Order>
+{
+    fn from_iter<T: IntoIterator<Item = bool>>(iter: T) -> Self {
+        DecodedBits(Bits::from_iter(iter), PhantomData, PhantomData)
+    }
+}
+
+impl<Store: BitStore, Order: BitOrder> codec::Decode for DecodedBits<Store, Order> {
+    fn decode<I: Input>(input: &mut I) -> Result<Self, codec::Error> {
+        /// Equivalent of `BitSlice::MAX_BITS` on 32bit machine.
+        const ARCH32BIT_BITSLICE_MAX_BITS: u32 = 0x1fff_ffff;
+
+        let Compact(bits) = <Compact<u32>>::decode(input)?;
+        // Otherwise it is impossible to store it on 32bit machine.
+        if bits > ARCH32BIT_BITSLICE_MAX_BITS {
+            return Err("Attempt to decode a BitVec with too many bits".into())
+        }
+        // NOTE: Replace with `bits.div_ceil(Store::BITS)` if `int_roundings` is stabilised
+        let elements = (bits / Store::BITS) + u32::from(bits % Store::BITS != 0);
+        let bytes_in_elem = Store::BITS.saturating_div(u8::BITS);
+        let bytes_needed = (elements * bytes_in_elem) as usize;
+
+        // NOTE: We could reduce allocations if it would be possible to directly
+        // decode from an `Input` type using a custom format (rather than default <u8, Lsb0>)
+        // for the `Bits` type.
+        let mut storage = codec::Encode::encode(&Compact(bits));
+        let prefix_len = storage.len();
+        storage.reserve_exact(bytes_needed);
+        storage.extend(vec![0; bytes_needed]);
+        input.read(&mut storage[prefix_len..])?;
+
+        let decoder =
+            scale_bits::decode_using_format_from(&storage, bit_format::<Store, Order>())?;
+        let bits = decoder.collect::<Result<Vec<_>, _>>()?;
+        let bits = Bits::from_iter(bits);
+
+        Ok(DecodedBits(bits, PhantomData, PhantomData))
+    }
+}
+
+impl<Store: BitStore, Order: BitOrder> codec::Encode for DecodedBits<Store, Order> {
+    fn size_hint(&self) -> usize {
+        self.0.size_hint()
+    }
+
+    fn encoded_size(&self) -> usize {
+        self.0.encoded_size()
+    }
+
+    fn encode(&self) -> Vec<u8> {
+        scale_bits::encode_using_format(self.0.iter(), bit_format::<Store, Order>())
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    use core::fmt::Debug;
+
+    use bitvec::vec::BitVec;
+    use codec::Decode as _;
+
+    // NOTE: We don't use `bitvec::order` types in our implementation, since we
+    // don't want to depend on `bitvec`. Rather than reimplementing the unsafe
+    // trait on our types here for testing purposes, we simply convert and
+    // delegate to `bitvec`'s own types.
+    trait ToBitVec {
+        type Order: bitvec::order::BitOrder;
+    }
+    impl ToBitVec for Lsb0 {
+        type Order = bitvec::order::Lsb0;
+    }
+    impl ToBitVec for Msb0 {
+        type Order = bitvec::order::Msb0;
+    }
+
+    fn scales_like_bitvec_and_roundtrips<
+        'a,
+        Store: BitStore + bitvec::store::BitStore + PartialEq,
+        Order: BitOrder + ToBitVec + Debug + PartialEq,
+    >(
+        input: impl IntoIterator<Item = &'a bool>,
+    ) where
+        BitVec<Store, <Order as ToBitVec>::Order>: codec::Encode + codec::Decode,
+    {
+        let input: Vec<_> = input.into_iter().copied().collect();
+
+        let decoded_bits = DecodedBits::<Store, Order>::from_iter(input.clone());
+        let bitvec = BitVec::<Store, <Order as ToBitVec>::Order>::from_iter(input);
+
+        let decoded_bits_encoded = codec::Encode::encode(&decoded_bits);
+        let bitvec_encoded = codec::Encode::encode(&bitvec);
+        assert_eq!(decoded_bits_encoded, bitvec_encoded);
+
+        let decoded_bits_decoded =
+            DecodedBits::<Store, Order>::decode(&mut &decoded_bits_encoded[..])
+                .expect("SCALE-encoding DecodedBits to roundtrip");
+        let bitvec_decoded =
+            BitVec::<Store, <Order as ToBitVec>::Order>::decode(&mut &bitvec_encoded[..])
+                .expect("SCALE-encoding BitVec to roundtrip");
+        assert_eq!(decoded_bits, decoded_bits_decoded);
+        assert_eq!(bitvec, bitvec_decoded);
+    }
+
+    #[test]
+    fn decoded_bitvec_scales_and_roundtrips() {
+        let test_cases = [
+            vec![],
+            vec![true],
+            vec![false],
+            vec![true, false, true],
+            vec![true, false, true, false, false, false, false, false, true],
+            [vec![true; 5], vec![false; 5], vec![true; 1], vec![false; 3]].concat(),
+            [vec![true; 9], vec![false; 9], vec![true; 9], vec![false; 9]].concat(),
+        ];
+
+        for test_case in &test_cases {
+            scales_like_bitvec_and_roundtrips::<u8, Lsb0>(test_case);
+            scales_like_bitvec_and_roundtrips::<u16, Lsb0>(test_case);
+            scales_like_bitvec_and_roundtrips::<u32, Lsb0>(test_case);
+            scales_like_bitvec_and_roundtrips::<u64, Lsb0>(test_case);
+            scales_like_bitvec_and_roundtrips::<u8, Msb0>(test_case);
+            scales_like_bitvec_and_roundtrips::<u16, Msb0>(test_case);
+            scales_like_bitvec_and_roundtrips::<u32, Msb0>(test_case);
+            scales_like_bitvec_and_roundtrips::<u64, Msb0>(test_case);
+        }
+    }
+}
@@ -4,6 +4,8 @@

 //! Miscellaneous utility helpers.

+pub mod bits;
+
 use codec::{
    Decode,
    DecodeAll,
@@ -38145,9 +38145,9 @@ pub mod api {
                    Debug,
                )]
                pub struct AvailabilityBitfield(
-                    pub  ::subxt::ext::bitvec::vec::BitVec<
+                    pub  ::subxt::utils::bits::DecodedBits<
                        ::core::primitive::u8,
-                        ::subxt::ext::bitvec::order::Lsb0,
+                        ::subxt::utils::bits::Lsb0,
                    >,
                );
                #[derive(
@@ -38163,9 +38163,9 @@ pub mod api {
                    pub validity_votes: ::std::vec::Vec<
                        runtime_types::polkadot_primitives::v2::ValidityAttestation,
                    >,
-                    pub validator_indices: ::subxt::ext::bitvec::vec::BitVec<
+                    pub validator_indices: ::subxt::utils::bits::DecodedBits<
                        ::core::primitive::u8,
-                        ::subxt::ext::bitvec::order::Lsb0,
+                        ::subxt::utils::bits::Lsb0,
                    >,
                }
                #[derive(
@@ -38255,13 +38255,13 @@ pub mod api {
                    Debug,
                )]
                pub struct DisputeState<_0> {
-                    pub validators_for: ::subxt::ext::bitvec::vec::BitVec<
+                    pub validators_for: ::subxt::utils::bits::DecodedBits<
                        ::core::primitive::u8,
-                        ::subxt::ext::bitvec::order::Lsb0,
+                        ::subxt::utils::bits::Lsb0,
                    >,
-                    pub validators_against: ::subxt::ext::bitvec::vec::BitVec<
+                    pub validators_against: ::subxt::utils::bits::DecodedBits<
                        ::core::primitive::u8,
-                        ::subxt::ext::bitvec::order::Lsb0,
+                        ::subxt::utils::bits::Lsb0,
                    >,
                    pub start: _0,
                    pub concluded_at: ::core::option::Option<_0>,
@@ -40088,13 +40088,13 @@ pub mod api {
                    pub hash: runtime_types::polkadot_core_primitives::CandidateHash,
                    pub descriptor:
                        runtime_types::polkadot_primitives::v2::CandidateDescriptor<_0>,
-                    pub availability_votes: ::subxt::ext::bitvec::vec::BitVec<
+                    pub availability_votes: ::subxt::utils::bits::DecodedBits<
                        ::core::primitive::u8,
-                        ::subxt::ext::bitvec::order::Lsb0,
+                        ::subxt::utils::bits::Lsb0,
                    >,
-                    pub backers: ::subxt::ext::bitvec::vec::BitVec<
+                    pub backers: ::subxt::utils::bits::DecodedBits<
                        ::core::primitive::u8,
-                        ::subxt::ext::bitvec::order::Lsb0,
+                        ::subxt::utils::bits::Lsb0,
                    >,
                    pub relay_parent_number: _1,
                    pub backed_in_number: _1,
@@ -40263,13 +40263,13 @@ pub mod api {
                    Debug,
                )]
                pub struct PvfCheckActiveVoteState<_0> {
-                    pub votes_accept: ::subxt::ext::bitvec::vec::BitVec<
+                    pub votes_accept: ::subxt::utils::bits::DecodedBits<
                        ::core::primitive::u8,
-                        ::subxt::ext::bitvec::order::Lsb0,
+                        ::subxt::utils::bits::Lsb0,
                    >,
-                    pub votes_reject: ::subxt::ext::bitvec::vec::BitVec<
+                    pub votes_reject: ::subxt::utils::bits::DecodedBits<
                        ::core::primitive::u8,
-                        ::subxt::ext::bitvec::order::Lsb0,
+                        ::subxt::utils::bits::Lsb0,
                    >,
                    pub age: _0,
                    pub created_at: _0,