Rework Subxt API to support offline and dynamic transactions (#593)

* WIP API changes * debug impls * Get main crate compiling with first round of changes * Some tidy up * Add WithExtrinsicParams, and have SubstrateConfig + PolkadotConfig, not DefaultConfig * move transaction into extrinsic folder * Add runtime updates back to OnlineClient * rework to be 'client first' to fit better with storage + events * add support for events to Client * tidy dupe trait bound * Wire storage into client, but need to remove static reliance * various tidy up and start stripping codegen to remove bits we dont need now * First pass updating calls and constants codegen * WIP storage client updates * First pass migrated runtime storage over to new format * pass over codegen to generate StorageAddresses and throw other stuff out * don't need a Call trait any more * shuffle things around a bit * Various proc_macro fixes to get 'cargo check' working * organise what's exposed from subxt * Get first example working; balance_transfer_with_params * get balance_transfer example compiling * get concurrent_storage_requests.rs example compiling * get fetch_all_accounts example compiling * get a bunch more of the examples compiling * almost get final example working; type mismatch to look into * wee tweaks * move StorageAddress to separate file * pass Defaultable/Iterable info to StorageAddress in codegen * fix storage validation ne, and partial run through example code * Remove static iteration and strip a generic param from everything * fix doc tests in subxt crate * update test utils and start fixing frame tests * fix frame staking tests * fix the rest of the test compile issues, Borrow on storage values * cargo fmt * remove extra logging during tests * Appease clippy and no more need for into_iter on events * cargo fmt * fix dryRun tests by waiting for blocks * wait for blocks instead of sleeping or other test hacks * cargo fmt * Fix doc links * Traitify StorageAddress * remove out-of-date doc comments * optimise decoding storage a little * cleanup tx stuff, trait for TxPayload, remove Err type param and decode at runtime * clippy fixes * fix doc links * fix doc example * constant address trait for consistency * fix a typo and remove EncodeWithMetadata stuff * Put EventDetails behind a proper interface and allow decoding into top level event, too * fix docs * tweak StorageAddress docs * re-export StorageAddress at root for consistency * fix clippy things * Add support for dynamic values * fix double encoding of storage map key after refactor * clippy fix * Fixes and add a dynamic usage example (needs new scale_value release) * bump scale_value version * cargo fmt * Tweak event bits * cargo fmt * Add a test and bump scale-value to 0.4.0 to support this * remove unnecessary vec from dynamic example * Various typo/grammar fixes Co-authored-by: Alexandru Vasile <60601340+lexnv@users.noreply.github.com> * Address PR nits * Undo accidental rename in changelog * Small PR nits/tidyups * fix tests; codegen change against latest substrate * tweak storage address util names * move error decoding to DecodeError and expose * impl some basic traits on the extrinsic param builder Co-authored-by: Alexandru Vasile <60601340+lexnv@users.noreply.github.com>
2026-06-15 08:01:09 +00:00 · 2022-08-08 11:55:20 +01:00
parent 7a09ac6cd7
commit e48f0e3b1d
84 changed files with 23097 additions and 35863 deletions
@@ -0,0 +1,48 @@
+// Copyright 2019-2022 Parity Technologies (UK) Ltd.
+// This file is dual-licensed as Apache-2.0 or GPL-3.0.
+// see LICENSE for license details.
+
+//! Types associated with accessing and working with storage items.
+
+mod storage_address;
+mod storage_client;
+mod storage_map_key;
+
+pub mod utils;
+
+pub use storage_client::{
+    KeyIter,
+    StorageClient,
+};
+
+// Re-export as this is used in the public API:
+pub use sp_core::storage::StorageKey;
+
+/// Types representing an address which describes where a storage
+/// entry lives and how to properly decode it.
+pub mod address {
+    pub use super::{
+        storage_address::{
+            dynamic,
+            dynamic_root,
+            DynamicStorageAddress,
+            StaticStorageAddress,
+            StorageAddress,
+            Yes,
+        },
+        storage_map_key::{
+            StorageHasher,
+            StorageMapKey,
+        },
+    };
+}
+
+// For consistency with other modules, also expose
+// the basic address stuff at the root of the module.
+pub use storage_address::{
+    dynamic,
+    dynamic_root,
+    DynamicStorageAddress,
+    StaticStorageAddress,
+    StorageAddress,
+};
@@ -0,0 +1,286 @@
+// Copyright 2019-2022 Parity Technologies (UK) Ltd.
+// This file is dual-licensed as Apache-2.0 or GPL-3.0.
+// see LICENSE for license details.
+
+use super::storage_map_key::StorageMapKey;
+use crate::{
+    dynamic::{
+        DecodedValue,
+        Value,
+    },
+    error::{
+        Error,
+        StorageAddressError,
+    },
+    metadata::{
+        DecodeWithMetadata,
+        EncodeWithMetadata,
+        Metadata,
+    },
+};
+use frame_metadata::StorageEntryType;
+use scale_info::TypeDef;
+use std::borrow::Cow;
+
+// We use this type a bunch, so export it from here.
+pub use frame_metadata::StorageHasher;
+
+/// This represents a storage address. Anything implementing this trait
+/// can be used to fetch and iterate over storage entries.
+pub trait StorageAddress {
+    /// The target type of the value that lives at this address.
+    type Target: DecodeWithMetadata;
+    /// Can an entry be fetched from this address?
+    /// Set this type to [`Yes`] to enable the corresponding calls to be made.
+    type IsFetchable;
+    /// Can a default entry be obtained from this address?
+    /// Set this type to [`Yes`] to enable the corresponding calls to be made.
+    type IsDefaultable;
+    /// Can this address be iterated over?
+    /// Set this type to [`Yes`] to enable the corresponding calls to be made.
+    type IsIterable;
+
+    /// The name of the pallet that the entry lives under.
+    fn pallet_name(&self) -> &str;
+
+    /// The name of the entry in a given pallet that the item is at.
+    fn entry_name(&self) -> &str;
+
+    /// Output the non-prefix bytes; that is, any additional bytes that need
+    /// to be appended to the key to dig into maps.
+    fn append_entry_bytes(
+        &self,
+        metadata: &Metadata,
+        bytes: &mut Vec<u8>,
+    ) -> Result<(), Error>;
+
+    /// An optional hash which, if present, will be checked against
+    /// the node metadata to confirm that the return type matches what
+    /// we are expecting.
+    fn validation_hash(&self) -> Option<[u8; 32]> {
+        None
+    }
+}
+
+/// Used to signal whether a [`StorageAddress`] can be iterated,
+/// fetched and returned with a default value in the type system.
+pub struct Yes;
+
+/// This represents a statically generated storage lookup address.
+pub struct StaticStorageAddress<ReturnTy, Fetchable, Defaultable, Iterable> {
+    pallet_name: &'static str,
+    entry_name: &'static str,
+    // How to access the specific value at that storage address.
+    storage_entry_keys: Vec<StorageMapKey>,
+    // Hash provided from static code for validation.
+    validation_hash: Option<[u8; 32]>,
+    _marker: std::marker::PhantomData<(ReturnTy, Fetchable, Defaultable, Iterable)>,
+}
+
+impl<ReturnTy, Fetchable, Defaultable, Iterable>
+    StaticStorageAddress<ReturnTy, Fetchable, Defaultable, Iterable>
+where
+    ReturnTy: DecodeWithMetadata,
+{
+    /// Create a new [`StaticStorageAddress`] that will be validated
+    /// against node metadata using the hash given.
+    pub fn new(
+        pallet_name: &'static str,
+        entry_name: &'static str,
+        storage_entry_keys: Vec<StorageMapKey>,
+        hash: [u8; 32],
+    ) -> Self {
+        Self {
+            pallet_name,
+            entry_name,
+            storage_entry_keys,
+            validation_hash: Some(hash),
+            _marker: std::marker::PhantomData,
+        }
+    }
+
+    /// Do not validate this storage entry prior to accessing it.
+    pub fn unvalidated(self) -> Self {
+        Self {
+            validation_hash: None,
+            ..self
+        }
+    }
+
+    /// Return bytes representing this storage entry.
+    pub fn to_bytes(&self) -> Vec<u8> {
+        let mut bytes = Vec::new();
+        super::utils::write_storage_address_root_bytes(self, &mut bytes);
+        for entry in &self.storage_entry_keys {
+            entry.to_bytes(&mut bytes);
+        }
+        bytes
+    }
+
+    /// Return bytes representing the root of this storage entry (ie a hash of
+    /// the pallet and entry name).
+    pub fn to_root_bytes(&self) -> Vec<u8> {
+        super::utils::storage_address_root_bytes(self)
+    }
+}
+
+impl<ReturnTy, Fetchable, Defaultable, Iterable> StorageAddress
+    for StaticStorageAddress<ReturnTy, Fetchable, Defaultable, Iterable>
+where
+    ReturnTy: DecodeWithMetadata,
+{
+    type Target = ReturnTy;
+    type IsDefaultable = Defaultable;
+    type IsIterable = Iterable;
+    type IsFetchable = Fetchable;
+
+    fn pallet_name(&self) -> &str {
+        self.pallet_name
+    }
+
+    fn entry_name(&self) -> &str {
+        self.entry_name
+    }
+
+    fn append_entry_bytes(
+        &self,
+        _metadata: &Metadata,
+        bytes: &mut Vec<u8>,
+    ) -> Result<(), Error> {
+        for entry in &self.storage_entry_keys {
+            entry.to_bytes(bytes);
+        }
+        Ok(())
+    }
+
+    fn validation_hash(&self) -> Option<[u8; 32]> {
+        self.validation_hash
+    }
+}
+
+/// This represents a dynamically generated storage address.
+pub struct DynamicStorageAddress<'a, Encodable> {
+    pallet_name: Cow<'a, str>,
+    entry_name: Cow<'a, str>,
+    storage_entry_keys: Vec<Encodable>,
+}
+
+/// Construct a new dynamic storage lookup to the root of some entry.
+pub fn dynamic_root<'a>(
+    pallet_name: impl Into<Cow<'a, str>>,
+    entry_name: impl Into<Cow<'a, str>>,
+) -> DynamicStorageAddress<'a, Value> {
+    DynamicStorageAddress {
+        pallet_name: pallet_name.into(),
+        entry_name: entry_name.into(),
+        storage_entry_keys: vec![],
+    }
+}
+
+/// Construct a new dynamic storage lookup.
+pub fn dynamic<'a, Encodable: EncodeWithMetadata>(
+    pallet_name: impl Into<Cow<'a, str>>,
+    entry_name: impl Into<Cow<'a, str>>,
+    storage_entry_keys: Vec<Encodable>,
+) -> DynamicStorageAddress<'a, Encodable> {
+    DynamicStorageAddress {
+        pallet_name: pallet_name.into(),
+        entry_name: entry_name.into(),
+        storage_entry_keys,
+    }
+}
+
+impl<'a, Encodable> StorageAddress for DynamicStorageAddress<'a, Encodable>
+where
+    Encodable: EncodeWithMetadata,
+{
+    type Target = DecodedValue;
+
+    // For dynamic types, we have no static guarantees about any of
+    // this stuff, so we just allow it and let it fail at runtime:
+    type IsFetchable = Yes;
+    type IsDefaultable = Yes;
+    type IsIterable = Yes;
+
+    fn pallet_name(&self) -> &str {
+        &self.pallet_name
+    }
+
+    fn entry_name(&self) -> &str {
+        &self.entry_name
+    }
+
+    fn append_entry_bytes(
+        &self,
+        metadata: &Metadata,
+        bytes: &mut Vec<u8>,
+    ) -> Result<(), Error> {
+        let pallet = metadata.pallet(&self.pallet_name)?;
+        let storage = pallet.storage(&self.entry_name)?;
+
+        match &storage.ty {
+            StorageEntryType::Plain(_) => {
+                if !self.storage_entry_keys.is_empty() {
+                    Err(StorageAddressError::WrongNumberOfKeys {
+                        expected: 0,
+                        actual: self.storage_entry_keys.len(),
+                    }
+                    .into())
+                } else {
+                    Ok(())
+                }
+            }
+            StorageEntryType::Map { hashers, key, .. } => {
+                let ty = metadata
+                    .resolve_type(key.id())
+                    .ok_or_else(|| StorageAddressError::TypeNotFound(key.id()))?;
+
+                // If the key is a tuple, we encode each value to the corresponding tuple type.
+                // If the key is not a tuple, encode a single value to the key type.
+                let type_ids = match ty.type_def() {
+                    TypeDef::Tuple(tuple) => {
+                        tuple.fields().iter().map(|f| f.id()).collect()
+                    }
+                    _other => {
+                        vec![key.id()]
+                    }
+                };
+
+                if type_ids.len() != self.storage_entry_keys.len() {
+                    return Err(StorageAddressError::WrongNumberOfKeys {
+                        expected: type_ids.len(),
+                        actual: self.storage_entry_keys.len(),
+                    }
+                    .into())
+                }
+
+                if hashers.len() == 1 {
+                    // One hasher; hash a tuple of all SCALE encoded bytes with the one hash function.
+                    let mut input = Vec::new();
+                    for (key, type_id) in self.storage_entry_keys.iter().zip(type_ids) {
+                        key.encode_with_metadata(type_id, metadata, &mut input)?;
+                    }
+                    super::storage_map_key::hash_bytes(&input, &hashers[0], bytes);
+                    Ok(())
+                } else if hashers.len() == type_ids.len() {
+                    // A hasher per field; encode and hash each field independently.
+                    for ((key, type_id), hasher) in
+                        self.storage_entry_keys.iter().zip(type_ids).zip(hashers)
+                    {
+                        let mut input = Vec::new();
+                        key.encode_with_metadata(type_id, metadata, &mut input)?;
+                        super::storage_map_key::hash_bytes(&input, hasher, bytes);
+                    }
+                    Ok(())
+                } else {
+                    // Mismatch; wrong number of hashers/fields.
+                    Err(StorageAddressError::WrongNumberOfHashers {
+                        hashers: hashers.len(),
+                        fields: type_ids.len(),
+                    }
+                    .into())
+                }
+            }
+        }
+    }
+}
@@ -0,0 +1,410 @@
+// Copyright 2019-2022 Parity Technologies (UK) Ltd.
+// This file is dual-licensed as Apache-2.0 or GPL-3.0.
+// see LICENSE for license details.
+
+use super::storage_address::{
+    StorageAddress,
+    Yes,
+};
+use crate::{
+    client::{
+        OfflineClientT,
+        OnlineClientT,
+    },
+    error::Error,
+    metadata::{
+        DecodeWithMetadata,
+        Metadata,
+    },
+    Config,
+};
+use derivative::Derivative;
+use frame_metadata::StorageEntryType;
+use scale_info::form::PortableForm;
+use sp_core::storage::{
+    StorageData,
+    StorageKey,
+};
+use std::{
+    future::Future,
+    marker::PhantomData,
+};
+
+/// Query the runtime storage.
+#[derive(Derivative)]
+#[derivative(Clone(bound = "Client: Clone"))]
+pub struct StorageClient<T, Client> {
+    client: Client,
+    _marker: PhantomData<T>,
+}
+
+impl<T, Client> StorageClient<T, Client> {
+    /// Create a new [`StorageClient`]
+    pub fn new(client: Client) -> Self {
+        Self {
+            client,
+            _marker: PhantomData,
+        }
+    }
+}
+
+impl<T, Client> StorageClient<T, Client>
+where
+    T: Config,
+    Client: OfflineClientT<T>,
+{
+    /// Run the validation logic against some storage address you'd like to access. Returns `Ok(())`
+    /// if the address is valid (or if it's not possible to check since the address has no validation hash).
+    /// Return an error if the address was not valid or something went wrong trying to validate it (ie
+    /// the pallet or storage entry in question do not exist at all).
+    pub fn validate<Address: StorageAddress>(
+        &self,
+        address: &Address,
+    ) -> Result<(), Error> {
+        if let Some(hash) = address.validation_hash() {
+            validate_storage(
+                address.pallet_name(),
+                address.entry_name(),
+                hash,
+                &self.client.metadata(),
+            )?;
+        }
+        Ok(())
+    }
+}
+
+impl<T, Client> StorageClient<T, Client>
+where
+    T: Config,
+    Client: OnlineClientT<T>,
+{
+    /// Fetch the raw encoded value at the address/key given.
+    pub fn fetch_raw<'a>(
+        &self,
+        key: &'a [u8],
+        hash: Option<T::Hash>,
+    ) -> impl Future<Output = Result<Option<Vec<u8>>, Error>> + 'a {
+        let client = self.client.clone();
+        // Ensure that the returned future doesn't have a lifetime tied to api.storage(),
+        // which is a temporary thing we'll be throwing away quickly:
+        async move {
+            let data = client.rpc().storage(key, hash).await?;
+            Ok(data.map(|d| d.0))
+        }
+    }
+
+    /// Fetch a decoded value from storage at a given address and optional block hash.
+    ///
+    /// # Example
+    ///
+    /// ```no_run
+    /// use subxt::{ PolkadotConfig, OnlineClient };
+    ///
+    /// #[subxt::subxt(runtime_metadata_path = "../artifacts/polkadot_metadata.scale")]
+    /// pub mod polkadot {}
+    ///
+    /// # #[tokio::main]
+    /// # async fn main() {
+    /// let api = OnlineClient::<PolkadotConfig>::new().await.unwrap();
+    ///
+    /// // Address to a storage entry we'd like to access.
+    /// let address = polkadot::storage().xcm_pallet().queries(&12345);
+    ///
+    /// // Fetch just the keys, returning up to 10 keys.
+    /// let value = api
+    ///     .storage()
+    ///     .fetch(&address, None)
+    ///     .await
+    ///     .unwrap();
+    ///
+    /// println!("Value: {:?}", value);
+    /// # }
+    /// ```
+    pub fn fetch<'a, Address>(
+        &self,
+        address: &'a Address,
+        hash: Option<T::Hash>,
+    ) -> impl Future<
+        Output = Result<Option<<Address::Target as DecodeWithMetadata>::Target>, Error>,
+    > + 'a
+    where
+        Address: StorageAddress<IsFetchable = Yes> + 'a,
+    {
+        let client = self.clone();
+        async move {
+            // Metadata validation checks whether the static address given
+            // is likely to actually correspond to a real storage entry or not.
+            // if not, it means static codegen doesn't line up with runtime
+            // metadata.
+            client.validate(address)?;
+
+            // Look up the return type ID to enable DecodeWithMetadata:
+            let metadata = client.client.metadata();
+            let lookup_bytes = super::utils::storage_address_bytes(address, &metadata)?;
+            if let Some(data) = client
+                .client
+                .storage()
+                .fetch_raw(&lookup_bytes, hash)
+                .await?
+            {
+                let val = <Address::Target as DecodeWithMetadata>::decode_storage_with_metadata(
+                    &mut &*data,
+                    address.pallet_name(),
+                    address.entry_name(),
+                    &metadata,
+                )?;
+                Ok(Some(val))
+            } else {
+                Ok(None)
+            }
+        }
+    }
+
+    /// Fetch a StorageKey that has a default value with an optional block hash.
+    pub fn fetch_or_default<'a, Address>(
+        &self,
+        address: &'a Address,
+        hash: Option<T::Hash>,
+    ) -> impl Future<Output = Result<<Address::Target as DecodeWithMetadata>::Target, Error>>
+           + 'a
+    where
+        Address: StorageAddress<IsFetchable = Yes, IsDefaultable = Yes> + 'a,
+    {
+        let client = self.client.clone();
+        async move {
+            let pallet_name = address.pallet_name();
+            let storage_name = address.entry_name();
+            // Metadata validation happens via .fetch():
+            if let Some(data) = client.storage().fetch(address, hash).await? {
+                Ok(data)
+            } else {
+                let metadata = client.metadata();
+
+                // We have to dig into metadata already, so no point using the optimised `decode_storage_with_metadata` call.
+                let pallet_metadata = metadata.pallet(pallet_name)?;
+                let storage_metadata = pallet_metadata.storage(storage_name)?;
+                let return_ty_id =
+                    return_type_from_storage_entry_type(&storage_metadata.ty);
+                let bytes = &mut &storage_metadata.default[..];
+
+                let val = <Address::Target as DecodeWithMetadata>::decode_with_metadata(
+                    bytes,
+                    return_ty_id,
+                    &metadata,
+                )?;
+                Ok(val)
+            }
+        }
+    }
+
+    /// Fetch up to `count` keys for a storage map in lexicographic order.
+    ///
+    /// Supports pagination by passing a value to `start_key`.
+    pub fn fetch_keys<'a>(
+        &self,
+        key: &'a [u8],
+        count: u32,
+        start_key: Option<&'a [u8]>,
+        hash: Option<T::Hash>,
+    ) -> impl Future<Output = Result<Vec<StorageKey>, Error>> + 'a {
+        let client = self.client.clone();
+        async move {
+            let keys = client
+                .rpc()
+                .storage_keys_paged(key, count, start_key, hash)
+                .await?;
+            Ok(keys)
+        }
+    }
+
+    /// Returns an iterator of key value pairs.
+    ///
+    /// ```no_run
+    /// use subxt::{ PolkadotConfig, OnlineClient };
+    ///
+    /// #[subxt::subxt(runtime_metadata_path = "../artifacts/polkadot_metadata.scale")]
+    /// pub mod polkadot {}
+    ///
+    /// # #[tokio::main]
+    /// # async fn main() {
+    /// let api = OnlineClient::<PolkadotConfig>::new().await.unwrap();
+    ///
+    /// // Address to the root of a storage entry that we'd like to iterate over.
+    /// let address = polkadot::storage().xcm_pallet().version_notifiers_root();
+    ///
+    /// // Iterate over keys and values at that address.
+    /// let mut iter = api
+    ///     .storage()
+    ///     .iter(address, 10, None)
+    ///     .await
+    ///     .unwrap();
+    ///
+    /// while let Some((key, value)) = iter.next().await.unwrap() {
+    ///     println!("Key: 0x{}", hex::encode(&key));
+    ///     println!("Value: {}", value);
+    /// }
+    /// # }
+    /// ```
+    pub fn iter<Address>(
+        &self,
+        address: Address,
+        page_size: u32,
+        hash: Option<T::Hash>,
+    ) -> impl Future<Output = Result<KeyIter<T, Client, Address::Target>, Error>> + 'static
+    where
+        Address: StorageAddress<IsIterable = Yes> + 'static,
+    {
+        let client = self.clone();
+        async move {
+            // Metadata validation checks whether the static address given
+            // is likely to actually correspond to a real storage entry or not.
+            // if not, it means static codegen doesn't line up with runtime
+            // metadata.
+            client.validate(&address)?;
+
+            // Fetch a concrete block hash to iterate over. We do this so that if new blocks
+            // are produced midway through iteration, we continue to iterate at the block
+            // we started with and not the new block.
+            let hash = if let Some(hash) = hash {
+                hash
+            } else {
+                client
+                    .client
+                    .rpc()
+                    .block_hash(None)
+                    .await?
+                    .expect("didn't pass a block number; qed")
+            };
+
+            let metadata = client.client.metadata();
+
+            // Look up the return type for flexible decoding. Do this once here to avoid
+            // potentially doing it every iteration if we used `decode_storage_with_metadata`
+            // in the iterator.
+            let return_type_id = lookup_storage_return_type(
+                &metadata,
+                address.pallet_name(),
+                address.entry_name(),
+            )?;
+
+            // The root pallet/entry bytes for this storage entry:
+            let address_root_bytes = super::utils::storage_address_root_bytes(&address);
+
+            Ok(KeyIter {
+                client,
+                address_root_bytes,
+                metadata,
+                return_type_id,
+                block_hash: hash,
+                count: page_size,
+                start_key: None,
+                buffer: Default::default(),
+                _marker: std::marker::PhantomData,
+            })
+        }
+    }
+}
+
+/// Iterates over key value pairs in a map.
+pub struct KeyIter<T: Config, Client, ReturnTy> {
+    client: StorageClient<T, Client>,
+    address_root_bytes: Vec<u8>,
+    return_type_id: u32,
+    metadata: Metadata,
+    count: u32,
+    block_hash: T::Hash,
+    start_key: Option<StorageKey>,
+    buffer: Vec<(StorageKey, StorageData)>,
+    _marker: std::marker::PhantomData<ReturnTy>,
+}
+
+impl<'a, T: Config, Client: OnlineClientT<T>, ReturnTy> KeyIter<T, Client, ReturnTy>
+where
+    T: Config,
+    Client: OnlineClientT<T>,
+    ReturnTy: DecodeWithMetadata,
+{
+    /// Returns the next key value pair from a map.
+    pub async fn next(
+        &mut self,
+    ) -> Result<Option<(StorageKey, ReturnTy::Target)>, Error> {
+        loop {
+            if let Some((k, v)) = self.buffer.pop() {
+                let val = ReturnTy::decode_with_metadata(
+                    &mut &v.0[..],
+                    self.return_type_id,
+                    &self.metadata,
+                )?;
+                return Ok(Some((k, val)))
+            } else {
+                let start_key = self.start_key.take();
+                let keys = self
+                    .client
+                    .fetch_keys(
+                        &self.address_root_bytes,
+                        self.count,
+                        start_key.as_ref().map(|k| &*k.0),
+                        Some(self.block_hash),
+                    )
+                    .await?;
+
+                if keys.is_empty() {
+                    return Ok(None)
+                }
+
+                self.start_key = keys.last().cloned();
+
+                let change_sets = self
+                    .client
+                    .client
+                    .rpc()
+                    .query_storage_at(keys.iter().map(|k| &*k.0), Some(self.block_hash))
+                    .await?;
+                for change_set in change_sets {
+                    for (k, v) in change_set.changes {
+                        if let Some(v) = v {
+                            self.buffer.push((k, v));
+                        }
+                    }
+                }
+                debug_assert_eq!(self.buffer.len(), keys.len());
+            }
+        }
+    }
+}
+
+/// Validate a storage entry against the metadata.
+fn validate_storage(
+    pallet_name: &str,
+    storage_name: &str,
+    hash: [u8; 32],
+    metadata: &Metadata,
+) -> Result<(), Error> {
+    let expected_hash = match metadata.storage_hash(pallet_name, storage_name) {
+        Ok(hash) => hash,
+        Err(e) => return Err(e.into()),
+    };
+    match expected_hash == hash {
+        true => Ok(()),
+        false => Err(crate::error::MetadataError::IncompatibleMetadata.into()),
+    }
+}
+
+/// look up a return type ID for some storage entry.
+fn lookup_storage_return_type(
+    metadata: &Metadata,
+    pallet: &str,
+    entry: &str,
+) -> Result<u32, Error> {
+    let storage_entry_type = &metadata.pallet(pallet)?.storage(entry)?.ty;
+
+    Ok(return_type_from_storage_entry_type(storage_entry_type))
+}
+
+/// Fetch the return type out of a [`StorageEntryType`].
+fn return_type_from_storage_entry_type(entry: &StorageEntryType<PortableForm>) -> u32 {
+    match entry {
+        StorageEntryType::Plain(ty) => ty.id(),
+        StorageEntryType::Map { value, .. } => value.id(),
+    }
+}
@@ -0,0 +1,53 @@
+// Copyright 2019-2022 Parity Technologies (UK) Ltd.
+// This file is dual-licensed as Apache-2.0 or GPL-3.0.
+// see LICENSE for license details.
+
+use codec::Encode;
+pub use sp_runtime::traits::SignedExtension;
+
+// We use this type a bunch, so export it from here.
+pub use frame_metadata::StorageHasher;
+
+/// Storage key for a Map.
+#[derive(Clone)]
+pub struct StorageMapKey {
+    value: Vec<u8>,
+    hasher: StorageHasher,
+}
+
+impl StorageMapKey {
+    /// Create a new [`StorageMapKey`] by pre-encoding static data and pairing it with a hasher.
+    pub fn new<Encodable: Encode>(
+        value: Encodable,
+        hasher: StorageHasher,
+    ) -> StorageMapKey {
+        Self {
+            value: value.encode(),
+            hasher,
+        }
+    }
+
+    /// Convert this [`StorageMapKey`] into bytes and append them to some existing bytes.
+    pub fn to_bytes(&self, bytes: &mut Vec<u8>) {
+        hash_bytes(&self.value, &self.hasher, bytes)
+    }
+}
+
+/// Take some SCALE encoded bytes and a [`StorageHasher`] and hash the bytes accordingly.
+pub(super) fn hash_bytes(input: &[u8], hasher: &StorageHasher, bytes: &mut Vec<u8>) {
+    match hasher {
+        StorageHasher::Identity => bytes.extend(input),
+        StorageHasher::Blake2_128 => bytes.extend(sp_core::blake2_128(input)),
+        StorageHasher::Blake2_128Concat => {
+            bytes.extend(sp_core::blake2_128(input));
+            bytes.extend(input);
+        }
+        StorageHasher::Blake2_256 => bytes.extend(sp_core::blake2_256(input)),
+        StorageHasher::Twox128 => bytes.extend(sp_core::twox_128(input)),
+        StorageHasher::Twox256 => bytes.extend(sp_core::twox_256(input)),
+        StorageHasher::Twox64Concat => {
+            bytes.extend(sp_core::twox_64(input));
+            bytes.extend(input);
+        }
+    }
+}
@@ -0,0 +1,42 @@
+// Copyright 2019-2022 Parity Technologies (UK) Ltd.
+// This file is dual-licensed as Apache-2.0 or GPL-3.0.
+// see LICENSE for license details.
+
+//! these utility methods complement the [`StorageAddress`] trait, but
+//! aren't things that should ever be overridden, and so don't exist on
+//! the trait itself.
+
+use super::StorageAddress;
+use crate::{
+    error::Error,
+    metadata::Metadata,
+};
+
+/// Return the root of a given [`StorageAddress`]: hash the pallet name and entry name
+/// and append those bytes to the output.
+pub fn write_storage_address_root_bytes<Address: StorageAddress>(
+    addr: &Address,
+    out: &mut Vec<u8>,
+) {
+    out.extend(&sp_core::twox_128(addr.pallet_name().as_bytes()));
+    out.extend(&sp_core::twox_128(addr.entry_name().as_bytes()));
+}
+
+/// Outputs the [`storage_address_root_bytes`] as well as any additional bytes that represent
+/// a lookup in a storage map at that location.
+pub fn storage_address_bytes<Address: StorageAddress>(
+    addr: &Address,
+    metadata: &Metadata,
+) -> Result<Vec<u8>, Error> {
+    let mut bytes = Vec::new();
+    write_storage_address_root_bytes(addr, &mut bytes);
+    addr.append_entry_bytes(metadata, &mut bytes)?;
+    Ok(bytes)
+}
+
+/// Outputs a vector containing the bytes written by [`write_storage_address_root_bytes`].
+pub fn storage_address_root_bytes<Address: StorageAddress>(addr: &Address) -> Vec<u8> {
+    let mut bytes = Vec::new();
+    write_storage_address_root_bytes(addr, &mut bytes);
+    bytes
+}