Add subxt-historic crate for accesing historic (non head-of-chain) blocks (#2040)

* WIP subxt-historic

* WIP subxt-historic

* WIP subxt-historic; flesh out basic foundations

* WIP filling in extrinsic decoding functionality

* iter and decode transaction extensions

* Fill in the Online/OfflineClient APIs and move more things to be part of the chain Config

* WIP storage

* clippy, fmt, finish extrinsics example

* prep for 0.0.1 release to claim crate name

* fix README link

* fmt

* WIP thinking about storage APIs

* WIP working out storage APIs

* Storage plain value fetching first pass

* WIP storage: first pass iterating over values done

* First apss finishing storage APIs

* fmt and clippy

* Create a storage example showing fetch and iteration

* Bump to frame-decode 0.9.0

* Bump subxt-historic to 0.0.3 for preview release

* Remove unused deps

* fix import

* clippy

* doc fixes

* tweak CI and fix some cargo hack findings

* Update README: subxt-historic is prerelease

historic/src/config/polkadot.rs

@@ -0,0 +1,88 @@
+use super::Config;
+use super::SubstrateConfig;
+use scale_info_legacy::{ChainTypeRegistry, TypeRegistrySet};
+use std::sync::Arc;
+
+/// Configuration that's suitable for the Polkadot Relay Chain
+pub struct PolkadotConfig(SubstrateConfig);
+
+impl PolkadotConfig {
+    /// Create a new PolkadotConfig.
+    pub fn new() -> Self {
+        let config = SubstrateConfig::new()
+            .set_legacy_types(frame_decode::legacy_types::polkadot::relay_chain());
+
+        // TODO: Set spec versions as well with known spec version changes, to speed
+        // up accessing historic blocks within the known ranges. For now, we just let
+        // the online client look these up on chain.
+
+        Self(config)
+    }
+
+    /// Set the metadata to be used for decoding blocks at the given spec versions.
+    pub fn set_metadata_for_spec_versions(
+        mut self,
+        ranges: impl Iterator<Item = (u32, frame_metadata::RuntimeMetadata)>,
+    ) -> Self {
+        self = Self(self.0.set_metadata_for_spec_versions(ranges));
+        self
+    }
+
+    /// Given an iterator of block ranges to spec version of the form `(start, end, spec_version)`, add them
+    /// to this configuration.
+    pub fn set_spec_version_for_block_ranges(
+        mut self,
+        ranges: impl Iterator<Item = (u64, u64, u32)>,
+    ) -> Self {
+        self = Self(self.0.set_spec_version_for_block_ranges(ranges));
+        self
+    }
+}
+
+/// This hands back the legacy types for the Polkadot Relay Chain, which is what [`PolkadotConfig`] uses internally.
+pub fn legacy_types() -> ChainTypeRegistry {
+    frame_decode::legacy_types::polkadot::relay_chain()
+}
+
+impl Default for PolkadotConfig {
+    fn default() -> Self {
+        Self::new()
+    }
+}
+
+impl Config for PolkadotConfig {
+    type Hash = <SubstrateConfig as Config>::Hash;
+
+    fn legacy_types_for_spec_version(&'_ self, spec_version: u32) -> TypeRegistrySet<'_> {
+        self.0.legacy_types_for_spec_version(spec_version)
+    }
+
+    fn spec_version_for_block_number(&self, block_number: u64) -> Option<u32> {
+        self.0.spec_version_for_block_number(block_number)
+    }
+
+    fn metadata_for_spec_version(
+        &self,
+        spec_version: u32,
+    ) -> Option<Arc<frame_metadata::RuntimeMetadata>> {
+        self.0.metadata_for_spec_version(spec_version)
+    }
+
+    fn set_metadata_for_spec_version(
+        &self,
+        spec_version: u32,
+        metadata: Arc<frame_metadata::RuntimeMetadata>,
+    ) {
+        self.0.set_metadata_for_spec_version(spec_version, metadata)
+    }
+
+    fn hash(s: &[u8]) -> <Self as Config>::Hash {
+        SubstrateConfig::hash(s)
+    }
+}
+
+impl subxt_rpcs::RpcConfig for PolkadotConfig {
+    type Hash = <SubstrateConfig as subxt_rpcs::RpcConfig>::Hash;
+    type Header = <SubstrateConfig as subxt_rpcs::RpcConfig>::Header;
+    type AccountId = <SubstrateConfig as subxt_rpcs::RpcConfig>::AccountId;
+}

historic/src/config/substrate.rs

@@ -0,0 +1,129 @@
+use super::Config;
+use crate::utils::RangeMap;
+use primitive_types::H256;
+use scale_info_legacy::{ChainTypeRegistry, TypeRegistrySet};
+use std::collections::HashMap;
+use std::sync::Arc;
+use std::sync::Mutex;
+
+/// Configuration that's suitable for standard Substrate chains (ie those
+/// that have not customized the block hash type).
+pub struct SubstrateConfig {
+    legacy_types: ChainTypeRegistry,
+    spec_version_for_block_number: RangeMap<u64, u32>,
+    metadata_for_spec_version: Mutex<HashMap<u32, Arc<frame_metadata::RuntimeMetadata>>>,
+}
+
+impl SubstrateConfig {
+    /// Create a new SubstrateConfig with no legacy types.
+    ///
+    /// Without any further configuration, this will only work with
+    /// the [`crate::client::OnlineClient`] for blocks that were produced by Runtimes
+    /// that emit metadata V14 or later.
+    ///
+    /// To support working at any block with the [`crate::client::OnlineClient`], you
+    /// must call [`SubstrateConfig::set_legacy_types`] with appropriate legacy type
+    /// definitions.
+    ///
+    /// To support working with the [`crate::client::OfflineClient`] at any block,
+    /// you must also call:
+    /// - [`SubstrateConfig::set_metadata_for_spec_versions`] to set the metadata to
+    ///   use at each spec version we might encounter.
+    /// - [`SubstrateConfig::set_spec_version_for_block_ranges`] to set the spec version
+    ///   to use for each range of blocks we might encounter.
+    pub fn new() -> Self {
+        Self {
+            legacy_types: ChainTypeRegistry::empty(),
+            spec_version_for_block_number: RangeMap::empty(),
+            metadata_for_spec_version: Mutex::new(HashMap::new()),
+        }
+    }
+
+    /// Set the legacy types to use for this configuration. This enables support for
+    /// blocks produced by Runtimes that emit metadata older than V14.
+    pub fn set_legacy_types(mut self, legacy_types: ChainTypeRegistry) -> Self {
+        self.legacy_types = legacy_types;
+        self
+    }
+
+    /// Set the metadata to be used for decoding blocks at the given spec versions.
+    pub fn set_metadata_for_spec_versions(
+        self,
+        ranges: impl Iterator<Item = (u32, frame_metadata::RuntimeMetadata)>,
+    ) -> Self {
+        let mut map = self.metadata_for_spec_version.lock().unwrap();
+        for (spec_version, metadata) in ranges {
+            map.insert(spec_version, Arc::new(metadata));
+        }
+        drop(map);
+        self
+    }
+
+    /// Given an iterator of block ranges to spec version of the form `(start, end, spec_version)`, add them
+    /// to this configuration.
+    pub fn set_spec_version_for_block_ranges(
+        mut self,
+        ranges: impl Iterator<Item = (u64, u64, u32)>,
+    ) -> Self {
+        let mut m = RangeMap::builder();
+        for (start, end, spec_version) in ranges {
+            m = m.add_range(start, end, spec_version);
+        }
+        self.spec_version_for_block_number = m.build();
+        self
+    }
+}
+
+impl Default for SubstrateConfig {
+    fn default() -> Self {
+        Self::new()
+    }
+}
+
+impl Config for SubstrateConfig {
+    type Hash = H256;
+
+    fn legacy_types_for_spec_version(&'_ self, spec_version: u32) -> TypeRegistrySet<'_> {
+        self.legacy_types.for_spec_version(spec_version as u64)
+    }
+
+    fn spec_version_for_block_number(&self, block_number: u64) -> Option<u32> {
+        self.spec_version_for_block_number
+            .get(block_number)
+            .copied()
+    }
+
+    fn metadata_for_spec_version(
+        &self,
+        spec_version: u32,
+    ) -> Option<Arc<frame_metadata::RuntimeMetadata>> {
+        self.metadata_for_spec_version
+            .lock()
+            .unwrap()
+            .get(&spec_version)
+            .cloned()
+    }
+
+    fn set_metadata_for_spec_version(
+        &self,
+        spec_version: u32,
+        metadata: Arc<frame_metadata::RuntimeMetadata>,
+    ) {
+        self.metadata_for_spec_version
+            .lock()
+            .unwrap()
+            .insert(spec_version, metadata);
+    }
+
+    fn hash(s: &[u8]) -> <Self as Config>::Hash {
+        sp_crypto_hashing::blake2_256(s).into()
+    }
+}
+
+impl subxt_rpcs::RpcConfig for SubstrateConfig {
+    type Hash = <Self as Config>::Hash;
+    // We don't use these types in any of the RPC methods we call,
+    // so don't bother setting them up:
+    type Header = ();
+    type AccountId = ();
+}