Make subxt-core ready for publishing (#1508)

* Move Extrinsic decoding things to subxt_core and various tidy-ups

* A couple more fixes and fmt

* first pass moving tx logic to subxt_core

* cargo fmt

* fix wasm example

* clippy

* more clippy

* WIP Adding examples and such

* Move storage functionality more fully to subxt_core and nice examples for storage and txs

* Add example for events

* consistify how addresses/payloads are exposed in subxt-core and add runtime API fns

* Add runtime API core example

* fmt

* remove scale-info patch

* Add a little to the top level docs

* swap args around

* clippy

* cargo fmt and fix wasm-example

* doc fixes

* no-std-ise new subxt-core additions

* alloc, not core

* more no-std fixes

* A couple more fixes

* Add back extrinsic decode test

core/src/storage/address.rs

@@ -1,7 +1,9 @@
-// Copyright 2019-2023 Parity Technologies (UK) Ltd.
+// Copyright 2019-2024 Parity Technologies (UK) Ltd.
 // This file is dual-licensed as Apache-2.0 or GPL-3.0.
 // see LICENSE for license details.
 
+//! Construct addresses to access storage entries with.
+
 use crate::{
     dynamic::DecodedValueThunk,
     error::{Error, MetadataError},
@@ -14,11 +16,12 @@ use alloc::borrow::{Cow, ToOwned};
 use alloc::string::String;
 use alloc::vec::Vec;
 
-use super::{storage_key::StorageHashers, StorageKey};
+// Re-export types used here:
+pub use super::storage_key::{StaticStorageKey, StorageHashers, StorageHashersIter, StorageKey};
 
 /// This represents a storage address. Anything implementing this trait
 /// can be used to fetch and iterate over storage entries.
-pub trait StorageAddress {
+pub trait AddressT {
     /// The target type of the value that lives at this address.
     type Target: DecodeWithMetadata;
     /// The keys type used to construct this address.
@@ -120,11 +123,11 @@ where
 
     /// Return bytes representing the root of this storage entry (a hash of the pallet and entry name).
     pub fn to_root_bytes(&self) -> Vec<u8> {
-        super::utils::storage_address_root_bytes(self)
+        super::get_address_root_bytes(self)
     }
 }
 
-impl<Keys, ReturnTy, Fetchable, Defaultable, Iterable> StorageAddress
+impl<Keys, ReturnTy, Fetchable, Defaultable, Iterable> AddressT
     for Address<Keys, ReturnTy, Fetchable, Defaultable, Iterable>
 where
     Keys: StorageKey,

core/src/storage/mod.rs

@@ -1,22 +1,142 @@
-// Copyright 2019-2023 Parity Technologies (UK) Ltd.
+// Copyright 2019-2024 Parity Technologies (UK) Ltd.
 // This file is dual-licensed as Apache-2.0 or GPL-3.0.
 // see LICENSE for license details.
 
-//! Types associated with accessing and working with storage items.
+//! Encode storage keys, decode storage values, and validate static storage addresses.
+//!
+//! # Example
+//!
+//! ```rust
+//! use subxt_signer::sr25519::dev;
+//! use subxt_macro::subxt;
+//! use subxt_core::storage;
+//! use subxt_core::metadata;
+//!
+//! // If we generate types without `subxt`, we need to point to `::subxt_core`:
+//! #[subxt(
+//!     crate = "::subxt_core",
+//!     runtime_metadata_path = "../artifacts/polkadot_metadata_small.scale",
+//! )]
+//! pub mod polkadot {}
+//!
+//! // Some metadata we'll use to work with storage entries:
+//! let metadata_bytes = include_bytes!("../../../artifacts/polkadot_metadata_small.scale");
+//! let metadata = metadata::decode_from(&metadata_bytes[..]).unwrap();
+//!
+//! // Build a storage query to access account information.
+//! let account = dev::alice().public_key().into();
+//! let address = polkadot::storage().system().account(&account);
+//!
+//! // We can validate that the address is compatible with the given metadata.
+//! storage::validate(&address, &metadata).unwrap();
+//!
+//! // Encode the address to bytes. These can be sent to a node to query the value.
+//! storage::get_address_bytes(&address, &metadata).unwrap();
+//!
+//! // If we were to obtain a value back from the node at that address, we could
+//! // then decode it using the same address and metadata like so:
+//! let value_bytes = hex::decode("00000000000000000100000000000000000064a7b3b6e00d0000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000080").unwrap();
+//! let value = storage::decode_value(&mut &*value_bytes, &address, &metadata).unwrap();
+//!
+//! println!("Alice's account info: {value:?}");
+//! ```
 
-mod storage_address;
 mod storage_key;
-pub mod utils;
+mod utils;
 
-/// Types representing an address which describes where a storage
-/// entry lives and how to properly decode it.
-pub mod address {
-    pub use super::storage_address::{dynamic, Address, DynamicAddress, StorageAddress};
-    pub use super::storage_key::{StaticStorageKey, StorageHashers, StorageKey};
+pub mod address;
+
+use crate::{error::MetadataError, metadata::DecodeWithMetadata, Error, Metadata};
+use address::AddressT;
+use alloc::vec::Vec;
+
+// This isn't a part of the public API, but expose here because it's useful in Subxt.
+#[doc(hidden)]
+pub use utils::lookup_storage_entry_details;
+
+/// When the provided `address` is statically generated via the `#[subxt]` macro, this validates
+/// that the shape of the storage value is the same as the shape expected by the static address.
+///
+/// When the provided `address` is dynamic (and thus does not come with any expectation of the
+/// shape of the constant value), this just returns `Ok(())`
+pub fn validate<Address: AddressT>(address: &Address, metadata: &Metadata) -> Result<(), Error> {
+    let Some(hash) = address.validation_hash() else {
+        return Ok(());
+    };
+
+    let pallet_name = address.pallet_name();
+    let entry_name = address.entry_name();
+
+    let pallet_metadata = metadata.pallet_by_name_err(pallet_name)?;
+
+    let Some(expected_hash) = pallet_metadata.storage_hash(entry_name) else {
+        return Err(MetadataError::IncompatibleCodegen.into());
+    };
+    if expected_hash != hash {
+        return Err(MetadataError::IncompatibleCodegen.into());
+    }
+    Ok(())
 }
 
-pub use storage_key::StorageKey;
+/// Given a storage address and some metadata, this encodes the address into bytes which can be
+/// handed to a node to retrieve the corresponding value.
+pub fn get_address_bytes<Address: AddressT>(
+    address: &Address,
+    metadata: &Metadata,
+) -> Result<Vec<u8>, Error> {
+    let mut bytes = Vec::new();
+    utils::write_storage_address_root_bytes(address, &mut bytes);
+    address.append_entry_bytes(metadata, &mut bytes)?;
+    Ok(bytes)
+}
 
-// For consistency with other modules, also expose
-// the basic address stuff at the root of the module.
-pub use storage_address::{dynamic, Address, DynamicAddress, StorageAddress};
+/// Given a storage address and some metadata, this encodes the root of the address (ie the pallet
+/// and storage entry part) into bytes. If the entry being addressed is inside a map, this returns
+/// the bytes needed to iterate over all of the entries within it.
+pub fn get_address_root_bytes<Address: AddressT>(address: &Address) -> Vec<u8> {
+    let mut bytes = Vec::new();
+    utils::write_storage_address_root_bytes(address, &mut bytes);
+    bytes
+}
+
+/// Given some storage value that we've retrieved from a node, the address used to retrieve it, and
+/// metadata from the node, this function attempts to decode the bytes into the target value specified
+/// by the address.
+pub fn decode_value<Address: AddressT>(
+    bytes: &mut &[u8],
+    address: &Address,
+    metadata: &Metadata,
+) -> Result<Address::Target, Error> {
+    let pallet_name = address.pallet_name();
+    let entry_name = address.entry_name();
+
+    let (_, entry_metadata) =
+        utils::lookup_storage_entry_details(pallet_name, entry_name, metadata)?;
+    let value_ty_id = match entry_metadata.entry_type() {
+        subxt_metadata::StorageEntryType::Plain(ty) => *ty,
+        subxt_metadata::StorageEntryType::Map { value_ty, .. } => *value_ty,
+    };
+
+    let val = Address::Target::decode_with_metadata(bytes, value_ty_id, metadata)?;
+    Ok(val)
+}
+
+/// Return the default value at a given storage address if one is available, or an error otherwise.
+pub fn default_value<Address: AddressT>(
+    address: &Address,
+    metadata: &Metadata,
+) -> Result<Address::Target, Error> {
+    let pallet_name = address.pallet_name();
+    let entry_name = address.entry_name();
+
+    let (_, entry_metadata) =
+        utils::lookup_storage_entry_details(pallet_name, entry_name, metadata)?;
+    let value_ty_id = match entry_metadata.entry_type() {
+        subxt_metadata::StorageEntryType::Plain(ty) => *ty,
+        subxt_metadata::StorageEntryType::Map { value_ty, .. } => *value_ty,
+    };
+
+    let default_bytes = entry_metadata.default_bytes();
+    let val = Address::Target::decode_with_metadata(&mut &*default_bytes, value_ty_id, metadata)?;
+    Ok(val)
+}

core/src/storage/storage_key.rs

@@ -1,3 +1,7 @@
+// Copyright 2019-2024 Parity Technologies (UK) Ltd.
+// This file is dual-licensed as Apache-2.0 or GPL-3.0.
+// see LICENSE for license details.
+
 use super::utils::hash_bytes;
 use crate::{
     error::{Error, MetadataError, StorageAddressError},

core/src/storage/utils.rs

@@ -1,50 +1,25 @@
-// Copyright 2019-2023 Parity Technologies (UK) Ltd.
+// Copyright 2019-2024 Parity Technologies (UK) Ltd.
 // This file is dual-licensed as Apache-2.0 or GPL-3.0.
 // see LICENSE for license details.
 
-//! these utility methods complement the [`StorageAddress`] trait, but
+//! these utility methods complement the [`AddressT`] trait, but
 //! aren't things that should ever be overridden, and so don't exist on
 //! the trait itself.
 
-use crate::error::MetadataError;
-use crate::metadata::{DecodeWithMetadata, MetadataExt};
-use alloc::vec::Vec;
-use subxt_metadata::PalletMetadata;
-use subxt_metadata::{StorageEntryMetadata, StorageHasher};
-
-use super::StorageAddress;
-use crate::{error::Error, metadata::Metadata};
+use super::address::AddressT;
+use crate::error::{Error, MetadataError};
+use crate::metadata::Metadata;
 use alloc::borrow::ToOwned;
+use alloc::vec::Vec;
+use subxt_metadata::{PalletMetadata, StorageEntryMetadata, StorageHasher};
 
-/// Return the root of a given [`StorageAddress`]: hash the pallet name and entry name
+/// Return the root of a given [`AddressT`]: hash the pallet name and entry name
 /// and append those bytes to the output.
-pub fn write_storage_address_root_bytes<Address: StorageAddress>(
-    addr: &Address,
-    out: &mut Vec<u8>,
-) {
+pub fn write_storage_address_root_bytes<Address: AddressT>(addr: &Address, out: &mut Vec<u8>) {
     out.extend(sp_crypto_hashing::twox_128(addr.pallet_name().as_bytes()));
     out.extend(sp_crypto_hashing::twox_128(addr.entry_name().as_bytes()));
 }
 
-/// Outputs the [`storage_address_root_bytes`] as well as any additional bytes that represent
-/// a lookup in a storage map at that location.
-pub fn storage_address_bytes<Address: StorageAddress>(
-    addr: &Address,
-    metadata: &Metadata,
-) -> Result<Vec<u8>, Error> {
-    let mut bytes = Vec::new();
-    write_storage_address_root_bytes(addr, &mut bytes);
-    addr.append_entry_bytes(metadata, &mut bytes)?;
-    Ok(bytes)
-}
-
-/// Outputs a vector containing the bytes written by [`write_storage_address_root_bytes`].
-pub fn storage_address_root_bytes<Address: StorageAddress>(addr: &Address) -> Vec<u8> {
-    let mut bytes = Vec::new();
-    write_storage_address_root_bytes(addr, &mut bytes);
-    bytes
-}
-
 /// Take some SCALE encoded bytes and a [`StorageHasher`] and hash the bytes accordingly.
 pub fn hash_bytes(input: &[u8], hasher: StorageHasher, bytes: &mut Vec<u8>) {
     match hasher {
@@ -65,10 +40,10 @@ pub fn hash_bytes(input: &[u8], hasher: StorageHasher, bytes: &mut Vec<u8>) {
 }
 
 /// Return details about the given storage entry.
-pub fn lookup_entry_details<'a>(
+pub fn lookup_storage_entry_details<'a>(
     pallet_name: &str,
     entry_name: &str,
-    metadata: &'a subxt_metadata::Metadata,
+    metadata: &'a Metadata,
 ) -> Result<(PalletMetadata<'a>, &'a StorageEntryMetadata), Error> {
     let pallet_metadata = metadata.pallet_by_name_err(pallet_name)?;
     let storage_metadata = pallet_metadata
@@ -79,40 +54,3 @@ pub fn lookup_entry_details<'a>(
         .ok_or_else(|| MetadataError::StorageEntryNotFound(entry_name.to_owned()))?;
     Ok((pallet_metadata, storage_entry))
 }
-
-/// Validate a storage address against the metadata.
-pub fn validate_storage_address<Address: StorageAddress>(
-    address: &Address,
-    pallet: PalletMetadata<'_>,
-) -> Result<(), Error> {
-    if let Some(hash) = address.validation_hash() {
-        validate_storage(pallet, address.entry_name(), hash)?;
-    }
-    Ok(())
-}
-
-/// Validate a storage entry against the metadata.
-fn validate_storage(
-    pallet: PalletMetadata<'_>,
-    storage_name: &str,
-    hash: [u8; 32],
-) -> Result<(), Error> {
-    let Some(expected_hash) = pallet.storage_hash(storage_name) else {
-        return Err(MetadataError::IncompatibleCodegen.into());
-    };
-    if expected_hash != hash {
-        return Err(MetadataError::IncompatibleCodegen.into());
-    }
-    Ok(())
-}
-
-/// Given some bytes, a pallet and storage name, decode the response.
-pub fn decode_storage_with_metadata<T: DecodeWithMetadata>(
-    bytes: &mut &[u8],
-    metadata: &Metadata,
-    storage_metadata: &StorageEntryMetadata,
-) -> Result<T, Error> {
-    let return_ty = storage_metadata.entry_type().value_ty();
-    let val = T::decode_with_metadata(bytes, return_ty, metadata)?;
-    Ok(val)
-}