v0.50.0: Integrate frame-decode, redo storage APIs and break up Error. (#2100)

* WIP integrating new frame-decode and working out new storage APIS

* WIP: first pass adding new storage things to subxt-core

* Second pass over Address type and start impl in Subxt

* WIP new storage APIs

* WIP New storage APIs roughly completed, lots of errors still

* Remove PlainorMap enum; plain and map values now use same struct to simplify usage

* Begin 'fixing' errors

* WIP splitting errors and tidying payload/address traits

* Get subxt-core compiling

* Small fixes in subxt-core and remove metadata mod

* subxt-core: cargo check --all-targets passes

* Fix test

* WIP starting to update subxt from subxt-core changes

* WIP splitting up subxt errors into smaller variants

* WIP errors: add DispatchError errors

* Port new Storage APIs to subxt-core

* cargo check -p subxt passes

* Quick-fix errors in subxt-cli (explore subcommand)

* fmt

* Finish fixing codegen up and start fixing examples

* get Subxt examples compiling and bytes_at for constants

* Add some arcs to limit lifetimes in subxt/subxt-core storage APIs

* A little Arcing to allow more method chaining in Storage APIs, aligning with Subxt

* Update codegen test

* cargo check --all-targets passing

* cargo check --features 'unstable-light-client' passing

* clippy

* Remove unused dep in subxt

* use published frame-decode

* fix wasm-example

* Add new tx extension to fix daily tests

* Remove unused subxt_core::dynamic::DecodedValue type

* Update book to match changes

* Update docs to fix more broken bits

* Add missing docs

* fmt

* allow larger result errs for now

* Add missing alloc imports in subxt-core

* Fix doc tests and fix bug getting constant info

* Fix V14 -> Metadata transform for storage & constants

* Fix parachain example

* Fix FFI example

* BlockLength decodes t ostruct, not u128

* use fetch/iter shorthands rather than entry in most storage tests

* Fix some integration tests

* Fix Runtime codegen tests

* Expose the dynamic custom_value selecter and use in a UI test

* Update codegen metadata

* Tidy CLI storage query and support (str,str) as a storage address

* Add (str,str) as valid constant address too

* Show string tuple in constants example

* Via the magic of traits, avoid needing any clones of queries/addresses and accept references to them

* clippy

subxt/examples/block_decoding_dynamic.rs

@@ -21,10 +21,10 @@ async fn main() -> Result<(), Box<dyn std::error::Error>> {
                 continue; // we do not look at inherents in this example
             };
 
-            let meta = ext.extrinsic_metadata()?;
-            let fields = ext.field_values()?;
+            // Decode the fields into our dynamic Value type to display:
+            let fields = ext.decode_as_fields::<scale_value::Value>()?;
 
-            println!("  {}/{}", meta.pallet.name(), meta.variant.name);
+            println!("  {}/{}", ext.pallet_name(), ext.call_name());
             println!("    Transaction Extensions:");
             for signed_ext in transaction_extensions.iter() {
                 // We only want to take a look at these 3 signed extensions, because the others all just have unit fields.

subxt/examples/blocks_subscribing.rs

@@ -42,7 +42,7 @@ async fn main() -> Result<(), Box<dyn std::error::Error>> {
                 let evt = evt?;
                 let pallet_name = evt.pallet_name();
                 let event_name = evt.variant_name();
-                let event_values = evt.field_values()?;
+                let event_values = evt.decode_as_fields::<scale_value::Value>()?;
 
                 println!("        {pallet_name}_{event_name}");
                 println!("          {event_values}");

subxt/examples/constants_dynamic.rs

@@ -1,4 +1,5 @@
 #![allow(missing_docs)]
+use subxt::dynamic::Value;
 use subxt::{OnlineClient, PolkadotConfig};
 
 #[tokio::main]
@@ -6,13 +7,20 @@ async fn main() -> Result<(), Box<dyn std::error::Error>> {
     // Create a client to use:
     let api = OnlineClient::<PolkadotConfig>::new().await?;
 
-    // A dynamic query to obtain some constant:
-    let constant_query = subxt::dynamic::constant("System", "BlockLength");
+    // We can query a constant by providing a tuple of the pallet and constant name. The return type
+    // will be `Value` if we pass this query:
+    let constant_query = ("System", "BlockLength");
+    let _value = api.constants().at(&constant_query)?;
 
-    // Obtain the value:
+    // Or we can use the library function to query a constant, which allows us to pass a generic type
+    // that Subxt will attempt to decode the constant into:
+    let constant_query = subxt::dynamic::constant::<Value>("System", "BlockLength");
     let value = api.constants().at(&constant_query)?;
 
-    println!("Constant bytes: {:?}", value.encoded());
-    println!("Constant value: {}", value.to_value()?);
+    // Or we can obtain the bytes for the constant, using either form of query.
+    let bytes = api.constants().bytes_at(&constant_query)?;
+
+    println!("Constant bytes: {:?}", bytes);
+    println!("Constant value: {}", value);
     Ok(())
 }

subxt/examples/constants_static.rs

@@ -15,6 +15,10 @@ async fn main() -> Result<(), Box<dyn std::error::Error>> {
     // Obtain the value:
     let value = api.constants().at(&constant_query)?;
 
+    // Or obtain the bytes:
+    let bytes = api.constants().bytes_at(&constant_query)?;
+
+    println!("Encoded block length: {bytes:?}");
     println!("Block length: {value:?}");
     Ok(())
 }

subxt/examples/events.rs

@@ -19,7 +19,7 @@ async fn main() -> Result<(), Box<dyn std::error::Error>> {
 
         let pallet = event.pallet_name();
         let variant = event.variant_name();
-        let field_values = event.field_values()?;
+        let field_values = event.decode_as_fields::<scale_value::Value>()?;
 
         println!("{pallet}::{variant}: {field_values}");
     }

subxt/examples/runtime_apis_dynamic.rs

@@ -1,5 +1,5 @@
 #![allow(missing_docs)]
-use subxt::dynamic::Value;
+use subxt::utils::AccountId32;
 use subxt::{OnlineClient, config::PolkadotConfig};
 use subxt_signer::sr25519::dev;
 
@@ -8,14 +8,14 @@ async fn main() -> Result<(), Box<dyn std::error::Error>> {
     // Create a client to use:
     let api = OnlineClient::<PolkadotConfig>::new().await?;
 
-    // Create a dynamically runtime API payload that calls the
-    // `AccountNonceApi_account_nonce` function.
-    let account = dev::alice().public_key();
-    let runtime_api_call = subxt::dynamic::runtime_api_call(
-        "AccountNonceApi",
-        "account_nonce",
-        vec![Value::from_bytes(account)],
-    );
+    // Create a "dynamic" runtime API payload that calls the
+    // `AccountNonceApi_account_nonce` function. We could use the
+    // `scale_value::Value` type as output, and a vec of those as inputs,
+    // but since we know the input + return types we can pass them directly.
+    // There is one input argument, so the inputs are a tuple of one element.
+    let account: AccountId32 = dev::alice().public_key().into();
+    let runtime_api_call =
+        subxt::dynamic::runtime_api_call::<_, u64>("AccountNonceApi", "account_nonce", (account,));
 
     // Submit the call to get back a result.
     let nonce = api
@@ -25,6 +25,6 @@ async fn main() -> Result<(), Box<dyn std::error::Error>> {
         .call(runtime_api_call)
         .await?;
 
-    println!("Account nonce: {:#?}", nonce.to_value());
+    println!("Account nonce: {:#?}", nonce);
     Ok(())
 }

subxt/examples/storage_fetch.rs

@@ -10,22 +10,23 @@ pub mod polkadot {}
 async fn main() -> Result<(), Box<dyn std::error::Error>> {
     // Create a new API client, configured to talk to Polkadot nodes.
     let api = OnlineClient::<PolkadotConfig>::new().await?;
+    let account = dev::alice().public_key().into();
 
     // Build a storage query to access account information.
-    let account = dev::alice().public_key().into();
-    let storage_query = polkadot::storage().system().account(account);
+    let storage_query = polkadot::storage().system().account();
 
-    // Use that query to `fetch` a result. This returns an `Option<_>`, which will be
-    // `None` if no value exists at the given address. You can also use `fetch_default`
-    // where applicable, which will return the default value if none exists.
-    let result = api
-        .storage()
-        .at_latest()
+    // Use that query to access a storage entry, fetch a result and decode the value.
+    // The static address knows that fetching requires a tuple of one value, an
+    // AccountId32.
+    let client_at = api.storage().at_latest().await?;
+    let account_info = client_at
+        .entry(storage_query)?
+        .fetch((account,))
         .await?
-        .fetch(&storage_query)
-        .await?;
+        .decode()?;
 
-    let v = result.unwrap().data.free;
-    println!("Alice: {v}");
+    // The static address that we got from the subxt macro knows the expected input
+    // and return types, so it is decoded into a static type for us.
+    println!("Alice: {account_info:?}");
     Ok(())
 }

subxt/examples/storage_fetch_dynamic.rs

@@ -1,5 +1,6 @@
 #![allow(missing_docs)]
 use subxt::dynamic::{At, Value};
+use subxt::utils::AccountId32;
 use subxt::{OnlineClient, PolkadotConfig};
 use subxt_signer::sr25519::dev;
 
@@ -9,20 +10,25 @@ async fn main() -> Result<(), Box<dyn std::error::Error>> {
     let api = OnlineClient::<PolkadotConfig>::new().await?;
 
     // Build a dynamic storage query to access account information.
-    let account = dev::alice().public_key();
-    let storage_query =
-        subxt::dynamic::storage("System", "Account", vec![Value::from_bytes(account)]);
+    // here, we assume that there is one value to provide at this entry
+    // to access a value; an AccountId32. In this example we don't know the
+    // return type and so we set it to `Value`, which anything can decode into.
+    let account: AccountId32 = dev::alice().public_key().into();
+    let storage_query = subxt::dynamic::storage::<(AccountId32,), Value>("System", "Account");
 
-    // Use that query to `fetch` a result. Because the query is dynamic, we don't know what the result
-    // type will be either, and so we get a type back that can be decoded into a dynamic Value type.
-    let result = api
-        .storage()
-        .at_latest()
+    // Use that query to access a storage entry, fetch a result and decode the value.
+    let client_at = api.storage().at_latest().await?;
+    let account_info = client_at
+        .entry(storage_query)?
+        .fetch((account,))
         .await?
-        .fetch(&storage_query)
-        .await?;
-    let value = result.unwrap().to_value()?;
+        .decode()?;
 
-    println!("Alice has free balance: {:?}", value.at("data").at("free"));
+    // With out `Value` type we can dig in to find what we want using the `At`
+    // trait and `.at()` method that this provides on the Value.
+    println!(
+        "Alice has free balance: {}",
+        account_info.at("data").at("free").unwrap()
+    );
     Ok(())
 }

subxt/examples/storage_iterating.rs

@@ -1,7 +1,9 @@
 #![allow(missing_docs)]
+use subxt::ext::futures::StreamExt;
 use subxt::{OnlineClient, PolkadotConfig};
 
-#[subxt::subxt(runtime_metadata_path = "../artifacts/polkadot_metadata_full.scale")]
+// Generate an interface that we can use from the node's metadata.
+#[subxt::subxt(runtime_metadata_path = "../artifacts/polkadot_metadata_small.scale")]
 pub mod polkadot {}
 
 #[tokio::main]
@@ -9,17 +11,31 @@ async fn main() -> Result<(), Box<dyn std::error::Error>> {
     // Create a new API client, configured to talk to Polkadot nodes.
     let api = OnlineClient::<PolkadotConfig>::new().await?;
 
-    // Build a storage query to iterate over account information.
-    let storage_query = polkadot::storage().system().account_iter();
+    // Build a storage query to access account information. Same as if we were
+    // fetching a single value from this entry.
+    let storage_query = polkadot::storage().system().account();
 
-    // Get back an iterator of results (here, we are fetching 10 items at
-    // a time from the node, but we always iterate over one at a time).
-    let mut results = api.storage().at_latest().await?.iter(storage_query).await?;
+    // Use that query to access a storage entry, iterate over it and decode values.
+    let client_at = api.storage().at_latest().await?;
 
-    while let Some(Ok(kv)) = results.next().await {
-        println!("Keys decoded: {:?}", kv.keys);
-        println!("Key: 0x{}", hex::encode(&kv.key_bytes));
-        println!("Value: {:?}", kv.value);
+    // We provide an empty tuple when iterating. If the storage entry had been an N map with
+    // multiple keys, then we could provide any prefix of those keys to iterate over. This is
+    // statically type checked, so only a valid number/type of keys in the tuple is accepted.
+    let mut values = client_at.entry(storage_query)?.iter(()).await?;
+
+    while let Some(kv) = values.next().await {
+        let kv = kv?;
+
+        // The key decodes into the type that the static address knows about, in this case a
+        // tuple of one entry, because the only part of the key that we can decode is the
+        // AccountId32 for each user.
+        let (account_id32,) = kv.key()?.decode()?;
+
+        // The value decodes into a statically generated type which holds account information.
+        let value = kv.value().decode()?;
+
+        let value_data = value.data;
+        println!("{account_id32}:\n  {value_data:?}");
     }
 
     Ok(())

subxt/examples/storage_iterating_dynamic.rs

@@ -1,23 +1,41 @@
 #![allow(missing_docs)]
-use subxt::{OnlineClient, PolkadotConfig, dynamic::Value};
+use subxt::ext::futures::StreamExt;
+use subxt::utils::AccountId32;
+use subxt::{
+    OnlineClient, PolkadotConfig,
+    dynamic::{At, Value},
+};
 
 #[tokio::main]
 async fn main() -> Result<(), Box<dyn std::error::Error>> {
     // Create a new API client, configured to talk to Polkadot nodes.
     let api = OnlineClient::<PolkadotConfig>::new().await?;
 
-    // Build a dynamic storage query to iterate account information.
-    // With a dynamic query, we can just provide an empty vector as the keys to iterate over all entries.
-    let keys: Vec<Value> = vec![];
-    let storage_query = subxt::dynamic::storage("System", "Account", keys);
+    // Build a dynamic storage query to access account information.
+    // here, we assume that there is one value to provide at this entry
+    // to access a value; an AccountId32. In this example we don't know the
+    // return type and so we set it to `Value`, which anything can decode into.
+    let storage_query = subxt::dynamic::storage::<(AccountId32,), Value>("System", "Account");
 
-    // Use that query to return an iterator over the results.
-    let mut results = api.storage().at_latest().await?.iter(storage_query).await?;
+    // Use that query to access a storage entry, iterate over it and decode values.
+    let client_at = api.storage().at_latest().await?;
+    let mut values = client_at.entry(storage_query)?.iter(()).await?;
 
-    while let Some(Ok(kv)) = results.next().await {
-        println!("Keys decoded: {:?}", kv.keys);
-        println!("Key: 0x{}", hex::encode(&kv.key_bytes));
-        println!("Value: {:?}", kv.value.to_value()?);
+    while let Some(kv) = values.next().await {
+        let kv = kv?;
+
+        // The key decodes into the first type we provided in the address. Since there's just
+        // one key, it is a tuple of one entry, an AccountId32. If we didn't know how many
+        // keys or their type, we could set the key to `Vec<Value>` instead.
+        let (account_id32,) = kv.key()?.decode()?;
+
+        // The value decodes into the second type we provided in the address. In this example,
+        // we just decode it into our `Value` type and then look at the "data" field in this
+        // (which implicitly assumes we get a struct shaped thing back with such a field).
+        let value = kv.value().decode()?;
+
+        let value_data = value.at("data").unwrap();
+        println!("{account_id32}:\n  {value_data}");
     }
 
     Ok(())

subxt/examples/storage_iterating_partial.rs

@@ -1,77 +0,0 @@
-#![allow(missing_docs)]
-use polkadot::multisig::events::NewMultisig;
-use polkadot::runtime_types::{
-    frame_system::pallet::Call, rococo_runtime::RuntimeCall, sp_weights::weight_v2::Weight,
-};
-use subxt::utils::AccountId32;
-use subxt::{OnlineClient, PolkadotConfig};
-use subxt_signer::sr25519::{Keypair, dev};
-
-#[subxt::subxt(runtime_metadata_path = "../artifacts/polkadot_metadata_full.scale")]
-pub mod polkadot {}
-
-#[tokio::main]
-async fn main() -> Result<(), Box<dyn std::error::Error>> {
-    // Create a new API client, configured to talk to Polkadot nodes.
-    let api = OnlineClient::<PolkadotConfig>::new().await?;
-
-    // Prepare the chain to have 3 open multisig requests (2 of them are alice + bob):
-    let alice_signer = dev::alice();
-    let bob = AccountId32(dev::bob().public_key().0);
-    let charlie = AccountId32(dev::charlie().public_key().0);
-
-    let new_multisig_1 = submit_remark_as_multi(&alice_signer, &bob, b"Hello", &api).await?;
-    let new_multisig_2 = submit_remark_as_multi(&alice_signer, &bob, b"Hi", &api).await?;
-    let new_multisig_3 = submit_remark_as_multi(&alice_signer, &charlie, b"Hello", &api).await?;
-
-    // Note: the NewMultisig event contains the multisig address we need to use for the storage queries:
-    assert_eq!(new_multisig_1.multisig, new_multisig_2.multisig);
-    assert_ne!(new_multisig_1.multisig, new_multisig_3.multisig);
-
-    // Build a storage query to iterate over open multisig extrinsics from
-    // new_multisig_1.multisig which is the AccountId of the alice + bob multisig account
-    let alice_bob_account_id = new_multisig_1.multisig;
-    let storage_query = polkadot::storage()
-        .multisig()
-        .multisigs_iter1(alice_bob_account_id);
-
-    // Get back an iterator of results.
-    let mut results = api.storage().at_latest().await?.iter(storage_query).await?;
-
-    while let Some(Ok(kv)) = results.next().await {
-        println!("Keys decoded: {:?}", kv.keys);
-        println!("Key: 0x{}", hex::encode(&kv.key_bytes));
-        println!("Value: {:?}", kv.value);
-    }
-    Ok(())
-}
-
-async fn submit_remark_as_multi(
-    signer: &Keypair,
-    other: &AccountId32,
-    remark: &[u8],
-    api: &OnlineClient<PolkadotConfig>,
-) -> Result<NewMultisig, Box<dyn std::error::Error>> {
-    let multisig_remark_tx = polkadot::tx().multisig().as_multi(
-        2,
-        vec![other.clone()],
-        None,
-        RuntimeCall::System(Call::remark {
-            remark: remark.to_vec(),
-        }),
-        Weight {
-            ref_time: 0,
-            proof_size: 0,
-        },
-    );
-    let events = api
-        .tx()
-        .sign_and_submit_then_watch_default(&multisig_remark_tx, signer)
-        .await?
-        .wait_for_finalized_success()
-        .await?;
-    let new_multisig = events
-        .find_first::<polkadot::multisig::events::NewMultisig>()?
-        .expect("should contain event");
-    Ok(new_multisig)
-}