Rework Subxt API to support offline and dynamic transactions (#593)

* WIP API changes

* debug impls

* Get main crate compiling with first round of changes

* Some tidy up

* Add WithExtrinsicParams, and have SubstrateConfig + PolkadotConfig, not DefaultConfig

* move transaction into extrinsic folder

* Add runtime updates back to OnlineClient

* rework to be 'client first' to fit better with storage + events

* add support for events to Client

* tidy dupe trait bound

* Wire storage into client, but need to remove static reliance

* various tidy up and start stripping codegen to remove bits we dont need now

* First pass updating calls and constants codegen

* WIP storage client updates

* First pass migrated runtime storage over to new format

* pass over codegen to generate StorageAddresses and throw other stuff out

* don't need a Call trait any more

* shuffle things around a bit

* Various proc_macro fixes to get 'cargo check' working

* organise what's exposed from subxt

* Get first example working; balance_transfer_with_params

* get balance_transfer example compiling

* get concurrent_storage_requests.rs example compiling

* get fetch_all_accounts example compiling

* get a bunch more of the examples compiling

* almost get final example working; type mismatch to look into

* wee tweaks

* move StorageAddress to separate file

* pass Defaultable/Iterable info to StorageAddress in codegen

* fix storage validation ne, and partial run through example code

* Remove static iteration and strip a generic param from everything

* fix doc tests in subxt crate

* update test utils and start fixing frame tests

* fix frame staking tests

* fix the rest of the test compile issues, Borrow on storage values

* cargo fmt

* remove extra logging during tests

* Appease clippy and no more need for into_iter on events

* cargo fmt

* fix dryRun tests by waiting for blocks

* wait for blocks instead of sleeping or other test hacks

* cargo fmt

* Fix doc links

* Traitify StorageAddress

* remove out-of-date doc comments

* optimise decoding storage a little

* cleanup tx stuff, trait for TxPayload, remove Err type param and decode at runtime

* clippy fixes

* fix doc links

* fix doc example

* constant address trait for consistency

* fix a typo and remove EncodeWithMetadata stuff

* Put EventDetails behind a proper interface and allow decoding into top level event, too

* fix docs

* tweak StorageAddress docs

* re-export StorageAddress at root for consistency

* fix clippy things

* Add support for dynamic values

* fix double encoding of storage map key after refactor

* clippy fix

* Fixes and add a dynamic usage example (needs new scale_value release)

* bump scale_value version

* cargo fmt

* Tweak event bits

* cargo fmt

* Add a test and bump scale-value to 0.4.0 to support this

* remove unnecessary vec from dynamic example

* Various typo/grammar fixes

Co-authored-by: Alexandru Vasile <60601340+lexnv@users.noreply.github.com>

* Address PR nits

* Undo accidental rename in changelog

* Small PR nits/tidyups

* fix tests; codegen change against latest substrate

* tweak storage address util names

* move error decoding to DecodeError and expose

* impl some basic traits on the extrinsic param builder

Co-authored-by: Alexandru Vasile <60601340+lexnv@users.noreply.github.com>

testing/integration-tests/src/storage/mod.rs

@@ -3,47 +3,48 @@
 // see LICENSE for license details.
 
 use crate::{
-    node_runtime::{
-        self,
-        DispatchError,
-    },
+    node_runtime,
     pair_signer,
     test_context,
+    utils::wait_for_blocks,
 };
 use sp_keyring::AccountKeyring;
 
 #[tokio::test]
-async fn storage_plain_lookup() -> Result<(), subxt::Error<DispatchError>> {
+async fn storage_plain_lookup() -> Result<(), subxt::Error> {
     let ctx = test_context().await;
+    let api = ctx.client();
 
     // Look up a plain value. Wait long enough that we don't get the genesis block data,
     // because it may have no storage associated with it.
-    tokio::time::sleep(std::time::Duration::from_secs(6)).await;
-    let entry = ctx.api.storage().timestamp().now(None).await?;
+    wait_for_blocks(&api).await;
+
+    let addr = node_runtime::storage().timestamp().now();
+    let entry = api.storage().fetch_or_default(&addr, None).await?;
     assert!(entry > 0);
 
     Ok(())
 }
 
 #[tokio::test]
-async fn storage_map_lookup() -> Result<(), subxt::Error<DispatchError>> {
+async fn storage_map_lookup() -> Result<(), subxt::Error> {
     let ctx = test_context().await;
+    let api = ctx.client();
 
     let signer = pair_signer(AccountKeyring::Alice.pair());
     let alice = AccountKeyring::Alice.to_account_id();
 
     // Do some transaction to bump the Alice nonce to 1:
-    ctx.api
-        .tx()
-        .system()
-        .remark(vec![1, 2, 3, 4, 5])?
-        .sign_and_submit_then_watch_default(&signer)
+    let remark_tx = node_runtime::tx().system().remark(vec![1, 2, 3, 4, 5]);
+    api.tx()
+        .sign_and_submit_then_watch_default(&remark_tx, &signer)
         .await?
         .wait_for_finalized_success()
         .await?;
 
     // Look up the nonce for the user (we expect it to be 1).
-    let entry = ctx.api.storage().system().account(&alice, None).await?;
+    let nonce_addr = node_runtime::storage().system().account(&alice);
+    let entry = api.storage().fetch_or_default(&nonce_addr, None).await?;
     assert_eq!(entry.nonce, 1);
 
     Ok(())
@@ -54,23 +55,15 @@ async fn storage_map_lookup() -> Result<(), subxt::Error<DispatchError>> {
 // treated as a StorageKey (ie we should hash both values together with one hasher, rather
 // than hash both values separately, or ignore the second value).
 #[tokio::test]
-async fn storage_n_mapish_key_is_properly_created(
-) -> Result<(), subxt::Error<DispatchError>> {
+async fn storage_n_mapish_key_is_properly_created() -> Result<(), subxt::Error> {
     use codec::Encode;
-    use node_runtime::{
-        runtime_types::sp_core::crypto::KeyTypeId,
-        session::storage::KeyOwner,
-    };
-    use subxt::{
-        storage::StorageKeyPrefix,
-        StorageEntry,
-    };
+    use node_runtime::runtime_types::sp_core::crypto::KeyTypeId;
 
     // This is what the generated code hashes a `session().key_owner(..)` key into:
-    let actual_key_bytes = KeyOwner(&KeyTypeId([1, 2, 3, 4]), &[5u8, 6, 7, 8])
-        .key()
-        .final_key(StorageKeyPrefix::new::<KeyOwner>())
-        .0;
+    let actual_key_bytes = node_runtime::storage()
+        .session()
+        .key_owner(KeyTypeId([1, 2, 3, 4]), [5u8, 6, 7, 8])
+        .to_bytes();
 
     // Let's manually hash to what we assume it should be and compare:
     let expected_key_bytes = {
@@ -89,8 +82,9 @@ async fn storage_n_mapish_key_is_properly_created(
 }
 
 #[tokio::test]
-async fn storage_n_map_storage_lookup() -> Result<(), subxt::Error<DispatchError>> {
+async fn storage_n_map_storage_lookup() -> Result<(), subxt::Error> {
     let ctx = test_context().await;
+    let api = ctx.client();
 
     // Boilerplate; we create a new asset class with ID 99, and then
     // we "approveTransfer" of some of this asset class. This gives us an
@@ -98,30 +92,27 @@ async fn storage_n_map_storage_lookup() -> Result<(), subxt::Error<DispatchError
     let signer = pair_signer(AccountKeyring::Alice.pair());
     let alice = AccountKeyring::Alice.to_account_id();
     let bob = AccountKeyring::Bob.to_account_id();
-    ctx.api
-        .tx()
+
+    let tx1 = node_runtime::tx()
         .assets()
-        .create(99, alice.clone().into(), 1)?
-        .sign_and_submit_then_watch_default(&signer)
+        .create(99, alice.clone().into(), 1);
+    let tx2 = node_runtime::tx()
+        .assets()
+        .approve_transfer(99, bob.clone().into(), 123);
+    api.tx()
+        .sign_and_submit_then_watch_default(&tx1, &signer)
         .await?
         .wait_for_finalized_success()
         .await?;
-    ctx.api
-        .tx()
-        .assets()
-        .approve_transfer(99, bob.clone().into(), 123)?
-        .sign_and_submit_then_watch_default(&signer)
+    api.tx()
+        .sign_and_submit_then_watch_default(&tx2, &signer)
         .await?
         .wait_for_finalized_success()
         .await?;
 
     // The actual test; look up this approval in storage:
-    let entry = ctx
-        .api
-        .storage()
-        .assets()
-        .approvals(&99, &alice, &bob, None)
-        .await?;
+    let addr = node_runtime::storage().assets().approvals(99, &alice, &bob);
+    let entry = api.storage().fetch(&addr, None).await?;
     assert_eq!(entry.map(|a| a.amount), Some(123));
     Ok(())
 }