Rework Subxt API to support offline and dynamic transactions (#593)

* WIP API changes

* debug impls

* Get main crate compiling with first round of changes

* Some tidy up

* Add WithExtrinsicParams, and have SubstrateConfig + PolkadotConfig, not DefaultConfig

* move transaction into extrinsic folder

* Add runtime updates back to OnlineClient

* rework to be 'client first' to fit better with storage + events

* add support for events to Client

* tidy dupe trait bound

* Wire storage into client, but need to remove static reliance

* various tidy up and start stripping codegen to remove bits we dont need now

* First pass updating calls and constants codegen

* WIP storage client updates

* First pass migrated runtime storage over to new format

* pass over codegen to generate StorageAddresses and throw other stuff out

* don't need a Call trait any more

* shuffle things around a bit

* Various proc_macro fixes to get 'cargo check' working

* organise what's exposed from subxt

* Get first example working; balance_transfer_with_params

* get balance_transfer example compiling

* get concurrent_storage_requests.rs example compiling

* get fetch_all_accounts example compiling

* get a bunch more of the examples compiling

* almost get final example working; type mismatch to look into

* wee tweaks

* move StorageAddress to separate file

* pass Defaultable/Iterable info to StorageAddress in codegen

* fix storage validation ne, and partial run through example code

* Remove static iteration and strip a generic param from everything

* fix doc tests in subxt crate

* update test utils and start fixing frame tests

* fix frame staking tests

* fix the rest of the test compile issues, Borrow on storage values

* cargo fmt

* remove extra logging during tests

* Appease clippy and no more need for into_iter on events

* cargo fmt

* fix dryRun tests by waiting for blocks

* wait for blocks instead of sleeping or other test hacks

* cargo fmt

* Fix doc links

* Traitify StorageAddress

* remove out-of-date doc comments

* optimise decoding storage a little

* cleanup tx stuff, trait for TxPayload, remove Err type param and decode at runtime

* clippy fixes

* fix doc links

* fix doc example

* constant address trait for consistency

* fix a typo and remove EncodeWithMetadata stuff

* Put EventDetails behind a proper interface and allow decoding into top level event, too

* fix docs

* tweak StorageAddress docs

* re-export StorageAddress at root for consistency

* fix clippy things

* Add support for dynamic values

* fix double encoding of storage map key after refactor

* clippy fix

* Fixes and add a dynamic usage example (needs new scale_value release)

* bump scale_value version

* cargo fmt

* Tweak event bits

* cargo fmt

* Add a test and bump scale-value to 0.4.0 to support this

* remove unnecessary vec from dynamic example

* Various typo/grammar fixes

Co-authored-by: Alexandru Vasile <60601340+lexnv@users.noreply.github.com>

* Address PR nits

* Undo accidental rename in changelog

* Small PR nits/tidyups

* fix tests; codegen change against latest substrate

* tweak storage address util names

* move error decoding to DecodeError and expose

* impl some basic traits on the extrinsic param builder

Co-authored-by: Alexandru Vasile <60601340+lexnv@users.noreply.github.com>

testing/integration-tests/src/frame/balances.rs

@@ -4,10 +4,10 @@
 
 use crate::{
     node_runtime::{
+        self,
         balances,
         runtime_types,
         system,
-        DispatchError,
     },
     pair_signer,
     test_context,
@@ -22,32 +22,36 @@ use sp_runtime::{
     AccountId32,
     MultiAddress,
 };
-use subxt::Error;
+use subxt::error::{
+    DispatchError,
+    Error,
+};
 
 #[tokio::test]
-async fn tx_basic_transfer() -> Result<(), subxt::Error<DispatchError>> {
+async fn tx_basic_transfer() -> Result<(), subxt::Error> {
     let alice = pair_signer(AccountKeyring::Alice.pair());
     let bob = pair_signer(AccountKeyring::Bob.pair());
     let bob_address = bob.account_id().clone().into();
-    let cxt = test_context().await;
-    let api = &cxt.api;
+    let ctx = test_context().await;
+    let api = ctx.client();
+
+    let alice_account_addr = node_runtime::storage().system().account(alice.account_id());
+    let bob_account_addr = node_runtime::storage().system().account(bob.account_id());
 
     let alice_pre = api
         .storage()
-        .system()
-        .account(alice.account_id(), None)
+        .fetch_or_default(&alice_account_addr, None)
         .await?;
     let bob_pre = api
         .storage()
-        .system()
-        .account(bob.account_id(), None)
+        .fetch_or_default(&bob_account_addr, None)
         .await?;
 
+    let tx = node_runtime::tx().balances().transfer(bob_address, 10_000);
+
     let events = api
         .tx()
-        .balances()
-        .transfer(bob_address, 10_000)?
-        .sign_and_submit_then_watch_default(&alice)
+        .sign_and_submit_then_watch_default(&tx, &alice)
         .await?
         .wait_for_finalized_success()
         .await?;
@@ -69,13 +73,11 @@ async fn tx_basic_transfer() -> Result<(), subxt::Error<DispatchError>> {
 
     let alice_post = api
         .storage()
-        .system()
-        .account(alice.account_id(), None)
+        .fetch_or_default(&alice_account_addr, None)
         .await?;
     let bob_post = api
         .storage()
-        .system()
-        .account(bob.account_id(), None)
+        .fetch_or_default(&bob_account_addr, None)
         .await?;
 
     assert!(alice_pre.data.free - 10_000 >= alice_post.data.free);
@@ -84,26 +86,118 @@ async fn tx_basic_transfer() -> Result<(), subxt::Error<DispatchError>> {
 }
 
 #[tokio::test]
-async fn multiple_transfers_work_nonce_incremented(
-) -> Result<(), subxt::Error<DispatchError>> {
+async fn tx_dynamic_transfer() -> Result<(), subxt::Error> {
+    use subxt::ext::scale_value::{
+        At,
+        Composite,
+        Value,
+    };
+
+    let alice = pair_signer(AccountKeyring::Alice.pair());
+    let bob = pair_signer(AccountKeyring::Bob.pair());
+    let ctx = test_context().await;
+    let api = ctx.client();
+
+    let alice_account_addr = subxt::dynamic::storage(
+        "System",
+        "Account",
+        vec![Value::from_bytes(&alice.account_id())],
+    );
+    let bob_account_addr = subxt::dynamic::storage(
+        "System",
+        "Account",
+        vec![Value::from_bytes(&bob.account_id())],
+    );
+
+    let alice_pre = api
+        .storage()
+        .fetch_or_default(&alice_account_addr, None)
+        .await?;
+    let bob_pre = api
+        .storage()
+        .fetch_or_default(&bob_account_addr, None)
+        .await?;
+
+    let tx = subxt::dynamic::tx(
+        "Balances",
+        "transfer",
+        vec![
+            Value::unnamed_variant("Id", vec![Value::from_bytes(&bob.account_id())]),
+            Value::u128(10_000u128),
+        ],
+    );
+
+    let events = api
+        .tx()
+        .sign_and_submit_then_watch_default(&tx, &alice)
+        .await?
+        .wait_for_finalized_success()
+        .await?;
+
+    let event_fields = events
+        .iter()
+        .filter_map(|ev| ev.ok())
+        .find(|ev| ev.pallet_name() == "Balances" && ev.variant_name() == "Transfer")
+        .expect("Failed to find Transfer event")
+        .field_values()
+        .map_context(|_| ());
+
+    let expected_fields = Composite::Named(vec![
+        (
+            "from".into(),
+            Value::unnamed_composite(vec![Value::from_bytes(&alice.account_id())]),
+        ),
+        (
+            "to".into(),
+            Value::unnamed_composite(vec![Value::from_bytes(&bob.account_id())]),
+        ),
+        ("amount".into(), Value::u128(10_000)),
+    ]);
+    assert_eq!(event_fields, expected_fields);
+
+    let alice_post = api
+        .storage()
+        .fetch_or_default(&alice_account_addr, None)
+        .await?;
+    let bob_post = api
+        .storage()
+        .fetch_or_default(&bob_account_addr, None)
+        .await?;
+
+    let alice_pre_free = alice_pre.at("data").at("free").unwrap().as_u128().unwrap();
+    let alice_post_free = alice_post.at("data").at("free").unwrap().as_u128().unwrap();
+
+    let bob_pre_free = bob_pre.at("data").at("free").unwrap().as_u128().unwrap();
+    let bob_post_free = bob_post.at("data").at("free").unwrap().as_u128().unwrap();
+
+    assert!(alice_pre_free - 10_000 >= alice_post_free);
+    assert_eq!(bob_pre_free + 10_000, bob_post_free);
+
+    Ok(())
+}
+
+#[tokio::test]
+async fn multiple_transfers_work_nonce_incremented() -> Result<(), subxt::Error> {
     let alice = pair_signer(AccountKeyring::Alice.pair());
     let bob = pair_signer(AccountKeyring::Bob.pair());
     let bob_address: MultiAddress<AccountId32, u32> = bob.account_id().clone().into();
-    let cxt = test_context().await;
-    let api = &cxt.api;
+    let ctx = test_context().await;
+    let api = ctx.client();
+
+    let bob_account_addr = node_runtime::storage().system().account(bob.account_id());
 
     let bob_pre = api
         .storage()
-        .system()
-        .account(bob.account_id(), None)
+        .fetch_or_default(&bob_account_addr, None)
         .await?;
 
+    let tx = node_runtime::tx()
+        .balances()
+        .transfer(bob_address.clone(), 10_000);
     for _ in 0..3 {
         api
             .tx()
-            .balances()
-            .transfer(bob_address.clone(), 10_000)?
-            .sign_and_submit_then_watch_default(&alice)
+            .sign_and_submit_then_watch_default(&tx, &alice)
             .await?
             .wait_for_in_block() // Don't need to wait for finalization; this is quicker.
             .await?
@@ -113,8 +207,7 @@ async fn multiple_transfers_work_nonce_incremented(
 
     let bob_post = api
         .storage()
-        .system()
-        .account(bob.account_id(), None)
+        .fetch_or_default(&bob_account_addr, None)
         .await?;
 
     assert_eq!(bob_pre.data.free + 30_000, bob_post.data.free);
@@ -123,45 +216,40 @@ async fn multiple_transfers_work_nonce_incremented(
 
 #[tokio::test]
 async fn storage_total_issuance() {
-    let cxt = test_context().await;
-    let total_issuance = cxt
-        .api
-        .storage()
-        .balances()
-        .total_issuance(None)
-        .await
-        .unwrap();
+    let ctx = test_context().await;
+    let api = ctx.client();
+
+    let addr = node_runtime::storage().balances().total_issuance();
+    let total_issuance = api.storage().fetch_or_default(&addr, None).await.unwrap();
     assert_ne!(total_issuance, 0);
 }
 
 #[tokio::test]
-async fn storage_balance_lock() -> Result<(), subxt::Error<DispatchError>> {
+async fn storage_balance_lock() -> Result<(), subxt::Error> {
     let bob = pair_signer(AccountKeyring::Bob.pair());
     let charlie = AccountKeyring::Charlie.to_account_id();
-    let cxt = test_context().await;
+    let ctx = test_context().await;
+    let api = ctx.client();
 
-    cxt.api
-        .tx()
-        .staking()
-        .bond(
-            charlie.into(),
-            100_000_000_000_000,
-            runtime_types::pallet_staking::RewardDestination::Stash,
-        )?
-        .sign_and_submit_then_watch_default(&bob)
+    let tx = node_runtime::tx().staking().bond(
+        charlie.into(),
+        100_000_000_000_000,
+        runtime_types::pallet_staking::RewardDestination::Stash,
+    );
+
+    api.tx()
+        .sign_and_submit_then_watch_default(&tx, &bob)
         .await?
         .wait_for_finalized_success()
         .await?
         .find_first::<system::events::ExtrinsicSuccess>()?
         .expect("No ExtrinsicSuccess Event found");
 
-    let locked_account = AccountKeyring::Bob.to_account_id();
-    let locks = cxt
-        .api
-        .storage()
+    let locks_addr = node_runtime::storage()
         .balances()
-        .locks(&locked_account, None)
-        .await?;
+        .locks(&AccountKeyring::Bob.to_account_id());
+
+    let locks = api.storage().fetch_or_default(&locks_addr, None).await?;
 
     assert_eq!(
         locks.0,
@@ -177,38 +265,37 @@ async fn storage_balance_lock() -> Result<(), subxt::Error<DispatchError>> {
 
 #[tokio::test]
 async fn transfer_error() {
-    tracing_subscriber::fmt::try_init().ok();
     let alice = pair_signer(AccountKeyring::Alice.pair());
     let alice_addr = alice.account_id().clone().into();
     let hans = pair_signer(Pair::generate().0);
     let hans_address = hans.account_id().clone().into();
     let ctx = test_context().await;
+    let api = ctx.client();
 
-    ctx.api
-        .tx()
+    let to_hans_tx = node_runtime::tx()
         .balances()
-        .transfer(hans_address, 100_000_000_000_000_000)
-        .unwrap()
-        .sign_and_submit_then_watch_default(&alice)
+        .transfer(hans_address, 100_000_000_000_000_000);
+    let to_alice_tx = node_runtime::tx()
+        .balances()
+        .transfer(alice_addr, 100_000_000_000_000_000);
+
+    api.tx()
+        .sign_and_submit_then_watch_default(&to_hans_tx, &alice)
         .await
         .unwrap()
         .wait_for_finalized_success()
         .await
         .unwrap();
 
-    let res = ctx
-        .api
+    let res = api
         .tx()
-        .balances()
-        .transfer(alice_addr, 100_000_000_000_000_000)
-        .unwrap()
-        .sign_and_submit_then_watch_default(&hans)
+        .sign_and_submit_then_watch_default(&to_alice_tx, &hans)
         .await
         .unwrap()
         .wait_for_finalized_success()
         .await;
 
-    if let Err(Error::Module(err)) = res {
+    if let Err(Error::Runtime(DispatchError::Module(err))) = res {
         assert_eq!(err.pallet, "Balances");
         assert_eq!(err.error, "InsufficientBalance");
     } else {
@@ -218,19 +305,17 @@ async fn transfer_error() {
 
 #[tokio::test]
 async fn transfer_implicit_subscription() {
-    tracing_subscriber::fmt::try_init().ok();
     let alice = pair_signer(AccountKeyring::Alice.pair());
     let bob = AccountKeyring::Bob.to_account_id();
     let bob_addr = bob.clone().into();
-    let cxt = test_context().await;
+    let ctx = test_context().await;
+    let api = ctx.client();
 
-    let event = cxt
-        .api
+    let to_bob_tx = node_runtime::tx().balances().transfer(bob_addr, 10_000);
+
+    let event = api
         .tx()
-        .balances()
-        .transfer(bob_addr, 10_000)
-        .unwrap()
-        .sign_and_submit_then_watch_default(&alice)
+        .sign_and_submit_then_watch_default(&to_bob_tx, &alice)
         .await
         .unwrap()
         .wait_for_finalized_success()
@@ -252,19 +337,19 @@ async fn transfer_implicit_subscription() {
 
 #[tokio::test]
 async fn constant_existential_deposit() {
-    let cxt = test_context().await;
-    let locked_metadata = cxt.client().metadata();
-    let metadata = locked_metadata.read();
+    let ctx = test_context().await;
+    let api = ctx.client();
+
+    // get and decode constant manually via metadata:
+    let metadata = api.metadata();
     let balances_metadata = metadata.pallet("Balances").unwrap();
     let constant_metadata = balances_metadata.constant("ExistentialDeposit").unwrap();
     let existential_deposit = u128::decode(&mut &constant_metadata.value[..]).unwrap();
     assert_eq!(existential_deposit, 100_000_000_000_000);
-    assert_eq!(
-        existential_deposit,
-        cxt.api
-            .constants()
-            .balances()
-            .existential_deposit()
-            .unwrap()
-    );
+
+    // constant address for API access:
+    let addr = node_runtime::constants().balances().existential_deposit();
+
+    // Make sure thetwo are identical:
+    assert_eq!(existential_deposit, api.constants().at(&addr).unwrap());
 }