Rework Subxt API to support offline and dynamic transactions (#593)

* WIP API changes

* debug impls

* Get main crate compiling with first round of changes

* Some tidy up

* Add WithExtrinsicParams, and have SubstrateConfig + PolkadotConfig, not DefaultConfig

* move transaction into extrinsic folder

* Add runtime updates back to OnlineClient

* rework to be 'client first' to fit better with storage + events

* add support for events to Client

* tidy dupe trait bound

* Wire storage into client, but need to remove static reliance

* various tidy up and start stripping codegen to remove bits we dont need now

* First pass updating calls and constants codegen

* WIP storage client updates

* First pass migrated runtime storage over to new format

* pass over codegen to generate StorageAddresses and throw other stuff out

* don't need a Call trait any more

* shuffle things around a bit

* Various proc_macro fixes to get 'cargo check' working

* organise what's exposed from subxt

* Get first example working; balance_transfer_with_params

* get balance_transfer example compiling

* get concurrent_storage_requests.rs example compiling

* get fetch_all_accounts example compiling

* get a bunch more of the examples compiling

* almost get final example working; type mismatch to look into

* wee tweaks

* move StorageAddress to separate file

* pass Defaultable/Iterable info to StorageAddress in codegen

* fix storage validation ne, and partial run through example code

* Remove static iteration and strip a generic param from everything

* fix doc tests in subxt crate

* update test utils and start fixing frame tests

* fix frame staking tests

* fix the rest of the test compile issues, Borrow on storage values

* cargo fmt

* remove extra logging during tests

* Appease clippy and no more need for into_iter on events

* cargo fmt

* fix dryRun tests by waiting for blocks

* wait for blocks instead of sleeping or other test hacks

* cargo fmt

* Fix doc links

* Traitify StorageAddress

* remove out-of-date doc comments

* optimise decoding storage a little

* cleanup tx stuff, trait for TxPayload, remove Err type param and decode at runtime

* clippy fixes

* fix doc links

* fix doc example

* constant address trait for consistency

* fix a typo and remove EncodeWithMetadata stuff

* Put EventDetails behind a proper interface and allow decoding into top level event, too

* fix docs

* tweak StorageAddress docs

* re-export StorageAddress at root for consistency

* fix clippy things

* Add support for dynamic values

* fix double encoding of storage map key after refactor

* clippy fix

* Fixes and add a dynamic usage example (needs new scale_value release)

* bump scale_value version

* cargo fmt

* Tweak event bits

* cargo fmt

* Add a test and bump scale-value to 0.4.0 to support this

* remove unnecessary vec from dynamic example

* Various typo/grammar fixes

Co-authored-by: Alexandru Vasile <60601340+lexnv@users.noreply.github.com>

* Address PR nits

* Undo accidental rename in changelog

* Small PR nits/tidyups

* fix tests; codegen change against latest substrate

* tweak storage address util names

* move error decoding to DecodeError and expose

* impl some basic traits on the extrinsic param builder

Co-authored-by: Alexandru Vasile <60601340+lexnv@users.noreply.github.com>

testing/integration-tests/src/frame/staking.rs

@@ -4,12 +4,12 @@
 
 use crate::{
     node_runtime::{
+        self,
         runtime_types::pallet_staking::{
             RewardDestination,
             ValidatorPrefs,
         },
         staking,
-        DispatchError,
     },
     pair_signer,
     test_context,
@@ -20,7 +20,10 @@ use sp_core::{
     Pair,
 };
 use sp_keyring::AccountKeyring;
-use subxt::Error;
+use subxt::error::{
+    DispatchError,
+    Error,
+};
 
 /// Helper function to generate a crypto pair from seed
 fn get_from_seed(seed: &str) -> sr25519::Pair {
@@ -37,14 +40,17 @@ fn default_validator_prefs() -> ValidatorPrefs {
 
 #[tokio::test]
 async fn validate_with_controller_account() {
-    let alice = pair_signer(AccountKeyring::Alice.pair());
     let ctx = test_context().await;
-    ctx.api
-        .tx()
+    let api = ctx.client();
+
+    let alice = pair_signer(AccountKeyring::Alice.pair());
+
+    let tx = node_runtime::tx()
         .staking()
-        .validate(default_validator_prefs())
-        .unwrap()
-        .sign_and_submit_then_watch_default(&alice)
+        .validate(default_validator_prefs());
+
+    api.tx()
+        .sign_and_submit_then_watch_default(&tx, &alice)
         .await
         .unwrap()
         .wait_for_finalized_success()
@@ -53,19 +59,23 @@ async fn validate_with_controller_account() {
 }
 
 #[tokio::test]
-async fn validate_not_possible_for_stash_account() -> Result<(), Error<DispatchError>> {
-    let alice_stash = pair_signer(get_from_seed("Alice//stash"));
+async fn validate_not_possible_for_stash_account() -> Result<(), Error> {
     let ctx = test_context().await;
-    let announce_validator = ctx
-        .api
-        .tx()
+    let api = ctx.client();
+
+    let alice_stash = pair_signer(get_from_seed("Alice//stash"));
+
+    let tx = node_runtime::tx()
         .staking()
-        .validate(default_validator_prefs())?
-        .sign_and_submit_then_watch_default(&alice_stash)
+        .validate(default_validator_prefs());
+
+    let announce_validator = api
+        .tx()
+        .sign_and_submit_then_watch_default(&tx, &alice_stash)
         .await?
         .wait_for_finalized_success()
         .await;
-    assert_matches!(announce_validator, Err(Error::Module(err)) => {
+    assert_matches!(announce_validator, Err(Error::Runtime(DispatchError::Module(err))) => {
         assert_eq!(err.pallet, "Staking");
         assert_eq!(err.error, "NotController");
     });
@@ -74,16 +84,18 @@ async fn validate_not_possible_for_stash_account() -> Result<(), Error<DispatchE
 
 #[tokio::test]
 async fn nominate_with_controller_account() {
+    let ctx = test_context().await;
+    let api = ctx.client();
+
     let alice = pair_signer(AccountKeyring::Alice.pair());
     let bob = pair_signer(AccountKeyring::Bob.pair());
-    let ctx = test_context().await;
 
-    ctx.api
-        .tx()
+    let tx = node_runtime::tx()
         .staking()
-        .nominate(vec![bob.account_id().clone().into()])
-        .unwrap()
-        .sign_and_submit_then_watch_default(&alice)
+        .nominate(vec![bob.account_id().clone().into()]);
+
+    api.tx()
+        .sign_and_submit_then_watch_default(&tx, &alice)
         .await
         .unwrap()
         .wait_for_finalized_success()
@@ -92,22 +104,26 @@ async fn nominate_with_controller_account() {
 }
 
 #[tokio::test]
-async fn nominate_not_possible_for_stash_account() -> Result<(), Error<DispatchError>> {
+async fn nominate_not_possible_for_stash_account() -> Result<(), Error> {
+    let ctx = test_context().await;
+    let api = ctx.client();
+
     let alice_stash = pair_signer(get_from_seed("Alice//stash"));
     let bob = pair_signer(AccountKeyring::Bob.pair());
-    let ctx = test_context().await;
 
-    let nomination = ctx
-        .api
-        .tx()
+    let tx = node_runtime::tx()
         .staking()
-        .nominate(vec![bob.account_id().clone().into()])?
-        .sign_and_submit_then_watch_default(&alice_stash)
-        .await?
+        .nominate(vec![bob.account_id().clone().into()]);
+
+    let nomination = api
+        .tx()
+        .sign_and_submit_then_watch_default(&tx, &alice_stash)
+        .await
+        .unwrap()
         .wait_for_finalized_success()
         .await;
 
-    assert_matches!(nomination, Err(Error::Module(err)) => {
+    assert_matches!(nomination, Err(Error::Runtime(DispatchError::Module(err))) => {
         assert_eq!(err.pallet, "Staking");
         assert_eq!(err.error, "NotController");
     });
@@ -115,52 +131,45 @@ async fn nominate_not_possible_for_stash_account() -> Result<(), Error<DispatchE
 }
 
 #[tokio::test]
-async fn chill_works_for_controller_only() -> Result<(), Error<DispatchError>> {
+async fn chill_works_for_controller_only() -> Result<(), Error> {
+    let ctx = test_context().await;
+    let api = ctx.client();
+
     let alice_stash = pair_signer(get_from_seed("Alice//stash"));
     let bob_stash = pair_signer(get_from_seed("Bob//stash"));
     let alice = pair_signer(AccountKeyring::Alice.pair());
-    let ctx = test_context().await;
 
     // this will fail the second time, which is why this is one test, not two
-    ctx.api
-        .tx()
+    let nominate_tx = node_runtime::tx()
         .staking()
-        .nominate(vec![bob_stash.account_id().clone().into()])?
-        .sign_and_submit_then_watch_default(&alice)
+        .nominate(vec![bob_stash.account_id().clone().into()]);
+    api.tx()
+        .sign_and_submit_then_watch_default(&nominate_tx, &alice)
         .await?
         .wait_for_finalized_success()
         .await?;
 
-    let ledger = ctx
-        .api
-        .storage()
-        .staking()
-        .ledger(alice.account_id(), None)
-        .await?
-        .unwrap();
+    let ledger_addr = node_runtime::storage().staking().ledger(alice.account_id());
+    let ledger = api.storage().fetch(&ledger_addr, None).await?.unwrap();
     assert_eq!(alice_stash.account_id(), &ledger.stash);
 
-    let chill = ctx
-        .api
+    let chill_tx = node_runtime::tx().staking().chill();
+
+    let chill = api
         .tx()
-        .staking()
-        .chill()?
-        .sign_and_submit_then_watch_default(&alice_stash)
+        .sign_and_submit_then_watch_default(&chill_tx, &alice_stash)
         .await?
         .wait_for_finalized_success()
         .await;
 
-    assert_matches!(chill, Err(Error::Module(err)) => {
+    assert_matches!(chill, Err(Error::Runtime(DispatchError::Module(err))) => {
         assert_eq!(err.pallet, "Staking");
         assert_eq!(err.error, "NotController");
     });
 
-    let is_chilled = ctx
-        .api
+    let is_chilled = api
         .tx()
-        .staking()
-        .chill()?
-        .sign_and_submit_then_watch_default(&alice)
+        .sign_and_submit_then_watch_default(&chill_tx, &alice)
         .await?
         .wait_for_finalized_success()
         .await?
@@ -171,43 +180,35 @@ async fn chill_works_for_controller_only() -> Result<(), Error<DispatchError>> {
 }
 
 #[tokio::test]
-async fn tx_bond() -> Result<(), Error<DispatchError>> {
-    let alice = pair_signer(AccountKeyring::Alice.pair());
+async fn tx_bond() -> Result<(), Error> {
     let ctx = test_context().await;
+    let api = ctx.client();
 
-    let bond = ctx
-        .api
+    let alice = pair_signer(AccountKeyring::Alice.pair());
+
+    let bond_tx = node_runtime::tx().staking().bond(
+        AccountKeyring::Bob.to_account_id().into(),
+        100_000_000_000_000,
+        RewardDestination::Stash,
+    );
+
+    let bond = api
         .tx()
-        .staking()
-        .bond(
-            AccountKeyring::Bob.to_account_id().into(),
-            100_000_000_000_000,
-            RewardDestination::Stash,
-        )
-        .unwrap()
-        .sign_and_submit_then_watch_default(&alice)
+        .sign_and_submit_then_watch_default(&bond_tx, &alice)
         .await?
         .wait_for_finalized_success()
         .await;
 
     assert!(bond.is_ok());
 
-    let bond_again = ctx
-        .api
+    let bond_again = api
         .tx()
-        .staking()
-        .bond(
-            AccountKeyring::Bob.to_account_id().into(),
-            100_000_000_000_000,
-            RewardDestination::Stash,
-        )
-        .unwrap()
-        .sign_and_submit_then_watch_default(&alice)
+        .sign_and_submit_then_watch_default(&bond_tx, &alice)
         .await?
         .wait_for_finalized_success()
         .await;
 
-    assert_matches!(bond_again, Err(Error::Module(err)) => {
+    assert_matches!(bond_again, Err(Error::Runtime(DispatchError::Module(err))) => {
         assert_eq!(err.pallet, "Staking");
         assert_eq!(err.error, "AlreadyBonded");
     });
@@ -215,35 +216,37 @@ async fn tx_bond() -> Result<(), Error<DispatchError>> {
 }
 
 #[tokio::test]
-async fn storage_history_depth() -> Result<(), Error<DispatchError>> {
+async fn storage_history_depth() -> Result<(), Error> {
     let ctx = test_context().await;
-    let history_depth = ctx.api.storage().staking().history_depth(None).await?;
+    let api = ctx.client();
+    let history_depth_addr = node_runtime::storage().staking().history_depth();
+    let history_depth = api
+        .storage()
+        .fetch_or_default(&history_depth_addr, None)
+        .await?;
     assert_eq!(history_depth, 84);
     Ok(())
 }
 
 #[tokio::test]
-async fn storage_current_era() -> Result<(), Error<DispatchError>> {
+async fn storage_current_era() -> Result<(), Error> {
     let ctx = test_context().await;
-    let _current_era = ctx
-        .api
+    let api = ctx.client();
+    let current_era_addr = node_runtime::storage().staking().current_era();
+    let _current_era = api
         .storage()
-        .staking()
-        .current_era(None)
+        .fetch(&current_era_addr, None)
         .await?
         .expect("current era always exists");
     Ok(())
 }
 
 #[tokio::test]
-async fn storage_era_reward_points() -> Result<(), Error<DispatchError>> {
-    let cxt = test_context().await;
-    let current_era_result = cxt
-        .api
-        .storage()
-        .staking()
-        .eras_reward_points(&0, None)
-        .await;
+async fn storage_era_reward_points() -> Result<(), Error> {
+    let ctx = test_context().await;
+    let api = ctx.client();
+    let reward_points_addr = node_runtime::storage().staking().eras_reward_points(&0);
+    let current_era_result = api.storage().fetch(&reward_points_addr, None).await;
     assert!(current_era_result.is_ok());
 
     Ok(())