Rework Subxt API to support offline and dynamic transactions (#593)

* WIP API changes

* debug impls

* Get main crate compiling with first round of changes

* Some tidy up

* Add WithExtrinsicParams, and have SubstrateConfig + PolkadotConfig, not DefaultConfig

* move transaction into extrinsic folder

* Add runtime updates back to OnlineClient

* rework to be 'client first' to fit better with storage + events

* add support for events to Client

* tidy dupe trait bound

* Wire storage into client, but need to remove static reliance

* various tidy up and start stripping codegen to remove bits we dont need now

* First pass updating calls and constants codegen

* WIP storage client updates

* First pass migrated runtime storage over to new format

* pass over codegen to generate StorageAddresses and throw other stuff out

* don't need a Call trait any more

* shuffle things around a bit

* Various proc_macro fixes to get 'cargo check' working

* organise what's exposed from subxt

* Get first example working; balance_transfer_with_params

* get balance_transfer example compiling

* get concurrent_storage_requests.rs example compiling

* get fetch_all_accounts example compiling

* get a bunch more of the examples compiling

* almost get final example working; type mismatch to look into

* wee tweaks

* move StorageAddress to separate file

* pass Defaultable/Iterable info to StorageAddress in codegen

* fix storage validation ne, and partial run through example code

* Remove static iteration and strip a generic param from everything

* fix doc tests in subxt crate

* update test utils and start fixing frame tests

* fix frame staking tests

* fix the rest of the test compile issues, Borrow on storage values

* cargo fmt

* remove extra logging during tests

* Appease clippy and no more need for into_iter on events

* cargo fmt

* fix dryRun tests by waiting for blocks

* wait for blocks instead of sleeping or other test hacks

* cargo fmt

* Fix doc links

* Traitify StorageAddress

* remove out-of-date doc comments

* optimise decoding storage a little

* cleanup tx stuff, trait for TxPayload, remove Err type param and decode at runtime

* clippy fixes

* fix doc links

* fix doc example

* constant address trait for consistency

* fix a typo and remove EncodeWithMetadata stuff

* Put EventDetails behind a proper interface and allow decoding into top level event, too

* fix docs

* tweak StorageAddress docs

* re-export StorageAddress at root for consistency

* fix clippy things

* Add support for dynamic values

* fix double encoding of storage map key after refactor

* clippy fix

* Fixes and add a dynamic usage example (needs new scale_value release)

* bump scale_value version

* cargo fmt

* Tweak event bits

* cargo fmt

* Add a test and bump scale-value to 0.4.0 to support this

* remove unnecessary vec from dynamic example

* Various typo/grammar fixes

Co-authored-by: Alexandru Vasile <60601340+lexnv@users.noreply.github.com>

* Address PR nits

* Undo accidental rename in changelog

* Small PR nits/tidyups

* fix tests; codegen change against latest substrate

* tweak storage address util names

* move error decoding to DecodeError and expose

* impl some basic traits on the extrinsic param builder

Co-authored-by: Alexandru Vasile <60601340+lexnv@users.noreply.github.com>

testing/integration-tests/src/metadata/validation.rs

@@ -3,6 +3,7 @@
 // see LICENSE for license details.
 
 use crate::{
+    node_runtime,
     test_context,
     TestContext,
 };
@@ -28,72 +29,57 @@ use scale_info::{
     TypeInfo,
 };
 use subxt::{
-    ClientBuilder,
-    DefaultConfig,
     Metadata,
-    SubstrateExtrinsicParams,
+    OfflineClient,
+    SubstrateConfig,
 };
 
-use crate::utils::node_runtime;
-
-type RuntimeApi =
-    node_runtime::RuntimeApi<DefaultConfig, SubstrateExtrinsicParams<DefaultConfig>>;
-
-async fn metadata_to_api(metadata: RuntimeMetadataV14, cxt: &TestContext) -> RuntimeApi {
+async fn metadata_to_api(
+    metadata: RuntimeMetadataV14,
+    ctx: &TestContext,
+) -> OfflineClient<SubstrateConfig> {
     let prefixed = RuntimeMetadataPrefixed::from(metadata);
     let metadata = Metadata::try_from(prefixed).unwrap();
 
-    ClientBuilder::new()
-            .set_url(cxt.node_proc.ws_url().to_string())
-            .set_metadata(metadata)
-            .build()
-            .await
-            .unwrap()
-            .to_runtime_api::<node_runtime::RuntimeApi<
-                DefaultConfig,
-                SubstrateExtrinsicParams<DefaultConfig>,
-            >>()
+    OfflineClient::new(
+        ctx.client().genesis_hash(),
+        ctx.client().runtime_version(),
+        metadata,
+    )
 }
 
 #[tokio::test]
 async fn full_metadata_check() {
-    let cxt = test_context().await;
-    let api = &cxt.api;
+    let ctx = test_context().await;
+    let api = ctx.client();
 
     // Runtime metadata is identical to the metadata used during API generation.
-    assert!(api.validate_metadata().is_ok());
+    assert!(node_runtime::validate_codegen(&api).is_ok());
 
     // Modify the metadata.
-    let mut metadata: RuntimeMetadataV14 = {
-        let locked_client_metadata = api.client.metadata();
-        let client_metadata = locked_client_metadata.read();
-        client_metadata.runtime_metadata().clone()
-    };
+    let mut metadata: RuntimeMetadataV14 = api.metadata().runtime_metadata().clone();
     metadata.pallets[0].name = "NewPallet".to_string();
 
-    let new_api = metadata_to_api(metadata, &cxt).await;
+    let api = metadata_to_api(metadata, &ctx).await;
     assert_eq!(
-        new_api
-            .validate_metadata()
+        node_runtime::validate_codegen(&api)
             .expect_err("Validation should fail for incompatible metadata"),
-        ::subxt::MetadataError::IncompatibleMetadata
+        ::subxt::error::MetadataError::IncompatibleMetadata
     );
 }
 
 #[tokio::test]
 async fn constant_values_are_not_validated() {
-    let cxt = test_context().await;
-    let api = &cxt.api;
+    let ctx = test_context().await;
+    let api = ctx.client();
 
-    // Ensure that `ExistentialDeposit` is compatible before altering the metadata.
-    assert!(cxt.api.constants().balances().existential_deposit().is_ok());
+    let deposit_addr = node_runtime::constants().balances().existential_deposit();
+
+    // Retrieve existential deposit to validate it and confirm that it's OK.
+    assert!(api.constants().at(&deposit_addr).is_ok());
 
     // Modify the metadata.
-    let mut metadata: RuntimeMetadataV14 = {
-        let locked_client_metadata = api.client.metadata();
-        let client_metadata = locked_client_metadata.read();
-        client_metadata.runtime_metadata().clone()
-    };
+    let mut metadata: RuntimeMetadataV14 = api.metadata().runtime_metadata().clone();
 
     let mut existential = metadata
         .pallets
@@ -108,13 +94,10 @@ async fn constant_values_are_not_validated() {
     // Modifying a constant value should not lead to an error:
     existential.value = vec![0u8; 32];
 
-    let new_api = metadata_to_api(metadata, &cxt).await;
+    let api = metadata_to_api(metadata, &ctx).await;
 
-    assert!(new_api.validate_metadata().is_ok());
-    assert!(new_api.constants().balances().existential_deposit().is_ok());
-
-    // Other constant validation should not be impacted.
-    assert!(new_api.constants().balances().max_locks().is_ok());
+    assert!(node_runtime::validate_codegen(&api).is_ok());
+    assert!(api.constants().at(&deposit_addr).is_ok());
 }
 
 fn default_pallet() -> PalletMetadata {
@@ -143,11 +126,15 @@ fn pallets_to_metadata(pallets: Vec<PalletMetadata>) -> RuntimeMetadataV14 {
 
 #[tokio::test]
 async fn calls_check() {
-    let cxt = test_context().await;
+    let ctx = test_context().await;
+    let api = ctx.client();
+
+    let unbond_tx = node_runtime::tx().staking().unbond(123_456_789_012_345);
+    let withdraw_unbonded_addr = node_runtime::tx().staking().withdraw_unbonded(10);
 
     // Ensure that `Unbond` and `WinthdrawUnbonded` calls are compatible before altering the metadata.
-    assert!(cxt.api.tx().staking().unbond(123_456_789_012_345).is_ok());
-    assert!(cxt.api.tx().staking().withdraw_unbonded(10).is_ok());
+    assert!(api.tx().validate(&unbond_tx).is_ok());
+    assert!(api.tx().validate(&withdraw_unbonded_addr).is_ok());
 
     // Reconstruct the `Staking` call as is.
     struct CallRec;
@@ -181,9 +168,11 @@ async fn calls_check() {
         ..default_pallet()
     };
     let metadata = pallets_to_metadata(vec![pallet]);
-    let new_api = metadata_to_api(metadata, &cxt).await;
-    assert!(new_api.tx().staking().unbond(123_456_789_012_345).is_ok());
-    assert!(new_api.tx().staking().withdraw_unbonded(10).is_ok());
+    let api = metadata_to_api(metadata, &ctx).await;
+
+    // The calls should still be valid with this new type info:
+    assert!(api.tx().validate(&unbond_tx).is_ok());
+    assert!(api.tx().validate(&withdraw_unbonded_addr).is_ok());
 
     // Change `Unbond` call but leave the rest as is.
     struct CallRecSecond;
@@ -216,31 +205,24 @@ async fn calls_check() {
         ..default_pallet()
     };
     let metadata = pallets_to_metadata(vec![pallet]);
-    let new_api = metadata_to_api(metadata, &cxt).await;
+    let api = metadata_to_api(metadata, &ctx).await;
+
     // Unbond call should fail, while withdraw_unbonded remains compatible.
-    assert!(new_api.tx().staking().unbond(123_456_789_012_345).is_err());
-    assert!(new_api.tx().staking().withdraw_unbonded(10).is_ok());
+    assert!(api.tx().validate(&unbond_tx).is_err());
+    assert!(api.tx().validate(&withdraw_unbonded_addr).is_ok());
 }
 
 #[tokio::test]
 async fn storage_check() {
-    let cxt = test_context().await;
+    let ctx = test_context().await;
+    let api = ctx.client();
+
+    let tx_count_addr = node_runtime::storage().system().extrinsic_count();
+    let tx_len_addr = node_runtime::storage().system().all_extrinsics_len();
 
     // Ensure that `ExtrinsicCount` and `EventCount` storages are compatible before altering the metadata.
-    assert!(cxt
-        .api
-        .storage()
-        .system()
-        .extrinsic_count(None)
-        .await
-        .is_ok());
-    assert!(cxt
-        .api
-        .storage()
-        .system()
-        .all_extrinsics_len(None)
-        .await
-        .is_ok());
+    assert!(api.storage().validate(&tx_count_addr).is_ok());
+    assert!(api.storage().validate(&tx_len_addr).is_ok());
 
     // Reconstruct the storage.
     let storage = PalletStorageMetadata {
@@ -268,19 +250,11 @@ async fn storage_check() {
         ..default_pallet()
     };
     let metadata = pallets_to_metadata(vec![pallet]);
-    let new_api = metadata_to_api(metadata, &cxt).await;
-    assert!(new_api
-        .storage()
-        .system()
-        .extrinsic_count(None)
-        .await
-        .is_ok());
-    assert!(new_api
-        .storage()
-        .system()
-        .all_extrinsics_len(None)
-        .await
-        .is_ok());
+    let api = metadata_to_api(metadata, &ctx).await;
+
+    // The addresses should still validate:
+    assert!(api.storage().validate(&tx_count_addr).is_ok());
+    assert!(api.storage().validate(&tx_len_addr).is_ok());
 
     // Reconstruct the storage while modifying ExtrinsicCount.
     let storage = PalletStorageMetadata {
@@ -309,17 +283,9 @@ async fn storage_check() {
         ..default_pallet()
     };
     let metadata = pallets_to_metadata(vec![pallet]);
-    let new_api = metadata_to_api(metadata, &cxt).await;
-    assert!(new_api
-        .storage()
-        .system()
-        .extrinsic_count(None)
-        .await
-        .is_err());
-    assert!(new_api
-        .storage()
-        .system()
-        .all_extrinsics_len(None)
-        .await
-        .is_ok());
+    let api = metadata_to_api(metadata, &ctx).await;
+
+    // The count route should fail now; the other will be ok still.
+    assert!(api.storage().validate(&tx_count_addr).is_err());
+    assert!(api.storage().validate(&tx_len_addr).is_ok());
 }