Rework Subxt API to support offline and dynamic transactions (#593)

* WIP API changes

* debug impls

* Get main crate compiling with first round of changes

* Some tidy up

* Add WithExtrinsicParams, and have SubstrateConfig + PolkadotConfig, not DefaultConfig

* move transaction into extrinsic folder

* Add runtime updates back to OnlineClient

* rework to be 'client first' to fit better with storage + events

* add support for events to Client

* tidy dupe trait bound

* Wire storage into client, but need to remove static reliance

* various tidy up and start stripping codegen to remove bits we dont need now

* First pass updating calls and constants codegen

* WIP storage client updates

* First pass migrated runtime storage over to new format

* pass over codegen to generate StorageAddresses and throw other stuff out

* don't need a Call trait any more

* shuffle things around a bit

* Various proc_macro fixes to get 'cargo check' working

* organise what's exposed from subxt

* Get first example working; balance_transfer_with_params

* get balance_transfer example compiling

* get concurrent_storage_requests.rs example compiling

* get fetch_all_accounts example compiling

* get a bunch more of the examples compiling

* almost get final example working; type mismatch to look into

* wee tweaks

* move StorageAddress to separate file

* pass Defaultable/Iterable info to StorageAddress in codegen

* fix storage validation ne, and partial run through example code

* Remove static iteration and strip a generic param from everything

* fix doc tests in subxt crate

* update test utils and start fixing frame tests

* fix frame staking tests

* fix the rest of the test compile issues, Borrow on storage values

* cargo fmt

* remove extra logging during tests

* Appease clippy and no more need for into_iter on events

* cargo fmt

* fix dryRun tests by waiting for blocks

* wait for blocks instead of sleeping or other test hacks

* cargo fmt

* Fix doc links

* Traitify StorageAddress

* remove out-of-date doc comments

* optimise decoding storage a little

* cleanup tx stuff, trait for TxPayload, remove Err type param and decode at runtime

* clippy fixes

* fix doc links

* fix doc example

* constant address trait for consistency

* fix a typo and remove EncodeWithMetadata stuff

* Put EventDetails behind a proper interface and allow decoding into top level event, too

* fix docs

* tweak StorageAddress docs

* re-export StorageAddress at root for consistency

* fix clippy things

* Add support for dynamic values

* fix double encoding of storage map key after refactor

* clippy fix

* Fixes and add a dynamic usage example (needs new scale_value release)

* bump scale_value version

* cargo fmt

* Tweak event bits

* cargo fmt

* Add a test and bump scale-value to 0.4.0 to support this

* remove unnecessary vec from dynamic example

* Various typo/grammar fixes

Co-authored-by: Alexandru Vasile <60601340+lexnv@users.noreply.github.com>

* Address PR nits

* Undo accidental rename in changelog

* Small PR nits/tidyups

* fix tests; codegen change against latest substrate

* tweak storage address util names

* move error decoding to DecodeError and expose

* impl some basic traits on the extrinsic param builder

Co-authored-by: Alexandru Vasile <60601340+lexnv@users.noreply.github.com>

testing/integration-tests/src/utils/context.rs

@@ -10,20 +10,14 @@ pub(crate) use crate::{
 use sp_core::sr25519::Pair;
 use sp_keyring::AccountKeyring;
 use subxt::{
-    Client,
-    DefaultConfig,
-    PairSigner,
-    SubstrateExtrinsicParams,
+    tx::PairSigner,
+    SubstrateConfig,
 };
 
 /// substrate node should be installed on the $PATH
 const SUBSTRATE_NODE_PATH: &str = "substrate";
 
-pub type NodeRuntimeParams = SubstrateExtrinsicParams<DefaultConfig>;
-
-pub async fn test_node_process_with(
-    key: AccountKeyring,
-) -> TestNodeProcess<DefaultConfig> {
+pub async fn test_context_with(key: AccountKeyring) -> TestContext {
     let path = std::env::var("SUBSTRATE_NODE_PATH").unwrap_or_else(|_| {
         if which::which(SUBSTRATE_NODE_PATH).is_err() {
             panic!("A substrate binary should be installed on your path for integration tests. \
@@ -32,35 +26,19 @@ pub async fn test_node_process_with(
         SUBSTRATE_NODE_PATH.to_string()
     });
 
-    let proc = TestNodeProcess::<DefaultConfig>::build(path.as_str())
+    let proc = TestContext::build(path.as_str())
         .with_authority(key)
-        .spawn::<DefaultConfig>()
+        .spawn::<SubstrateConfig>()
         .await;
     proc.unwrap()
 }
 
-pub async fn test_node_process() -> TestNodeProcess<DefaultConfig> {
-    test_node_process_with(AccountKeyring::Alice).await
-}
-
-pub struct TestContext {
-    pub node_proc: TestNodeProcess<DefaultConfig>,
-    pub api: node_runtime::RuntimeApi<DefaultConfig, NodeRuntimeParams>,
-}
-
-impl TestContext {
-    pub fn client(&self) -> &Client<DefaultConfig> {
-        &self.api.client
-    }
-}
+pub type TestContext = TestNodeProcess<SubstrateConfig>;
 
 pub async fn test_context() -> TestContext {
-    tracing_subscriber::fmt::try_init().ok();
-    let node_proc = test_node_process_with(AccountKeyring::Alice).await;
-    let api = node_proc.client().clone().to_runtime_api();
-    TestContext { node_proc, api }
+    test_context_with(AccountKeyring::Alice).await
 }
 
-pub fn pair_signer(pair: Pair) -> PairSigner<DefaultConfig, Pair> {
+pub fn pair_signer(pair: Pair) -> PairSigner<SubstrateConfig, Pair> {
     PairSigner::new(pair)
 }

testing/integration-tests/src/utils/mod.rs

@@ -4,6 +4,8 @@
 
 mod context;
 mod node_proc;
+mod wait_for_blocks;
 
 pub use context::*;
 pub use node_proc::TestNodeProcess;
+pub use wait_for_blocks::wait_for_blocks;

testing/integration-tests/src/utils/node_proc.rs

@@ -16,16 +16,14 @@ use std::{
     process,
 };
 use subxt::{
-    Client,
-    ClientBuilder,
     Config,
+    OnlineClient,
 };
 
 /// Spawn a local substrate node for testing subxt.
 pub struct TestNodeProcess<R: Config> {
     proc: process::Child,
-    client: Client<R>,
-    ws_url: String,
+    client: OnlineClient<R>,
 }
 
 impl<R> Drop for TestNodeProcess<R>
@@ -61,13 +59,8 @@ where
     }
 
     /// Returns the subxt client connected to the running node.
-    pub fn client(&self) -> &Client<R> {
-        &self.client
-    }
-
-    /// Returns the address to which the client is connected.
-    pub fn ws_url(&self) -> &str {
-        &self.ws_url
+    pub fn client(&self) -> OnlineClient<R> {
+        self.client.clone()
     }
 }
 
@@ -129,15 +122,9 @@ impl TestNodeProcessBuilder {
         let ws_url = format!("ws://127.0.0.1:{}", ws_port);
 
         // Connect to the node with a subxt client:
-        let client = ClientBuilder::new().set_url(ws_url.clone()).build().await;
+        let client = OnlineClient::from_url(ws_url.clone()).await;
         match client {
-            Ok(client) => {
-                Ok(TestNodeProcess {
-                    proc,
-                    client,
-                    ws_url,
-                })
-            }
+            Ok(client) => Ok(TestNodeProcess { proc, client }),
             Err(err) => {
                 let err = format!("Failed to connect to node rpc at {}: {}", ws_url, err);
                 tracing::error!("{}", err);

testing/integration-tests/src/utils/wait_for_blocks.rs

@@ -0,0 +1,17 @@
+// Copyright 2019-2022 Parity Technologies (UK) Ltd.
+// This file is dual-licensed as Apache-2.0 or GPL-3.0.
+// see LICENSE for license details.
+
+use subxt::{
+    client::OnlineClientT,
+    Config,
+};
+
+/// Wait for blocks to be produced before running tests. Waiting for two blocks
+/// (the genesis block and another one) seems to be enough to allow tests
+/// like `dry_run_passes` to work properly.
+pub async fn wait_for_blocks<C: Config>(api: &impl OnlineClientT<C>) {
+    let mut sub = api.rpc().subscribe_blocks().await.unwrap();
+    sub.next().await;
+    sub.next().await;
+}