Introduce Backend trait to allow different RPC (or other) backends to be implemented (#1126)

* WIP backend trait

* WIP converting higher level stuff to using Backend impl

* more implementing new backend trait, mainly storage focused

* Get core code compiling with new backend bits

* subxt crate checks passing

* fix tests

* cargo fmt

* clippy/fixes

* merging and other fixes

* fix test

* fix lightclient code

* Fix some broken doc links

* another book link fix

* fix broken test when moving default_rpc_client

* fix dry_run test

* fix more tests; lightclient and wasm

* fix wasm tests

* fix some doc examples

* use next() instead of next_item()

* missing next_item() -> next()s

* move legacy RPc methods to LegacyRpcMethods type to host generic param instead of RpcClient

* standardise on all RpcClient types prefixed with Rpc, and 'raw' trait types prefixed with RawRpc so it's less ocnfusing which is which

* rename fixes

* doc fixes

* Add back system_dryRun RPC method and rename tx.dry_run() to tx.validate(), to signal that the calls are different

* Add a test that we return the correct extrinsic hash from submit()

* add TransactionValid details back, and protect against out of range bytes

* add test for decoding transaction validation from empty bytes

* fix clippy warning

subxt/examples/blocks_subscribing.rs

@@ -1,4 +1,3 @@
-use futures::StreamExt;
 use subxt::{OnlineClient, PolkadotConfig};
 
 #[subxt::subxt(runtime_metadata_path = "../artifacts/polkadot_metadata_small.scale")]
@@ -24,8 +23,8 @@ async fn main() -> Result<(), Box<dyn std::error::Error>> {
         println!("  Extrinsics:");
 
         // Log each of the extrinsic with it's associated events:
-        let body = block.body().await?;
-        for ext in body.extrinsics().iter() {
+        let extrinsics = block.extrinsics().await?;
+        for ext in extrinsics.iter() {
             let ext = ext?;
             let idx = ext.index();
             let events = ext.events().await?;

subxt/examples/setup_client_custom_rpc.rs

@@ -4,7 +4,7 @@ use std::{
     sync::{Arc, Mutex},
 };
 use subxt::{
-    rpc::{RawValue, RpcClientT, RpcFuture, RpcSubscription},
+    backend::rpc::{RawRpcFuture, RawRpcSubscription, RawValue, RpcClient, RpcClientT},
     OnlineClient, PolkadotConfig,
 };
 
@@ -22,7 +22,7 @@ impl RpcClientT for MyLoggingClient {
         &'a self,
         method: &'a str,
         params: Option<Box<RawValue>>,
-    ) -> RpcFuture<'a, Box<RawValue>> {
+    ) -> RawRpcFuture<'a, Box<RawValue>> {
         writeln!(
             self.log.lock().unwrap(),
             "{method}({})",
@@ -41,7 +41,7 @@ impl RpcClientT for MyLoggingClient {
         sub: &'a str,
         params: Option<Box<RawValue>>,
         unsub: &'a str,
-    ) -> RpcFuture<'a, RpcSubscription> {
+    ) -> RawRpcFuture<'a, RawRpcSubscription> {
         writeln!(
             self.log.lock().unwrap(),
             "{sub}({}) (unsub: {unsub})",
@@ -56,7 +56,10 @@ impl RpcClientT for MyLoggingClient {
         let stream = futures::stream::once(async move { Ok(res) });
         let stream: Pin<Box<dyn futures::Stream<Item = _> + Send>> = Box::pin(stream);
         // This subscription does not provide an ID.
-        Box::pin(std::future::ready(Ok(RpcSubscription { stream, id: None })))
+        Box::pin(std::future::ready(Ok(RawRpcSubscription {
+            stream,
+            id: None,
+        })))
     }
 }
 
@@ -64,14 +67,17 @@ impl RpcClientT for MyLoggingClient {
 async fn main() -> Result<(), Box<dyn std::error::Error>> {
     // Instantiate our replacement RPC client.
     let log = Arc::default();
-    let rpc_client = MyLoggingClient {
-        log: Arc::clone(&log),
+    let rpc_client = {
+        let inner = MyLoggingClient {
+            log: Arc::clone(&log),
+        };
+        RpcClient::new(inner)
     };
 
     // Pass this into our OnlineClient to instantiate it. This will lead to some
     // RPC calls being made to fetch chain details/metadata, which will immediately
     // fail..
-    let _ = OnlineClient::<PolkadotConfig>::from_rpc_client(Arc::new(rpc_client)).await;
+    let _ = OnlineClient::<PolkadotConfig>::from_rpc_client(rpc_client).await;
 
     // But, we can see that the calls were made via our custom RPC client:
     println!("Log of calls made:\n\n{}", log.lock().unwrap().as_str());

subxt/examples/setup_client_offline.rs

@@ -15,10 +15,9 @@ async fn main() -> Result<(), Box<dyn std::error::Error>> {
     };
 
     // 2. A runtime version (system_version constant on a Substrate node has these):
-    let runtime_version = subxt::rpc::types::RuntimeVersion {
+    let runtime_version = subxt::backend::RuntimeVersion {
         spec_version: 9370,
         transaction_version: 20,
-        other: Default::default(),
     };
 
     // 3. Metadata (I'll load it from the downloaded metadata, but you can use

subxt/examples/storage_iterating.rs

@@ -13,14 +13,9 @@ async fn main() -> Result<(), Box<dyn std::error::Error>> {
 
     // Get back an iterator of results (here, we are fetching 10 items at
     // a time from the node, but we always iterate over one at a time).
-    let mut results = api
-        .storage()
-        .at_latest()
-        .await?
-        .iter(storage_query, 10)
-        .await?;
+    let mut results = api.storage().at_latest().await?.iter(storage_query).await?;
 
-    while let Some((key, value)) = results.next().await? {
+    while let Some(Ok((key, value))) = results.next().await {
         println!("Key: 0x{}", hex::encode(&key));
         println!("Value: {:?}", value);
     }

subxt/examples/storage_iterating_dynamic.rs

@@ -11,14 +11,9 @@ async fn main() -> Result<(), Box<dyn std::error::Error>> {
     let storage_query = subxt::dynamic::storage("System", "Account", keys);
 
     // Use that query to return an iterator over the results.
-    let mut results = api
-        .storage()
-        .at_latest()
-        .await?
-        .iter(storage_query, 10)
-        .await?;
+    let mut results = api.storage().at_latest().await?.iter(storage_query).await?;
 
-    while let Some((key, value)) = results.next().await? {
+    while let Some(Ok((key, value))) = results.next().await {
         println!("Key: 0x{}", hex::encode(&key));
         println!("Value: {:?}", value.to_value()?);
     }

subxt/examples/storage_iterating_partial.rs

@@ -1,15 +1,13 @@
-use subxt::{OnlineClient, PolkadotConfig};
-
-#[subxt::subxt(runtime_metadata_path = "../artifacts/polkadot_metadata_full.scale")]
-pub mod polkadot {}
-
-use subxt::utils::AccountId32;
-use subxt_signer::sr25519::{dev, Keypair};
-
 use polkadot::multisig::events::NewMultisig;
 use polkadot::runtime_types::{
     frame_system::pallet::Call, polkadot_runtime::RuntimeCall, sp_weights::weight_v2::Weight,
 };
+use subxt::utils::AccountId32;
+use subxt::{OnlineClient, PolkadotConfig};
+use subxt_signer::sr25519::{dev, Keypair};
+
+#[subxt::subxt(runtime_metadata_path = "../artifacts/polkadot_metadata_full.scale")]
+pub mod polkadot {}
 
 #[tokio::main]
 async fn main() -> Result<(), Box<dyn std::error::Error>> {
@@ -24,6 +22,7 @@ async fn main() -> Result<(), Box<dyn std::error::Error>> {
     let new_multisig_1 = submit_remark_as_multi(&alice_signer, &bob, b"Hello", &api).await?;
     let new_multisig_2 = submit_remark_as_multi(&alice_signer, &bob, b"Hi", &api).await?;
     let new_multisig_3 = submit_remark_as_multi(&alice_signer, &charlie, b"Hello", &api).await?;
+
     // Note: the NewMultisig event contains the multisig address we need to use for the storage queries:
     assert_eq!(new_multisig_1.multisig, new_multisig_2.multisig);
     assert_ne!(new_multisig_1.multisig, new_multisig_3.multisig);
@@ -35,16 +34,10 @@ async fn main() -> Result<(), Box<dyn std::error::Error>> {
         .multisig()
         .multisigs_iter1(alice_bob_account_id);
 
-    // Get back an iterator of results (here, we are fetching 10 items at
-    // a time from the node, but we always iterate over one at a time).
-    let mut results = api
-        .storage()
-        .at_latest()
-        .await?
-        .iter(storage_query, 10)
-        .await?;
+    // Get back an iterator of results.
+    let mut results = api.storage().at_latest().await?.iter(storage_query).await?;
 
-    while let Some((key, value)) = results.next().await? {
+    while let Some(Ok((key, value))) = results.next().await {
         println!("Key: 0x{}", hex::encode(&key));
         println!("Value: {:?}", value);
     }

subxt/examples/tx_status_stream.rs

@@ -1,4 +1,3 @@
-use futures::StreamExt;
 use subxt::{tx::TxStatus, OnlineClient, PolkadotConfig};
 use subxt_signer::sr25519::dev;
 
@@ -26,7 +25,7 @@ async fn main() -> Result<(), Box<dyn std::error::Error>> {
     while let Some(status) = balance_transfer_progress.next().await {
         match status? {
             // It's finalized in a block!
-            TxStatus::Finalized(in_block) => {
+            TxStatus::InFinalizedBlock(in_block) => {
                 println!(
                     "Transaction {:?} is finalized in block {:?}",
                     in_block.extrinsic_hash(),