chainHead based backend implementation (#1161)

* add follow_stream impl

* follow_stream_unpin first draft

* add tests for follow_stream_unpin

* more tests and fixes for follow_stream_unpin

* first pass follow_stream_driver

* follow_stream_driver: add tests, fix things, buffer events from last finalized

* First pass finishing Backend impl

* Fix test compile issues

* clippy fixes

* clippy fix and consistify light_client

* revert lightclient tweak

* revert other lightclient thing

* cargo fmt

* start testing unstable backend behind feature flag

* more test fixes and move test-runtime metadata path just incase

* fix compile error

* ensure transaction progress stream actually used and fix another test

* cargo fmt

* CI tweak

* improve some comments and address some feedback bits

* update CI to use our own nightly binary

* wait for finalized block perhaps

testing/integration-tests/src/utils/node_proc.rs

@@ -3,6 +3,7 @@
 // see LICENSE for license details.
 
 use std::ffi::{OsStr, OsString};
+use std::sync::Arc;
 use substrate_runner::SubstrateNode;
 use subxt::{
     backend::{legacy, rpc, unstable},
@@ -118,9 +119,14 @@ impl TestNodeProcessBuilder {
         #[cfg(feature = "unstable-light-client")]
         let client = build_light_client(&proc).await;
 
-        // Connect to the node with a subxt client:
-        #[cfg(not(feature = "unstable-light-client"))]
-        let client = OnlineClient::from_url(ws_url.clone()).await;
+        #[cfg(feature = "unstable-backend-client")]
+        let client = build_unstable_client(&proc).await;
+
+        #[cfg(all(
+            not(feature = "unstable-light-client"),
+            not(feature = "unstable-backend-client")
+        ))]
+        let client = build_legacy_client(&proc).await;
 
         match client {
             Ok(client) => Ok(TestNodeProcess { proc, client }),
@@ -129,13 +135,59 @@ impl TestNodeProcessBuilder {
     }
 }
 
+#[cfg(all(
+    not(feature = "unstable-light-client"),
+    not(feature = "unstable-backend-client")
+))]
+async fn build_legacy_client<T: Config>(proc: &SubstrateNode) -> Result<OnlineClient<T>, String> {
+    let ws_url = format!("ws://127.0.0.1:{}", proc.ws_port());
+
+    let rpc_client = rpc::RpcClient::from_url(ws_url)
+        .await
+        .map_err(|e| format!("Cannot construct RPC client: {e}"))?;
+    let backend = legacy::LegacyBackend::new(rpc_client);
+    let client = OnlineClient::from_backend(Arc::new(backend))
+        .await
+        .map_err(|e| format!("Cannot construct OnlineClient from backend: {e}"))?;
+
+    Ok(client)
+}
+
+#[cfg(feature = "unstable-backend-client")]
+async fn build_unstable_client<T: Config>(proc: &SubstrateNode) -> Result<OnlineClient<T>, String> {
+    let ws_url = format!("ws://127.0.0.1:{}", proc.ws_port());
+
+    let rpc_client = rpc::RpcClient::from_url(ws_url)
+        .await
+        .map_err(|e| format!("Cannot construct RPC client: {e}"))?;
+
+    let (backend, mut driver) = unstable::UnstableBackend::builder().build(rpc_client);
+
+    // The unstable backend needs driving:
+    tokio::spawn(async move {
+        use futures::StreamExt;
+        while let Some(val) = driver.next().await {
+            if let Err(e) = val {
+                eprintln!("Error driving unstable backend: {e}");
+                break;
+            }
+        }
+    });
+
+    let client = OnlineClient::from_backend(Arc::new(backend))
+        .await
+        .map_err(|e| format!("Cannot construct OnlineClient from backend: {e}"))?;
+
+    Ok(client)
+}
+
 #[cfg(feature = "unstable-light-client")]
-async fn build_light_client<R: Config>(proc: &SubstrateNode) -> Result<LightClient<R>, String> {
+async fn build_light_client<T: Config>(proc: &SubstrateNode) -> Result<LightClient<T>, String> {
     // RPC endpoint.
     let ws_url = format!("ws://127.0.0.1:{}", proc.ws_port());
 
     // Step 1. Wait for a few blocks to be produced using the subxt client.
-    let client = OnlineClient::<R>::from_url(ws_url.clone())
+    let client = OnlineClient::<T>::from_url(ws_url.clone())
         .await
         .map_err(|err| format!("Failed to connect to node rpc at {ws_url}: {err}"))?;
 

testing/integration-tests/src/utils/wait_for_blocks.rs

@@ -4,23 +4,13 @@
 
 use subxt::{client::OnlineClientT, Config};
 
-/// Wait for blocks to be produced before running tests. Waiting for two blocks
-/// (the genesis block and another one) seems to be enough to allow tests
-/// like `validation_passes` to work properly.
-///
-/// If the "unstable-light-client" feature flag is enabled, this will wait for
-/// 5 blocks instead of two. The light client needs the extra blocks to avoid
-/// errors caused by loading information that is not available in the first 2 blocks
-/// (`Failed to load the block weight for block`).
+/// Wait for blocks to be produced before running tests. Specifically, we
+/// wait for one more finalized block to be produced, which is important because
+/// the first finalized block doesn't have much state etc associated with it.
 pub async fn wait_for_blocks<C: Config>(api: &impl OnlineClientT<C>) {
-    let mut sub = api.backend().stream_all_block_headers().await.unwrap();
+    let mut sub = api.blocks().subscribe_finalized().await.unwrap();
+    // The current finalized block:
     sub.next().await;
+    // The next one:
     sub.next().await;
-
-    #[cfg(feature = "unstable-light-client")]
-    {
-        sub.next().await;
-        sub.next().await;
-        sub.next().await;
-    }
 }