light-client: Add experimental light-client support (#965)

* rpc/types: Decode `SubstrateTxStatus` for substrate and smoldot Signed-off-by: Alexandru Vasile <alexandru.vasile@parity.io> * lightclient: Add light client Error Signed-off-by: Alexandru Vasile <alexandru.vasile@parity.io> * lightclient: Add background task to manage RPC responses Signed-off-by: Alexandru Vasile <alexandru.vasile@parity.io> * lightclient: Implement the light client RPC in subxt Signed-off-by: Alexandru Vasile <alexandru.vasile@parity.io> * subxt: Expose light client under experimental feature-flag Signed-off-by: Alexandru Vasile <alexandru.vasile@parity.io> * artifacts: Add development chain spec for local nodes Signed-off-by: Alexandru Vasile <alexandru.vasile@parity.io> * Update cargo lock Signed-off-by: Alexandru Vasile <alexandru.vasile@parity.io> * examples: Add light client example Signed-off-by: Alexandru Vasile <alexandru.vasile@parity.io> * Update sp-* crates and smoldot to use git with branch / rev Signed-off-by: Alexandru Vasile <alexandru.vasile@parity.io> * Apply cargo fmt Signed-off-by: Alexandru Vasile <alexandru.vasile@parity.io> * Fix clippy Signed-off-by: Alexandru Vasile <alexandru.vasile@parity.io> * Import hashmap entry Signed-off-by: Alexandru Vasile <alexandru.vasile@parity.io> * lightclient: Fetch spec only if jsonrpsee feature is enabled Signed-off-by: Alexandru Vasile <alexandru.vasile@parity.io> * Update subxt/src/rpc/lightclient/background.rs Co-authored-by: Niklas Adolfsson <niklasadolfsson1@gmail.com> * Fix typo Signed-off-by: Alexandru Vasile <alexandru.vasile@parity.io> * artifacts: Update dev chain spec Signed-off-by: Alexandru Vasile <alexandru.vasile@parity.io> * types: Handle storage replies from chainHead_storage Signed-off-by: Alexandru Vasile <alexandru.vasile@parity.io> * artifacts: Add polkadot spec Signed-off-by: Alexandru Vasile <alexandru.vasile@parity.io> * lightclient: Handle RPC error responses Signed-off-by: Alexandru Vasile <alexandru.vasile@parity.io> * examples: Tx basic with light client for local nodes Signed-off-by: Alexandru Vasile <alexandru.vasile@parity.io> * example: Light client coprehensive example for live chains Signed-off-by: Alexandru Vasile <alexandru.vasile@parity.io> * examples: Remove prior light client example Signed-off-by: Alexandru Vasile <alexandru.vasile@parity.io> * feature: Rename experimental to unstable Signed-off-by: Alexandru Vasile <alexandru.vasile@parity.io> * book: Add light client section Signed-off-by: Alexandru Vasile <alexandru.vasile@parity.io> * testing: Fix clippy Signed-off-by: Alexandru Vasile <alexandru.vasile@parity.io> * lightclient: Ignore validated events Signed-off-by: Alexandru Vasile <alexandru.vasile@parity.io> * Adjust tests for light-clients and normal clients Signed-off-by: Alexandru Vasile <alexandru.vasile@parity.io> * testing: Keep lightclient variant Signed-off-by: Alexandru Vasile <alexandru.vasile@parity.io> * Remove support for chainHead_storage for light client Signed-off-by: Alexandru Vasile <alexandru.vasile@parity.io> * Update light client to point to crates.io Signed-off-by: Alexandru Vasile <alexandru.vasile@parity.io> * Update sp-crates from crates.io Signed-off-by: Alexandru Vasile <alexandru.vasile@parity.io> * Replace Atomic with u64 Signed-off-by: Alexandru Vasile <alexandru.vasile@parity.io> * Add LightClientBuilder Signed-off-by: Alexandru Vasile <alexandru.vasile@parity.io> * Adjust chainspec with provided bootnodes Signed-off-by: Alexandru Vasile <alexandru.vasile@parity.io> * Add potential_relay_chains to light client builder Signed-off-by: Alexandru Vasile <alexandru.vasile@parity.io> * Move the light-client to the background task Signed-off-by: Alexandru Vasile <alexandru.vasile@parity.io> * Adjust tracing logs Signed-off-by: Alexandru Vasile <alexandru.vasile@parity.io> * Update book and example Signed-off-by: Alexandru Vasile <alexandru.vasile@parity.io> * Apply cargo fmt Signed-off-by: Alexandru Vasile <alexandru.vasile@parity.io> * Remove dev_spec.json artifact Signed-off-by: Alexandru Vasile <alexandru.vasile@parity.io> * Examples fix duplicate Cargo.toml Signed-off-by: Alexandru Vasile <alexandru.vasile@parity.io> * Use tracing_subscriber crate Signed-off-by: Alexandru Vasile <alexandru.vasile@parity.io> * Fix clippy for different features Signed-off-by: Alexandru Vasile <alexandru.vasile@parity.io> * Add comment about bootNodes Signed-off-by: Alexandru Vasile <alexandru.vasile@parity.io> * Add comment about tracing-sub dependency Signed-off-by: Alexandru Vasile <alexandru.vasile@parity.io> * Run integration-tests with light-client Signed-off-by: Alexandru Vasile <alexandru.vasile@parity.io> * Feature guard some incompatible tests Signed-off-by: Alexandru Vasile <alexandru.vasile@parity.io> * ci: Enable light-client tests under feature flag Signed-off-by: Alexandru Vasile <alexandru.vasile@parity.io> * ci: Fix git step name Signed-off-by: Alexandru Vasile <alexandru.vasile@parity.io> * Adjust flags for testing Signed-off-by: Alexandru Vasile <alexandru.vasile@parity.io> * Adjust warnings Signed-off-by: Alexandru Vasile <alexandru.vasile@parity.io> * Rename feature flag jsonrpsee-ws to jsonrpsee Signed-off-by: Alexandru Vasile <alexandru.vasile@parity.io> * Fix cargo check Signed-off-by: Alexandru Vasile <alexandru.vasile@parity.io> * ci: Run tests on just 2 threads Signed-off-by: Alexandru Vasile <alexandru.vasile@parity.io> * Move light-client to subxt/src/client Signed-off-by: Alexandru Vasile <alexandru.vasile@parity.io> * Adjust LightClientBuilder Signed-off-by: Alexandru Vasile <alexandru.vasile@parity.io> * Use ws_url to construct light client for testing Signed-off-by: Alexandru Vasile <alexandru.vasile@parity.io> * Refactor background Signed-off-by: Alexandru Vasile <alexandru.vasile@parity.io> * Address feedback Signed-off-by: Alexandru Vasile <alexandru.vasile@parity.io> * Remove polkadot.spec and trim sub_id Signed-off-by: Alexandru Vasile <alexandru.vasile@parity.io> * Wait for substrate to produce block before connecting light client Signed-off-by: Alexandru Vasile <alexandru.vasile@parity.io> * Adjust builder and tests Signed-off-by: Alexandru Vasile <alexandru.vasile@parity.io> * Apply fmt Signed-off-by: Alexandru Vasile <alexandru.vasile@parity.io> * ci: Use release for light client testing Signed-off-by: Alexandru Vasile <alexandru.vasile@parity.io> * Add single test for light-client Signed-off-by: Alexandru Vasile <alexandru.vasile@parity.io> * Wait for more blocks Signed-off-by: Alexandru Vasile <alexandru.vasile@parity.io> * Use polkadot endpoint for testing Signed-off-by: Alexandru Vasile <alexandru.vasile@parity.io> * Adjust cargo check Signed-off-by: Alexandru Vasile <alexandru.vasile@parity.io> * examples: Remove light client chain connection example Signed-off-by: Alexandru Vasile <alexandru.vasile@parity.io> * Adjust cargo.toml section for the old example Signed-off-by: Alexandru Vasile <alexandru.vasile@parity.io> * Adjust background task to use usize for subscription Id Signed-off-by: Alexandru Vasile <alexandru.vasile@parity.io> * Build bootnodes with serde_json::Value directly Signed-off-by: Alexandru Vasile <alexandru.vasile@parity.io> * Make channel between subxt user and subxt background unbounded Signed-off-by: Alexandru Vasile <alexandru.vasile@parity.io> * Update subxt/src/client/lightclient/builder.rs Co-authored-by: Niklas Adolfsson <niklasadolfsson1@gmail.com> * Switch to smoldot 0.6.0 from 0.5.0 Signed-off-by: Alexandru Vasile <alexandru.vasile@parity.io> * Move testing to `full_client` and `light_client` higher modules Signed-off-by: Alexandru Vasile <alexandru.vasile@parity.io> * Remove subscriptionID type Signed-off-by: Alexandru Vasile <alexandru.vasile@parity.io> * Remove subxt/integration-testing feature flag Signed-off-by: Alexandru Vasile <alexandru.vasile@parity.io> * Adjust wait_for_blocks documentation Signed-off-by: Alexandru Vasile <alexandru.vasile@parity.io> * Adjust utils import for testing Signed-off-by: Alexandru Vasile <alexandru.vasile@parity.io> * Remove into_iter from builder construction Signed-off-by: Alexandru Vasile <alexandru.vasile@parity.io> --------- Signed-off-by: Alexandru Vasile <alexandru.vasile@parity.io> Co-authored-by: Niklas Adolfsson <niklasadolfsson1@gmail.com>
2026-04-22 23:08:00 +00:00 · 2023-06-26 12:10:57 +03:00
parent 8413c4d2dd
commit ef89752904
42 changed files with 2352 additions and 147 deletions
@@ -0,0 +1,16 @@
+// Copyright 2019-2023 Parity Technologies (UK) Ltd.
+// This file is dual-licensed as Apache-2.0 or GPL-3.0.
+// see LICENSE for license details.
+
+#[cfg(test)]
+mod blocks;
+#[cfg(test)]
+mod client;
+#[cfg(test)]
+mod frame;
+#[cfg(test)]
+mod metadata;
+#[cfg(test)]
+mod runtime_api;
+#[cfg(test)]
+mod storage;
@@ -5,27 +5,28 @@
 #![deny(unused_crate_dependencies)]

 #[cfg(test)]
-mod codegen;
-#[cfg(test)]
-mod utils;
+pub mod utils;

 #[cfg(test)]
-mod blocks;
-#[cfg(test)]
-mod client;
-#[cfg(test)]
-mod frame;
-#[cfg(test)]
-mod metadata;
-#[cfg(test)]
-mod runtime_api;
-#[cfg(test)]
-mod storage;
+#[cfg_attr(test, allow(unused_imports))]
+use utils::*;
+
+#[cfg(all(test, not(feature = "unstable-light-client")))]
+mod full_client;
+
+#[cfg(all(test, feature = "unstable-light-client"))]
+mod light_client;

 #[cfg(test)]
 use test_runtime::node_runtime;
-#[cfg(test)]
-use utils::*;
+
+// These dependencies are used for the full client.
+#[cfg(all(test, not(feature = "unstable-light-client")))]
+use regex as _;
+#[cfg(all(test, not(feature = "unstable-light-client")))]
+use subxt_codegen as _;
+#[cfg(all(test, not(feature = "unstable-light-client")))]
+use syn as _;

 // We don't use this dependency, but it's here so that we
 // can enable logging easily if need be. Add this to a test
@@ -0,0 +1,197 @@
+// Copyright 2019-2023 Parity Technologies (UK) Ltd.
+// This file is dual-licensed as Apache-2.0 or GPL-3.0.
+// see LICENSE for license details.
+
+//! # Light Client Initialization and Testing
+//!
+//! The initialization process of the light client can be slow, especially when
+//! it needs to synchronize with a local running node for each individual
+//! #[tokio::test] in subxt. To optimize this process, a subset of tests is
+//! exposed to ensure the light client remains functional over time. Currently,
+//! these tests are placed under an unstable feature flag.
+//!
+//! Ideally, we would place the light client initialization in a shared static
+//! using `OnceCell`. However, during the initialization, tokio::spawn is used
+//! to multiplex between subxt requests and node responses. The #[tokio::test]
+//! macro internally creates a new Runtime for each individual test. This means
+//! that only the first test, which spawns the substrate binary and synchronizes
+//! the light client, would have access to the background task. The cleanup process
+//! would destroy the spawned background task, preventing subsequent tests from
+//! accessing it.
+//!
+//! To address this issue, we can consider creating a slim proc-macro that
+//! transforms the #[tokio::test] into a plain #[test] and runs all the tests
+//! on a shared tokio runtime. This approach would allow multiple tests to share
+//! the same background task, ensuring consistent access to the light client.
+//!
+//! For more context see: https://github.com/tokio-rs/tokio/issues/2374.
+//!
+
+use crate::utils::node_runtime;
+use codec::{Compact, Encode};
+use futures::StreamExt;
+use subxt::{
+    client::{LightClient, LightClientBuilder, OfflineClientT, OnlineClientT},
+    config::PolkadotConfig,
+    rpc::types::FollowEvent,
+};
+use subxt_metadata::Metadata;
+
+// We don't use these dependencies.
+use assert_matches as _;
+use frame_metadata as _;
+use hex as _;
+use regex as _;
+use scale_info as _;
+use sp_core as _;
+use sp_runtime as _;
+use subxt_codegen as _;
+use subxt_signer as _;
+use syn as _;
+use tracing as _;
+use wabt as _;
+
+type Client = LightClient<PolkadotConfig>;
+
+// Check that we can subscribe to non-finalized blocks.
+async fn non_finalized_headers_subscription(api: &Client) -> Result<(), subxt::Error> {
+    let mut sub = api.blocks().subscribe_best().await?;
+    let header = sub.next().await.unwrap()?;
+    let block_hash = header.hash();
+    let current_block_hash = api.rpc().block_hash(None).await?.unwrap();
+
+    assert_eq!(block_hash, current_block_hash);
+
+    let _block = sub.next().await.unwrap()?;
+    let _block = sub.next().await.unwrap()?;
+    let _block = sub.next().await.unwrap()?;
+
+    Ok(())
+}
+
+// Check that we can subscribe to finalized blocks.
+async fn finalized_headers_subscription(api: &Client) -> Result<(), subxt::Error> {
+    let mut sub = api.blocks().subscribe_finalized().await?;
+    let header = sub.next().await.unwrap()?;
+    let finalized_hash = api.rpc().finalized_head().await?;
+
+    assert_eq!(header.hash(), finalized_hash);
+
+    let _block = sub.next().await.unwrap()?;
+    let _block = sub.next().await.unwrap()?;
+    let _block = sub.next().await.unwrap()?;
+
+    Ok(())
+}
+
+// Check that we can subscribe to non-finalized blocks.
+async fn runtime_api_call(api: &Client) -> Result<(), subxt::Error> {
+    let mut sub = api.blocks().subscribe_best().await?;
+
+    let block = sub.next().await.unwrap()?;
+    let rt = block.runtime_api().await?;
+
+    // get metadata via state_call.
+    let (_, meta1) = rt
+        .call_raw::<(Compact<u32>, Metadata)>("Metadata_metadata", None)
+        .await?;
+
+    // get metadata via `state_getMetadata`.
+    let meta2 = api.rpc().metadata_legacy(None).await?;
+
+    // They should be the same.
+    assert_eq!(meta1.encode(), meta2.encode());
+
+    Ok(())
+}
+
+// Lookup for the `Timestamp::now` plain storage entry.
+async fn storage_plain_lookup(api: &Client) -> Result<(), subxt::Error> {
+    let addr = node_runtime::storage().timestamp().now();
+    let entry = api
+        .storage()
+        .at_latest()
+        .await?
+        .fetch_or_default(&addr)
+        .await?;
+    assert!(entry > 0);
+
+    Ok(())
+}
+
+// Subscribe to produced blocks using the `ChainHead` spec V2 and fetch the header of
+// just a few reported blocks.
+async fn follow_chain_head(api: &Client) -> Result<(), subxt::Error> {
+    let mut blocks = api.rpc().chainhead_unstable_follow(false).await?;
+    let sub_id = blocks
+        .subscription_id()
+        .expect("RPC provides a valid subscription id; qed")
+        .to_owned();
+
+    let event = blocks.next().await.unwrap()?;
+    if let FollowEvent::BestBlockChanged(best_block) = event {
+        let hash = best_block.best_block_hash;
+        let _header = api
+            .rpc()
+            .chainhead_unstable_header(sub_id.clone(), hash)
+            .await?
+            .unwrap();
+    }
+
+    let event = blocks.next().await.unwrap()?;
+    if let FollowEvent::BestBlockChanged(best_block) = event {
+        let hash = best_block.best_block_hash;
+        let _header = api
+            .rpc()
+            .chainhead_unstable_header(sub_id.clone(), hash)
+            .await?
+            .unwrap();
+    }
+    Ok(())
+}
+
+// Make a dynamic constant query for `System::BlockLenght`.
+async fn dynamic_constant_query(api: &Client) -> Result<(), subxt::Error> {
+    let constant_query = subxt::dynamic::constant("System", "BlockLength");
+    let _value = api.constants().at(&constant_query)?;
+
+    Ok(())
+}
+
+// Fetch a few all events from the latest block and decode them dynamically.
+async fn dynamic_events(api: &Client) -> Result<(), subxt::Error> {
+    let events = api.events().at_latest().await?;
+
+    for event in events.iter() {
+        let _event = event?;
+    }
+
+    Ok(())
+}
+
+// Make a few raw RPC calls to the chain.
+async fn various_rpc_calls(api: &Client) -> Result<(), subxt::Error> {
+    let _system_chain = api.rpc().system_chain().await?;
+    let _system_name = api.rpc().system_name().await?;
+    let _finalized_hash = api.rpc().finalized_head().await?;
+
+    Ok(())
+}
+
+#[tokio::test]
+async fn light_client_testing() -> Result<(), subxt::Error> {
+    let api: LightClient<PolkadotConfig> = LightClientBuilder::new()
+        .build_from_url("wss://rpc.polkadot.io:443")
+        .await?;
+
+    non_finalized_headers_subscription(&api).await?;
+    finalized_headers_subscription(&api).await?;
+    runtime_api_call(&api).await?;
+    storage_plain_lookup(&api).await?;
+    follow_chain_head(&api).await?;
+    dynamic_constant_query(&api).await?;
+    dynamic_events(&api).await?;
+    various_rpc_calls(&api).await?;
+
+    Ok(())
+}
@@ -2,7 +2,7 @@
 // This file is dual-licensed as Apache-2.0 or GPL-3.0.
 // see LICENSE for license details.

-pub(crate) use crate::{node_runtime, TestNodeProcess};
+pub(crate) use crate::{node_runtime, utils::TestNodeProcess};

 use subxt::SubstrateConfig;

@@ -6,11 +6,19 @@ use std::ffi::{OsStr, OsString};
 use substrate_runner::SubstrateNode;
 use subxt::{Config, OnlineClient};

+#[cfg(feature = "unstable-light-client")]
+use subxt::client::{LightClient, LightClientBuilder};
+
 /// Spawn a local substrate node for testing subxt.
 pub struct TestNodeProcess<R: Config> {
    // Keep a handle to the node; once it's dropped the node is killed.
    _proc: SubstrateNode,
+
+    #[cfg(not(feature = "unstable-light-client"))]
    client: OnlineClient<R>,
+
+    #[cfg(feature = "unstable-light-client")]
+    client: LightClient<R>,
 }

 impl<R> TestNodeProcess<R>
@@ -26,9 +34,16 @@ where
    }

    /// Returns the subxt client connected to the running node.
+    #[cfg(not(feature = "unstable-light-client"))]
    pub fn client(&self) -> OnlineClient<R> {
        self.client.clone()
    }
+
+    /// Returns the subxt client connected to the running node.
+    #[cfg(feature = "unstable-light-client")]
+    pub fn client(&self) -> LightClient<R> {
+        self.client.clone()
+    }
 }

 /// Construct a test node process.
@@ -71,8 +86,13 @@ impl TestNodeProcessBuilder {
        let proc = node_builder.spawn().map_err(|e| e.to_string())?;
        let ws_url = format!("ws://127.0.0.1:{}", proc.ws_port());

+        #[cfg(feature = "unstable-light-client")]
+        let client = build_light_client(&proc).await;
+
        // Connect to the node with a subxt client:
+        #[cfg(not(feature = "unstable-light-client"))]
        let client = OnlineClient::from_url(ws_url.clone()).await;
+
        match client {
            Ok(client) => Ok(TestNodeProcess {
                _proc: proc,
@@ -82,3 +102,30 @@ impl TestNodeProcessBuilder {
        }
    }
 }
+
+#[cfg(feature = "unstable-light-client")]
+async fn build_light_client<R: Config>(proc: &SubstrateNode) -> Result<LightClient<R>, String> {
+    // RPC endpoint.
+    let ws_url = format!("ws://127.0.0.1:{}", proc.ws_port());
+
+    // Step 1. Wait for a few blocks to be produced using the subxt client.
+    let client = OnlineClient::<R>::from_url(ws_url.clone())
+        .await
+        .map_err(|err| format!("Failed to connect to node rpc at {ws_url}: {err}"))?;
+
+    super::wait_for_blocks(&client).await;
+
+    // Step 2. Construct the light client.
+    // P2p bootnode.
+    let bootnode = format!(
+        "/ip4/127.0.0.1/tcp/{}/p2p/{}",
+        proc.p2p_port(),
+        proc.p2p_address()
+    );
+
+    LightClientBuilder::new()
+        .bootnodes([bootnode.as_str()])
+        .build_from_url(ws_url.as_str())
+        .await
+        .map_err(|e| format!("Failed to construct light client {}", e.to_string()))
+}
@@ -7,8 +7,20 @@ use subxt::{client::OnlineClientT, Config};
 /// Wait for blocks to be produced before running tests. Waiting for two blocks
 /// (the genesis block and another one) seems to be enough to allow tests
 /// like `dry_run_passes` to work properly.
+///
+/// If the "unstable-light-client" feature flag is enabled, this will wait for
+/// 5 blocks instead of two. The light client needs the extra blocks to avoid
+/// errors caused by loading information that is not available in the first 2 blocks
+/// (`Failed to load the block weight for block`).
 pub async fn wait_for_blocks<C: Config>(api: &impl OnlineClientT<C>) {
    let mut sub = api.rpc().subscribe_all_block_headers().await.unwrap();
    sub.next().await;
    sub.next().await;
+
+    #[cfg(feature = "unstable-light-client")]
+    {
+        sub.next().await;
+        sub.next().await;
+        sub.next().await;
+    }
 }