rpc: backpressured RPC server (bump jsonrpsee 0.20) (#1313)

This is a rather big change in jsonrpsee, the major things in this bump are: - Server backpressure (the subscription impls are modified to deal with that) - Allow custom error types / return types (remove jsonrpsee::core::Error and jsonrpee::core::CallError) - Bug fixes (graceful shutdown in particular not used by substrate anyway) - Less dependencies for the clients in particular - Return type requires Clone in method call responses - Moved to tokio channels - Async subscription API (not used in this PR) Major changes in this PR: - The subscriptions are now bounded and if subscription can't keep up with the server it is dropped - CLI: add parameter to configure the jsonrpc server bounded message buffer (default is 64) - Add our own subscription helper to deal with the unbounded streams in substrate The most important things in this PR to review is the added helpers functions in `substrate/client/rpc/src/utils.rs` and the rest is pretty much chore. Regarding the "bounded buffer limit" it may cause the server to handle the JSON-RPC calls slower than before. The message size limit is bounded by "--rpc-response-size" thus "by default 10MB * 64 = 640MB" but the subscription message size is not covered by this limit and could be capped as well. Hopefully the last release prior to 1.0, sorry in advance for a big PR Previous attempt: https://github.com/paritytech/substrate/pull/13992 Resolves https://github.com/paritytech/polkadot-sdk/issues/748, resolves https://github.com/paritytech/polkadot-sdk/issues/627
2026-06-16 20:11:03 +00:00 · 2024-01-23 09:55:13 +01:00
parent 76c37c930b
commit e16ef0861f
117 changed files with 1245 additions and 1090 deletions
@@ -24,32 +24,23 @@ mod utils;
 #[cfg(test)]
 mod tests;

-use std::sync::Arc;
-
 use crate::SubscriptionTaskExecutor;
-
-use jsonrpsee::{
-	core::{async_trait, server::rpc_module::SubscriptionSink, Error as JsonRpseeError, RpcResult},
-	types::SubscriptionResult,
+use jsonrpsee::{core::async_trait, PendingSubscriptionSink};
+use sc_client_api::{
+	Backend, BlockBackend, BlockchainEvents, ExecutorProvider, ProofProvider, StorageProvider,
 };
-
 use sc_rpc_api::DenyUnsafe;
+use sp_api::{CallApiAt, Metadata, ProvideRuntimeApi};
+use sp_blockchain::{HeaderBackend, HeaderMetadata};
 use sp_core::{
 	storage::{PrefixedStorageKey, StorageChangeSet, StorageData, StorageKey},
 	Bytes,
 };
 use sp_runtime::traits::Block as BlockT;
 use sp_version::RuntimeVersion;
+use std::sync::Arc;

-use sp_api::{CallApiAt, Metadata, ProvideRuntimeApi};
-
-use self::error::Error;
-
-use sc_client_api::{
-	Backend, BlockBackend, BlockchainEvents, ExecutorProvider, ProofProvider, StorageProvider,
-};
 pub use sc_rpc_api::{child_state::*, state::*};
-use sp_blockchain::{HeaderBackend, HeaderMetadata};

 const STORAGE_KEYS_PAGED_MAX_COUNT: u32 = 1000;

@@ -158,10 +149,15 @@ where
 	) -> Result<sp_rpc::tracing::TraceBlockResponse, Error>;

 	/// New runtime version subscription
-	fn subscribe_runtime_version(&self, sink: SubscriptionSink);
+	fn subscribe_runtime_version(&self, pending: PendingSubscriptionSink);

 	/// New storage subscription
-	fn subscribe_storage(&self, sink: SubscriptionSink, keys: Option<Vec<StorageKey>>);
+	fn subscribe_storage(
+		&self,
+		pending: PendingSubscriptionSink,
+		keys: Option<Vec<StorageKey>>,
+		deny_unsafe: DenyUnsafe,
+	);
 }

 /// Create new state API that works on full node.
@@ -207,7 +203,12 @@ where
 	Block: BlockT + 'static,
 	Client: Send + Sync + 'static,
 {
-	fn call(&self, method: String, data: Bytes, block: Option<Block::Hash>) -> RpcResult<Bytes> {
+	fn call(
+		&self,
+		method: String,
+		data: Bytes,
+		block: Option<Block::Hash>,
+	) -> Result<Bytes, Error> {
 		self.backend.call(block, method, data).map_err(Into::into)
 	}

@@ -215,7 +216,7 @@ where
 		&self,
 		key_prefix: StorageKey,
 		block: Option<Block::Hash>,
-	) -> RpcResult<Vec<StorageKey>> {
+	) -> Result<Vec<StorageKey>, Error> {
 		self.backend.storage_keys(block, key_prefix).map_err(Into::into)
 	}

@@ -223,7 +224,7 @@ where
 		&self,
 		key_prefix: StorageKey,
 		block: Option<Block::Hash>,
-	) -> RpcResult<Vec<(StorageKey, StorageData)>> {
+	) -> Result<Vec<(StorageKey, StorageData)>, Error> {
 		self.deny_unsafe.check_if_safe()?;
 		self.backend.storage_pairs(block, key_prefix).map_err(Into::into)
 	}
@@ -234,12 +235,9 @@ where
 		count: u32,
 		start_key: Option<StorageKey>,
 		block: Option<Block::Hash>,
-	) -> RpcResult<Vec<StorageKey>> {
+	) -> Result<Vec<StorageKey>, Error> {
 		if count > STORAGE_KEYS_PAGED_MAX_COUNT {
-			return Err(JsonRpseeError::from(Error::InvalidCount {
-				value: count,
-				max: STORAGE_KEYS_PAGED_MAX_COUNT,
-			}))
+			return Err(Error::InvalidCount { value: count, max: STORAGE_KEYS_PAGED_MAX_COUNT })
 		}
 		self.backend
 			.storage_keys_paged(block, prefix, count, start_key)
@@ -250,7 +248,7 @@ where
 		&self,
 		key: StorageKey,
 		block: Option<Block::Hash>,
-	) -> RpcResult<Option<StorageData>> {
+	) -> Result<Option<StorageData>, Error> {
 		self.backend.storage(block, key).map_err(Into::into)
 	}

@@ -258,7 +256,7 @@ where
 		&self,
 		key: StorageKey,
 		block: Option<Block::Hash>,
-	) -> RpcResult<Option<Block::Hash>> {
+	) -> Result<Option<Block::Hash>, Error> {
 		self.backend.storage_hash(block, key).map_err(Into::into)
 	}

@@ -266,18 +264,18 @@ where
 		&self,
 		key: StorageKey,
 		block: Option<Block::Hash>,
-	) -> RpcResult<Option<u64>> {
+	) -> Result<Option<u64>, Error> {
 		self.backend
 			.storage_size(block, key, self.deny_unsafe)
 			.await
 			.map_err(Into::into)
 	}

-	fn metadata(&self, block: Option<Block::Hash>) -> RpcResult<Bytes> {
+	fn metadata(&self, block: Option<Block::Hash>) -> Result<Bytes, Error> {
 		self.backend.metadata(block).map_err(Into::into)
 	}

-	fn runtime_version(&self, at: Option<Block::Hash>) -> RpcResult<RuntimeVersion> {
+	fn runtime_version(&self, at: Option<Block::Hash>) -> Result<RuntimeVersion, Error> {
 		self.backend.runtime_version(at).map_err(Into::into)
 	}

@@ -286,7 +284,7 @@ where
 		keys: Vec<StorageKey>,
 		from: Block::Hash,
 		to: Option<Block::Hash>,
-	) -> RpcResult<Vec<StorageChangeSet<Block::Hash>>> {
+	) -> Result<Vec<StorageChangeSet<Block::Hash>>, Error> {
 		self.deny_unsafe.check_if_safe()?;
 		self.backend.query_storage(from, to, keys).map_err(Into::into)
 	}
@@ -295,7 +293,7 @@ where
 		&self,
 		keys: Vec<StorageKey>,
 		at: Option<Block::Hash>,
-	) -> RpcResult<Vec<StorageChangeSet<Block::Hash>>> {
+	) -> Result<Vec<StorageChangeSet<Block::Hash>>, Error> {
 		self.backend.query_storage_at(keys, at).map_err(Into::into)
 	}

@@ -303,7 +301,7 @@ where
 		&self,
 		keys: Vec<StorageKey>,
 		block: Option<Block::Hash>,
-	) -> RpcResult<ReadProof<Block::Hash>> {
+	) -> Result<ReadProof<Block::Hash>, Error> {
 		self.backend.read_proof(block, keys).map_err(Into::into)
 	}

@@ -318,32 +316,19 @@ where
 		targets: Option<String>,
 		storage_keys: Option<String>,
 		methods: Option<String>,
-	) -> RpcResult<sp_rpc::tracing::TraceBlockResponse> {
+	) -> Result<sp_rpc::tracing::TraceBlockResponse, Error> {
 		self.deny_unsafe.check_if_safe()?;
 		self.backend
 			.trace_block(block, targets, storage_keys, methods)
 			.map_err(Into::into)
 	}

-	fn subscribe_runtime_version(&self, sink: SubscriptionSink) -> SubscriptionResult {
-		self.backend.subscribe_runtime_version(sink);
-		Ok(())
+	fn subscribe_runtime_version(&self, pending: PendingSubscriptionSink) {
+		self.backend.subscribe_runtime_version(pending)
 	}

-	fn subscribe_storage(
-		&self,
-		mut sink: SubscriptionSink,
-		keys: Option<Vec<StorageKey>>,
-	) -> SubscriptionResult {
-		if keys.is_none() {
-			if let Err(err) = self.deny_unsafe.check_if_safe() {
-				let _ = sink.reject(JsonRpseeError::from(err));
-				return Ok(())
-			}
-		}
-
-		self.backend.subscribe_storage(sink, keys);
-		Ok(())
+	fn subscribe_storage(&self, pending: PendingSubscriptionSink, keys: Option<Vec<StorageKey>>) {
+		self.backend.subscribe_storage(pending, keys, self.deny_unsafe)
 	}
 }

@@ -430,7 +415,7 @@ where
 		storage_key: PrefixedStorageKey,
 		key_prefix: StorageKey,
 		block: Option<Block::Hash>,
-	) -> RpcResult<Vec<StorageKey>> {
+	) -> Result<Vec<StorageKey>, Error> {
 		self.backend.storage_keys(block, storage_key, key_prefix).map_err(Into::into)
 	}

@@ -441,7 +426,7 @@ where
 		count: u32,
 		start_key: Option<StorageKey>,
 		block: Option<Block::Hash>,
-	) -> RpcResult<Vec<StorageKey>> {
+	) -> Result<Vec<StorageKey>, Error> {
 		self.backend
 			.storage_keys_paged(block, storage_key, prefix, count, start_key)
 			.map_err(Into::into)
@@ -452,7 +437,7 @@ where
 		storage_key: PrefixedStorageKey,
 		key: StorageKey,
 		block: Option<Block::Hash>,
-	) -> RpcResult<Option<StorageData>> {
+	) -> Result<Option<StorageData>, Error> {
 		self.backend.storage(block, storage_key, key).map_err(Into::into)
 	}

@@ -461,7 +446,7 @@ where
 		storage_key: PrefixedStorageKey,
 		keys: Vec<StorageKey>,
 		block: Option<Block::Hash>,
-	) -> RpcResult<Vec<Option<StorageData>>> {
+	) -> Result<Vec<Option<StorageData>>, Error> {
 		self.backend.storage_entries(block, storage_key, keys).map_err(Into::into)
 	}

@@ -470,7 +455,7 @@ where
 		storage_key: PrefixedStorageKey,
 		key: StorageKey,
 		block: Option<Block::Hash>,
-	) -> RpcResult<Option<Block::Hash>> {
+	) -> Result<Option<Block::Hash>, Error> {
 		self.backend.storage_hash(block, storage_key, key).map_err(Into::into)
 	}

@@ -479,7 +464,7 @@ where
 		storage_key: PrefixedStorageKey,
 		key: StorageKey,
 		block: Option<Block::Hash>,
-	) -> RpcResult<Option<u64>> {
+	) -> Result<Option<u64>, Error> {
 		self.backend.storage_size(block, storage_key, key).map_err(Into::into)
 	}

@@ -488,7 +473,7 @@ where
 		child_storage_key: PrefixedStorageKey,
 		keys: Vec<StorageKey>,
 		block: Option<Block::Hash>,
-	) -> RpcResult<ReadProof<Block::Hash>> {
+	) -> Result<ReadProof<Block::Hash>, Error> {
 		self.backend
 			.read_child_proof(block, child_storage_key, keys)
 			.map_err(Into::into)