rpc: backpressured RPC server (bump jsonrpsee 0.20) (#1313)

This is a rather big change in jsonrpsee, the major things in this bump
are:
- Server backpressure (the subscription impls are modified to deal with
that)
- Allow custom error types / return types (remove jsonrpsee::core::Error
and jsonrpee::core::CallError)
- Bug fixes (graceful shutdown in particular not used by substrate
anyway)
   - Less dependencies for the clients in particular
   - Return type requires Clone in method call responses
   - Moved to tokio channels
   - Async subscription API (not used in this PR)

Major changes in this PR:
- The subscriptions are now bounded and if subscription can't keep up
with the server it is dropped
- CLI: add parameter to configure the jsonrpc server bounded message
buffer (default is 64)
- Add our own subscription helper to deal with the unbounded streams in
substrate

The most important things in this PR to review is the added helpers
functions in `substrate/client/rpc/src/utils.rs` and the rest is pretty
much chore.

Regarding the "bounded buffer limit" it may cause the server to handle
the JSON-RPC calls
slower than before.

The message size limit is bounded by "--rpc-response-size" thus "by
default 10MB * 64 = 640MB"
but the subscription message size is not covered by this limit and could
be capped as well.

Hopefully the last release prior to 1.0, sorry in advance for a big PR

Previous attempt: https://github.com/paritytech/substrate/pull/13992

Resolves https://github.com/paritytech/polkadot-sdk/issues/748, resolves
https://github.com/paritytech/polkadot-sdk/issues/627
This commit is contained in:
Niklas Adolfsson
2024-01-23 09:55:13 +01:00
committed by GitHub
parent 76c37c930b
commit e16ef0861f
117 changed files with 1245 additions and 1090 deletions
+28 -25
View File
@@ -25,13 +25,13 @@ use super::{
error::{Error, Result},
ChildStateBackend, StateBackend,
};
use crate::{DenyUnsafe, SubscriptionTaskExecutor};
use futures::{future, stream, FutureExt, StreamExt};
use jsonrpsee::{
core::{async_trait, Error as JsonRpseeError},
SubscriptionSink,
use crate::{
utils::{pipe_from_stream, spawn_subscription_task},
DenyUnsafe, SubscriptionTaskExecutor,
};
use futures::{future, stream, StreamExt};
use jsonrpsee::{core::async_trait, types::ErrorObject, PendingSubscriptionSink};
use sc_client_api::{
Backend, BlockBackend, BlockchainEvents, CallExecutor, ExecutorProvider, ProofProvider,
StorageProvider,
@@ -371,9 +371,7 @@ where
.map_err(client_err)
}
fn subscribe_runtime_version(&self, mut sink: SubscriptionSink) {
let client = self.client.clone();
fn subscribe_runtime_version(&self, pending: PendingSubscriptionSink) {
let initial = match self
.block_or_best(None)
.and_then(|block| self.client.runtime_version_at(block).map_err(Into::into))
@@ -381,12 +379,13 @@ where
{
Ok(initial) => initial,
Err(e) => {
let _ = sink.reject(JsonRpseeError::from(e));
spawn_subscription_task(&self.executor, pending.reject(e));
return
},
};
let mut previous_version = initial.clone();
let client = self.client.clone();
// A stream of new versions
let version_stream = client
@@ -406,24 +405,33 @@ where
});
let stream = futures::stream::once(future::ready(initial)).chain(version_stream);
let fut = async move {
sink.pipe_from_stream(stream).await;
};
self.executor.spawn("substrate-rpc-subscription", Some("rpc"), fut.boxed());
spawn_subscription_task(&self.executor, pipe_from_stream(pending, stream));
}
fn subscribe_storage(&self, mut sink: SubscriptionSink, keys: Option<Vec<StorageKey>>) {
fn subscribe_storage(
&self,
pending: PendingSubscriptionSink,
keys: Option<Vec<StorageKey>>,
deny_unsafe: DenyUnsafe,
) {
if keys.is_none() {
if let Err(err) = deny_unsafe.check_if_safe() {
spawn_subscription_task(&self.executor, pending.reject(ErrorObject::from(err)));
return
}
}
let stream = match self.client.storage_changes_notification_stream(keys.as_deref(), None) {
Ok(stream) => stream,
Err(blockchain_err) => {
let _ = sink.reject(JsonRpseeError::from(Error::Client(Box::new(blockchain_err))));
spawn_subscription_task(
&self.executor,
pending.reject(Error::Client(Box::new(blockchain_err))),
);
return
},
};
// initial values
let initial = stream::iter(keys.map(|keys| {
let block = self.client.info().best_hash;
let changes = keys
@@ -436,7 +444,6 @@ where
StorageChangeSet { block, changes }
}));
// let storage_stream = stream.map(|(block, changes)| StorageChangeSet {
let storage_stream = stream.map(|storage_notif| StorageChangeSet {
block: storage_notif.block,
changes: storage_notif
@@ -450,11 +457,7 @@ where
.chain(storage_stream)
.filter(|storage| future::ready(!storage.changes.is_empty()));
let fut = async move {
sink.pipe_from_stream(stream).await;
};
self.executor.spawn("substrate-rpc-subscription", Some("rpc"), fut.boxed());
spawn_subscription_task(&self.executor, pipe_from_stream(pending, stream));
}
fn trace_block(