rpc: backpressured RPC server (bump jsonrpsee 0.20) (#1313)

This is a rather big change in jsonrpsee, the major things in this bump
are:
- Server backpressure (the subscription impls are modified to deal with
that)
- Allow custom error types / return types (remove jsonrpsee::core::Error
and jsonrpee::core::CallError)
- Bug fixes (graceful shutdown in particular not used by substrate
anyway)
   - Less dependencies for the clients in particular
   - Return type requires Clone in method call responses
   - Moved to tokio channels
   - Async subscription API (not used in this PR)

Major changes in this PR:
- The subscriptions are now bounded and if subscription can't keep up
with the server it is dropped
- CLI: add parameter to configure the jsonrpc server bounded message
buffer (default is 64)
- Add our own subscription helper to deal with the unbounded streams in
substrate

The most important things in this PR to review is the added helpers
functions in `substrate/client/rpc/src/utils.rs` and the rest is pretty
much chore.

Regarding the "bounded buffer limit" it may cause the server to handle
the JSON-RPC calls
slower than before.

The message size limit is bounded by "--rpc-response-size" thus "by
default 10MB * 64 = 640MB"
but the subscription message size is not covered by this limit and could
be capped as well.

Hopefully the last release prior to 1.0, sorry in advance for a big PR

Previous attempt: https://github.com/paritytech/substrate/pull/13992

Resolves https://github.com/paritytech/polkadot-sdk/issues/748, resolves
https://github.com/paritytech/polkadot-sdk/issues/627

substrate/client/rpc-api/src/state/error.rs

@@ -18,10 +18,8 @@
 
 //! State RPC errors.
 
-use jsonrpsee::{
-	core::Error as JsonRpseeError,
-	types::error::{CallError, ErrorObject},
-};
+use jsonrpsee::types::error::{ErrorObject, ErrorObjectOwned};
+
 /// State RPC Result type.
 pub type Result<T> = std::result::Result<T, Error>;
 
@@ -57,16 +55,14 @@ pub enum Error {
 /// Base code for all state errors.
 const BASE_ERROR: i32 = crate::error::base::STATE;
 
-impl From<Error> for JsonRpseeError {
-	fn from(e: Error) -> Self {
+impl From<Error> for ErrorObjectOwned {
+	fn from(e: Error) -> ErrorObjectOwned {
 		match e {
 			Error::InvalidBlockRange { .. } =>
-				CallError::Custom(ErrorObject::owned(BASE_ERROR + 1, e.to_string(), None::<()>))
-					.into(),
+				ErrorObject::owned(BASE_ERROR + 1, e.to_string(), None::<()>),
 			Error::InvalidCount { .. } =>
-				CallError::Custom(ErrorObject::owned(BASE_ERROR + 2, e.to_string(), None::<()>))
-					.into(),
-			e => Self::to_call_error(e),
+				ErrorObject::owned(BASE_ERROR + 2, e.to_string(), None::<()>),
+			e => ErrorObject::owned(BASE_ERROR + 3, e.to_string(), None::<()>),
 		}
 	}
 }

substrate/client/rpc-api/src/state/helpers.rs

@@ -22,7 +22,7 @@ use serde::{Deserialize, Serialize};
 use sp_core::Bytes;
 
 /// ReadProof struct returned by the RPC
-#[derive(Debug, PartialEq, Serialize, Deserialize)]
+#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)]
 #[serde(rename_all = "camelCase")]
 pub struct ReadProof<Hash> {
 	/// Block hash used to generate the proof

substrate/client/rpc-api/src/state/mod.rs

@@ -18,7 +18,7 @@
 
 //! Substrate state API.
 
-use jsonrpsee::{core::RpcResult, proc_macros::rpc};
+use jsonrpsee::proc_macros::rpc;
 use sp_core::{
 	storage::{StorageChangeSet, StorageData, StorageKey},
 	Bytes,
@@ -29,18 +29,23 @@ pub mod error;
 pub mod helpers;
 
 pub use self::helpers::ReadProof;
+pub use error::Error;
 
 /// Substrate state API
 #[rpc(client, server)]
 pub trait StateApi<Hash> {
 	/// Call a method from the runtime API at a block's state.
 	#[method(name = "state_call", aliases = ["state_callAt"], blocking)]
-	fn call(&self, name: String, bytes: Bytes, hash: Option<Hash>) -> RpcResult<Bytes>;
+	fn call(&self, name: String, bytes: Bytes, hash: Option<Hash>) -> Result<Bytes, Error>;
 
 	/// Returns the keys with prefix, leave empty to get all the keys.
 	#[method(name = "state_getKeys", blocking)]
 	#[deprecated(since = "2.0.0", note = "Please use `getKeysPaged` with proper paging support")]
-	fn storage_keys(&self, prefix: StorageKey, hash: Option<Hash>) -> RpcResult<Vec<StorageKey>>;
+	fn storage_keys(
+		&self,
+		prefix: StorageKey,
+		hash: Option<Hash>,
+	) -> Result<Vec<StorageKey>, Error>;
 
 	/// Returns the keys with prefix, leave empty to get all the keys
 	#[method(name = "state_getPairs", blocking)]
@@ -48,7 +53,7 @@ pub trait StateApi<Hash> {
 		&self,
 		prefix: StorageKey,
 		hash: Option<Hash>,
-	) -> RpcResult<Vec<(StorageKey, StorageData)>>;
+	) -> Result<Vec<(StorageKey, StorageData)>, Error>;
 
 	/// Returns the keys with prefix with pagination support.
 	/// Up to `count` keys will be returned.
@@ -60,27 +65,28 @@ pub trait StateApi<Hash> {
 		count: u32,
 		start_key: Option<StorageKey>,
 		hash: Option<Hash>,
-	) -> RpcResult<Vec<StorageKey>>;
+	) -> Result<Vec<StorageKey>, Error>;
 
 	/// Returns a storage entry at a specific block's state.
 	#[method(name = "state_getStorage", aliases = ["state_getStorageAt"], blocking)]
-	fn storage(&self, key: StorageKey, hash: Option<Hash>) -> RpcResult<Option<StorageData>>;
+	fn storage(&self, key: StorageKey, hash: Option<Hash>) -> Result<Option<StorageData>, Error>;
 
 	/// Returns the hash of a storage entry at a block's state.
 	#[method(name = "state_getStorageHash", aliases = ["state_getStorageHashAt"], blocking)]
-	fn storage_hash(&self, key: StorageKey, hash: Option<Hash>) -> RpcResult<Option<Hash>>;
+	fn storage_hash(&self, key: StorageKey, hash: Option<Hash>) -> Result<Option<Hash>, Error>;
 
 	/// Returns the size of a storage entry at a block's state.
 	#[method(name = "state_getStorageSize", aliases = ["state_getStorageSizeAt"])]
-	async fn storage_size(&self, key: StorageKey, hash: Option<Hash>) -> RpcResult<Option<u64>>;
+	async fn storage_size(&self, key: StorageKey, hash: Option<Hash>)
+		-> Result<Option<u64>, Error>;
 
 	/// Returns the runtime metadata as an opaque blob.
 	#[method(name = "state_getMetadata", blocking)]
-	fn metadata(&self, hash: Option<Hash>) -> RpcResult<Bytes>;
+	fn metadata(&self, hash: Option<Hash>) -> Result<Bytes, Error>;
 
 	/// Get the runtime version.
 	#[method(name = "state_getRuntimeVersion", aliases = ["chain_getRuntimeVersion"], blocking)]
-	fn runtime_version(&self, hash: Option<Hash>) -> RpcResult<RuntimeVersion>;
+	fn runtime_version(&self, hash: Option<Hash>) -> Result<RuntimeVersion, Error>;
 
 	/// Query historical storage entries (by key) starting from a block given as the second
 	/// parameter.
@@ -95,7 +101,7 @@ pub trait StateApi<Hash> {
 		keys: Vec<StorageKey>,
 		block: Hash,
 		hash: Option<Hash>,
-	) -> RpcResult<Vec<StorageChangeSet<Hash>>>;
+	) -> Result<Vec<StorageChangeSet<Hash>>, Error>;
 
 	/// Query storage entries (by key) at a block hash given as the second parameter.
 	/// NOTE: Each StorageChangeSet in the result corresponds to exactly one element --
@@ -105,11 +111,15 @@ pub trait StateApi<Hash> {
 		&self,
 		keys: Vec<StorageKey>,
 		at: Option<Hash>,
-	) -> RpcResult<Vec<StorageChangeSet<Hash>>>;
+	) -> Result<Vec<StorageChangeSet<Hash>>, Error>;
 
 	/// Returns proof of storage entries at a specific block's state.
 	#[method(name = "state_getReadProof", blocking)]
-	fn read_proof(&self, keys: Vec<StorageKey>, hash: Option<Hash>) -> RpcResult<ReadProof<Hash>>;
+	fn read_proof(
+		&self,
+		keys: Vec<StorageKey>,
+		hash: Option<Hash>,
+	) -> Result<ReadProof<Hash>, Error>;
 
 	/// New runtime version subscription
 	#[subscription(
@@ -288,5 +298,5 @@ pub trait StateApi<Hash> {
 		targets: Option<String>,
 		storage_keys: Option<String>,
 		methods: Option<String>,
-	) -> RpcResult<sp_rpc::tracing::TraceBlockResponse>;
+	) -> Result<sp_rpc::tracing::TraceBlockResponse, Error>;
 }