pezkuwichain/pezkuwi-subxt
1
0
Fork 0
mirror of https://github.com/pezkuwichain/pezkuwi-subxt.git synced 2026-07-18 07:05:42 +00:00
Code Issues Packages Projects Releases Wiki Activity

rpc: backpressured RPC server (bump jsonrpsee 0.20) (#1313)

Browse Source 
This is a rather big change in jsonrpsee, the major things in this bump
are:
- Server backpressure (the subscription impls are modified to deal with
that)
- Allow custom error types / return types (remove jsonrpsee::core::Error
and jsonrpee::core::CallError)
- Bug fixes (graceful shutdown in particular not used by substrate
anyway)
   - Less dependencies for the clients in particular
   - Return type requires Clone in method call responses
   - Moved to tokio channels
   - Async subscription API (not used in this PR)

Major changes in this PR:
- The subscriptions are now bounded and if subscription can't keep up
with the server it is dropped
- CLI: add parameter to configure the jsonrpc server bounded message
buffer (default is 64)
- Add our own subscription helper to deal with the unbounded streams in
substrate

The most important things in this PR to review is the added helpers
functions in `substrate/client/rpc/src/utils.rs` and the rest is pretty
much chore.

Regarding the "bounded buffer limit" it may cause the server to handle
the JSON-RPC calls
slower than before.

The message size limit is bounded by "--rpc-response-size" thus "by
default 10MB * 64 = 640MB"
but the subscription message size is not covered by this limit and could
be capped as well.

Hopefully the last release prior to 1.0, sorry in advance for a big PR

Previous attempt: https://github.com/paritytech/substrate/pull/13992

Resolves https://github.com/paritytech/polkadot-sdk/issues/748, resolves
https://github.com/paritytech/polkadot-sdk/issues/627
This commit is contained in:
Niklas Adolfsson
2024-01-23 09:55:13 +01:00
committed by GitHub
parent 76c37c930b
commit e16ef0861f
117 changed files with 1245 additions and 1090 deletions
Download Patch File Download Diff File Expand all files Collapse all files
substrate/client/rpc-spec-v2/src/chain_head/tests.rs
+33 -34
View File
@@ -27,10 +27,8 @@ use assert_matches::assert_matches;
use codec::{Decode, Encode};
use futures::Future;
use jsonrpsee::{
core::{error::Error, server::rpc_module::Subscription as RpcSubscription},
rpc_params,
types::error::CallError,
RpcModule,
core::{error::Error, server::Subscription as RpcSubscription},
rpc_params, RpcModule,
};
use sc_block_builder::BlockBuilderBuilder;
use sc_client_api::ChildInfo;
@@ -120,7 +118,7 @@ async fn setup_api() -> (
)
.into_rpc();
let mut sub = api.subscribe("chainHead_unstable_follow", [true]).await.unwrap();
let mut sub = api.subscribe_unbounded("chainHead_unstable_follow", [true]).await.unwrap();
let sub_id = sub.subscription_id();
let sub_id = serde_json::to_string(&sub_id).unwrap();
@@ -171,7 +169,7 @@ async fn follow_subscription_produces_blocks() {
.into_rpc();
let finalized_hash = client.info().finalized_hash;
let mut sub = api.subscribe("chainHead_unstable_follow", [false]).await.unwrap();
let mut sub = api.subscribe_unbounded("chainHead_unstable_follow", [false]).await.unwrap();
// Initialized must always be reported first.
let event: FollowEvent<String> = get_next_event(&mut sub).await;
@@ -239,7 +237,7 @@ async fn follow_with_runtime() {
.into_rpc();
let finalized_hash = client.info().finalized_hash;
let mut sub = api.subscribe("chainHead_unstable_follow", [true]).await.unwrap();
let mut sub = api.subscribe_unbounded("chainHead_unstable_follow", [true]).await.unwrap();
// Initialized must always be reported first.
let event: FollowEvent<String> = get_next_event(&mut sub).await;
@@ -361,7 +359,7 @@ async fn get_header() {
.await
.unwrap_err();
assert_matches!(err,
Error::Call(CallError::Custom(ref err)) if err.code() == super::error::rpc_spec_v2::INVALID_BLOCK_ERROR && err.message() == "Invalid block hash"
Error::Call(err) if err.code() == super::error::rpc_spec_v2::INVALID_BLOCK_ERROR && err.message() == "Invalid block hash"
);
// Obtain the valid header.
@@ -390,7 +388,7 @@ async fn get_body() {
.await
.unwrap_err();
assert_matches!(err,
Error::Call(CallError::Custom(ref err)) if err.code() == super::error::rpc_spec_v2::INVALID_BLOCK_ERROR && err.message() == "Invalid block hash"
Error::Call(err) if err.code() == super::error::rpc_spec_v2::INVALID_BLOCK_ERROR && err.message() == "Invalid block hash"
);
// Valid call.
@@ -475,7 +473,7 @@ async fn call_runtime() {
.await
.unwrap_err();
assert_matches!(err,
Error::Call(CallError::Custom(ref err)) if err.code() == super::error::rpc_spec_v2::INVALID_BLOCK_ERROR && err.message() == "Invalid block hash"
Error::Call(err) if err.code() == super::error::rpc_spec_v2::INVALID_BLOCK_ERROR && err.message() == "Invalid block hash"
);
// Pass an invalid parameters that cannot be decode.
@@ -488,7 +486,7 @@ async fn call_runtime() {
.await
.unwrap_err();
assert_matches!(err,
Error::Call(CallError::Custom(ref err)) if err.code() == super::error::json_rpc_spec::INVALID_PARAM_ERROR && err.message().contains("Invalid parameter")
Error::Call(err) if err.code() == super::error::json_rpc_spec::INVALID_PARAM_ERROR && err.message().contains("Invalid parameter")
);
// Valid call.
@@ -550,7 +548,7 @@ async fn call_runtime_without_flag() {
)
.into_rpc();
let mut sub = api.subscribe("chainHead_unstable_follow", [false]).await.unwrap();
let mut sub = api.subscribe_unbounded("chainHead_unstable_follow", [false]).await.unwrap();
let sub_id = sub.subscription_id();
let sub_id = serde_json::to_string(&sub_id).unwrap();
@@ -591,7 +589,7 @@ async fn call_runtime_without_flag() {
.unwrap_err();
assert_matches!(err,
Error::Call(CallError::Custom(ref err)) if err.code() == super::error::rpc_spec_v2::INVALID_RUNTIME_CALL && err.message().contains("subscription was started with `withRuntime` set to `false`")
Error::Call(err) if err.code() == super::error::rpc_spec_v2::INVALID_RUNTIME_CALL && err.message().contains("subscription was started with `withRuntime` set to `false`")
);
}
@@ -629,7 +627,7 @@ async fn get_storage_hash() {
.await
.unwrap_err();
assert_matches!(err,
Error::Call(CallError::Custom(ref err)) if err.code() == super::error::rpc_spec_v2::INVALID_BLOCK_ERROR && err.message() == "Invalid block hash"
Error::Call(err) if err.code() == super::error::rpc_spec_v2::INVALID_BLOCK_ERROR && err.message() == "Invalid block hash"
);
// Valid call without storage at the key.
@@ -897,7 +895,7 @@ async fn get_storage_value() {
.await
.unwrap_err();
assert_matches!(err,
Error::Call(CallError::Custom(ref err)) if err.code() == super::error::rpc_spec_v2::INVALID_BLOCK_ERROR && err.message() == "Invalid block hash"
Error::Call(err) if err.code() == super::error::rpc_spec_v2::INVALID_BLOCK_ERROR && err.message() == "Invalid block hash"
);
// Valid call without storage at the key.
@@ -1209,11 +1207,12 @@ async fn separate_operation_ids_for_subscriptions() {
.into_rpc();
// Create two separate subscriptions.
let mut sub_first = api.subscribe("chainHead_unstable_follow", [true]).await.unwrap();
let mut sub_first = api.subscribe_unbounded("chainHead_unstable_follow", [true]).await.unwrap();
let sub_id_first = sub_first.subscription_id();
let sub_id_first = serde_json::to_string(&sub_id_first).unwrap();
let mut sub_second = api.subscribe("chainHead_unstable_follow", [true]).await.unwrap();
let mut sub_second =
api.subscribe_unbounded("chainHead_unstable_follow", [true]).await.unwrap();
let sub_id_second = sub_second.subscription_id();
let sub_id_second = serde_json::to_string(&sub_id_second).unwrap();
@@ -1341,7 +1340,7 @@ async fn follow_generates_initial_blocks() {
let block_2_f_hash = block_2_f.header.hash();
client.import(BlockOrigin::Own, block_2_f.clone()).await.unwrap();
let mut sub = api.subscribe("chainHead_unstable_follow", [false]).await.unwrap();
let mut sub = api.subscribe_unbounded("chainHead_unstable_follow", [false]).await.unwrap();
// Initialized must always be reported first.
let event: FollowEvent<String> = get_next_event(&mut sub).await;
@@ -1450,7 +1449,7 @@ async fn follow_exceeding_pinned_blocks() {
)
.into_rpc();
let mut sub = api.subscribe("chainHead_unstable_follow", [false]).await.unwrap();
let mut sub = api.subscribe_unbounded("chainHead_unstable_follow", [false]).await.unwrap();
let block = BlockBuilderBuilder::new(&*client)
.on_parent_block(client.chain_info().genesis_hash)
@@ -1526,7 +1525,7 @@ async fn follow_with_unpin() {
)
.into_rpc();
let mut sub = api.subscribe("chainHead_unstable_follow", [false]).await.unwrap();
let mut sub = api.subscribe_unbounded("chainHead_unstable_follow", [false]).await.unwrap();
let sub_id = sub.subscription_id();
let sub_id = serde_json::to_string(&sub_id).unwrap();
@@ -1572,7 +1571,7 @@ async fn follow_with_unpin() {
.await
.unwrap_err();
assert_matches!(err,
Error::Call(CallError::Custom(ref err)) if err.code() == super::error::rpc_spec_v2::INVALID_BLOCK_ERROR && err.message() == "Invalid block hash"
Error::Call(err) if err.code() == super::error::rpc_spec_v2::INVALID_BLOCK_ERROR && err.message() == "Invalid block hash"
);
// To not exceed the number of pinned blocks, we need to unpin before the next import.
@@ -1637,7 +1636,7 @@ async fn follow_with_multiple_unpin_hashes() {
)
.into_rpc();
let mut sub = api.subscribe("chainHead_unstable_follow", [false]).await.unwrap();
let mut sub = api.subscribe_unbounded("chainHead_unstable_follow", [false]).await.unwrap();
let sub_id = sub.subscription_id();
let sub_id = serde_json::to_string(&sub_id).unwrap();
@@ -1721,7 +1720,7 @@ async fn follow_with_multiple_unpin_hashes() {
.await
.unwrap_err();
assert_matches!(err,
Error::Call(CallError::Custom(ref err)) if err.code() == super::error::rpc_spec_v2::INVALID_BLOCK_ERROR && err.message() == "Invalid block hash"
Error::Call(err) if err.code() == super::error::rpc_spec_v2::INVALID_BLOCK_ERROR && err.message() == "Invalid block hash"
);
let _res: () = api
@@ -1738,7 +1737,7 @@ async fn follow_with_multiple_unpin_hashes() {
.await
.unwrap_err();
assert_matches!(err,
Error::Call(CallError::Custom(ref err)) if err.code() == super::error::rpc_spec_v2::INVALID_BLOCK_ERROR && err.message() == "Invalid block hash"
Error::Call(err) if err.code() == super::error::rpc_spec_v2::INVALID_BLOCK_ERROR && err.message() == "Invalid block hash"
);
// Unpin multiple blocks.
@@ -1756,7 +1755,7 @@ async fn follow_with_multiple_unpin_hashes() {
.await
.unwrap_err();
assert_matches!(err,
Error::Call(CallError::Custom(ref err)) if err.code() == super::error::rpc_spec_v2::INVALID_BLOCK_ERROR && err.message() == "Invalid block hash"
Error::Call(err) if err.code() == super::error::rpc_spec_v2::INVALID_BLOCK_ERROR && err.message() == "Invalid block hash"
);
let err = api
@@ -1767,7 +1766,7 @@ async fn follow_with_multiple_unpin_hashes() {
.await
.unwrap_err();
assert_matches!(err,
Error::Call(CallError::Custom(ref err)) if err.code() == super::error::rpc_spec_v2::INVALID_BLOCK_ERROR && err.message() == "Invalid block hash"
Error::Call(err) if err.code() == super::error::rpc_spec_v2::INVALID_BLOCK_ERROR && err.message() == "Invalid block hash"
);
}
@@ -1791,7 +1790,7 @@ async fn follow_prune_best_block() {
.into_rpc();
let finalized_hash = client.info().finalized_hash;
let mut sub = api.subscribe("chainHead_unstable_follow", [false]).await.unwrap();
let mut sub = api.subscribe_unbounded("chainHead_unstable_follow", [false]).await.unwrap();
// Initialized must always be reported first.
let event: FollowEvent<String> = get_next_event(&mut sub).await;
@@ -2051,7 +2050,7 @@ async fn follow_forks_pruned_block() {
// Block 2_f and 3_f are not pruned, pruning happens at height (N - 1).
client.finalize_block(block_3_hash, None).unwrap();
let mut sub = api.subscribe("chainHead_unstable_follow", [false]).await.unwrap();
let mut sub = api.subscribe_unbounded("chainHead_unstable_follow", [false]).await.unwrap();
// Initialized must always be reported first.
let event: FollowEvent<String> = get_next_event(&mut sub).await;
@@ -2205,7 +2204,7 @@ async fn follow_report_multiple_pruned_block() {
let block_3_f = block_builder.build().unwrap().block;
let block_3_f_hash = block_3_f.hash();
client.import(BlockOrigin::Own, block_3_f.clone()).await.unwrap();
let mut sub = api.subscribe("chainHead_unstable_follow", [false]).await.unwrap();
let mut sub = api.subscribe_unbounded("chainHead_unstable_follow", [false]).await.unwrap();
// Initialized must always be reported first.
let event: FollowEvent<String> = get_next_event(&mut sub).await;
@@ -2388,7 +2387,7 @@ async fn pin_block_references() {
}
}
let mut sub = api.subscribe("chainHead_unstable_follow", [false]).await.unwrap();
let mut sub = api.subscribe_unbounded("chainHead_unstable_follow", [false]).await.unwrap();
let sub_id = sub.subscription_id();
let sub_id = serde_json::to_string(&sub_id).unwrap();
@@ -2520,7 +2519,7 @@ async fn follow_finalized_before_new_block() {
let block_1_hash = block_1.header.hash();
client.import(BlockOrigin::Own, block_1.clone()).await.unwrap();
let mut sub = api.subscribe("chainHead_unstable_follow", [false]).await.unwrap();
let mut sub = api.subscribe_unbounded("chainHead_unstable_follow", [false]).await.unwrap();
// Trigger the `FinalizedNotification` for block 1 before the `BlockImportNotification`, and
// expect for the `chainHead` to generate `NewBlock`, `BestBlock` and `Finalized` events.
@@ -2622,7 +2621,7 @@ async fn ensure_operation_limits_works() {
)
.into_rpc();
let mut sub = api.subscribe("chainHead_unstable_follow", [true]).await.unwrap();
let mut sub = api.subscribe_unbounded("chainHead_unstable_follow", [true]).await.unwrap();
let sub_id = sub.subscription_id();
let sub_id = serde_json::to_string(&sub_id).unwrap();
@@ -2726,7 +2725,7 @@ async fn check_continue_operation() {
)
.into_rpc();
let mut sub = api.subscribe("chainHead_unstable_follow", [true]).await.unwrap();
let mut sub = api.subscribe_unbounded("chainHead_unstable_follow", [true]).await.unwrap();
let sub_id = sub.subscription_id();
let sub_id = serde_json::to_string(&sub_id).unwrap();
@@ -2908,7 +2907,7 @@ async fn stop_storage_operation() {
)
.into_rpc();
let mut sub = api.subscribe("chainHead_unstable_follow", [true]).await.unwrap();
let mut sub = api.subscribe_unbounded("chainHead_unstable_follow", [true]).await.unwrap();
let sub_id = sub.subscription_id();
let sub_id = serde_json::to_string(&sub_id).unwrap();