rpc server with HTTP/WS on the same socket (#12663)

* jsonrpsee v0.16

add backwards compatibility

run old http server on http only

* cargo fmt

* update jsonrpsee 0.16.1

* less verbose cors log

* fix nit in log: WS -> HTTP

* revert needless changes in Cargo.lock

* remove unused features in tower

* fix nits; add client-core feature

* jsonrpsee v0.16.2

substrate/utils/frame/remote-externalities/src/lib.rs

@@ -39,7 +39,9 @@ use std::{
 	path::{Path, PathBuf},
 	sync::Arc,
 };
-use substrate_rpc_client::{rpc_params, ws_client, ChainApi, ClientT, StateApi, WsClient};
+use substrate_rpc_client::{
+	rpc_params, ws_client, BatchRequestBuilder, ChainApi, ClientT, StateApi, WsClient,
+};
 
 type KeyValue = (StorageKey, StorageData);
 type TopKeyValues = Vec<KeyValue>;
@@ -334,33 +336,50 @@ where
 		log::debug!(target: LOG_TARGET, "Querying a total of {} keys", keys.len());
 
 		let mut key_values: Vec<KeyValue> = vec![];
+		let mut batch_success = true;
+
 		let client = self.as_online().rpc_client();
 		for chunk_keys in keys.chunks(BATCH_SIZE) {
-			let batch = chunk_keys
-				.iter()
-				.cloned()
-				.map(|key| ("state_getStorage", rpc_params![key, at]))
-				.collect::<Vec<_>>();
+			let mut batch = BatchRequestBuilder::new();
 
-			let values = client.batch_request::<Option<StorageData>>(batch).await.map_err(|e| {
-				log::error!(
-					target: LOG_TARGET,
-					"failed to execute batch: {:?}. Error: {:?}",
-					chunk_keys.iter().map(HexDisplay::from).collect::<Vec<_>>(),
-					e
-				);
-				"batch failed."
-			})?;
+			for key in chunk_keys.iter() {
+				batch
+					.insert("state_getStorage", rpc_params![key, at])
+					.map_err(|_| "Invalid batch params")?;
+			}
 
-			assert_eq!(chunk_keys.len(), values.len());
+			let batch_response =
+				client.batch_request::<Option<StorageData>>(batch).await.map_err(|e| {
+					log::error!(
+						target: LOG_TARGET,
+						"failed to execute batch: {:?}. Error: {:?}",
+						chunk_keys.iter().map(HexDisplay::from).collect::<Vec<_>>(),
+						e
+					);
+					"batch failed."
+				})?;
+
+			assert_eq!(chunk_keys.len(), batch_response.len());
+
+			for (key, maybe_value) in chunk_keys.into_iter().zip(batch_response) {
+				match maybe_value {
+					Ok(Some(v)) => {
+						key_values.push((key.clone(), v));
+					},
+					Ok(None) => {
+						log::warn!(
+							target: LOG_TARGET,
+							"key {:?} had none corresponding value.",
+							&key
+						);
+						key_values.push((key.clone(), StorageData(vec![])));
+					},
+					Err(e) => {
+						log::error!(target: LOG_TARGET, "key {:?} failed: {:?}", &key, e);
+						batch_success = false;
+					},
+				};
 
-			for (idx, key) in chunk_keys.iter().enumerate() {
-				let maybe_value = values[idx].clone();
-				let value = maybe_value.unwrap_or_else(|| {
-					log::warn!(target: LOG_TARGET, "key {:?} had none corresponding value.", &key);
-					StorageData(vec![])
-				});
-				key_values.push((key.clone(), value));
 				if key_values.len() % (10 * BATCH_SIZE) == 0 {
 					let ratio: f64 = key_values.len() as f64 / keys_count as f64;
 					log::debug!(
@@ -374,7 +393,11 @@ where
 			}
 		}
 
-		Ok(key_values)
+		if batch_success {
+			Ok(key_values)
+		} else {
+			Err("batch failed.")
+		}
 	}
 
 	/// Get the values corresponding to `child_keys` at the given `prefixed_top_key`.
@@ -385,12 +408,14 @@ where
 		at: B::Hash,
 	) -> Result<Vec<KeyValue>, &'static str> {
 		let mut child_kv_inner = vec![];
+		let mut batch_success = true;
+
 		for batch_child_key in child_keys.chunks(BATCH_SIZE) {
-			let batch_request = batch_child_key
-				.iter()
-				.cloned()
-				.map(|key| {
-					(
+			let mut batch_request = BatchRequestBuilder::new();
+
+			for key in batch_child_key {
+				batch_request
+					.insert(
 						"childstate_getStorage",
 						rpc_params![
 							PrefixedStorageKey::new(prefixed_top_key.as_ref().to_vec()),
@@ -398,8 +423,8 @@ where
 							at
 						],
 					)
-				})
-				.collect::<Vec<_>>();
+					.map_err(|_| "Invalid batch params")?;
+			}
 
 			let batch_response = self
 				.as_online()
@@ -418,17 +443,32 @@ where
 
 			assert_eq!(batch_child_key.len(), batch_response.len());
 
-			for (idx, key) in batch_child_key.iter().enumerate() {
-				let maybe_value = batch_response[idx].clone();
-				let value = maybe_value.unwrap_or_else(|| {
-					log::warn!(target: LOG_TARGET, "key {:?} had none corresponding value.", &key);
-					StorageData(vec![])
-				});
-				child_kv_inner.push((key.clone(), value));
+			for (key, maybe_value) in batch_child_key.iter().zip(batch_response) {
+				match maybe_value {
+					Ok(Some(v)) => {
+						child_kv_inner.push((key.clone(), v));
+					},
+					Ok(None) => {
+						log::warn!(
+							target: LOG_TARGET,
+							"key {:?} had none corresponding value.",
+							&key
+						);
+						child_kv_inner.push((key.clone(), StorageData(vec![])));
+					},
+					Err(e) => {
+						log::error!(target: LOG_TARGET, "key {:?} failed: {:?}", &key, e);
+						batch_success = false;
+					},
+				};
 			}
 		}
 
-		Ok(child_kv_inner)
+		if batch_success {
+			Ok(child_kv_inner)
+		} else {
+			Err("batch failed.")
+		}
 	}
 
 	pub(crate) async fn rpc_child_get_keys(

substrate/utils/frame/rpc/client/Cargo.toml

@@ -12,7 +12,7 @@ description = "Shared JSON-RPC client"
 targets = ["x86_64-unknown-linux-gnu"]
 
 [dependencies]
-jsonrpsee = { version = "0.15.1", features = ["ws-client"] }
+jsonrpsee = { version = "0.16.2", features = ["ws-client"] }
 sc-rpc-api = { version = "0.10.0-dev", path = "../../../../client/rpc-api" }
 async-trait = "0.1.57"
 serde = "1"

substrate/utils/frame/rpc/client/src/lib.rs

@@ -43,7 +43,10 @@ use sp_runtime::traits::{Block as BlockT, Header as HeaderT};
 use std::collections::VecDeque;
 
 pub use jsonrpsee::{
-	core::client::{ClientT, Subscription, SubscriptionClientT},
+	core::{
+		client::{ClientT, Subscription, SubscriptionClientT},
+		params::BatchRequestBuilder,
+	},
 	rpc_params,
 	ws_client::{WsClient, WsClientBuilder},
 };

substrate/utils/frame/rpc/state-trie-migration-rpc/Cargo.toml

@@ -25,7 +25,7 @@ sp-state-machine = { path = "../../../../primitives/state-machine" }
 sp-trie = { path = "../../../../primitives/trie" }
 trie-db = "0.24.0"
 
-jsonrpsee = { version = "0.15.1", features = ["server", "macros"] }
+jsonrpsee = { version = "0.16.2", features = ["client-core", "server", "macros"] }
 
 # Substrate Dependencies
 sc-client-api = { version = "4.0.0-dev", path = "../../../../client/api" }

substrate/utils/frame/rpc/support/Cargo.toml

@@ -17,7 +17,7 @@ targets = ["x86_64-unknown-linux-gnu"]
 [dependencies]
 codec = { package = "parity-scale-codec", version = "3.0.0" }
 futures = "0.3.21"
-jsonrpsee = { version = "0.15.1", features = ["jsonrpsee-types"] }
+jsonrpsee = { version = "0.16.2", features = ["jsonrpsee-types"] }
 serde = "1"
 frame-support = { version = "4.0.0-dev", path = "../../../../frame/support" }
 sc-rpc-api = { version = "0.10.0-dev", path = "../../../../client/rpc-api" }
@@ -25,7 +25,7 @@ sp-storage = { version = "7.0.0", path = "../../../../primitives/storage" }
 
 [dev-dependencies]
 scale-info = "2.1.1"
-jsonrpsee = { version = "0.15.1", features = ["ws-client", "jsonrpsee-types"] }
+jsonrpsee = { version = "0.16.2", features = ["ws-client", "jsonrpsee-types"] }
 tokio = "1.22.0"
 sp-core = { version = "7.0.0", path = "../../../../primitives/core" }
 sp-runtime = { version = "7.0.0", path = "../../../../primitives/runtime" }

substrate/utils/frame/rpc/system/Cargo.toml

@@ -15,7 +15,7 @@ targets = ["x86_64-unknown-linux-gnu"]
 [dependencies]
 serde_json = "1"
 codec = { package = "parity-scale-codec", version = "3.0.0" }
-jsonrpsee = { version = "0.15.1", features = ["server"] }
+jsonrpsee = { version = "0.16.2", features = ["client-core", "server", "macros"] }
 futures = "0.3.21"
 log = "0.4.17"
 frame-system-rpc-runtime-api = { version = "4.0.0-dev", path = "../../../../frame/system/rpc/runtime-api" }