frame/utils: introduce substrate-rpc-client crate for RPC utils (#12212)

* hack together a PoC

* Update utils/frame/rpc-utils/Cargo.toml

Co-authored-by: Kian Paimani <5588131+kianenigma@users.noreply.github.com>

* Update utils/frame/rpc-utils/src/lib.rs

Co-authored-by: Kian Paimani <5588131+kianenigma@users.noreply.github.com>

* rpc_utils -> substrate_rpc_client

* try runtime: remove keep connection

* make CI happy

* cargo fmt

* fix ci

* update lock file

* fix

* fix

Co-authored-by: Kian Paimani <5588131+kianenigma@users.noreply.github.com>
Co-authored-by: kianenigma <kian@parity.io>

substrate/utils/frame/remote-externalities/src/lib.rs

@@ -22,13 +22,6 @@
 
 use codec::{Decode, Encode};
 
-use jsonrpsee::{
-	core::{client::ClientT, Error as RpcError},
-	proc_macros::rpc,
-	rpc_params,
-	ws_client::{WsClient, WsClientBuilder},
-};
-
 use log::*;
 use serde::de::DeserializeOwned;
 use sp_core::{
@@ -46,8 +39,7 @@ use std::{
 	path::{Path, PathBuf},
 	sync::Arc,
 };
-
-pub mod rpc_api;
+use substrate_rpc_client::{rpc_params, ws_client, ChainApi, ClientT, StateApi, WsClient};
 
 type KeyValue = (StorageKey, StorageData);
 type TopKeyValues = Vec<KeyValue>;
@@ -58,40 +50,6 @@ const DEFAULT_TARGET: &str = "wss://rpc.polkadot.io:443";
 const BATCH_SIZE: usize = 1000;
 const PAGE: u32 = 1000;
 
-#[rpc(client)]
-pub trait RpcApi<Hash> {
-	#[method(name = "childstate_getKeys")]
-	fn child_get_keys(
-		&self,
-		child_key: PrefixedStorageKey,
-		prefix: StorageKey,
-		hash: Option<Hash>,
-	) -> Result<Vec<StorageKey>, RpcError>;
-
-	#[method(name = "childstate_getStorage")]
-	fn child_get_storage(
-		&self,
-		child_key: PrefixedStorageKey,
-		prefix: StorageKey,
-		hash: Option<Hash>,
-	) -> Result<StorageData, RpcError>;
-
-	#[method(name = "state_getStorage")]
-	fn get_storage(&self, prefix: StorageKey, hash: Option<Hash>) -> Result<StorageData, RpcError>;
-
-	#[method(name = "state_getKeysPaged")]
-	fn get_keys_paged(
-		&self,
-		prefix: Option<StorageKey>,
-		count: u32,
-		start_key: Option<StorageKey>,
-		hash: Option<Hash>,
-	) -> Result<Vec<StorageKey>, RpcError>;
-
-	#[method(name = "chain_getFinalizedHead")]
-	fn finalized_head(&self) -> Result<Hash, RpcError>;
-}
-
 /// The execution mode.
 #[derive(Clone)]
 pub enum Mode<B: BlockT> {
@@ -140,14 +98,10 @@ impl Transport {
 		if let Self::Uri(uri) = self {
 			log::debug!(target: LOG_TARGET, "initializing remote client to {:?}", uri);
 
-			let ws_client = WsClientBuilder::default()
-				.max_request_body_size(u32::MAX)
-				.build(&uri)
-				.await
-				.map_err(|e| {
-					log::error!(target: LOG_TARGET, "error: {:?}", e);
-					"failed to build ws client"
-				})?;
+			let ws_client = ws_client(uri).await.map_err(|e| {
+				log::error!(target: LOG_TARGET, "error: {:?}", e);
+				"failed to build ws client"
+			})?;
 
 			*self = Self::RemoteClient(Arc::new(ws_client))
 		}
@@ -258,7 +212,7 @@ pub struct Builder<B: BlockT> {
 
 // NOTE: ideally we would use `DefaultNoBound` here, but not worth bringing in frame-support for
 // that.
-impl<B: BlockT + DeserializeOwned> Default for Builder<B> {
+impl<B: BlockT> Default for Builder<B> {
 	fn default() -> Self {
 		Self {
 			mode: Default::default(),
@@ -272,7 +226,7 @@ impl<B: BlockT + DeserializeOwned> Default for Builder<B> {
 }
 
 // Mode methods
-impl<B: BlockT + DeserializeOwned> Builder<B> {
+impl<B: BlockT> Builder<B> {
 	fn as_online(&self) -> &OnlineConfig<B> {
 		match &self.mode {
 			Mode::Online(config) => config,
@@ -291,26 +245,38 @@ impl<B: BlockT + DeserializeOwned> Builder<B> {
 }
 
 // RPC methods
-impl<B: BlockT + DeserializeOwned> Builder<B> {
+impl<B: BlockT> Builder<B>
+where
+	B::Hash: DeserializeOwned,
+	B::Header: DeserializeOwned,
+{
 	async fn rpc_get_storage(
 		&self,
 		key: StorageKey,
 		maybe_at: Option<B::Hash>,
 	) -> Result<StorageData, &'static str> {
 		trace!(target: LOG_TARGET, "rpc: get_storage");
-		self.as_online().rpc_client().get_storage(key, maybe_at).await.map_err(|e| {
-			error!(target: LOG_TARGET, "Error = {:?}", e);
-			"rpc get_storage failed."
-		})
+		match self.as_online().rpc_client().storage(key, maybe_at).await {
+			Ok(Some(res)) => Ok(res),
+			Ok(None) => Err("get_storage not found"),
+			Err(e) => {
+				error!(target: LOG_TARGET, "Error = {:?}", e);
+				Err("rpc get_storage failed.")
+			},
+		}
 	}
 
 	/// Get the latest finalized head.
 	async fn rpc_get_head(&self) -> Result<B::Hash, &'static str> {
 		trace!(target: LOG_TARGET, "rpc: finalized_head");
-		self.as_online().rpc_client().finalized_head().await.map_err(|e| {
-			error!(target: LOG_TARGET, "Error = {:?}", e);
-			"rpc finalized_head failed."
-		})
+
+		// sadly this pretty much unreadable...
+		ChainApi::<(), _, B::Header, ()>::finalized_head(self.as_online().rpc_client())
+			.await
+			.map_err(|e| {
+				error!(target: LOG_TARGET, "Error = {:?}", e);
+				"rpc finalized_head failed."
+			})
 	}
 
 	/// Get all the keys at `prefix` at `hash` using the paged, safe RPC methods.
@@ -325,7 +291,7 @@ impl<B: BlockT + DeserializeOwned> Builder<B> {
 			let page = self
 				.as_online()
 				.rpc_client()
-				.get_keys_paged(Some(prefix.clone()), PAGE, last_key.clone(), Some(at))
+				.storage_keys_paged(Some(prefix.clone()), PAGE, last_key.clone(), Some(at))
 				.await
 				.map_err(|e| {
 					error!(target: LOG_TARGET, "Error = {:?}", e);
@@ -471,19 +437,19 @@ impl<B: BlockT + DeserializeOwned> Builder<B> {
 		child_prefix: StorageKey,
 		at: B::Hash,
 	) -> Result<Vec<StorageKey>, &'static str> {
-		let child_keys = self
-			.as_online()
-			.rpc_client()
-			.child_get_keys(
-				PrefixedStorageKey::new(prefixed_top_key.as_ref().to_vec()),
-				child_prefix,
-				Some(at),
-			)
-			.await
-			.map_err(|e| {
-				error!(target: LOG_TARGET, "Error = {:?}", e);
-				"rpc child_get_keys failed."
-			})?;
+		// This is deprecated and will generate a warning which causes the CI to fail.
+		#[allow(warnings)]
+		let child_keys = substrate_rpc_client::ChildStateApi::storage_keys(
+			self.as_online().rpc_client(),
+			PrefixedStorageKey::new(prefixed_top_key.as_ref().to_vec()),
+			child_prefix,
+			Some(at),
+		)
+		.await
+		.map_err(|e| {
+			error!(target: LOG_TARGET, "Error = {:?}", e);
+			"rpc child_get_keys failed."
+		})?;
 
 		debug!(
 			target: LOG_TARGET,
@@ -497,7 +463,11 @@ impl<B: BlockT + DeserializeOwned> Builder<B> {
 }
 
 // Internal methods
-impl<B: BlockT + DeserializeOwned> Builder<B> {
+impl<B: BlockT> Builder<B>
+where
+	B::Hash: DeserializeOwned,
+	B::Header: DeserializeOwned,
+{
 	/// Save the given data to the top keys snapshot.
 	fn save_top_snapshot(&self, data: &[KeyValue], path: &PathBuf) -> Result<(), &'static str> {
 		let mut path = path.clone();
@@ -726,12 +696,13 @@ impl<B: BlockT + DeserializeOwned> Builder<B> {
 
 		let child_kv = match self.mode.clone() {
 			Mode::Online(_) => self.load_child_remote_and_maybe_save(&top_kv).await?,
-			Mode::OfflineOrElseOnline(offline_config, _) =>
+			Mode::OfflineOrElseOnline(offline_config, _) => {
 				if let Ok(kv) = self.load_child_snapshot(&offline_config.state_snapshot.path) {
 					kv
 				} else {
 					self.load_child_remote_and_maybe_save(&top_kv).await?
-				},
+				}
+			},
 			Mode::Offline(ref config) => self
 				.load_child_snapshot(&config.state_snapshot.path)
 				.map_err(|why| {
@@ -749,7 +720,7 @@ impl<B: BlockT + DeserializeOwned> Builder<B> {
 }
 
 // Public methods
-impl<B: BlockT + DeserializeOwned> Builder<B> {
+impl<B: BlockT> Builder<B> {
 	/// Create a new builder.
 	pub fn new() -> Self {
 		Default::default()
@@ -824,7 +795,13 @@ impl<B: BlockT + DeserializeOwned> Builder<B> {
 		}
 		self
 	}
+}
 
+// Public methods
+impl<B: BlockT> Builder<B>
+where
+	B::Header: DeserializeOwned,
+{
 	/// Build the test externalities.
 	pub async fn build(self) -> Result<TestExternalities, &'static str> {
 		let state_version = self.state_version;