[try-runtime-cli] Offchain worker support (#8966)

* make remote-ext work with ws and safe RPCs

* Update docs.

* Update utils/frame/remote-externalities/Cargo.toml

Co-authored-by: Niklas Adolfsson <niklasadolfsson1@gmail.com>

* Fix test

* Update lock file

* Update utils/frame/remote-externalities/src/lib.rs

Co-authored-by: Bastian Köcher <bkchr@users.noreply.github.com>

* Fix build again.

* checkpoint, merging the paged rpc now

* revert lifetime stuff

* WIP: remote client init not working

* Small cleanups

* use jsonrpsee alpha.7

* WIP

* Executiing without errors

* Reorg & cleanup

* Trivial cleaning

* Add txpool & keystore extension

* Small cleaning

* More :cleaning

* Flags: page-size, override-code

* WIP

* Apply suggestions from code review

Co-authored-by: Kian Paimani <5588131+kianenigma@users.noreply.github.com>

* Remove heap_pages

* Dry code extraction from state

* Formatting

* More formatting

* Add issue todo

* Use jsonrpsee 0.2.0

* Try trigger gitlab

* Fix "block_import_works" test

* fix native_big_block_import_fails_on_fallback test

* fix commit should work

* Rewrite UI tests

* Revert "Rewrite UI tests"

This reverts commit ada7f670f701c21fb399946a3f6918453f537bcb.

* try again with UI

* Use const for legacy heap pages val

* Move parse module to its own file

* Move rpc_api module to its own file

* Apply suggestions from code review

Co-authored-by: Peter Goodspeed-Niklaus <coriolinus@users.noreply.github.com>

* trait names: Block, not B

* Corect HEAP_PAGES_TEST_LEGACY export

* Update utils/frame/remote-externalities/src/rpc_api.rs

Co-authored-by: Kian Paimani <5588131+kianenigma@users.noreply.github.com>

* Revert test_ext heap_page insert; adjust storage root instead

* Doc comments for try_runtime::cli::Command

* TryRuntime stub

* trailing comma

* Remove unused dev dep in frame-executive

* Improve parse::hash variable name & error index

* Use Result for rpc_api fns

* Richer err messagges

* Remove HEAP_PAGE_TEST_LEGACY

* Update bin/node/executor/tests/basic.rs

Co-authored-by: kianenigma <kian@parity.io>
Co-authored-by: Kian Paimani <5588131+kianenigma@users.noreply.github.com>
Co-authored-by: Niklas Adolfsson <niklasadolfsson1@gmail.com>
Co-authored-by: Bastian Köcher <bkchr@users.noreply.github.com>
Co-authored-by: Peter Goodspeed-Niklaus <coriolinus@users.noreply.github.com>

substrate/utils/frame/remote-externalities/Cargo.toml

@@ -20,8 +20,8 @@ hex = "0.4.0"
 env_logger = "0.8.2"
 log = "0.4.11"
 codec = { package = "parity-scale-codec", version = "2.0.0" }
-
 serde_json = "1.0"
+serde = "1.0.0"
 
 sp-io = { version = "3.0.0", path = "../../../primitives/io" }
 sp-core = { version = "3.0.0", path = "../../../primitives/core" }

substrate/utils/frame/remote-externalities/src/lib.rs

@@ -34,9 +34,11 @@ use sp_core::{
 use codec::{Encode, Decode};
 use sp_runtime::traits::Block as BlockT;
 use jsonrpsee_ws_client::{
-	WsClientBuilder, WsClient, v2::params::JsonRpcParams, traits::Client,
+	WsClientBuilder, WsClient, v2::params::JsonRpcParams,
 };
 
+pub mod rpc_api;
+
 type KeyPair = (StorageKey, StorageData);
 
 const LOG_TARGET: &str = "remote-ext";
@@ -72,7 +74,7 @@ impl<B: BlockT> Default for Mode<B> {
 	}
 }
 
-/// configuration of the online execution.
+/// Configuration of the offline execution.
 ///
 /// A state snapshot config must be present.
 #[derive(Clone)]
@@ -81,7 +83,7 @@ pub struct OfflineConfig {
 	pub state_snapshot: SnapshotConfig,
 }
 
-/// Description of the transport protocol.
+/// Description of the transport protocol (for online execution).
 #[derive(Debug)]
 pub struct Transport {
 	uri: String,
@@ -115,10 +117,17 @@ pub struct OnlineConfig<B: BlockT> {
 	pub transport: Transport,
 }
 
+impl<B: BlockT> OnlineConfig<B> {
+	/// Return rpc (ws) client.
+	fn rpc_client(&self) -> &WsClient {
+		self.transport.client.as_ref().expect("ws client must have been initialized by now; qed.")
+	}
+}
+
 impl<B: BlockT> Default for OnlineConfig<B> {
 	fn default() -> Self {
 		Self {
-			transport: Transport { uri: DEFAULT_TARGET.to_string(), client: None },
+			transport: Transport { uri: DEFAULT_TARGET.to_owned(), client: None },
 			at: None,
 			state_snapshot: None,
 			modules: vec![],
@@ -126,12 +135,6 @@ impl<B: BlockT> Default for OnlineConfig<B> {
 	}
 }
 
-impl<B: BlockT> OnlineConfig<B> {
-	/// Return rpc (ws) client.
-	fn rpc_client(&self) -> &WsClient {
-		self.transport.client.as_ref().expect("ws client must have been initialized by now; qed.")
-	}
-}
 
 /// Configuration of the state snapshot.
 #[derive(Clone)]
@@ -189,6 +192,7 @@ impl<B: BlockT> Builder<B> {
 
 // RPC methods
 impl<B: BlockT> Builder<B> {
+	/// Get the latest finalized head.
 	async fn rpc_get_head(&self) -> Result<B::Hash, &'static str> {
 		trace!(target: LOG_TARGET, "rpc: finalized_head");
 		RpcApi::<B>::finalized_head(self.as_online().rpc_client()).await.map_err(|e| {
@@ -250,6 +254,7 @@ impl<B: BlockT> Builder<B> {
 		prefix: StorageKey,
 		at: B::Hash,
 	) -> Result<Vec<KeyPair>, &'static str> {
+		use jsonrpsee_ws_client::traits::Client;
 		use serde_json::to_value;
 		let keys = self.get_keys_paged(prefix, at).await?;
 		let keys_count = keys.len();
@@ -438,8 +443,10 @@ impl<B: BlockT> Builder<B> {
 		info!(target: LOG_TARGET, "injecting a total of {} keys", kv.len());
 		for (k, v) in kv {
 			let (k, v) = (k.0, v.0);
+			// Insert the key,value pair into the test trie backend
 			ext.insert(k, v);
 		}
+
 		Ok(ext)
 	}
 }

substrate/utils/frame/remote-externalities/src/rpc_api.rs

@@ -0,0 +1,53 @@
+// This file is part of Substrate.
+
+// Copyright (C) 2021 Parity Technologies (UK) Ltd.
+// SPDX-License-Identifier: Apache-2.0
+
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// 	http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+//! WS RPC API for one off RPC calls to a substrate node.
+// TODO: Consolidate one off RPC calls https://github.com/paritytech/substrate/issues/8988
+
+use super::*;
+
+/// Get the header of the block identified by `at`
+pub async fn get_header<B: BlockT, S: AsRef<str>>(from: S, at: B::Hash) -> Result<B::Header, String>
+where
+	B::Header: serde::de::DeserializeOwned,
+{
+	use jsonrpsee_ws_client::traits::Client;
+	let at = serde_json::to_value(at)
+		.map_err(|e| format!("Block hash could not be converted to JSON due to {:?}", e))?;
+	let params = vec![at];
+	let client = WsClientBuilder::default()
+		.max_request_body_size(u32::MAX)
+		.build(from.as_ref())
+		.await
+		.map_err(|e| format!("`WsClientBuilder` failed to build do to {:?}", e))?;
+	client.request::<B::Header>("chain_getHeader", JsonRpcParams::Array(params))
+		.await
+		.map_err(|e| format!("chain_getHeader request failed due to {:?}", e))
+}
+
+/// Get the finalized head
+pub async fn get_finalized_head<B: BlockT, S: AsRef<str>>(from: S) -> Result<B::Hash, String> {
+	use jsonrpsee_ws_client::traits::Client;
+	let client = WsClientBuilder::default()
+		.max_request_body_size(u32::MAX)
+		.build(from.as_ref())
+		.await
+		.map_err(|e| format!("`WsClientBuilder` failed to build do to {:?}", e))?;
+	client.request::<B::Hash>("chain_getFinalizedHead", JsonRpcParams::NoParams)
+		.await
+		.map_err(|e| format!("chain_getFinalizedHead request failed due to {:?}", e))
+}