Migration testing runtime API/Bot (#8038)

* A clean new attempt

* Checkpoint to move remote.

* A lot of dependency wiring to make it feature gated.

* bad macro, bad macro.

* Undo the DB mess.

* Update frame/support/src/traits.rs

Co-authored-by: Alexander Popiak <alexander.popiak@parity.io>

* Apply suggestions from code review

Co-authored-by: Alexander Popiak <alexander.popiak@parity.io>

* unbreak the build

* Update frame/try-runtime/src/lib.rs

Co-authored-by: Bastian Köcher <bkchr@users.noreply.github.com>

* Update utils/frame/try-runtime/cli/Cargo.toml

Co-authored-by: Shawn Tabrizi <shawntabrizi@gmail.com>

* Update frame/try-runtime/Cargo.toml

Co-authored-by: Shawn Tabrizi <shawntabrizi@gmail.com>

* Address most review grumbles.

* Fix build

* Add some comments

* Remove allowing one pallet at a time.

* More grumbles.

* relocate remote-ext

* Fix build

Co-authored-by: Alexander Popiak <alexander.popiak@parity.io>
Co-authored-by: Bastian Köcher <bkchr@users.noreply.github.com>
Co-authored-by: Shawn Tabrizi <shawntabrizi@gmail.com>

substrate/.gitignore

@@ -23,3 +23,4 @@ rls*.log
 **/hfuzz_workspace/
 .cargo/
 .cargo-remote.toml
+*.bin

substrate/Cargo.lock

@@ -1861,6 +1861,17 @@ dependencies = [
  "sp-api",
 ]
 
+[[package]]
+name = "frame-try-runtime"
+version = "0.9.0"
+dependencies = [
+ "frame-support",
+ "parity-scale-codec",
+ "sp-api",
+ "sp-runtime",
+ "sp-std",
+]
+
 [[package]]
 name = "fs-swap"
 version = "0.2.5"
@@ -3949,6 +3960,7 @@ dependencies = [
  "substrate-build-script-utils",
  "substrate-frame-cli",
  "tempfile",
+ "try-runtime-cli",
  "wasm-bindgen",
  "wasm-bindgen-futures",
 ]
@@ -4072,6 +4084,7 @@ dependencies = [
  "frame-system",
  "frame-system-benchmarking",
  "frame-system-rpc-runtime-api",
+ "frame-try-runtime",
  "hex-literal",
  "node-primitives",
  "pallet-assets",
@@ -6372,6 +6385,24 @@ dependencies = [
  "winapi 0.3.9",
 ]
 
+[[package]]
+name = "remote-externalities"
+version = "0.9.0"
+dependencies = [
+ "async-std",
+ "bincode",
+ "env_logger 0.8.2",
+ "futures 0.1.30",
+ "hex-literal",
+ "jsonrpc-core-client",
+ "log",
+ "sc-rpc",
+ "sc-rpc-api",
+ "sp-core",
+ "sp-io",
+ "tokio 0.1.22",
+]
+
 [[package]]
 name = "remove_dir_all"
 version = "0.5.3"
@@ -9898,6 +9929,27 @@ version = "0.2.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "59547bce71d9c38b83d9c0e92b6066c4253371f15005def0c30d9657f50c7642"
 
+[[package]]
+name = "try-runtime-cli"
+version = "0.9.0"
+dependencies = [
+ "frame-try-runtime",
+ "log",
+ "parity-scale-codec",
+ "remote-externalities",
+ "sc-cli",
+ "sc-client-api",
+ "sc-executor",
+ "sc-service",
+ "sp-api",
+ "sp-blockchain",
+ "sp-core",
+ "sp-externalities",
+ "sp-runtime",
+ "sp-state-machine",
+ "structopt",
+]
+
 [[package]]
 name = "trybuild"
 version = "1.0.39"

substrate/Cargo.toml

@@ -73,6 +73,7 @@ members = [
 	"frame/contracts/rpc",
 	"frame/contracts/rpc/runtime-api",
 	"frame/democracy",
+	"frame/try-runtime",
 	"frame/elections",
 	"frame/example",
 	"frame/example-offchain-worker",
@@ -185,7 +186,9 @@ members = [
 	"utils/build-script-utils",
 	"utils/fork-tree",
 	"utils/frame/benchmarking-cli",
+	"utils/frame/remote-externalities",
 	"utils/frame/frame-utilities-cli",
+	"utils/frame/try-runtime/cli",
 	"utils/frame/rpc/support",
 	"utils/frame/rpc/system",
 	"utils/prometheus",

substrate/bin/node/cli/Cargo.toml

@@ -99,6 +99,7 @@ node-executor = { version = "2.0.0", path = "../executor" }
 sc-cli = { version = "0.9.0", optional = true, path = "../../../client/cli" }
 frame-benchmarking-cli = { version = "3.0.0", optional = true, path = "../../../utils/frame/benchmarking-cli" }
 node-inspect = { version = "0.8.0", optional = true, path = "../inspect" }
+try-runtime-cli = { version = "0.9.0", optional = true, path = "../../../utils/frame/try-runtime/cli" }
 
 # WASM-specific dependencies
 wasm-bindgen = { version = "0.2.57", optional = true }
@@ -133,6 +134,7 @@ node-inspect = { version = "0.8.0", optional = true, path = "../inspect" }
 frame-benchmarking-cli = { version = "3.0.0", optional = true, path = "../../../utils/frame/benchmarking-cli" }
 substrate-build-script-utils = { version = "3.0.0", optional = true, path = "../../../utils/build-script-utils" }
 substrate-frame-cli = { version = "3.0.0", optional = true, path = "../../../utils/frame/frame-utilities-cli" }
+try-runtime-cli = { version = "0.9.0", optional = true, path = "../../../utils/frame/try-runtime/cli" }
 
 [build-dependencies.sc-cli]
 version = "0.9.0"
@@ -157,8 +159,15 @@ cli = [
 	"sc-finality-grandpa-warp-sync",
 	"structopt",
 	"substrate-build-script-utils",
+	"try-runtime-cli",
 ]
 runtime-benchmarks = [
 	"node-runtime/runtime-benchmarks",
 	"frame-benchmarking-cli",
 ]
+# Enable features that allow the runtime to be tried and debugged. Name might be subject to change
+# in the near future.
+try-runtime = [
+	"node-runtime/try-runtime",
+	"try-runtime-cli",
+]

substrate/bin/node/cli/src/cli.rs

@@ -47,6 +47,11 @@ pub enum Subcommand {
 	#[structopt(name = "benchmark", about = "Benchmark runtime pallets.")]
 	Benchmark(frame_benchmarking_cli::BenchmarkCmd),
 
+	/// Try some experimental command on the runtime. This includes migration and runtime-upgrade
+	/// testing.
+	#[cfg(feature = "try-runtime")]
+	TryRuntime(try_runtime_cli::TryRuntimeCmd),
+
 	/// Verify a signature for a message, provided on STDIN, with a given (public or secret) key.
 	Verify(VerifyCmd),
 

substrate/bin/node/cli/src/command.rs

@@ -149,5 +149,20 @@ pub fn run() -> Result<()> {
 				Ok((cmd.run(client, backend), task_manager))
 			})
 		},
+		#[cfg(feature = "try-runtime")]
+		Some(Subcommand::TryRuntime(cmd)) => {
+			let runner = cli.create_runner(cmd)?;
+			runner.async_run(|config| {
+				// we don't need any of the components of new_partial, just a runtime, or a task
+				// manager to do `async_run`.
+				let registry = config.prometheus_config.as_ref().map(|cfg| &cfg.registry);
+				let task_manager = sc_service::TaskManager::new(
+					config.task_executor.clone(),
+					registry,
+				).unwrap();
+
+				Ok((cmd.run::<Block, Executor>(config), task_manager))
+			})
+		}
 	}
 }

substrate/bin/node/runtime/Cargo.toml

@@ -43,6 +43,7 @@ frame-support = { version = "3.0.0", default-features = false, path = "../../../
 frame-system = { version = "3.0.0", default-features = false, path = "../../../frame/system" }
 frame-system-benchmarking = { version = "3.0.0", default-features = false, path = "../../../frame/system/benchmarking", optional = true }
 frame-system-rpc-runtime-api = { version = "3.0.0", default-features = false, path = "../../../frame/system/rpc/runtime-api/" }
+frame-try-runtime = { version = "0.9.0", default-features = false, path = "../../../frame/try-runtime", optional = true }
 pallet-assets = { version = "3.0.0", default-features = false, path = "../../../frame/assets" }
 pallet-authority-discovery = { version = "3.0.0", default-features = false, path = "../../../frame/authority-discovery" }
 pallet-authorship = { version = "3.0.0", default-features = false, path = "../../../frame/authorship" }
@@ -186,3 +187,7 @@ runtime-benchmarks = [
 	"frame-system-benchmarking",
 	"hex-literal",
 ]
+try-runtime = [
+	"frame-executive/try-runtime",
+	"frame-try-runtime",
+]

substrate/bin/node/runtime/src/lib.rs

@@ -1080,7 +1080,14 @@ pub type SignedPayload = generic::SignedPayload<Call, SignedExtra>;
 /// Extrinsic type that has already been checked.
 pub type CheckedExtrinsic = generic::CheckedExtrinsic<AccountId, Call, SignedExtra>;
 /// Executive: handles dispatch to the various modules.
-pub type Executive = frame_executive::Executive<Runtime, Block, frame_system::ChainContext<Runtime>, Runtime, AllModules>;
+pub type Executive = frame_executive::Executive<
+	Runtime,
+	Block,
+	frame_system::ChainContext<Runtime>,
+	Runtime,
+	AllModules,
+	(),
+>;
 
 /// MMR helper types.
 mod mmr {
@@ -1325,15 +1332,24 @@ impl_runtime_apis! {
 		}
 	}
 
+	#[cfg(feature = "try-runtime")]
+	impl frame_try_runtime::TryRuntime<Block> for Runtime {
+		fn on_runtime_upgrade() -> Result<(Weight, Weight), sp_runtime::RuntimeString> {
+			frame_support::debug::RuntimeLogger::init();
+			let weight = Executive::try_runtime_upgrade()?;
+			Ok((weight, RuntimeBlockWeights::get().max_block))
+		}
+	}
+
 	#[cfg(feature = "runtime-benchmarks")]
 	impl frame_benchmarking::Benchmark<Block> for Runtime {
 		fn dispatch_benchmark(
 			config: frame_benchmarking::BenchmarkConfig
 		) -> Result<Vec<frame_benchmarking::BenchmarkBatch>, sp_runtime::RuntimeString> {
 			use frame_benchmarking::{Benchmarking, BenchmarkBatch, add_benchmark, TrackedStorageKey};
-			// Trying to add benchmarks directly to the Session Pallet caused cyclic dependency issues.
-			// To get around that, we separated the Session benchmarks into its own crate, which is why
-			// we need these two lines below.
+			// Trying to add benchmarks directly to the Session Pallet caused cyclic dependency
+			// issues. To get around that, we separated the Session benchmarks into its own crate,
+			// which is why we need these two lines below.
 			use pallet_session_benchmarking::Module as SessionBench;
 			use pallet_offences_benchmarking::Module as OffencesBench;
 			use frame_system_benchmarking::Module as SystemBench;

substrate/client/service/src/task_manager/mod.rs

@@ -234,7 +234,7 @@ pub struct TaskManager {
 impl TaskManager {
 	/// If a Prometheus registry is passed, it will be used to report statistics about the
 	/// service tasks.
-	pub(super) fn new(
+	pub fn new(
 		executor: TaskExecutor,
 		prometheus_registry: Option<&Registry>,
 	) -> Result<Self, PrometheusError> {

substrate/client/state-db/src/lib.rs

@@ -24,12 +24,12 @@
 //! Canonicalization window tracks a tree of blocks identified by header hash. The in-memory
 //! overlay allows to get any node that was inserted in any of the blocks within the window.
 //! The tree is journaled to the backing database and rebuilt on startup.
-//! Canonicalization function selects one root from the top of the tree and discards all other roots and
-//! their subtrees.
+//! Canonicalization function selects one root from the top of the tree and discards all other roots
+//! and their subtrees.
 //!
 //! # Pruning.
-//! See `RefWindow` for pruning algorithm details. `StateDb` prunes on each canonicalization until pruning
-//! constraints are satisfied.
+//! See `RefWindow` for pruning algorithm details. `StateDb` prunes on each canonicalization until
+//! pruning constraints are satisfied.
 
 mod noncanonical;
 mod pruning;

substrate/frame/executive/Cargo.toml

@@ -47,3 +47,6 @@ std = [
 	"sp-tracing/std",
 	"sp-std/std",
 ]
+try-runtime = [
+	"frame-support/try-runtime"
+]

substrate/frame/executive/src/lib.rs

@@ -44,7 +44,8 @@
 //!
 //! ## Usage
 //!
-//! The default Substrate node template declares the [`Executive`](./struct.Executive.html) type in its library.
+//! The default Substrate node template declares the [`Executive`](./struct.Executive.html) type in
+//! its library.
 //!
 //! ### Example
 //!
@@ -185,26 +186,58 @@ where
 }
 
 impl<
-	System: frame_system::Config,
-	Block: traits::Block<Header=System::Header, Hash=System::Hash>,
-	Context: Default,
-	UnsignedValidator,
-	AllModules:
-		OnRuntimeUpgrade +
-		OnInitialize<System::BlockNumber> +
-		OnFinalize<System::BlockNumber> +
-		OffchainWorker<System::BlockNumber>,
-	COnRuntimeUpgrade: OnRuntimeUpgrade,
-> Executive<System, Block, Context, UnsignedValidator, AllModules, COnRuntimeUpgrade>
+		System: frame_system::Config,
+		Block: traits::Block<Header = System::Header, Hash = System::Hash>,
+		Context: Default,
+		UnsignedValidator,
+		AllModules: OnRuntimeUpgrade
+			+ OnInitialize<System::BlockNumber>
+			+ OnFinalize<System::BlockNumber>
+			+ OffchainWorker<System::BlockNumber>,
+		COnRuntimeUpgrade: OnRuntimeUpgrade,
+	> Executive<System, Block, Context, UnsignedValidator, AllModules, COnRuntimeUpgrade>
 where
 	Block::Extrinsic: Checkable<Context> + Codec,
-	CheckedOf<Block::Extrinsic, Context>:
-		Applyable +
-		GetDispatchInfo,
-	CallOf<Block::Extrinsic, Context>: Dispatchable<Info=DispatchInfo, PostInfo=PostDispatchInfo>,
+	CheckedOf<Block::Extrinsic, Context>: Applyable + GetDispatchInfo,
+	CallOf<Block::Extrinsic, Context>:
+		Dispatchable<Info = DispatchInfo, PostInfo = PostDispatchInfo>,
 	OriginOf<Block::Extrinsic, Context>: From<Option<System::AccountId>>,
-	UnsignedValidator: ValidateUnsigned<Call=CallOf<Block::Extrinsic, Context>>,
+	UnsignedValidator: ValidateUnsigned<Call = CallOf<Block::Extrinsic, Context>>,
 {
+	/// Execute all `OnRuntimeUpgrade` of this runtime, and return the aggregate weight.
+	pub fn execute_on_runtime_upgrade() -> frame_support::weights::Weight {
+		let mut weight = 0;
+		weight = weight.saturating_add(
+			<frame_system::Module<System> as OnRuntimeUpgrade>::on_runtime_upgrade(),
+		);
+		weight = weight.saturating_add(COnRuntimeUpgrade::on_runtime_upgrade());
+		weight = weight.saturating_add(<AllModules as OnRuntimeUpgrade>::on_runtime_upgrade());
+
+		weight
+	}
+
+	/// Execute all `OnRuntimeUpgrade` of this runtime, including the pre and post migration checks.
+	///
+	/// This should only be used for testing.
+	#[cfg(feature = "try-runtime")]
+	pub fn try_runtime_upgrade() -> Result<frame_support::weights::Weight, &'static str> {
+		<
+			(frame_system::Module::<System>, COnRuntimeUpgrade, AllModules)
+			as
+			OnRuntimeUpgrade
+		>::pre_upgrade()?;
+
+		let weight = Self::execute_on_runtime_upgrade();
+
+		<
+			(frame_system::Module::<System>, COnRuntimeUpgrade, AllModules)
+			as
+			OnRuntimeUpgrade
+		>::post_upgrade()?;
+
+		Ok(weight)
+	}
+
 	/// Start the execution of a particular block.
 	pub fn initialize_block(header: &System::Header) {
 		sp_io::init_tracing();
@@ -234,10 +267,7 @@ where
 	) {
 		let mut weight = 0;
 		if Self::runtime_upgraded() {
-			// System is not part of `AllModules`, so we need to call this manually.
-			weight = weight.saturating_add(<frame_system::Module::<System> as OnRuntimeUpgrade>::on_runtime_upgrade());
-			weight = weight.saturating_add(COnRuntimeUpgrade::on_runtime_upgrade());
-			weight = weight.saturating_add(<AllModules as OnRuntimeUpgrade>::on_runtime_upgrade());
+			weight = weight.saturating_add(Self::execute_on_runtime_upgrade());
 		}
 		<frame_system::Module<System>>::initialize(
 			block_number,

substrate/frame/support/Cargo.toml

@@ -60,3 +60,4 @@ std = [
 nightly = []
 strict = []
 runtime-benchmarks = []
+try-runtime = []

substrate/frame/support/src/traits.rs

@@ -1547,7 +1547,25 @@ pub trait OnRuntimeUpgrade {
 	/// block local data are not accessible.
 	///
 	/// Return the non-negotiable weight consumed for runtime upgrade.
-	fn on_runtime_upgrade() -> crate::weights::Weight { 0 }
+	fn on_runtime_upgrade() -> crate::weights::Weight {
+		0
+	}
+
+	/// Execute some pre-checks prior to a runtime upgrade.
+	///
+	/// This hook is never meant to be executed on-chain but is meant to be used by testing tools.
+	#[cfg(feature = "try-runtime")]
+	fn pre_upgrade() -> Result<(), &'static str> {
+		Ok(())
+	}
+
+	/// Execute some post-checks after a runtime upgrade.
+	///
+	/// This hook is never meant to be executed on-chain but is meant to be used by testing tools.
+	#[cfg(feature = "try-runtime")]
+	fn post_upgrade() -> Result<(), &'static str> {
+		Ok(())
+	}
 }
 
 #[impl_for_tuples(30)]
@@ -1557,6 +1575,20 @@ impl OnRuntimeUpgrade for Tuple {
 		for_tuples!( #( weight = weight.saturating_add(Tuple::on_runtime_upgrade()); )* );
 		weight
 	}
+
+	#[cfg(feature = "try-runtime")]
+	fn pre_upgrade() -> Result<(), &'static str> {
+		let mut result = Ok(());
+		for_tuples!( #( result = result.and(Tuple::pre_upgrade()); )* );
+		result
+	}
+
+	#[cfg(feature = "try-runtime")]
+	fn post_upgrade() -> Result<(), &'static str> {
+		let mut result = Ok(());
+		for_tuples!( #( result = result.and(Tuple::post_upgrade()); )* );
+		result
+	}
 }
 
 /// Off-chain computation trait.

substrate/frame/try-runtime/Cargo.toml

@@ -0,0 +1,31 @@
+[package]
+name = "frame-try-runtime"
+version = "0.9.0"
+authors = ["Parity Technologies <admin@parity.io>"]
+edition = "2018"
+license = "Apache-2.0"
+homepage = "https://substrate.dev"
+repository = "https://github.com/paritytech/substrate/"
+description = "FRAME pallet for democracy"
+readme = "README.md"
+
+[package.metadata.docs.rs]
+targets = ["x86_64-unknown-linux-gnu"]
+
+[dependencies]
+codec = { package = "parity-scale-codec", version = "2.0.0", default-features = false }
+
+sp-api = { version = "3.0.0", path = "../../primitives/api", default-features = false }
+sp-std = { version = "3.0.0", path = "../../primitives/std" , default-features = false }
+sp-runtime = { version = "3.0.0", path = "../../primitives/runtime" , default-features = false }
+
+frame-support = { version = "3.0.0", path = "../support", default-features = false }
+
+[features]
+default = [ "std" ]
+std = [
+	"sp-api/std",
+	"sp-std/std",
+	"sp-runtime/std",
+	"frame-support/std",
+]

substrate/frame/try-runtime/src/lib.rs

@@ -0,0 +1,37 @@
+// This file is part of Substrate.
+
+// Copyright (C) 2021 Parity Technologies (UK) Ltd.
+// SPDX-License-Identifier: Apache-2.0
+
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// 	http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+//! Supporting types for try-runtime, testing and dry-running commands.
+
+#![cfg_attr(not(feature = "std"), no_std)]
+
+use sp_std::prelude::*;
+use frame_support::weights::Weight;
+
+sp_api::decl_runtime_apis! {
+	/// Runtime api for testing the execution of a runtime upgrade.
+	pub trait TryRuntime {
+		/// dry-run runtime upgrades, returning the total weight consumed.
+		///
+		/// This should do EXACTLY the same operations as the runtime would have done in the case of
+		/// a runtime upgrade (e.g. pallet ordering must be the same)
+		///
+		/// Returns the consumed weight of the migration in case of a successful one, combined with
+		/// the total allowed block weight of the runtime.
+		fn on_runtime_upgrade() -> Result<(Weight, Weight), sp_runtime::RuntimeString>;
+	}
+}

substrate/primitives/core/src/hexdisplay.rs

@@ -71,6 +71,12 @@ impl AsBytesRef for sp_std::vec::Vec<u8> {
 	fn as_bytes_ref(&self) -> &[u8] { &self }
 }
 
+impl AsBytesRef for sp_storage::StorageKey {
+	fn as_bytes_ref(&self) -> &[u8] {
+		self.as_ref()
+	}
+}
+
 macro_rules! impl_non_endians {
 	( $( $t:ty ),* ) => { $(
 		impl AsBytesRef for $t {

substrate/primitives/state-machine/src/testing.rs

@@ -48,20 +48,22 @@ pub struct TestExternalities<H: Hasher, N: ChangesTrieBlockNumber = u64>
 where
 	H::Out: codec::Codec + Ord,
 {
+	/// The overlay changed storage.
 	overlay: OverlayedChanges,
 	offchain_db: TestPersistentOffchainDB,
-	storage_transaction_cache: StorageTransactionCache<
-		<InMemoryBackend<H> as Backend<H>>::Transaction, H, N
-	>,
-	backend: InMemoryBackend<H>,
+	storage_transaction_cache:
+		StorageTransactionCache<<InMemoryBackend<H> as Backend<H>>::Transaction, H, N>,
+	/// Storage backend.
+	pub backend: InMemoryBackend<H>,
 	changes_trie_config: Option<ChangesTrieConfiguration>,
 	changes_trie_storage: ChangesTrieInMemoryStorage<H, N>,
-	extensions: Extensions,
+	/// Extensions.
+	pub extensions: Extensions,
 }
 
 impl<H: Hasher, N: ChangesTrieBlockNumber> TestExternalities<H, N>
-	where
-		H::Out: Ord + 'static + codec::Codec
+where
+	H::Out: Ord + 'static + codec::Codec,
 {
 	/// Get externalities implementation.
 	pub fn ext(&mut self) -> Ext<H, N, InMemoryBackend<H>> {

substrate/primitives/storage/src/lib.rs

@@ -31,10 +31,15 @@ use codec::{Encode, Decode};
 #[derive(PartialEq, Eq, RuntimeDebug)]
 #[cfg_attr(feature = "std", derive(Serialize, Deserialize, Hash, PartialOrd, Ord, Clone))]
 pub struct StorageKey(
-	#[cfg_attr(feature = "std", serde(with="impl_serde::serialize"))]
-	pub Vec<u8>,
+	#[cfg_attr(feature = "std", serde(with = "impl_serde::serialize"))] pub Vec<u8>,
 );
 
+impl AsRef<[u8]> for StorageKey {
+	fn as_ref(&self) -> &[u8] {
+		self.0.as_ref()
+	}
+}
+
 /// Storage key with read/write tracking information.
 #[derive(PartialEq, Eq, RuntimeDebug, Clone, Encode, Decode)]
 #[cfg_attr(feature = "std", derive(Hash, PartialOrd, Ord))]

substrate/utils/frame/remote-externalities/Cargo.toml

@@ -0,0 +1,34 @@
+[package]
+name = "remote-externalities"
+version = "0.9.0"
+authors = ["Parity Technologies <admin@parity.io>"]
+edition = "2018"
+license = "Apache-2.0"
+homepage = "https://substrate.dev"
+repository = "https://github.com/paritytech/substrate/"
+description = "An externalities provided environemnt that can load itself from remote nodes or cache files"
+readme = "README.md"
+
+[package.metadata.docs.rs]
+targets = ["x86_64-unknown-linux-gnu"]
+
+[dependencies]
+jsonrpc-core-client = { version = "15.1.0", features = ["http"] }
+sc-rpc-api = { version = "0.9.0", path = "../../../client/rpc-api" }
+sc-rpc = { version = "3.0.0", path = "../../../client/rpc" }
+futures = "0.1.29"
+
+hex-literal = "0.3.1"
+env_logger = "0.8.2"
+log = "0.4.11"
+bincode = "1.3.1"
+tokio = "0.1.22"
+
+sp-io = { version = "3.0.0", path = "../../../primitives/io" }
+sp-core = { version = "3.0.0", path = "../../../primitives/core" }
+
+[dev-dependencies]
+async-std = { version = "1.6.5", features = ["attributes"] }
+
+[features]
+remote-test = []

substrate/utils/frame/remote-externalities/src/lib.rs

@@ -0,0 +1,454 @@
+// This file is part of Substrate.
+
+// Copyright (C) 2020-2021 Parity Technologies (UK) Ltd.
+// SPDX-License-Identifier: Apache-2.0
+
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// 	http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+//! # Remote Externalities
+//!
+//! An equivalent of `sp_io::TestExternalities` that can load its state from a remote substrate
+//! based chain, or a local cache file.
+//!
+//! #### Runtime to Test Against
+//!
+//! While not absolutely necessary, you most likely need a `Runtime` equivalent in your test setup
+//! through which you can infer storage types. There are two options here:
+//!
+//! 1. Build a mock runtime, similar how to you would build one in a pallet test (see example
+//!    below). The very important point here is that this mock needs to hold real values for types
+//!    that matter for you, based on the chain of interest. Some typical ones are:
+//!
+//! - `sp_runtime::AccountId32` as `AccountId`.
+//! - `u32` as `BlockNumber`.
+//! - `u128` as Balance.
+//!
+//! Once you have your `Runtime`, you can use it for storage type resolution and do things like
+//! `<my_pallet::Pallet<Runtime>>::storage_getter()` or `<my_pallet::StorageItem<Runtime>>::get()`.
+//!
+//! 2. Or, you can use a real runtime.
+//!
+//! ### Example
+//!
+//! With a test runtime
+//!
+//! ```ignore
+//! use remote_externalities::Builder;
+//!
+//! #[derive(Clone, Eq, PartialEq, Debug, Default)]
+//! pub struct TestRuntime;
+//!
+//! use frame_system as system;
+//! impl_outer_origin! {
+//!     pub enum Origin for TestRuntime {}
+//! }
+//!
+//! impl frame_system::Config for TestRuntime {
+//!     ..
+//!     // we only care about these two for now. The rest can be mock. The block number type of
+//!     // kusama is u32.
+//!     type BlockNumber = u32;
+//!     type Header = Header;
+//!     ..
+//! }
+//!
+//! #[test]
+//! fn test_runtime_works() {
+//!     let hash: Hash =
+//!         hex!["f9a4ce984129569f63edc01b1c13374779f9384f1befd39931ffdcc83acf63a7"].into();
+//!     let parent: Hash =
+//!         hex!["540922e96a8fcaf945ed23c6f09c3e189bd88504ec945cc2171deaebeaf2f37e"].into();
+//!     Builder::new()
+//!         .at(hash)
+//!         .module("System")
+//!         .build()
+//!         .execute_with(|| {
+//!             assert_eq!(
+//!                 // note: the hash corresponds to 3098546. We can check only the parent.
+//!                 // https://polkascan.io/kusama/block/3098546
+//!                 <frame_system::Module<Runtime>>::block_hash(3098545u32),
+//!                 parent,
+//!             )
+//!         });
+//! }
+//! ```
+//!
+//! Or with the real kusama runtime.
+//!
+//! ```ignore
+//! use remote_externalities::Builder;
+//! use kusama_runtime::Runtime;
+//!
+//! #[test]
+//! fn test_runtime_works() {
+//!     let hash: Hash =
+//!         hex!["f9a4ce984129569f63edc01b1c13374779f9384f1befd39931ffdcc83acf63a7"].into();
+//!     Builder::new()
+//!         .at(hash)
+//!         .module("Staking")
+//!         .build()
+//!         .execute_with(|| assert_eq!(<pallet_staking::Module<Runtime>>::validator_count(), 400));
+//! }
+//! ```
+
+use std::{
+	fs,
+	path::{Path, PathBuf},
+};
+use log::*;
+use sp_core::{hashing::twox_128};
+pub use sp_io::TestExternalities;
+use sp_core::{
+	hexdisplay::HexDisplay,
+	storage::{StorageKey, StorageData},
+};
+use futures::future::Future;
+
+type KeyPair = (StorageKey, StorageData);
+type Number = u32;
+type Hash = sp_core::H256;
+// TODO: make these two generic.
+
+const LOG_TARGET: &'static str = "remote-ext";
+
+/// The execution mode.
+#[derive(Clone)]
+pub enum Mode {
+	/// Online.
+	Online(OnlineConfig),
+	/// Offline. Uses a cached file and needs not any client config.
+	Offline(OfflineConfig),
+}
+
+/// configuration of the online execution.
+///
+/// A cache config must be present.
+#[derive(Clone)]
+pub struct OfflineConfig {
+	/// The configuration of the cache file to use. It must be present.
+	pub cache: CacheConfig,
+}
+
+/// Configuration of the online execution.
+///
+/// A cache config may be present and will be written to in that case.
+#[derive(Clone)]
+pub struct OnlineConfig {
+	/// The HTTP uri to use.
+	pub uri: String,
+	/// The block number at which to connect. Will be latest finalized head if not provided.
+	pub at: Option<Hash>,
+	/// An optional cache file to WRITE to, not for reading. Not cached if set to `None`.
+	pub cache: Option<CacheConfig>,
+	/// The modules to scrape. If empty, entire chain state will be scraped.
+	pub modules: Vec<String>,
+}
+
+impl Default for OnlineConfig {
+	fn default() -> Self {
+		Self {
+			uri: "http://localhost:9933".into(),
+			at: None,
+			cache: None,
+			modules: Default::default(),
+		}
+	}
+}
+
+/// Configuration of the cache.
+#[derive(Clone)]
+pub struct CacheConfig {
+	// TODO: I could mix these two into one filed, but I think separate is better bc one can be
+	// configurable while one not.
+	/// File name.
+	pub name: String,
+	/// Base directory.
+	pub directory: String,
+}
+
+impl Default for CacheConfig {
+	fn default() -> Self {
+		Self { name: "CACHE".into(), directory: ".".into() }
+	}
+}
+
+impl CacheConfig {
+	fn path(&self) -> PathBuf {
+		Path::new(&self.directory).join(self.name.clone())
+	}
+}
+
+/// Builder for remote-externalities.
+pub struct Builder {
+	inject: Vec<KeyPair>,
+	mode: Mode,
+	chain: String,
+}
+
+impl Default for Builder {
+	fn default() -> Self {
+		Self {
+			inject: Default::default(),
+			mode: Mode::Online(OnlineConfig {
+				at: None,
+				uri: "http://localhost:9933".into(),
+				cache: None,
+				modules: Default::default(),
+			}),
+			chain: "UNSET".into(),
+		}
+	}
+}
+
+// Mode methods
+impl Builder {
+	fn as_online(&self) -> &OnlineConfig {
+		match &self.mode {
+			Mode::Online(config) => &config,
+			_ => panic!("Unexpected mode: Online"),
+		}
+	}
+
+	fn as_online_mut(&mut self) -> &mut OnlineConfig {
+		match &mut self.mode {
+			Mode::Online(config) => config,
+			_ => panic!("Unexpected mode: Online"),
+		}
+	}
+}
+
+// RPC methods
+impl Builder {
+	async fn rpc_get_head(&self) -> Hash {
+		let mut rt = tokio::runtime::Runtime::new().expect("Unable to create a runtime");
+		let uri = self.as_online().uri.clone();
+		rt.block_on::<_, _, ()>(futures::lazy(move || {
+			trace!(target: LOG_TARGET, "rpc: finalized_head");
+			let client: sc_rpc_api::chain::ChainClient<Number, Hash, (), ()> =
+				jsonrpc_core_client::transports::http::connect(&uri).wait().unwrap();
+			Ok(client.finalized_head().wait().unwrap())
+		}))
+		.unwrap()
+	}
+
+	/// Relay the request to `state_getPairs` rpc endpoint.
+	///
+	/// Note that this is an unsafe RPC.
+	async fn rpc_get_pairs(&self, prefix: StorageKey, at: Hash) -> Vec<KeyPair> {
+		let mut rt = tokio::runtime::Runtime::new().expect("Unable to create a runtime");
+		let uri = self.as_online().uri.clone();
+		rt.block_on::<_, _, ()>(futures::lazy(move || {
+			trace!(target: LOG_TARGET, "rpc: storage_pairs: {:?} / {:?}", prefix, at);
+			let client: sc_rpc_api::state::StateClient<Hash> =
+				jsonrpc_core_client::transports::http::connect(&uri).wait().unwrap();
+			Ok(client.storage_pairs(prefix, Some(at)).wait().unwrap())
+		}))
+		.unwrap()
+	}
+
+	/// Get the chain name.
+	async fn chain_name(&self) -> String {
+		let mut rt = tokio::runtime::Runtime::new().expect("Unable to create a runtime");
+		let uri = self.as_online().uri.clone();
+		rt.block_on::<_, _, ()>(futures::lazy(move || {
+			trace!(target: LOG_TARGET, "rpc: system_chain");
+			let client: sc_rpc_api::system::SystemClient<(), ()> =
+				jsonrpc_core_client::transports::http::connect(&uri).wait().unwrap();
+			Ok(client.system_chain().wait().unwrap())
+		}))
+		.unwrap()
+	}
+}
+
+// Internal methods
+impl Builder {
+	/// Save the given data as cache.
+	fn save_cache(&self, data: &[KeyPair], path: &Path) {
+		let bdata = bincode::serialize(data).unwrap();
+		info!(target: LOG_TARGET, "writing to cache file {:?}", path);
+		fs::write(path, bdata).unwrap();
+	}
+
+	/// initialize `Self` from cache. Panics if the file does not exist.
+	fn load_cache(&self, path: &Path) -> Vec<KeyPair> {
+		info!(target: LOG_TARGET, "scraping keypairs from cache {:?}", path,);
+		let bytes = fs::read(path).unwrap();
+		bincode::deserialize(&bytes[..]).unwrap()
+	}
+
+	/// Build `Self` from a network node denoted by `uri`.
+	async fn load_remote(&self) -> Vec<KeyPair> {
+		let config = self.as_online();
+		let at = self.as_online().at.unwrap().clone();
+		info!(target: LOG_TARGET, "scraping keypairs from remote node {} @ {:?}", config.uri, at);
+
+		let keys_and_values = if config.modules.len() > 0 {
+			let mut filtered_kv = vec![];
+			for f in config.modules.iter() {
+				let hashed_prefix = StorageKey(twox_128(f.as_bytes()).to_vec());
+				let module_kv = self.rpc_get_pairs(hashed_prefix.clone(), at).await;
+				info!(
+					target: LOG_TARGET,
+					"downloaded data for module {} (count: {} / prefix: {:?}).",
+					f,
+					module_kv.len(),
+					HexDisplay::from(&hashed_prefix),
+				);
+				filtered_kv.extend(module_kv);
+			}
+			filtered_kv
+		} else {
+			info!(target: LOG_TARGET, "downloading data for all modules.");
+			self.rpc_get_pairs(StorageKey(vec![]), at).await.into_iter().collect::<Vec<_>>()
+		};
+
+		keys_and_values
+	}
+
+	async fn init_remote_client(&mut self) {
+		self.as_online_mut().at = Some(self.rpc_get_head().await);
+		self.chain = self.chain_name().await;
+	}
+
+	async fn pre_build(mut self) -> Vec<KeyPair> {
+		let mut base_kv = match self.mode.clone() {
+			Mode::Offline(config) => self.load_cache(&config.cache.path()),
+			Mode::Online(config) => {
+				self.init_remote_client().await;
+				let kp = self.load_remote().await;
+				if let Some(c) = config.cache {
+					self.save_cache(&kp, &c.path());
+				}
+				kp
+			}
+		};
+
+		info!(
+			target: LOG_TARGET,
+			"extending externalities with {} manually injected keys",
+			self.inject.len()
+		);
+		base_kv.extend(self.inject.clone());
+		base_kv
+	}
+}
+
+// Public methods
+impl Builder {
+	/// Create a new builder.
+	pub fn new() -> Self {
+		Default::default()
+	}
+
+	/// Inject a manual list of key and values to the storage.
+	pub fn inject(mut self, injections: &[KeyPair]) -> Self {
+		for i in injections {
+			self.inject.push(i.clone());
+		}
+		self
+	}
+
+	/// Configure a cache to be used.
+	pub fn mode(mut self, mode: Mode) -> Self {
+		self.mode = mode;
+		self
+	}
+
+	/// Build the test externalities.
+	pub async fn build(self) -> TestExternalities {
+		let kv = self.pre_build().await;
+		let mut ext = TestExternalities::new_empty();
+
+		info!(target: LOG_TARGET, "injecting a total of {} keys", kv.len());
+		for (k, v) in kv {
+			let (k, v) = (k.0, v.0);
+			ext.insert(k, v);
+		}
+		ext
+	}
+}
+
+#[cfg(test)]
+mod tests {
+	use super::*;
+
+	fn init_logger() {
+		let _ = env_logger::Builder::from_default_env()
+			.format_module_path(false)
+			.format_level(true)
+			.try_init();
+	}
+
+	#[async_std::test]
+	#[cfg(feature = "remote-test")]
+	async fn can_build_one_pallet() {
+		init_logger();
+		Builder::new()
+			.mode(Mode::Online(OnlineConfig {
+				modules: vec!["Proxy".into()],
+				..Default::default()
+			}))
+			.build()
+			.await
+			.execute_with(|| {});
+	}
+
+	#[async_std::test]
+	async fn can_load_cache() {
+		init_logger();
+		Builder::new()
+			.mode(Mode::Offline(OfflineConfig {
+				cache: CacheConfig { name: "proxy_test".into(), ..Default::default() },
+			}))
+			.build()
+			.await
+			.execute_with(|| {});
+	}
+
+	#[async_std::test]
+	#[cfg(feature = "remote-test")]
+	async fn can_create_cache() {
+		init_logger();
+		Builder::new()
+			.mode(Mode::Online(OnlineConfig {
+				cache: Some(CacheConfig {
+					name: "test_cache_to_remove.bin".into(),
+					..Default::default()
+				}),
+				..Default::default()
+			}))
+			.build()
+			.await
+			.execute_with(|| {});
+
+		let to_delete = std::fs::read_dir(CacheConfig::default().directory)
+			.unwrap()
+			.into_iter()
+			.map(|d| d.unwrap())
+			.filter(|p| p.path().extension().unwrap_or_default() == "bin")
+			.collect::<Vec<_>>();
+
+		assert!(to_delete.len() > 0);
+
+		for d in to_delete {
+			std::fs::remove_file(d.path()).unwrap();
+		}
+	}
+
+	#[async_std::test]
+	#[cfg(feature = "remote-test")]
+	async fn can_build_all() {
+		init_logger();
+		Builder::new().build().await.execute_with(|| {});
+	}
+}

substrate/utils/frame/try-runtime/cli/Cargo.toml

@@ -0,0 +1,32 @@
+[package]
+name = "try-runtime-cli"
+version = "0.9.0"
+authors = ["Parity Technologies <admin@parity.io>"]
+edition = "2018"
+license = "Apache-2.0"
+homepage = "https://substrate.dev"
+repository = "https://github.com/paritytech/substrate/"
+description = "Cli command runtime testing and dry-running"
+readme = "README.md"
+
+[package.metadata.docs.rs]
+targets = ["x86_64-unknown-linux-gnu"]
+
+[dependencies]
+log = "0.4.8"
+parity-scale-codec = { version = "2.0.0" }
+
+sc-service = { version = "0.9.0", default-features = false, path = "../../../../client/service" }
+sc-cli = { version = "0.9.0", path = "../../../../client/cli" }
+sc-executor = { path = "../../../../client/executor" }
+sc-client-api = { version = "3.0.0", path = "../../../../client/api" }
+structopt = "0.3.8"
+sp-state-machine = { version = "0.9.0", path = "../../../../primitives/state-machine" }
+sp-api = { version = "3.0.0", path = "../../../../primitives/api" }
+sp-blockchain = { version = "3.0.0", path = "../../../../primitives/blockchain" }
+sp-runtime = { version = "3.0.0", path = "../../../../primitives/runtime" }
+sp-externalities = { version = "0.9.0", path = "../../../../primitives/externalities" }
+sp-core = { version = "3.0.0", path = "../../../../primitives/core" }
+frame-try-runtime = { version = "0.9.0", path = "../../../../frame/try-runtime" }
+
+remote-externalities = { path = "../../remote-externalities" }

substrate/utils/frame/try-runtime/cli/src/lib.rs

@@ -0,0 +1,178 @@
+// This file is part of Substrate.
+
+// Copyright (C) 2021 Parity Technologies (UK) Ltd.
+// SPDX-License-Identifier: Apache-2.0
+
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// 	http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+//! `Structopt`-ready struct for `try-runtime`.
+
+use parity_scale_codec::Decode;
+use std::{fmt::Debug, str::FromStr};
+use sc_service::Configuration;
+use sc_cli::{CliConfiguration, ExecutionStrategy, WasmExecutionMethod};
+use sc_executor::NativeExecutor;
+use sc_service::NativeExecutionDispatch;
+use sp_state_machine::StateMachine;
+use sp_runtime::traits::{Block as BlockT, NumberFor};
+use sp_core::storage::{StorageData, StorageKey, well_known_keys};
+
+/// Various commands to try out the new runtime, over configurable states.
+///
+/// For now this only assumes running the `on_runtime_upgrade` hooks.
+#[derive(Debug, structopt::StructOpt)]
+pub struct TryRuntimeCmd {
+	/// The shared parameters
+	#[allow(missing_docs)]
+	#[structopt(flatten)]
+	pub shared_params: sc_cli::SharedParams,
+
+	/// The state to use to run the migration. Should be a valid FILE or HTTP URI.
+	#[structopt(short, long, default_value = "http://localhost:9933")]
+	pub state: State,
+
+	/// The execution strategy that should be used for benchmarks
+	#[structopt(
+		long = "execution",
+		value_name = "STRATEGY",
+		possible_values = &ExecutionStrategy::variants(),
+		case_insensitive = true,
+		default_value = "Native",
+	)]
+	pub execution: ExecutionStrategy,
+
+	/// Method for executing Wasm runtime code.
+	#[structopt(
+		long = "wasm-execution",
+		value_name = "METHOD",
+		possible_values = &WasmExecutionMethod::enabled_variants(),
+		case_insensitive = true,
+		default_value = "Interpreted"
+	)]
+	pub wasm_method: WasmExecutionMethod,
+}
+
+/// The state to use for a migration dry-run.
+#[derive(Debug)]
+pub enum State {
+	/// A snapshot. Inner value is a file path.
+	Snap(String),
+
+	/// A live chain. Inner value is the HTTP uri.
+	Live(String),
+}
+
+impl FromStr for State {
+	type Err = &'static str;
+	fn from_str(s: &str) -> Result<Self, Self::Err> {
+		match s.get(..7) {
+			// could use Url crate as well, but lets keep it simple for now.
+			Some("http://") => Ok(State::Live(s.to_string())),
+			Some("file://") => s
+				.split("//")
+				.collect::<Vec<_>>()
+				.get(1)
+				.map(|s| State::Snap(s.to_string()))
+				.ok_or("invalid file URI"),
+			_ => Err("invalid format. Must be a valid HTTP or File URI"),
+		}
+	}
+}
+
+impl TryRuntimeCmd {
+	pub async fn run<B, ExecDispatch>(&self, config: Configuration) -> sc_cli::Result<()>
+	where
+		B: BlockT,
+		ExecDispatch: NativeExecutionDispatch + 'static,
+	{
+		let spec = config.chain_spec;
+		let genesis_storage = spec.build_storage()?;
+
+		let code = StorageData(
+			genesis_storage
+				.top
+				.get(well_known_keys::CODE)
+				.expect("code key must exist in genesis storage; qed")
+				.to_vec(),
+		);
+		let code_key = StorageKey(well_known_keys::CODE.to_vec());
+
+		let wasm_method = self.wasm_method;
+		let execution = self.execution;
+
+		let mut changes = Default::default();
+		// don't really care about these -- use the default values.
+		let max_runtime_instances = config.max_runtime_instances;
+		let heap_pages = config.default_heap_pages;
+		let executor = NativeExecutor::<ExecDispatch>::new(
+			wasm_method.into(),
+			heap_pages,
+			max_runtime_instances,
+		);
+
+		let ext = {
+			use remote_externalities::{Builder, Mode, CacheConfig, OfflineConfig, OnlineConfig};
+			let builder = match &self.state {
+				State::Snap(file_path) => Builder::new().mode(Mode::Offline(OfflineConfig {
+					cache: CacheConfig { name: file_path.into(), ..Default::default() },
+				})),
+				State::Live(http_uri) => Builder::new().mode(Mode::Online(OnlineConfig {
+					uri: http_uri.into(),
+					..Default::default()
+				})),
+			};
+
+			// inject the code into this ext.
+			builder.inject(&[(code_key, code)]).build().await
+		};
+
+		let encoded_result = StateMachine::<_, _, NumberFor<B>, _>::new(
+			&ext.backend,
+			None,
+			&mut changes,
+			&executor,
+			"TryRuntime_on_runtime_upgrade",
+			&[],
+			ext.extensions,
+			&sp_state_machine::backend::BackendRuntimeCode::new(&ext.backend)
+				.runtime_code()?,
+			sp_core::testing::TaskExecutor::new(),
+		)
+		.execute(execution.into())
+		.map_err(|e| format!("failed to execute 'TryRuntime_on_runtime_upgrade' due to {:?}", e))?;
+
+		let (weight, total_weight) = <(u64, u64) as Decode>::decode(&mut &*encoded_result)
+			.map_err(|e| format!("failed to decode output due to {:?}", e))?;
+		log::info!(
+			"try-runtime executed without errors. Consumed weight = {}, total weight = {} ({})",
+			weight,
+			total_weight,
+			weight as f64 / total_weight as f64
+		);
+
+		Ok(())
+	}
+}
+
+impl CliConfiguration for TryRuntimeCmd {
+	fn shared_params(&self) -> &sc_cli::SharedParams {
+		&self.shared_params
+	}
+
+	fn chain_id(&self, _is_dev: bool) -> sc_cli::Result<String> {
+		Ok(match self.shared_params.chain {
+			Some(ref chain) => chain.clone(),
+			None => "dev".into(),
+		})
+	}
+}