From 16a27c28a92ef09e547550b6bb78f132856b5a97 Mon Sep 17 00:00:00 2001 From: Kian Paimani <5588131+kianenigma@users.noreply.github.com> Date: Fri, 19 Feb 2021 14:52:09 +0000 Subject: [PATCH] Migration testing runtime API/Bot (#8038) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * A clean new attempt * Checkpoint to move remote. * A lot of dependency wiring to make it feature gated. * bad macro, bad macro. * Undo the DB mess. * Update frame/support/src/traits.rs Co-authored-by: Alexander Popiak * Apply suggestions from code review Co-authored-by: Alexander Popiak * unbreak the build * Update frame/try-runtime/src/lib.rs Co-authored-by: Bastian Köcher * Update utils/frame/try-runtime/cli/Cargo.toml Co-authored-by: Shawn Tabrizi * Update frame/try-runtime/Cargo.toml Co-authored-by: Shawn Tabrizi * Address most review grumbles. * Fix build * Add some comments * Remove allowing one pallet at a time. * More grumbles. * relocate remote-ext * Fix build Co-authored-by: Alexander Popiak Co-authored-by: Bastian Köcher Co-authored-by: Shawn Tabrizi --- substrate/.gitignore | 1 + substrate/Cargo.lock | 52 ++ substrate/Cargo.toml | 3 + substrate/bin/node/cli/Cargo.toml | 9 + substrate/bin/node/cli/src/cli.rs | 5 + substrate/bin/node/cli/src/command.rs | 15 + substrate/bin/node/runtime/Cargo.toml | 5 + substrate/bin/node/runtime/src/lib.rs | 24 +- .../client/service/src/task_manager/mod.rs | 2 +- substrate/client/state-db/src/lib.rs | 8 +- substrate/frame/executive/Cargo.toml | 3 + substrate/frame/executive/src/lib.rs | 72 ++- substrate/frame/support/Cargo.toml | 1 + substrate/frame/support/src/traits.rs | 34 +- substrate/frame/try-runtime/Cargo.toml | 31 ++ substrate/frame/try-runtime/src/lib.rs | 37 ++ substrate/primitives/core/src/hexdisplay.rs | 6 + .../primitives/state-machine/src/testing.rs | 16 +- substrate/primitives/storage/src/lib.rs | 9 +- .../frame/remote-externalities/Cargo.toml | 34 ++ .../frame/remote-externalities/proxy_test | Bin 0 -> 26476 bytes .../frame/remote-externalities/src/lib.rs | 454 ++++++++++++++++++ .../utils/frame/try-runtime/cli/Cargo.toml | 32 ++ .../utils/frame/try-runtime/cli/src/lib.rs | 178 +++++++ 24 files changed, 991 insertions(+), 40 deletions(-) create mode 100644 substrate/frame/try-runtime/Cargo.toml create mode 100644 substrate/frame/try-runtime/src/lib.rs create mode 100644 substrate/utils/frame/remote-externalities/Cargo.toml create mode 100644 substrate/utils/frame/remote-externalities/proxy_test create mode 100644 substrate/utils/frame/remote-externalities/src/lib.rs create mode 100644 substrate/utils/frame/try-runtime/cli/Cargo.toml create mode 100644 substrate/utils/frame/try-runtime/cli/src/lib.rs diff --git a/substrate/.gitignore b/substrate/.gitignore index c8f1ea9567..ce302c74e1 100644 --- a/substrate/.gitignore +++ b/substrate/.gitignore @@ -23,3 +23,4 @@ rls*.log **/hfuzz_workspace/ .cargo/ .cargo-remote.toml +*.bin diff --git a/substrate/Cargo.lock b/substrate/Cargo.lock index 58c6baeb23..dea99ed63b 100644 --- a/substrate/Cargo.lock +++ b/substrate/Cargo.lock @@ -1861,6 +1861,17 @@ dependencies = [ "sp-api", ] +[[package]] +name = "frame-try-runtime" +version = "0.9.0" +dependencies = [ + "frame-support", + "parity-scale-codec", + "sp-api", + "sp-runtime", + "sp-std", +] + [[package]] name = "fs-swap" version = "0.2.5" @@ -3949,6 +3960,7 @@ dependencies = [ "substrate-build-script-utils", "substrate-frame-cli", "tempfile", + "try-runtime-cli", "wasm-bindgen", "wasm-bindgen-futures", ] @@ -4072,6 +4084,7 @@ dependencies = [ "frame-system", "frame-system-benchmarking", "frame-system-rpc-runtime-api", + "frame-try-runtime", "hex-literal", "node-primitives", "pallet-assets", @@ -6372,6 +6385,24 @@ dependencies = [ "winapi 0.3.9", ] +[[package]] +name = "remote-externalities" +version = "0.9.0" +dependencies = [ + "async-std", + "bincode", + "env_logger 0.8.2", + "futures 0.1.30", + "hex-literal", + "jsonrpc-core-client", + "log", + "sc-rpc", + "sc-rpc-api", + "sp-core", + "sp-io", + "tokio 0.1.22", +] + [[package]] name = "remove_dir_all" version = "0.5.3" @@ -9898,6 +9929,27 @@ version = "0.2.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "59547bce71d9c38b83d9c0e92b6066c4253371f15005def0c30d9657f50c7642" +[[package]] +name = "try-runtime-cli" +version = "0.9.0" +dependencies = [ + "frame-try-runtime", + "log", + "parity-scale-codec", + "remote-externalities", + "sc-cli", + "sc-client-api", + "sc-executor", + "sc-service", + "sp-api", + "sp-blockchain", + "sp-core", + "sp-externalities", + "sp-runtime", + "sp-state-machine", + "structopt", +] + [[package]] name = "trybuild" version = "1.0.39" diff --git a/substrate/Cargo.toml b/substrate/Cargo.toml index 38b3a2bdcf..adc8960ffd 100644 --- a/substrate/Cargo.toml +++ b/substrate/Cargo.toml @@ -73,6 +73,7 @@ members = [ "frame/contracts/rpc", "frame/contracts/rpc/runtime-api", "frame/democracy", + "frame/try-runtime", "frame/elections", "frame/example", "frame/example-offchain-worker", @@ -185,7 +186,9 @@ members = [ "utils/build-script-utils", "utils/fork-tree", "utils/frame/benchmarking-cli", + "utils/frame/remote-externalities", "utils/frame/frame-utilities-cli", + "utils/frame/try-runtime/cli", "utils/frame/rpc/support", "utils/frame/rpc/system", "utils/prometheus", diff --git a/substrate/bin/node/cli/Cargo.toml b/substrate/bin/node/cli/Cargo.toml index 6162726c89..4aa73e2f70 100644 --- a/substrate/bin/node/cli/Cargo.toml +++ b/substrate/bin/node/cli/Cargo.toml @@ -99,6 +99,7 @@ node-executor = { version = "2.0.0", path = "../executor" } sc-cli = { version = "0.9.0", optional = true, path = "../../../client/cli" } frame-benchmarking-cli = { version = "3.0.0", optional = true, path = "../../../utils/frame/benchmarking-cli" } node-inspect = { version = "0.8.0", optional = true, path = "../inspect" } +try-runtime-cli = { version = "0.9.0", optional = true, path = "../../../utils/frame/try-runtime/cli" } # WASM-specific dependencies wasm-bindgen = { version = "0.2.57", optional = true } @@ -133,6 +134,7 @@ node-inspect = { version = "0.8.0", optional = true, path = "../inspect" } frame-benchmarking-cli = { version = "3.0.0", optional = true, path = "../../../utils/frame/benchmarking-cli" } substrate-build-script-utils = { version = "3.0.0", optional = true, path = "../../../utils/build-script-utils" } substrate-frame-cli = { version = "3.0.0", optional = true, path = "../../../utils/frame/frame-utilities-cli" } +try-runtime-cli = { version = "0.9.0", optional = true, path = "../../../utils/frame/try-runtime/cli" } [build-dependencies.sc-cli] version = "0.9.0" @@ -157,8 +159,15 @@ cli = [ "sc-finality-grandpa-warp-sync", "structopt", "substrate-build-script-utils", + "try-runtime-cli", ] runtime-benchmarks = [ "node-runtime/runtime-benchmarks", "frame-benchmarking-cli", ] +# Enable features that allow the runtime to be tried and debugged. Name might be subject to change +# in the near future. +try-runtime = [ + "node-runtime/try-runtime", + "try-runtime-cli", +] diff --git a/substrate/bin/node/cli/src/cli.rs b/substrate/bin/node/cli/src/cli.rs index 63a07e00e2..9b80a3e345 100644 --- a/substrate/bin/node/cli/src/cli.rs +++ b/substrate/bin/node/cli/src/cli.rs @@ -47,6 +47,11 @@ pub enum Subcommand { #[structopt(name = "benchmark", about = "Benchmark runtime pallets.")] Benchmark(frame_benchmarking_cli::BenchmarkCmd), + /// Try some experimental command on the runtime. This includes migration and runtime-upgrade + /// testing. + #[cfg(feature = "try-runtime")] + TryRuntime(try_runtime_cli::TryRuntimeCmd), + /// Verify a signature for a message, provided on STDIN, with a given (public or secret) key. Verify(VerifyCmd), diff --git a/substrate/bin/node/cli/src/command.rs b/substrate/bin/node/cli/src/command.rs index 461930a613..d3689bdcd6 100644 --- a/substrate/bin/node/cli/src/command.rs +++ b/substrate/bin/node/cli/src/command.rs @@ -149,5 +149,20 @@ pub fn run() -> Result<()> { Ok((cmd.run(client, backend), task_manager)) }) }, + #[cfg(feature = "try-runtime")] + Some(Subcommand::TryRuntime(cmd)) => { + let runner = cli.create_runner(cmd)?; + runner.async_run(|config| { + // we don't need any of the components of new_partial, just a runtime, or a task + // manager to do `async_run`. + let registry = config.prometheus_config.as_ref().map(|cfg| &cfg.registry); + let task_manager = sc_service::TaskManager::new( + config.task_executor.clone(), + registry, + ).unwrap(); + + Ok((cmd.run::(config), task_manager)) + }) + } } } diff --git a/substrate/bin/node/runtime/Cargo.toml b/substrate/bin/node/runtime/Cargo.toml index f77a16a10f..1a55efbf85 100644 --- a/substrate/bin/node/runtime/Cargo.toml +++ b/substrate/bin/node/runtime/Cargo.toml @@ -43,6 +43,7 @@ frame-support = { version = "3.0.0", default-features = false, path = "../../../ frame-system = { version = "3.0.0", default-features = false, path = "../../../frame/system" } frame-system-benchmarking = { version = "3.0.0", default-features = false, path = "../../../frame/system/benchmarking", optional = true } frame-system-rpc-runtime-api = { version = "3.0.0", default-features = false, path = "../../../frame/system/rpc/runtime-api/" } +frame-try-runtime = { version = "0.9.0", default-features = false, path = "../../../frame/try-runtime", optional = true } pallet-assets = { version = "3.0.0", default-features = false, path = "../../../frame/assets" } pallet-authority-discovery = { version = "3.0.0", default-features = false, path = "../../../frame/authority-discovery" } pallet-authorship = { version = "3.0.0", default-features = false, path = "../../../frame/authorship" } @@ -186,3 +187,7 @@ runtime-benchmarks = [ "frame-system-benchmarking", "hex-literal", ] +try-runtime = [ + "frame-executive/try-runtime", + "frame-try-runtime", +] diff --git a/substrate/bin/node/runtime/src/lib.rs b/substrate/bin/node/runtime/src/lib.rs index 86e3075c3a..fb2b189e2d 100644 --- a/substrate/bin/node/runtime/src/lib.rs +++ b/substrate/bin/node/runtime/src/lib.rs @@ -1080,7 +1080,14 @@ pub type SignedPayload = generic::SignedPayload; /// Extrinsic type that has already been checked. pub type CheckedExtrinsic = generic::CheckedExtrinsic; /// Executive: handles dispatch to the various modules. -pub type Executive = frame_executive::Executive, Runtime, AllModules>; +pub type Executive = frame_executive::Executive< + Runtime, + Block, + frame_system::ChainContext, + Runtime, + AllModules, + (), +>; /// MMR helper types. mod mmr { @@ -1325,15 +1332,24 @@ impl_runtime_apis! { } } + #[cfg(feature = "try-runtime")] + impl frame_try_runtime::TryRuntime for Runtime { + fn on_runtime_upgrade() -> Result<(Weight, Weight), sp_runtime::RuntimeString> { + frame_support::debug::RuntimeLogger::init(); + let weight = Executive::try_runtime_upgrade()?; + Ok((weight, RuntimeBlockWeights::get().max_block)) + } + } + #[cfg(feature = "runtime-benchmarks")] impl frame_benchmarking::Benchmark for Runtime { fn dispatch_benchmark( config: frame_benchmarking::BenchmarkConfig ) -> Result, sp_runtime::RuntimeString> { use frame_benchmarking::{Benchmarking, BenchmarkBatch, add_benchmark, TrackedStorageKey}; - // Trying to add benchmarks directly to the Session Pallet caused cyclic dependency issues. - // To get around that, we separated the Session benchmarks into its own crate, which is why - // we need these two lines below. + // Trying to add benchmarks directly to the Session Pallet caused cyclic dependency + // issues. To get around that, we separated the Session benchmarks into its own crate, + // which is why we need these two lines below. use pallet_session_benchmarking::Module as SessionBench; use pallet_offences_benchmarking::Module as OffencesBench; use frame_system_benchmarking::Module as SystemBench; diff --git a/substrate/client/service/src/task_manager/mod.rs b/substrate/client/service/src/task_manager/mod.rs index 652e5d4439..02d83e6dce 100644 --- a/substrate/client/service/src/task_manager/mod.rs +++ b/substrate/client/service/src/task_manager/mod.rs @@ -234,7 +234,7 @@ pub struct TaskManager { impl TaskManager { /// If a Prometheus registry is passed, it will be used to report statistics about the /// service tasks. - pub(super) fn new( + pub fn new( executor: TaskExecutor, prometheus_registry: Option<&Registry>, ) -> Result { diff --git a/substrate/client/state-db/src/lib.rs b/substrate/client/state-db/src/lib.rs index dd2baf9d18..1f73f3cca3 100644 --- a/substrate/client/state-db/src/lib.rs +++ b/substrate/client/state-db/src/lib.rs @@ -24,12 +24,12 @@ //! Canonicalization window tracks a tree of blocks identified by header hash. The in-memory //! overlay allows to get any node that was inserted in any of the blocks within the window. //! The tree is journaled to the backing database and rebuilt on startup. -//! Canonicalization function selects one root from the top of the tree and discards all other roots and -//! their subtrees. +//! Canonicalization function selects one root from the top of the tree and discards all other roots +//! and their subtrees. //! //! # Pruning. -//! See `RefWindow` for pruning algorithm details. `StateDb` prunes on each canonicalization until pruning -//! constraints are satisfied. +//! See `RefWindow` for pruning algorithm details. `StateDb` prunes on each canonicalization until +//! pruning constraints are satisfied. mod noncanonical; mod pruning; diff --git a/substrate/frame/executive/Cargo.toml b/substrate/frame/executive/Cargo.toml index 31f1f34174..7ef00e7ff7 100644 --- a/substrate/frame/executive/Cargo.toml +++ b/substrate/frame/executive/Cargo.toml @@ -47,3 +47,6 @@ std = [ "sp-tracing/std", "sp-std/std", ] +try-runtime = [ + "frame-support/try-runtime" +] diff --git a/substrate/frame/executive/src/lib.rs b/substrate/frame/executive/src/lib.rs index b31f40dc28..924adea95f 100644 --- a/substrate/frame/executive/src/lib.rs +++ b/substrate/frame/executive/src/lib.rs @@ -44,7 +44,8 @@ //! //! ## Usage //! -//! The default Substrate node template declares the [`Executive`](./struct.Executive.html) type in its library. +//! The default Substrate node template declares the [`Executive`](./struct.Executive.html) type in +//! its library. //! //! ### Example //! @@ -185,26 +186,58 @@ where } impl< - System: frame_system::Config, - Block: traits::Block, - Context: Default, - UnsignedValidator, - AllModules: - OnRuntimeUpgrade + - OnInitialize + - OnFinalize + - OffchainWorker, - COnRuntimeUpgrade: OnRuntimeUpgrade, -> Executive + System: frame_system::Config, + Block: traits::Block
, + Context: Default, + UnsignedValidator, + AllModules: OnRuntimeUpgrade + + OnInitialize + + OnFinalize + + OffchainWorker, + COnRuntimeUpgrade: OnRuntimeUpgrade, + > Executive where Block::Extrinsic: Checkable + Codec, - CheckedOf: - Applyable + - GetDispatchInfo, - CallOf: Dispatchable, + CheckedOf: Applyable + GetDispatchInfo, + CallOf: + Dispatchable, OriginOf: From>, - UnsignedValidator: ValidateUnsigned>, + UnsignedValidator: ValidateUnsigned>, { + /// Execute all `OnRuntimeUpgrade` of this runtime, and return the aggregate weight. + pub fn execute_on_runtime_upgrade() -> frame_support::weights::Weight { + let mut weight = 0; + weight = weight.saturating_add( + as OnRuntimeUpgrade>::on_runtime_upgrade(), + ); + weight = weight.saturating_add(COnRuntimeUpgrade::on_runtime_upgrade()); + weight = weight.saturating_add(::on_runtime_upgrade()); + + weight + } + + /// Execute all `OnRuntimeUpgrade` of this runtime, including the pre and post migration checks. + /// + /// This should only be used for testing. + #[cfg(feature = "try-runtime")] + pub fn try_runtime_upgrade() -> Result { + < + (frame_system::Module::, COnRuntimeUpgrade, AllModules) + as + OnRuntimeUpgrade + >::pre_upgrade()?; + + let weight = Self::execute_on_runtime_upgrade(); + + < + (frame_system::Module::, COnRuntimeUpgrade, AllModules) + as + OnRuntimeUpgrade + >::post_upgrade()?; + + Ok(weight) + } + /// Start the execution of a particular block. pub fn initialize_block(header: &System::Header) { sp_io::init_tracing(); @@ -234,10 +267,7 @@ where ) { let mut weight = 0; if Self::runtime_upgraded() { - // System is not part of `AllModules`, so we need to call this manually. - weight = weight.saturating_add( as OnRuntimeUpgrade>::on_runtime_upgrade()); - weight = weight.saturating_add(COnRuntimeUpgrade::on_runtime_upgrade()); - weight = weight.saturating_add(::on_runtime_upgrade()); + weight = weight.saturating_add(Self::execute_on_runtime_upgrade()); } >::initialize( block_number, diff --git a/substrate/frame/support/Cargo.toml b/substrate/frame/support/Cargo.toml index 8edf1ff6dd..b77907721b 100644 --- a/substrate/frame/support/Cargo.toml +++ b/substrate/frame/support/Cargo.toml @@ -60,3 +60,4 @@ std = [ nightly = [] strict = [] runtime-benchmarks = [] +try-runtime = [] diff --git a/substrate/frame/support/src/traits.rs b/substrate/frame/support/src/traits.rs index 106ec10c6c..602bc1aa1a 100644 --- a/substrate/frame/support/src/traits.rs +++ b/substrate/frame/support/src/traits.rs @@ -1547,7 +1547,25 @@ pub trait OnRuntimeUpgrade { /// block local data are not accessible. /// /// Return the non-negotiable weight consumed for runtime upgrade. - fn on_runtime_upgrade() -> crate::weights::Weight { 0 } + fn on_runtime_upgrade() -> crate::weights::Weight { + 0 + } + + /// Execute some pre-checks prior to a runtime upgrade. + /// + /// This hook is never meant to be executed on-chain but is meant to be used by testing tools. + #[cfg(feature = "try-runtime")] + fn pre_upgrade() -> Result<(), &'static str> { + Ok(()) + } + + /// Execute some post-checks after a runtime upgrade. + /// + /// This hook is never meant to be executed on-chain but is meant to be used by testing tools. + #[cfg(feature = "try-runtime")] + fn post_upgrade() -> Result<(), &'static str> { + Ok(()) + } } #[impl_for_tuples(30)] @@ -1557,6 +1575,20 @@ impl OnRuntimeUpgrade for Tuple { for_tuples!( #( weight = weight.saturating_add(Tuple::on_runtime_upgrade()); )* ); weight } + + #[cfg(feature = "try-runtime")] + fn pre_upgrade() -> Result<(), &'static str> { + let mut result = Ok(()); + for_tuples!( #( result = result.and(Tuple::pre_upgrade()); )* ); + result + } + + #[cfg(feature = "try-runtime")] + fn post_upgrade() -> Result<(), &'static str> { + let mut result = Ok(()); + for_tuples!( #( result = result.and(Tuple::post_upgrade()); )* ); + result + } } /// Off-chain computation trait. diff --git a/substrate/frame/try-runtime/Cargo.toml b/substrate/frame/try-runtime/Cargo.toml new file mode 100644 index 0000000000..9c1919d380 --- /dev/null +++ b/substrate/frame/try-runtime/Cargo.toml @@ -0,0 +1,31 @@ +[package] +name = "frame-try-runtime" +version = "0.9.0" +authors = ["Parity Technologies "] +edition = "2018" +license = "Apache-2.0" +homepage = "https://substrate.dev" +repository = "https://github.com/paritytech/substrate/" +description = "FRAME pallet for democracy" +readme = "README.md" + +[package.metadata.docs.rs] +targets = ["x86_64-unknown-linux-gnu"] + +[dependencies] +codec = { package = "parity-scale-codec", version = "2.0.0", default-features = false } + +sp-api = { version = "3.0.0", path = "../../primitives/api", default-features = false } +sp-std = { version = "3.0.0", path = "../../primitives/std" , default-features = false } +sp-runtime = { version = "3.0.0", path = "../../primitives/runtime" , default-features = false } + +frame-support = { version = "3.0.0", path = "../support", default-features = false } + +[features] +default = [ "std" ] +std = [ + "sp-api/std", + "sp-std/std", + "sp-runtime/std", + "frame-support/std", +] diff --git a/substrate/frame/try-runtime/src/lib.rs b/substrate/frame/try-runtime/src/lib.rs new file mode 100644 index 0000000000..dcd3a47878 --- /dev/null +++ b/substrate/frame/try-runtime/src/lib.rs @@ -0,0 +1,37 @@ +// This file is part of Substrate. + +// Copyright (C) 2021 Parity Technologies (UK) Ltd. +// SPDX-License-Identifier: Apache-2.0 + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! Supporting types for try-runtime, testing and dry-running commands. + +#![cfg_attr(not(feature = "std"), no_std)] + +use sp_std::prelude::*; +use frame_support::weights::Weight; + +sp_api::decl_runtime_apis! { + /// Runtime api for testing the execution of a runtime upgrade. + pub trait TryRuntime { + /// dry-run runtime upgrades, returning the total weight consumed. + /// + /// This should do EXACTLY the same operations as the runtime would have done in the case of + /// a runtime upgrade (e.g. pallet ordering must be the same) + /// + /// Returns the consumed weight of the migration in case of a successful one, combined with + /// the total allowed block weight of the runtime. + fn on_runtime_upgrade() -> Result<(Weight, Weight), sp_runtime::RuntimeString>; + } +} diff --git a/substrate/primitives/core/src/hexdisplay.rs b/substrate/primitives/core/src/hexdisplay.rs index 304b665a72..e590eec0e5 100644 --- a/substrate/primitives/core/src/hexdisplay.rs +++ b/substrate/primitives/core/src/hexdisplay.rs @@ -71,6 +71,12 @@ impl AsBytesRef for sp_std::vec::Vec { fn as_bytes_ref(&self) -> &[u8] { &self } } +impl AsBytesRef for sp_storage::StorageKey { + fn as_bytes_ref(&self) -> &[u8] { + self.as_ref() + } +} + macro_rules! impl_non_endians { ( $( $t:ty ),* ) => { $( impl AsBytesRef for $t { diff --git a/substrate/primitives/state-machine/src/testing.rs b/substrate/primitives/state-machine/src/testing.rs index a6f9d06824..5f10fc0a27 100644 --- a/substrate/primitives/state-machine/src/testing.rs +++ b/substrate/primitives/state-machine/src/testing.rs @@ -48,20 +48,22 @@ pub struct TestExternalities where H::Out: codec::Codec + Ord, { + /// The overlay changed storage. overlay: OverlayedChanges, offchain_db: TestPersistentOffchainDB, - storage_transaction_cache: StorageTransactionCache< - as Backend>::Transaction, H, N - >, - backend: InMemoryBackend, + storage_transaction_cache: + StorageTransactionCache< as Backend>::Transaction, H, N>, + /// Storage backend. + pub backend: InMemoryBackend, changes_trie_config: Option, changes_trie_storage: ChangesTrieInMemoryStorage, - extensions: Extensions, + /// Extensions. + pub extensions: Extensions, } impl TestExternalities - where - H::Out: Ord + 'static + codec::Codec +where + H::Out: Ord + 'static + codec::Codec, { /// Get externalities implementation. pub fn ext(&mut self) -> Ext> { diff --git a/substrate/primitives/storage/src/lib.rs b/substrate/primitives/storage/src/lib.rs index 1e9f976607..1016b73eb1 100644 --- a/substrate/primitives/storage/src/lib.rs +++ b/substrate/primitives/storage/src/lib.rs @@ -31,10 +31,15 @@ use codec::{Encode, Decode}; #[derive(PartialEq, Eq, RuntimeDebug)] #[cfg_attr(feature = "std", derive(Serialize, Deserialize, Hash, PartialOrd, Ord, Clone))] pub struct StorageKey( - #[cfg_attr(feature = "std", serde(with="impl_serde::serialize"))] - pub Vec, + #[cfg_attr(feature = "std", serde(with = "impl_serde::serialize"))] pub Vec, ); +impl AsRef<[u8]> for StorageKey { + fn as_ref(&self) -> &[u8] { + self.0.as_ref() + } +} + /// Storage key with read/write tracking information. #[derive(PartialEq, Eq, RuntimeDebug, Clone, Encode, Decode)] #[cfg_attr(feature = "std", derive(Hash, PartialOrd, Ord))] diff --git a/substrate/utils/frame/remote-externalities/Cargo.toml b/substrate/utils/frame/remote-externalities/Cargo.toml new file mode 100644 index 0000000000..41a3b26217 --- /dev/null +++ b/substrate/utils/frame/remote-externalities/Cargo.toml @@ -0,0 +1,34 @@ +[package] +name = "remote-externalities" +version = "0.9.0" +authors = ["Parity Technologies "] +edition = "2018" +license = "Apache-2.0" +homepage = "https://substrate.dev" +repository = "https://github.com/paritytech/substrate/" +description = "An externalities provided environemnt that can load itself from remote nodes or cache files" +readme = "README.md" + +[package.metadata.docs.rs] +targets = ["x86_64-unknown-linux-gnu"] + +[dependencies] +jsonrpc-core-client = { version = "15.1.0", features = ["http"] } +sc-rpc-api = { version = "0.9.0", path = "../../../client/rpc-api" } +sc-rpc = { version = "3.0.0", path = "../../../client/rpc" } +futures = "0.1.29" + +hex-literal = "0.3.1" +env_logger = "0.8.2" +log = "0.4.11" +bincode = "1.3.1" +tokio = "0.1.22" + +sp-io = { version = "3.0.0", path = "../../../primitives/io" } +sp-core = { version = "3.0.0", path = "../../../primitives/core" } + +[dev-dependencies] +async-std = { version = "1.6.5", features = ["attributes"] } + +[features] +remote-test = [] diff --git a/substrate/utils/frame/remote-externalities/proxy_test b/substrate/utils/frame/remote-externalities/proxy_test new file mode 100644 index 0000000000000000000000000000000000000000..adb93f5ba270c3e3740394f183d96bc54a3211f9 GIT binary patch literal 26476 zcmd_zJC^Q9k{n=?;%M9h7k~)F15ihx#7zVu0Fm;ngR9~4_<45aOBwPov;V3(lQZ2_ zeQ#AJ{(y(OxtY1&|Nfu;@%6v{`0Ms>>DbQIkG-^>``GTM9&>xfwqN5Jd*0JJj=JV^ z<=&rk)F-dwZh82dbDKxq`gWC^(>(kuTiV9BeX~^4IBT6fUt?|i>iNmX)9#~etIhMM zXB$25eXKj3dwupgw$^K_?SFqf*>;qBUw51Jx{veP+B4eI%gFnE9HU;TweqBPUUxfd zyZy`Qe$K6zQt!3Zr`}uM{`jid??1)!ls%O*G@mEm z`OJISSDyLa_Uc#fR<3<1R^=!^EU!LoqejaDt_Gi1VanF^m$8(;?x>nmp zulMtKuygtCyz+C*EAKfuul3yPIlOJ3ZyU#RU452Oa!Fs$;}|I=f!~|ku;o10b&OK> zy{`K_i=VMp8mpdnD_bhZd7tGv_w&d#J!kQ2f5Le@dEU*t*=8%H?=3w~OG(6B=Uuk_ z+#cbbZb4LXzw0^gv7Kk%k37bC=H#GUmeO2R7x4Oe^RAIj!ElW0sQH-ZQ+JUssLIt+ zEop7ue^Dwlzgw*`?WXsy^iOU%I4X`L_Y3%)Y6_>q@=p9L)z=l zm1-|Xc}5jtDK$6mS}u*l-SgK-+e>luZ5``8gm1g|*{?Pn#-5In_pR-z_2wUY9+{~f zGMDI-i_POe_rJS}V+mGI=Ozy+?>vUZ6m=fdT|E}C_ zzw81%eI~hlS3CBRwy)>$)3g1IwAY=7va(-KIp)3Ac`ffR^&=-)rXK21>-{KdpaY30 zI@WWp{ke;ZGxix(XUf@qKhvd<-no3edB0clB8JvhRL(rkJ-03Gm%=HGxB1j`s2JtA z?`=rw+r9RyUqu*l>t?>{*Ze)ZF&gPld>n-yZXTQ}%s(j^S=JefBr|&+x~5vzjv~#Yd)W6Ynw_SFO=zpDgv|3*4!`jU5Bxd(`IOC|Ty&)&1R zntR_?5p<>8^F0@A+uRi4Gk)?vyAEfUe;jJ)waW1M>UjxnZ$^ZTTS0lwS#y*Rus^>;q!)b~s+(2U0HnH}gYPNt~wbLP6P-U53k1X)^b(JLg`zVs!T-14@giKEpfz{=zP7HL!W&;}ql zCh#2S=$3h+qbmV$%PtD_m~*?+@eEIJ$e$7yjB!Y(@23%<>Q%kg4yNn*<(*x7cu3c` zl*5u^HFY|={{4_$pd+`pq#0jQ~o zUzc|DoKPAD=)CYA`CCpa*t$#7%h-~Eo;tBFBD~9Tb#Jcr-L~tJJC&YGRc(sujd+2% z9M^`(c|VXB#_Vx6@cd|q&FwzZPnn6YIqj$1%V$oDVw;`|fBAW@-_I)k`g{ChId^GC zE-uGD)%546NVDX21|>0Xx%S?pZBh&6WR(qLs1^pd(e)Zzy7FlGK~W%YQvY^`tM32M zrs#X_XTm8-7IQPUOU2yLu-h#VBuv+4t5eOpP(K?wB*@Ay==-nOwY~5TkKi|D8-!WL zQ#UyDgvQGHM?FHxB8g6U>8yJf)3C4rKNnUQ-vA&_7ZjVNXn`2V*O)z`z0h@@K<=TU zMM{&MQ&Dj@EgBwR)8dX>Yjhr{W!X%0=>5`ErEwtdarwGCEHE&T<%BB%130sxpla`R z*YocM3yiMz$PyY_a-UQRy+c`7-kx)`mViycyl2?8UHu#LLXnWJ5n7-ya(5ungZe~p z9S0I;caS^675m>WMStKa^=i4tA>WX4a1t4j4&FU8=s#iy!ljWvrrsZop0F`428`4t zP~`dG~mdq3A#a-KEUsWRw!po|-|Afg=0=X#zx^-5ZgNI~evP!y6`&QpD;Ywdwzfjq@XC+_8Fjojr>sug)D{T? z3I-gd9x}|MyZJni7e<=r>Mj$qTiM5~yL{rv%fY`Cc^e#49fRw- zfrP{gw=}QO`u^=aTvLaL@j-Jtm_pERB(sX=+Y0e-^T{DWN=i5+H!aQ`r|*kcqGHu^GJub{;CQp zzIkdEwE8$_5O34mHsy2(LMK3dqTjf;TAqxbma!zjKnQ<)z3@ExBb98PMjzSy*Q!7a zSb$Hs9F11FOfjO+6pVOZyR>}5vdYQ1AHlvy{sMl6Cbf>QnMfDMl5hrFg^Mp+PA58b zNhKIXsr^P-tc-Lc6C*}MWxWWP#K08L=*xLH#(j;8`Wh%1mPiOSgpMcnPm(<~{k}Cs zC9AGrED!mURGTXPDX5e8uh(TFk4EFJI(P_uAPGgkt9WN6>B=I==UoYXui6cH6g{Jt z1Q?vsTl?SZ)cnQ24Bi~u*N)()c;d4|`aQ*P>OqRGMt0?bXfkR?RE9tvB1&38J}ckk zfdgi$%p@oLUXay;BZTd*mFTqIURJ7tk z#jDzfm}#mrLhwEIUx|Rj#e(*P$onL80}G*hOZ^Z(brI6jctH}kHi@fZJ-imR9~iIW zcy4OVP0KyV5Z+CJ_rK;@e8W1mVp!wJSQG!0I2w0P3J_OH4m%6&Rbn`Csa~HG(l@j% z?%Mp44PKXLJ8IYKn>i#X?`nH)#5WugtWH1>-4rrU52FFaTMn-qt8_a;G@s5O(Zp$K zVH4q$NMEzTFGkw0X|amPV7|yOhAa;X?9bUl3)cWLxzBWO3dYh+wKt5*8;|vPV z7{aHOuQB^~RLHt&Pg`MD*#?P>>x5S7*uujfR>&YI?o`Qqu|Jcn=YnHT^6iE z(xRex2AKP^dcH2F9ebXIF7fn?lfIUZyDLpB3tUYXOWUSEg?sMd=OA5hw0ZDdGd2Mo zNUWnkHS9xGz%qT)ae@?5j?r@!lcC&c%Z}n0H1?z5of#lPT1C1s5rU6_1T~= zH9iEIPF8`#BtQCCsyZr!5o+Z|0+59Y`k1qYNOUGy3S&TFBnf(==*mCP)eGZ;6LLQq zz6UquatDJ!vI9$4kNBba+bEKcCh>7?+5&PBc&sJ|QjnGf+TcVKaNyD9`o{V=GePgt zQS*4hG*(M_-A35ZO=Ru|g(m)nXHeb?D zeW73DQ(6NX+Bv?hm7@ZsUU1B~=Y1ln@D1K0?;WZeU0 zy!c#sn740ENI(O%Be&=`5f1?=UA@${?1m`U;j!S9m0kTZ*5dKKm=D5IfT<84A1dSP z;)4%b$hLr~$a54gJ4%cI%*;CLAe>P^LU@=fpQDu|%0Ut)HFgsDfQ5P^468G!Ecx|o ze4ykix?RsbBu~_*OU5cOM{2w$T8ZHF)LJ|(U>b@Q{2Y9Sp@1S*{|uJ&L^MoIyiW_Z zld^W6UYf8jjl~khUE0pxOp!%Z?0z{ENn#q}N|C7CB?Jm4LY{qn-ZlT~m>Ju|<9IX# zz`65CpaupT1fC?~{l?xdui?92dWFZkKmFb@P{WK9W)8Efe?gBd?pc* z*%4K2Tm$DQx`Lvfg5sh zItAx1Tf86b>o`xt8sfTKovy-3PF6wo0d&YTqMr+)+Ouw>*S3yHC)6T{58TEARCQHM z*<2HbIR@bDsK1`Ujk&i3S=EaB^)kk{icsEVu-u4L(fztGHfF8nkwA3Z2G0%glSlh~Ag> z?}c7|DmH{FD#C>m&AKi_{X)=+u1e2<;NJLGv_R(B4vY2-2oi84CvGXFaZO4MV7Scwj(w zQhsW_!mr5q$DQP#;2+>X*A*eiv`J-Yr-5U5ao_`tHn52@goa<8EhGS09y11oV+aVG zXP-uGQe9vYgRR)vzkW6d%?LTr5MUHDdbauL2R-yQ*BI*LeH(oCJNT1E%QoXjk;K$d z`?;?^GxqCH275lQ0ah?8C&pYA{2oqlswk?XFa!n(!b_onp$(@KgjFx~06q+ZSqvG3 z$p^`XhR)@Gd`w?y#(r(S;K$$T17-Z)q{5HC)5kM~pBhbwmi_EsKV#uH{h!9gkrpu9 zrBq?|F7sqZOqa_z^eQ#m-1HnV@rgL5B*;dI8H$-le+n&Db8kd;O%3XE?tk_AYn!C4K*bIDG-3XvKp4D6Wb0hFbiy1OO7#ae#c87V2OsCxDq^O`tcu|a5|A9k zQEIvfAzmnzn|=W&-D7kd=+!MFv+$o`qRRd*2A}^7ewGv((j0}@Fep_4_%J8w!y=(B z(*`Reo~l!M$o8xZY>8-EqqyM_%2iXdog|#gFe2?Xe>;ypeJWCkBuL}L(wSroD`ZTZ zJ!-UQkO$Gfi>O3!XXZpzgWXtBm&hQ`dw{%-e+vRyQ$Sb`uM%S{T}p={BElt0n1SxX z^p7)S0Wjf4v>7pcuZo)Vq5>il5pyC;7uLSj&I34%~0D7z|Doty1y$0YvmpYFZ6G!6+p?gcUV`EE03RfL1 z17&j^hBsJ=!~Mf8!2SPilfi#%_QO`25YeYI3XPpavl3fYGlpUv2p0pVM0uJIpxliFK}!0Y){Ku8WLYD-Wg2XMpnT5yM1; z^s#ir@iFWO5Hj(PzJLARMv&L_76t}jTJkhP(i`Af`Po*JZ+Z5vmZTkd3$KgJ8DnhB zF3^kNSv`%+tgYeGW}&uWO0uP?OUMRlC=u@9$JYz$db42 zII~c3%bzjj3S5T~y{|F*ciaJYV^WRD z7&I^(#-wCF1Q;4>q2W(uA7O`S)`3v$82G6v7!MFsAFaLD!|Uyh^>K)T@q+s6d2lHe zi8z3LEJ{q+0WidN0&XNrJ@k%}JW>**GkjKKpw0-qK3*&JmZ7XMiYjDXl#t2D4S*j3{pL@B7qru|L!b zWo?=W511t;rUq(tOLr~-qPhij{HF#Aa80JS1gj-zaf@|3VwA(S#`s&LCvfeDGvJaL zl!CI6L-hiTdEsd_^PoLsEz?8sGz1wK?92f!#e>HFTRD*ZIW+7)p5tfT1NH|K&ArZV zO%9HsYls28Yv3;C%WM%u7HLOfZYMTuFlktq=b3H#z&lOw^zA%;_+ISai?F2Fqgn0B zwS$M*k(h-#hk10`I%1uYac?rQbpZ^r=HyrqEdPv6gT$=gcrR~O^w|dG$$Td=%|xsw zHr7hS@)c7lTstvia)yF{_wpp^Z=XI0<|#J7+Zs}<4O=wtzPk? zn&&_!BAjRiXaSZ3_lQ+%Xtl!K1nA?ae|lexPb-e|7ISYFu~;n;8yeY@jCl(Ic1C^x zT&k_Gu_nQ!7w43CM#b|}F?am&96x(6+>^ZN3tgq5G@Of#bcd}Nj38*%DHJ>ZVvj|Y zOpI6>-p7gVM2K>LgB7+fU*w%PqZ|iRSKG}*r*1*0b z{2R{!;TqM;rVGphxgTIpEDMpP{Kqx5`Z~ml(98-&2jeH7y>5FND98yV?9>R6w%2QmFWda`5O^oZJNB%GHRb2$A!kMAieY1GvivA3 z5Ex~x(n2gf=jpv9F46L1jxW`yC#AKAyt|KZd zI|eIx2(Ai^H5}Y@0=Jt0Vt=*%5i$SV9i=Bcz?q81pk zG6@Dm2Qpe;*)`R%ffwEP8_!|wOSCOg0I(G%+?&C`#G+YDn|r7zNe|OodBi5;e_%Bx~Tbv-SqY(xe*;rq=Bq~P*5%p~e6I(L86fsI#wQPk=(6U0~ zgg<{Tx+de*Q7LR9W!V*~(wJeAx9nw78)xjuC3V#lIqk%Bj|Pr-Q3TdA=h)Dcufpcw)gL_hA&E%=4=2=OcS z0C*V|7LZ;glQ$CxW9{3j*SB3=G;Y#A3g@=f)56lQPduRu#v*^C=s$lis#Q$$$liuL zgkmF*K9X^-~NQGwJ0ez3@{3SK?VQ51R;Zod5~e>a@CbvVJM|mSVXZWdo2`iD(QKc|KQ3CH z9A%c<4}=ctolV1J0o_B@!@{dYTALU##rwua#$PxO=o1mDoVpKh`w4&^Ym;p*pj*XG zG(U^rLEMB-PWJb(TWa=VqCBKvT$(?j!Fv%S(7^waTdVZMD zt_m~8RLtAxf=q5+&XuO$#(*dTbqvmIv7_wg@d3){qW9SRwCvWAu}R37Kl$5ew2fUt p#^SFqX&9c6*Bw`EWr32VKMjsu$nhNTtv|y?__%zZ{QT$N{U5`qT;l)$ literal 0 HcmV?d00001 diff --git a/substrate/utils/frame/remote-externalities/src/lib.rs b/substrate/utils/frame/remote-externalities/src/lib.rs new file mode 100644 index 0000000000..6c8b49c7c8 --- /dev/null +++ b/substrate/utils/frame/remote-externalities/src/lib.rs @@ -0,0 +1,454 @@ +// This file is part of Substrate. + +// Copyright (C) 2020-2021 Parity Technologies (UK) Ltd. +// SPDX-License-Identifier: Apache-2.0 + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! # Remote Externalities +//! +//! An equivalent of `sp_io::TestExternalities` that can load its state from a remote substrate +//! based chain, or a local cache file. +//! +//! #### Runtime to Test Against +//! +//! While not absolutely necessary, you most likely need a `Runtime` equivalent in your test setup +//! through which you can infer storage types. There are two options here: +//! +//! 1. Build a mock runtime, similar how to you would build one in a pallet test (see example +//! below). The very important point here is that this mock needs to hold real values for types +//! that matter for you, based on the chain of interest. Some typical ones are: +//! +//! - `sp_runtime::AccountId32` as `AccountId`. +//! - `u32` as `BlockNumber`. +//! - `u128` as Balance. +//! +//! Once you have your `Runtime`, you can use it for storage type resolution and do things like +//! `>::storage_getter()` or `>::get()`. +//! +//! 2. Or, you can use a real runtime. +//! +//! ### Example +//! +//! With a test runtime +//! +//! ```ignore +//! use remote_externalities::Builder; +//! +//! #[derive(Clone, Eq, PartialEq, Debug, Default)] +//! pub struct TestRuntime; +//! +//! use frame_system as system; +//! impl_outer_origin! { +//! pub enum Origin for TestRuntime {} +//! } +//! +//! impl frame_system::Config for TestRuntime { +//! .. +//! // we only care about these two for now. The rest can be mock. The block number type of +//! // kusama is u32. +//! type BlockNumber = u32; +//! type Header = Header; +//! .. +//! } +//! +//! #[test] +//! fn test_runtime_works() { +//! let hash: Hash = +//! hex!["f9a4ce984129569f63edc01b1c13374779f9384f1befd39931ffdcc83acf63a7"].into(); +//! let parent: Hash = +//! hex!["540922e96a8fcaf945ed23c6f09c3e189bd88504ec945cc2171deaebeaf2f37e"].into(); +//! Builder::new() +//! .at(hash) +//! .module("System") +//! .build() +//! .execute_with(|| { +//! assert_eq!( +//! // note: the hash corresponds to 3098546. We can check only the parent. +//! // https://polkascan.io/kusama/block/3098546 +//! >::block_hash(3098545u32), +//! parent, +//! ) +//! }); +//! } +//! ``` +//! +//! Or with the real kusama runtime. +//! +//! ```ignore +//! use remote_externalities::Builder; +//! use kusama_runtime::Runtime; +//! +//! #[test] +//! fn test_runtime_works() { +//! let hash: Hash = +//! hex!["f9a4ce984129569f63edc01b1c13374779f9384f1befd39931ffdcc83acf63a7"].into(); +//! Builder::new() +//! .at(hash) +//! .module("Staking") +//! .build() +//! .execute_with(|| assert_eq!(>::validator_count(), 400)); +//! } +//! ``` + +use std::{ + fs, + path::{Path, PathBuf}, +}; +use log::*; +use sp_core::{hashing::twox_128}; +pub use sp_io::TestExternalities; +use sp_core::{ + hexdisplay::HexDisplay, + storage::{StorageKey, StorageData}, +}; +use futures::future::Future; + +type KeyPair = (StorageKey, StorageData); +type Number = u32; +type Hash = sp_core::H256; +// TODO: make these two generic. + +const LOG_TARGET: &'static str = "remote-ext"; + +/// The execution mode. +#[derive(Clone)] +pub enum Mode { + /// Online. + Online(OnlineConfig), + /// Offline. Uses a cached file and needs not any client config. + Offline(OfflineConfig), +} + +/// configuration of the online execution. +/// +/// A cache config must be present. +#[derive(Clone)] +pub struct OfflineConfig { + /// The configuration of the cache file to use. It must be present. + pub cache: CacheConfig, +} + +/// Configuration of the online execution. +/// +/// A cache config may be present and will be written to in that case. +#[derive(Clone)] +pub struct OnlineConfig { + /// The HTTP uri to use. + pub uri: String, + /// The block number at which to connect. Will be latest finalized head if not provided. + pub at: Option, + /// An optional cache file to WRITE to, not for reading. Not cached if set to `None`. + pub cache: Option, + /// The modules to scrape. If empty, entire chain state will be scraped. + pub modules: Vec, +} + +impl Default for OnlineConfig { + fn default() -> Self { + Self { + uri: "http://localhost:9933".into(), + at: None, + cache: None, + modules: Default::default(), + } + } +} + +/// Configuration of the cache. +#[derive(Clone)] +pub struct CacheConfig { + // TODO: I could mix these two into one filed, but I think separate is better bc one can be + // configurable while one not. + /// File name. + pub name: String, + /// Base directory. + pub directory: String, +} + +impl Default for CacheConfig { + fn default() -> Self { + Self { name: "CACHE".into(), directory: ".".into() } + } +} + +impl CacheConfig { + fn path(&self) -> PathBuf { + Path::new(&self.directory).join(self.name.clone()) + } +} + +/// Builder for remote-externalities. +pub struct Builder { + inject: Vec, + mode: Mode, + chain: String, +} + +impl Default for Builder { + fn default() -> Self { + Self { + inject: Default::default(), + mode: Mode::Online(OnlineConfig { + at: None, + uri: "http://localhost:9933".into(), + cache: None, + modules: Default::default(), + }), + chain: "UNSET".into(), + } + } +} + +// Mode methods +impl Builder { + fn as_online(&self) -> &OnlineConfig { + match &self.mode { + Mode::Online(config) => &config, + _ => panic!("Unexpected mode: Online"), + } + } + + fn as_online_mut(&mut self) -> &mut OnlineConfig { + match &mut self.mode { + Mode::Online(config) => config, + _ => panic!("Unexpected mode: Online"), + } + } +} + +// RPC methods +impl Builder { + async fn rpc_get_head(&self) -> Hash { + let mut rt = tokio::runtime::Runtime::new().expect("Unable to create a runtime"); + let uri = self.as_online().uri.clone(); + rt.block_on::<_, _, ()>(futures::lazy(move || { + trace!(target: LOG_TARGET, "rpc: finalized_head"); + let client: sc_rpc_api::chain::ChainClient = + jsonrpc_core_client::transports::http::connect(&uri).wait().unwrap(); + Ok(client.finalized_head().wait().unwrap()) + })) + .unwrap() + } + + /// Relay the request to `state_getPairs` rpc endpoint. + /// + /// Note that this is an unsafe RPC. + async fn rpc_get_pairs(&self, prefix: StorageKey, at: Hash) -> Vec { + let mut rt = tokio::runtime::Runtime::new().expect("Unable to create a runtime"); + let uri = self.as_online().uri.clone(); + rt.block_on::<_, _, ()>(futures::lazy(move || { + trace!(target: LOG_TARGET, "rpc: storage_pairs: {:?} / {:?}", prefix, at); + let client: sc_rpc_api::state::StateClient = + jsonrpc_core_client::transports::http::connect(&uri).wait().unwrap(); + Ok(client.storage_pairs(prefix, Some(at)).wait().unwrap()) + })) + .unwrap() + } + + /// Get the chain name. + async fn chain_name(&self) -> String { + let mut rt = tokio::runtime::Runtime::new().expect("Unable to create a runtime"); + let uri = self.as_online().uri.clone(); + rt.block_on::<_, _, ()>(futures::lazy(move || { + trace!(target: LOG_TARGET, "rpc: system_chain"); + let client: sc_rpc_api::system::SystemClient<(), ()> = + jsonrpc_core_client::transports::http::connect(&uri).wait().unwrap(); + Ok(client.system_chain().wait().unwrap()) + })) + .unwrap() + } +} + +// Internal methods +impl Builder { + /// Save the given data as cache. + fn save_cache(&self, data: &[KeyPair], path: &Path) { + let bdata = bincode::serialize(data).unwrap(); + info!(target: LOG_TARGET, "writing to cache file {:?}", path); + fs::write(path, bdata).unwrap(); + } + + /// initialize `Self` from cache. Panics if the file does not exist. + fn load_cache(&self, path: &Path) -> Vec { + info!(target: LOG_TARGET, "scraping keypairs from cache {:?}", path,); + let bytes = fs::read(path).unwrap(); + bincode::deserialize(&bytes[..]).unwrap() + } + + /// Build `Self` from a network node denoted by `uri`. + async fn load_remote(&self) -> Vec { + let config = self.as_online(); + let at = self.as_online().at.unwrap().clone(); + info!(target: LOG_TARGET, "scraping keypairs from remote node {} @ {:?}", config.uri, at); + + let keys_and_values = if config.modules.len() > 0 { + let mut filtered_kv = vec![]; + for f in config.modules.iter() { + let hashed_prefix = StorageKey(twox_128(f.as_bytes()).to_vec()); + let module_kv = self.rpc_get_pairs(hashed_prefix.clone(), at).await; + info!( + target: LOG_TARGET, + "downloaded data for module {} (count: {} / prefix: {:?}).", + f, + module_kv.len(), + HexDisplay::from(&hashed_prefix), + ); + filtered_kv.extend(module_kv); + } + filtered_kv + } else { + info!(target: LOG_TARGET, "downloading data for all modules."); + self.rpc_get_pairs(StorageKey(vec![]), at).await.into_iter().collect::>() + }; + + keys_and_values + } + + async fn init_remote_client(&mut self) { + self.as_online_mut().at = Some(self.rpc_get_head().await); + self.chain = self.chain_name().await; + } + + async fn pre_build(mut self) -> Vec { + let mut base_kv = match self.mode.clone() { + Mode::Offline(config) => self.load_cache(&config.cache.path()), + Mode::Online(config) => { + self.init_remote_client().await; + let kp = self.load_remote().await; + if let Some(c) = config.cache { + self.save_cache(&kp, &c.path()); + } + kp + } + }; + + info!( + target: LOG_TARGET, + "extending externalities with {} manually injected keys", + self.inject.len() + ); + base_kv.extend(self.inject.clone()); + base_kv + } +} + +// Public methods +impl Builder { + /// Create a new builder. + pub fn new() -> Self { + Default::default() + } + + /// Inject a manual list of key and values to the storage. + pub fn inject(mut self, injections: &[KeyPair]) -> Self { + for i in injections { + self.inject.push(i.clone()); + } + self + } + + /// Configure a cache to be used. + pub fn mode(mut self, mode: Mode) -> Self { + self.mode = mode; + self + } + + /// Build the test externalities. + pub async fn build(self) -> TestExternalities { + let kv = self.pre_build().await; + let mut ext = TestExternalities::new_empty(); + + info!(target: LOG_TARGET, "injecting a total of {} keys", kv.len()); + for (k, v) in kv { + let (k, v) = (k.0, v.0); + ext.insert(k, v); + } + ext + } +} + +#[cfg(test)] +mod tests { + use super::*; + + fn init_logger() { + let _ = env_logger::Builder::from_default_env() + .format_module_path(false) + .format_level(true) + .try_init(); + } + + #[async_std::test] + #[cfg(feature = "remote-test")] + async fn can_build_one_pallet() { + init_logger(); + Builder::new() + .mode(Mode::Online(OnlineConfig { + modules: vec!["Proxy".into()], + ..Default::default() + })) + .build() + .await + .execute_with(|| {}); + } + + #[async_std::test] + async fn can_load_cache() { + init_logger(); + Builder::new() + .mode(Mode::Offline(OfflineConfig { + cache: CacheConfig { name: "proxy_test".into(), ..Default::default() }, + })) + .build() + .await + .execute_with(|| {}); + } + + #[async_std::test] + #[cfg(feature = "remote-test")] + async fn can_create_cache() { + init_logger(); + Builder::new() + .mode(Mode::Online(OnlineConfig { + cache: Some(CacheConfig { + name: "test_cache_to_remove.bin".into(), + ..Default::default() + }), + ..Default::default() + })) + .build() + .await + .execute_with(|| {}); + + let to_delete = std::fs::read_dir(CacheConfig::default().directory) + .unwrap() + .into_iter() + .map(|d| d.unwrap()) + .filter(|p| p.path().extension().unwrap_or_default() == "bin") + .collect::>(); + + assert!(to_delete.len() > 0); + + for d in to_delete { + std::fs::remove_file(d.path()).unwrap(); + } + } + + #[async_std::test] + #[cfg(feature = "remote-test")] + async fn can_build_all() { + init_logger(); + Builder::new().build().await.execute_with(|| {}); + } +} diff --git a/substrate/utils/frame/try-runtime/cli/Cargo.toml b/substrate/utils/frame/try-runtime/cli/Cargo.toml new file mode 100644 index 0000000000..592d0a5b99 --- /dev/null +++ b/substrate/utils/frame/try-runtime/cli/Cargo.toml @@ -0,0 +1,32 @@ +[package] +name = "try-runtime-cli" +version = "0.9.0" +authors = ["Parity Technologies "] +edition = "2018" +license = "Apache-2.0" +homepage = "https://substrate.dev" +repository = "https://github.com/paritytech/substrate/" +description = "Cli command runtime testing and dry-running" +readme = "README.md" + +[package.metadata.docs.rs] +targets = ["x86_64-unknown-linux-gnu"] + +[dependencies] +log = "0.4.8" +parity-scale-codec = { version = "2.0.0" } + +sc-service = { version = "0.9.0", default-features = false, path = "../../../../client/service" } +sc-cli = { version = "0.9.0", path = "../../../../client/cli" } +sc-executor = { path = "../../../../client/executor" } +sc-client-api = { version = "3.0.0", path = "../../../../client/api" } +structopt = "0.3.8" +sp-state-machine = { version = "0.9.0", path = "../../../../primitives/state-machine" } +sp-api = { version = "3.0.0", path = "../../../../primitives/api" } +sp-blockchain = { version = "3.0.0", path = "../../../../primitives/blockchain" } +sp-runtime = { version = "3.0.0", path = "../../../../primitives/runtime" } +sp-externalities = { version = "0.9.0", path = "../../../../primitives/externalities" } +sp-core = { version = "3.0.0", path = "../../../../primitives/core" } +frame-try-runtime = { version = "0.9.0", path = "../../../../frame/try-runtime" } + +remote-externalities = { path = "../../remote-externalities" } diff --git a/substrate/utils/frame/try-runtime/cli/src/lib.rs b/substrate/utils/frame/try-runtime/cli/src/lib.rs new file mode 100644 index 0000000000..92526379f4 --- /dev/null +++ b/substrate/utils/frame/try-runtime/cli/src/lib.rs @@ -0,0 +1,178 @@ +// This file is part of Substrate. + +// Copyright (C) 2021 Parity Technologies (UK) Ltd. +// SPDX-License-Identifier: Apache-2.0 + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! `Structopt`-ready struct for `try-runtime`. + +use parity_scale_codec::Decode; +use std::{fmt::Debug, str::FromStr}; +use sc_service::Configuration; +use sc_cli::{CliConfiguration, ExecutionStrategy, WasmExecutionMethod}; +use sc_executor::NativeExecutor; +use sc_service::NativeExecutionDispatch; +use sp_state_machine::StateMachine; +use sp_runtime::traits::{Block as BlockT, NumberFor}; +use sp_core::storage::{StorageData, StorageKey, well_known_keys}; + +/// Various commands to try out the new runtime, over configurable states. +/// +/// For now this only assumes running the `on_runtime_upgrade` hooks. +#[derive(Debug, structopt::StructOpt)] +pub struct TryRuntimeCmd { + /// The shared parameters + #[allow(missing_docs)] + #[structopt(flatten)] + pub shared_params: sc_cli::SharedParams, + + /// The state to use to run the migration. Should be a valid FILE or HTTP URI. + #[structopt(short, long, default_value = "http://localhost:9933")] + pub state: State, + + /// The execution strategy that should be used for benchmarks + #[structopt( + long = "execution", + value_name = "STRATEGY", + possible_values = &ExecutionStrategy::variants(), + case_insensitive = true, + default_value = "Native", + )] + pub execution: ExecutionStrategy, + + /// Method for executing Wasm runtime code. + #[structopt( + long = "wasm-execution", + value_name = "METHOD", + possible_values = &WasmExecutionMethod::enabled_variants(), + case_insensitive = true, + default_value = "Interpreted" + )] + pub wasm_method: WasmExecutionMethod, +} + +/// The state to use for a migration dry-run. +#[derive(Debug)] +pub enum State { + /// A snapshot. Inner value is a file path. + Snap(String), + + /// A live chain. Inner value is the HTTP uri. + Live(String), +} + +impl FromStr for State { + type Err = &'static str; + fn from_str(s: &str) -> Result { + match s.get(..7) { + // could use Url crate as well, but lets keep it simple for now. + Some("http://") => Ok(State::Live(s.to_string())), + Some("file://") => s + .split("//") + .collect::>() + .get(1) + .map(|s| State::Snap(s.to_string())) + .ok_or("invalid file URI"), + _ => Err("invalid format. Must be a valid HTTP or File URI"), + } + } +} + +impl TryRuntimeCmd { + pub async fn run(&self, config: Configuration) -> sc_cli::Result<()> + where + B: BlockT, + ExecDispatch: NativeExecutionDispatch + 'static, + { + let spec = config.chain_spec; + let genesis_storage = spec.build_storage()?; + + let code = StorageData( + genesis_storage + .top + .get(well_known_keys::CODE) + .expect("code key must exist in genesis storage; qed") + .to_vec(), + ); + let code_key = StorageKey(well_known_keys::CODE.to_vec()); + + let wasm_method = self.wasm_method; + let execution = self.execution; + + let mut changes = Default::default(); + // don't really care about these -- use the default values. + let max_runtime_instances = config.max_runtime_instances; + let heap_pages = config.default_heap_pages; + let executor = NativeExecutor::::new( + wasm_method.into(), + heap_pages, + max_runtime_instances, + ); + + let ext = { + use remote_externalities::{Builder, Mode, CacheConfig, OfflineConfig, OnlineConfig}; + let builder = match &self.state { + State::Snap(file_path) => Builder::new().mode(Mode::Offline(OfflineConfig { + cache: CacheConfig { name: file_path.into(), ..Default::default() }, + })), + State::Live(http_uri) => Builder::new().mode(Mode::Online(OnlineConfig { + uri: http_uri.into(), + ..Default::default() + })), + }; + + // inject the code into this ext. + builder.inject(&[(code_key, code)]).build().await + }; + + let encoded_result = StateMachine::<_, _, NumberFor, _>::new( + &ext.backend, + None, + &mut changes, + &executor, + "TryRuntime_on_runtime_upgrade", + &[], + ext.extensions, + &sp_state_machine::backend::BackendRuntimeCode::new(&ext.backend) + .runtime_code()?, + sp_core::testing::TaskExecutor::new(), + ) + .execute(execution.into()) + .map_err(|e| format!("failed to execute 'TryRuntime_on_runtime_upgrade' due to {:?}", e))?; + + let (weight, total_weight) = <(u64, u64) as Decode>::decode(&mut &*encoded_result) + .map_err(|e| format!("failed to decode output due to {:?}", e))?; + log::info!( + "try-runtime executed without errors. Consumed weight = {}, total weight = {} ({})", + weight, + total_weight, + weight as f64 / total_weight as f64 + ); + + Ok(()) + } +} + +impl CliConfiguration for TryRuntimeCmd { + fn shared_params(&self) -> &sc_cli::SharedParams { + &self.shared_params + } + + fn chain_id(&self, _is_dev: bool) -> sc_cli::Result { + Ok(match self.shared_params.chain { + Some(ref chain) => chain.clone(), + None => "dev".into(), + }) + } +}