try-runtime::fast-forward (#12896)

* try-runtime::fast-forward

* Revert un`pub`ing command's fields

* Handle storage change failure

* Adjust Substrate node

* Feature-gated imports

* doc link

* Feature-gated imports in node-template

* Move trait, blanket implementation and auxiliary functions to a new module

* Distinguish between plain babe+timestamp and substrate enhanced info

* Remove uncles inherents

* Missing argument

* Add doc comment about `blocktime_millis`

* Add licenses

substrate/utils/frame/try-runtime/cli/src/block_building_info.rs

@@ -0,0 +1,152 @@
+// This file is part of Substrate.
+
+// Copyright (C) 2021-2022 Parity Technologies (UK) Ltd.
+// SPDX-License-Identifier: Apache-2.0
+
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// 	http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use crate::BlockT;
+use parity_scale_codec::Encode;
+use sc_cli::Result;
+use sp_consensus_aura::{Slot, SlotDuration, AURA_ENGINE_ID};
+use sp_consensus_babe::{
+	digests::{PreDigest, SecondaryPlainPreDigest},
+	BABE_ENGINE_ID,
+};
+use sp_inherents::{InherentData, InherentDataProvider};
+use sp_runtime::{Digest, DigestItem};
+use sp_timestamp::TimestampInherentData;
+
+/// Something that can create inherent data providers and pre-runtime digest.
+///
+/// It is possible for the caller to provide custom arguments to the callee by setting the
+/// `ExtraArgs` generic parameter.
+///
+/// This module already provides some convenience implementation of this trait for closures. So, it
+/// should not be required to implement it directly.
+#[async_trait::async_trait]
+pub trait BlockBuildingInfoProvider<Block: BlockT, ExtraArgs = ()> {
+	type InherentDataProviders: InherentDataProvider;
+
+	async fn get_inherent_providers_and_pre_digest(
+		&self,
+		parent_hash: Block::Hash,
+		extra_args: ExtraArgs,
+	) -> Result<(Self::InherentDataProviders, Vec<DigestItem>)>;
+}
+
+#[async_trait::async_trait]
+impl<F, Block, IDP, ExtraArgs, Fut> BlockBuildingInfoProvider<Block, ExtraArgs> for F
+where
+	Block: BlockT,
+	F: Fn(Block::Hash, ExtraArgs) -> Fut + Sync + Send,
+	Fut: std::future::Future<Output = Result<(IDP, Vec<DigestItem>)>> + Send + 'static,
+	IDP: InherentDataProvider + 'static,
+	ExtraArgs: Send + 'static,
+{
+	type InherentDataProviders = IDP;
+
+	async fn get_inherent_providers_and_pre_digest(
+		&self,
+		parent: Block::Hash,
+		extra_args: ExtraArgs,
+	) -> Result<(Self::InherentDataProviders, Vec<DigestItem>)> {
+		(*self)(parent, extra_args).await
+	}
+}
+
+/// Provides [`BlockBuildingInfoProvider`] implementation for chains that include timestamp inherent
+/// and use Aura for a block production.
+///
+/// It depends only on the expected block production frequency, i.e. `blocktime_millis`.
+pub fn timestamp_with_aura_info<Block: BlockT>(
+	blocktime_millis: u64,
+) -> impl BlockBuildingInfoProvider<Block, Option<(InherentData, Digest)>> {
+	move |_, maybe_prev_info: Option<(InherentData, Digest)>| async move {
+		let timestamp_idp = match maybe_prev_info {
+			Some((inherent_data, _)) => sp_timestamp::InherentDataProvider::new(
+				inherent_data.timestamp_inherent_data().unwrap().unwrap() + blocktime_millis,
+			),
+			None => sp_timestamp::InherentDataProvider::from_system_time(),
+		};
+
+		let slot =
+			Slot::from_timestamp(*timestamp_idp, SlotDuration::from_millis(blocktime_millis));
+		let digest = vec![DigestItem::PreRuntime(AURA_ENGINE_ID, slot.encode())];
+
+		Ok((timestamp_idp, digest))
+	}
+}
+
+/// Provides [`BlockBuildingInfoProvider`] implementation for chains that include timestamp inherent
+/// and use Babe for a block production.
+///
+/// It depends only on the expected block production frequency, i.e. `blocktime_millis`.
+pub fn timestamp_with_babe_info<Block: BlockT>(
+	blocktime_millis: u64,
+) -> impl BlockBuildingInfoProvider<Block, Option<(InherentData, Digest)>> {
+	move |_, maybe_prev_info: Option<(InherentData, Digest)>| async move {
+		let timestamp_idp = match maybe_prev_info {
+			Some((inherent_data, _)) => sp_timestamp::InherentDataProvider::new(
+				inherent_data.timestamp_inherent_data().unwrap().unwrap() + blocktime_millis,
+			),
+			None => sp_timestamp::InherentDataProvider::from_system_time(),
+		};
+
+		let slot =
+			Slot::from_timestamp(*timestamp_idp, SlotDuration::from_millis(blocktime_millis));
+		let slot_idp = sp_consensus_babe::inherents::InherentDataProvider::new(slot);
+
+		let digest = vec![DigestItem::PreRuntime(
+			BABE_ENGINE_ID,
+			PreDigest::SecondaryPlain(SecondaryPlainPreDigest { slot, authority_index: 0 })
+				.encode(),
+		)];
+
+		Ok(((slot_idp, timestamp_idp), digest))
+	}
+}
+
+/// Provides [`BlockBuildingInfoProvider`] implementation for chains that use:
+///  - timestamp inherent,
+///  - Babe for a block production (inherent + digest),
+///  - uncles inherent,
+///  - storage proof inherent
+///
+/// It depends only on the expected block production frequency, i.e. `blocktime_millis`.
+pub fn substrate_info<Block: BlockT>(
+	blocktime_millis: u64,
+) -> impl BlockBuildingInfoProvider<Block, Option<(InherentData, Digest)>> {
+	move |_, maybe_prev_info: Option<(InherentData, Digest)>| async move {
+		let timestamp_idp = match maybe_prev_info {
+			Some((inherent_data, _)) => sp_timestamp::InherentDataProvider::new(
+				inherent_data.timestamp_inherent_data().unwrap().unwrap() + blocktime_millis,
+			),
+			None => sp_timestamp::InherentDataProvider::from_system_time(),
+		};
+
+		let slot =
+			Slot::from_timestamp(*timestamp_idp, SlotDuration::from_millis(blocktime_millis));
+		let slot_idp = sp_consensus_babe::inherents::InherentDataProvider::new(slot);
+
+		let storage_proof_idp = sp_transaction_storage_proof::InherentDataProvider::new(None);
+
+		let digest = vec![DigestItem::PreRuntime(
+			BABE_ENGINE_ID,
+			PreDigest::SecondaryPlain(SecondaryPlainPreDigest { slot, authority_index: 0 })
+				.encode(),
+		)];
+
+		Ok(((slot_idp, timestamp_idp, storage_proof_idp), digest))
+	}
+}

substrate/utils/frame/try-runtime/cli/src/commands/fast_forward.rs

@@ -0,0 +1,268 @@
+// This file is part of Substrate.
+
+// Copyright (C) 2021-2022 Parity Technologies (UK) Ltd.
+// SPDX-License-Identifier: Apache-2.0
+
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// 	http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use crate::{
+	block_building_info::BlockBuildingInfoProvider, build_executor, full_extensions,
+	rpc_err_handler, state_machine_call, BlockT, LiveState, SharedParams, State,
+};
+use parity_scale_codec::{Decode, Encode};
+use sc_cli::Result;
+use sc_executor::{sp_wasm_interface::HostFunctions, WasmExecutor};
+use serde::de::DeserializeOwned;
+use sp_core::H256;
+use sp_inherents::{InherentData, InherentDataProvider};
+use sp_io::TestExternalities;
+use sp_runtime::{
+	traits::{Header, NumberFor, One},
+	Digest,
+};
+use std::{fmt::Debug, str::FromStr};
+use substrate_rpc_client::{ws_client, ChainApi};
+
+/// Configurations of the [`crate::Command::FastForward`].
+#[derive(Debug, Clone, clap::Parser)]
+pub struct FastForwardCmd {
+	/// How many blocks should be processed. If `None`, then blocks will be produced and processed
+	/// in a loop.
+	#[arg(long)]
+	n_blocks: Option<u64>,
+
+	/// The state type to use.
+	#[command(subcommand)]
+	state: State,
+
+	/// The ws uri from which to fetch the block.
+	///
+	/// If `state` is `Live`, this is ignored. Otherwise, it must not be empty.
+	#[arg(long, value_parser = crate::parse::url)]
+	block_ws_uri: Option<String>,
+
+	/// Which try-state targets to execute when running this command.
+	///
+	/// Expected values:
+	/// - `all`
+	/// - `none`
+	/// - A comma separated list of pallets, as per pallet names in `construct_runtime!()` (e.g.
+	///   `Staking, System`).
+	/// - `rr-[x]` where `[x]` is a number. Then, the given number of pallets are checked in a
+	///   round-robin fashion.
+	#[arg(long, default_value = "all")]
+	try_state: frame_try_runtime::TryStateSelect,
+}
+
+impl FastForwardCmd {
+	fn block_ws_uri(&self) -> &str {
+		match self.state {
+			State::Live(LiveState { ref uri, .. }) => &uri,
+			_ => self
+				.block_ws_uri
+				.as_ref()
+				.expect("Either `--block-uri` must be provided, or state must be `live`"),
+		}
+	}
+}
+
+/// Read the block number corresponding to `hash` with an RPC call to `ws_uri`.
+async fn get_block_number<Block: BlockT>(
+	hash: Block::Hash,
+	ws_uri: &str,
+) -> Result<NumberFor<Block>>
+where
+	Block::Header: DeserializeOwned,
+{
+	let rpc = ws_client(ws_uri).await?;
+	Ok(ChainApi::<(), Block::Hash, Block::Header, ()>::header(&rpc, Some(hash))
+		.await
+		.map_err(rpc_err_handler)
+		.and_then(|maybe_header| maybe_header.ok_or("header_not_found").map(|h| *h.number()))?)
+}
+
+/// Call `method` with `data` and return the result. `externalities` will not change.
+async fn dry_run<T: Decode, Block: BlockT, HostFns: HostFunctions>(
+	externalities: &TestExternalities,
+	executor: &WasmExecutor<HostFns>,
+	method: &'static str,
+	data: &[u8],
+) -> Result<T> {
+	let (_, result) = state_machine_call::<Block, HostFns>(
+		externalities,
+		executor,
+		method,
+		data,
+		full_extensions(),
+	)?;
+
+	Ok(<T>::decode(&mut &*result)?)
+}
+
+/// Call `method` with `data` and actually save storage changes to `externalities`.
+async fn run<Block: BlockT, HostFns: HostFunctions>(
+	externalities: &mut TestExternalities,
+	executor: &WasmExecutor<HostFns>,
+	method: &'static str,
+	data: &[u8],
+) -> Result<()> {
+	let (mut changes, _) = state_machine_call::<Block, HostFns>(
+		externalities,
+		executor,
+		method,
+		data,
+		full_extensions(),
+	)?;
+
+	let storage_changes = changes.drain_storage_changes(
+		&externalities.backend,
+		&mut Default::default(),
+		externalities.state_version,
+	)?;
+
+	externalities
+		.backend
+		.apply_transaction(storage_changes.transaction_storage_root, storage_changes.transaction);
+
+	Ok(())
+}
+
+/// Produce next empty block.
+async fn next_empty_block<
+	Block: BlockT,
+	HostFns: HostFunctions,
+	BBIP: BlockBuildingInfoProvider<Block, Option<(InherentData, Digest)>>,
+>(
+	externalities: &mut TestExternalities,
+	executor: &WasmExecutor<HostFns>,
+	parent_height: NumberFor<Block>,
+	parent_hash: Block::Hash,
+	block_building_info_provider: &Option<BBIP>,
+	previous_block_building_info: Option<(InherentData, Digest)>,
+) -> Result<(Block, Option<(InherentData, Digest)>)> {
+	let (maybe_inherent_data, pre_digest) = match &block_building_info_provider {
+		None => (None, Default::default()),
+		Some(bbip) => {
+			let (inherent_data_provider, pre_digest) = bbip
+				.get_inherent_providers_and_pre_digest(parent_hash, previous_block_building_info)
+				.await?;
+			let inherent_data = inherent_data_provider
+				.create_inherent_data()
+				.await
+				.map_err(|e| sc_cli::Error::Input(format!("I don't know how to convert {e:?}")))?;
+
+			(Some(inherent_data), Digest { logs: pre_digest })
+		},
+	};
+
+	let header = Block::Header::new(
+		parent_height + One::one(),
+		Default::default(),
+		Default::default(),
+		parent_hash,
+		pre_digest.clone(),
+	);
+	let mut extrinsics = <Vec<Block::Extrinsic>>::new();
+
+	run::<Block, _>(externalities, executor, "Core_initialize_block", &header.encode()).await?;
+
+	if let Some(ref inherent_data) = maybe_inherent_data {
+		extrinsics = dry_run::<Vec<Block::Extrinsic>, Block, _>(
+			externalities,
+			executor,
+			"BlockBuilder_inherent_extrinsics",
+			&inherent_data.encode(),
+		)
+		.await?;
+	}
+
+	for xt in &extrinsics {
+		run::<Block, _>(externalities, executor, "BlockBuilder_apply_extrinsic", &xt.encode())
+			.await?;
+	}
+
+	let header = dry_run::<Block::Header, Block, _>(
+		externalities,
+		executor,
+		"BlockBuilder_finalize_block",
+		&[0u8; 0],
+	)
+	.await?;
+
+	run::<Block, _>(externalities, executor, "BlockBuilder_finalize_block", &[0u8; 0]).await?;
+
+	Ok((Block::new(header, extrinsics), (maybe_inherent_data.map(|id| (id, pre_digest)))))
+}
+
+pub(crate) async fn fast_forward<Block, HostFns, BBIP>(
+	shared: SharedParams,
+	command: FastForwardCmd,
+	block_building_info_provider: Option<BBIP>,
+) -> Result<()>
+where
+	Block: BlockT<Hash = H256> + DeserializeOwned,
+	Block::Hash: FromStr,
+	Block::Header: DeserializeOwned,
+	<Block::Hash as FromStr>::Err: Debug,
+	NumberFor<Block>: FromStr,
+	<NumberFor<Block> as FromStr>::Err: Debug,
+	HostFns: HostFunctions,
+	BBIP: BlockBuildingInfoProvider<Block, Option<(InherentData, Digest)>>,
+{
+	let executor = build_executor::<HostFns>(&shared);
+	let ext = command.state.into_ext::<Block, HostFns>(&shared, &executor, None, true).await?;
+
+	let mut last_block_hash = ext.block_hash;
+	let mut last_block_number =
+		get_block_number::<Block>(last_block_hash, command.block_ws_uri()).await?;
+	let mut prev_block_building_info = None;
+
+	let mut ext = ext.inner_ext;
+
+	for _ in 1..=command.n_blocks.unwrap_or(u64::MAX) {
+		// We are saving state before we overwrite it while producing new block.
+		let backend = ext.as_backend();
+
+		log::info!("Producing new empty block at height {:?}", last_block_number + One::one());
+
+		let (next_block, new_block_building_info) = next_empty_block::<Block, HostFns, BBIP>(
+			&mut ext,
+			&executor,
+			last_block_number,
+			last_block_hash,
+			&block_building_info_provider,
+			prev_block_building_info,
+		)
+		.await?;
+
+		log::info!("Produced a new block: {:?}", next_block.header());
+
+		// And now we restore previous state.
+		ext.backend = backend;
+
+		let state_root_check = true;
+		let signature_check = true;
+		let payload =
+			(next_block.clone(), state_root_check, signature_check, command.try_state.clone())
+				.encode();
+		run::<Block, _>(&mut ext, &executor, "TryRuntime_execute_block", &payload).await?;
+
+		log::info!("Executed the new block");
+
+		prev_block_building_info = new_block_building_info;
+		last_block_hash = next_block.hash();
+		last_block_number += One::one();
+	}
+
+	Ok(())
+}

substrate/utils/frame/try-runtime/cli/src/commands/mod.rs

@@ -17,6 +17,7 @@
 
 pub mod create_snapshot;
 pub mod execute_block;
+pub mod fast_forward;
 pub mod follow_chain;
 pub mod offchain_worker;
 pub mod on_runtime_upgrade;

substrate/utils/frame/try-runtime/cli/src/lib.rs

@@ -358,6 +358,7 @@
 
 #![cfg(feature = "try-runtime")]
 
+use crate::block_building_info::BlockBuildingInfoProvider;
 use parity_scale_codec::Decode;
 use remote_externalities::{
 	Builder, Mode, OfflineConfig, OnlineConfig, RemoteExternalities, SnapshotConfig,
@@ -381,15 +382,17 @@ use sp_core::{
 	twox_128, H256,
 };
 use sp_externalities::Extensions;
+use sp_inherents::InherentData;
 use sp_keystore::{testing::KeyStore, KeystoreExt};
 use sp_runtime::{
 	traits::{BlakeTwo256, Block as BlockT, NumberFor},
-	DeserializeOwned,
+	DeserializeOwned, Digest,
 };
 use sp_state_machine::{CompactProof, OverlayedChanges, StateMachine, TrieBackendBuilder};
 use sp_version::StateVersion;
 use std::{fmt::Debug, path::PathBuf, str::FromStr};
 
+pub mod block_building_info;
 pub mod commands;
 pub(crate) mod parse;
 pub(crate) const LOG_TARGET: &str = "try-runtime::cli";
@@ -445,6 +448,15 @@ pub enum Command {
 	/// tested has remained the same, otherwise block decoding might fail.
 	FollowChain(commands::follow_chain::FollowChainCmd),
 
+	/// Produce a series of empty, consecutive blocks and execute them one-by-one.
+	///
+	/// To compare it with [`Command::FollowChain`]:
+	///  - we don't have the delay of the original blocktime (for Polkadot 6s), but instead, we
+	///    execute every block immediately
+	///  - the only data that will be put into blocks are pre-runtime digest items and inherent
+	///    extrinsics; both things should be defined in your node CLI handling level
+	FastForward(commands::fast_forward::FastForwardCmd),
+
 	/// Create a new snapshot file.
 	CreateSnapshot(commands::create_snapshot::CreateSnapshotCmd),
 }
@@ -719,7 +731,10 @@ impl State {
 }
 
 impl TryRuntimeCmd {
-	pub async fn run<Block, HostFns>(&self) -> sc_cli::Result<()>
+	pub async fn run<Block, HostFns, BBIP>(
+		&self,
+		block_building_info_provider: Option<BBIP>,
+	) -> sc_cli::Result<()>
 	where
 		Block: BlockT<Hash = H256> + DeserializeOwned,
 		Block::Header: DeserializeOwned,
@@ -729,6 +744,7 @@ impl TryRuntimeCmd {
 		<NumberFor<Block> as TryInto<u64>>::Error: Debug,
 		NumberFor<Block>: FromStr,
 		HostFns: HostFunctions,
+		BBIP: BlockBuildingInfoProvider<Block, Option<(InherentData, Digest)>>,
 	{
 		match &self.command {
 			Command::OnRuntimeUpgrade(ref cmd) =>
@@ -755,6 +771,13 @@ impl TryRuntimeCmd {
 					cmd.clone(),
 				)
 				.await,
+			Command::FastForward(cmd) =>
+				commands::fast_forward::fast_forward::<Block, HostFns, BBIP>(
+					self.shared.clone(),
+					cmd.clone(),
+					block_building_info_provider,
+				)
+				.await,
 			Command::CreateSnapshot(cmd) =>
 				commands::create_snapshot::create_snapshot::<Block, HostFns>(
 					self.shared.clone(),