Fast sync (#8884)

* State sync

* Importing state fixes

* Bugfixes

* Sync with proof

* Status reporting

* Unsafe sync mode

* Sync test

* Cleanup

* Apply suggestions from code review

Co-authored-by: cheme <emericchevalier.pro@gmail.com>
Co-authored-by: Pierre Krieger <pierre.krieger1708@gmail.com>

* set_genesis_storage

* Extract keys from range proof

* Detect iter completion

* Download and import bodies with fast sync

* Replaced meta updates tuple with a struct

* Fixed reverting finalized state

* Reverted timeout

* Typo

* Doc

* Doc

* Fixed light client test

* Fixed error handling

* Tweaks

* More UpdateMeta changes

* Rename convert_transaction

* Apply suggestions from code review

Co-authored-by: Bastian Köcher <bkchr@users.noreply.github.com>

* Apply suggestions from code review

Co-authored-by: Bastian Köcher <bkchr@users.noreply.github.com>

* Code review suggestions

* Fixed count handling

Co-authored-by: cheme <emericchevalier.pro@gmail.com>
Co-authored-by: Pierre Krieger <pierre.krieger1708@gmail.com>
Co-authored-by: Bastian Köcher <bkchr@users.noreply.github.com>

substrate/primitives/consensus/common/src/block_import.rs

@@ -135,6 +135,43 @@ pub struct BlockCheckParams<Block: BlockT> {
 	pub import_existing: bool,
 }
 
+/// Precomputed storage.
+pub enum StorageChanges<Block: BlockT, Transaction> {
+	/// Changes coming from block execution.
+	Changes(sp_state_machine::StorageChanges<Transaction, HashFor<Block>, NumberFor<Block>>),
+	/// Whole new state.
+	Import(ImportedState<Block>),
+}
+
+/// Imported state data. A vector of key-value pairs that should form a trie.
+#[derive(PartialEq, Eq, Clone)]
+pub struct ImportedState<B: BlockT> {
+	/// Target block hash.
+	pub block: B::Hash,
+	/// State keys and values.
+	pub state: Vec<(Vec<u8>, Vec<u8>)>,
+}
+
+impl<B: BlockT> std::fmt::Debug for ImportedState<B> {
+	fn fmt(&self, fmt: &mut std::fmt::Formatter) -> std::fmt::Result {
+		fmt.debug_struct("ImportedState")
+			.field("block", &self.block)
+			.finish()
+	}
+}
+
+/// Defines how a new state is computed for a given imported block.
+pub enum StateAction<Block: BlockT, Transaction> {
+	/// Apply precomputed changes coming from block execution or state sync.
+	ApplyChanges(StorageChanges<Block, Transaction>),
+	/// Execute block body (required) and compute state.
+	Execute,
+	/// Execute block body if parent state is available and compute state.
+	ExecuteIfPossible,
+	/// Don't execute or import state.
+	Skip,
+}
+
 /// Data required to import a Block.
 #[non_exhaustive]
 pub struct BlockImportParams<Block: BlockT, Transaction> {
@@ -159,11 +196,8 @@ pub struct BlockImportParams<Block: BlockT, Transaction> {
 	pub post_digests: Vec<DigestItemFor<Block>>,
 	/// The body of the block.
 	pub body: Option<Vec<Block::Extrinsic>>,
-	/// The changes to the storage to create the state for the block. If this is `Some(_)`,
-	/// the block import will not need to re-execute the block for importing it.
-	pub storage_changes: Option<
-		sp_state_machine::StorageChanges<Transaction, HashFor<Block>, NumberFor<Block>>
-	>,
+	/// Specify how the new state is computed.
+	pub state_action: StateAction<Block, Transaction>,
 	/// Is this block finalized already?
 	/// `true` implies instant finality.
 	pub finalized: bool,
@@ -182,8 +216,6 @@ pub struct BlockImportParams<Block: BlockT, Transaction> {
 	/// to modify it. If `None` is passed all the way down to bottom block
 	/// importer, the import fails with an `IncompletePipeline` error.
 	pub fork_choice: Option<ForkChoiceStrategy>,
-	/// Allow importing the block skipping state verification if parent state is missing.
-	pub allow_missing_state: bool,
 	/// Re-validate existing block.
 	pub import_existing: bool,
 	/// Cached full header hash (with post-digests applied).
@@ -201,12 +233,11 @@ impl<Block: BlockT, Transaction> BlockImportParams<Block, Transaction> {
 			justifications: None,
 			post_digests: Vec::new(),
 			body: None,
-			storage_changes: None,
+			state_action: StateAction::Execute,
 			finalized: false,
 			intermediates: HashMap::new(),
 			auxiliary: Vec::new(),
 			fork_choice: None,
-			allow_missing_state: false,
 			import_existing: false,
 			post_hash: None,
 		}
@@ -237,20 +268,28 @@ impl<Block: BlockT, Transaction> BlockImportParams<Block, Transaction> {
 
 	/// Auxiliary function for "converting" the transaction type.
 	///
-	/// Actually this just sets `storage_changes` to `None` and makes rustc think that `Self` now
+	/// Actually this just sets `StorageChanges::Changes` to `None` and makes rustc think that `Self` now
 	/// uses a different transaction type.
-	pub fn convert_transaction<Transaction2>(self) -> BlockImportParams<Block, Transaction2> {
+	pub fn clear_storage_changes_and_mutate<Transaction2>(self) -> BlockImportParams<Block, Transaction2> {
+		// Preserve imported state.
+		let state_action = match self.state_action {
+			StateAction::ApplyChanges(StorageChanges::Import(state)) =>
+				StateAction::ApplyChanges(StorageChanges::Import(state)),
+			StateAction::ApplyChanges(StorageChanges::Changes(_)) => StateAction::Skip,
+			StateAction::Execute => StateAction::Execute,
+			StateAction::ExecuteIfPossible => StateAction::ExecuteIfPossible,
+			StateAction::Skip => StateAction::Skip,
+		};
 		BlockImportParams {
 			origin: self.origin,
 			header: self.header,
 			justifications: self.justifications,
 			post_digests: self.post_digests,
 			body: self.body,
-			storage_changes: None,
+			state_action,
 			finalized: self.finalized,
 			auxiliary: self.auxiliary,
 			intermediates: self.intermediates,
-			allow_missing_state: self.allow_missing_state,
 			fork_choice: self.fork_choice,
 			import_existing: self.import_existing,
 			post_hash: self.post_hash,

substrate/primitives/consensus/common/src/import_queue.rs

@@ -34,7 +34,7 @@ use crate::{
 	error::Error as ConsensusError,
 	block_import::{
 		BlockImport, BlockOrigin, BlockImportParams, ImportedAux, JustificationImport, ImportResult,
-		BlockCheckParams,
+		BlockCheckParams, ImportedState, StateAction,
 	},
 	metrics::Metrics,
 };
@@ -74,8 +74,12 @@ pub struct IncomingBlock<B: BlockT> {
 	pub origin: Option<Origin>,
 	/// Allow importing the block skipping state verification if parent state is missing.
 	pub allow_missing_state: bool,
+	/// Skip block exection and state verification.
+	pub skip_execution: bool,
 	/// Re-validate existing block.
 	pub import_existing: bool,
+	/// Do not compute new state, but rather set it to the given set.
+	pub state: Option<ImportedState<B>>,
 }
 
 /// Type of keys in the blockchain cache that consensus module could use for its needs.
@@ -264,9 +268,17 @@ pub(crate) async fn import_single_block_metered<B: BlockT, V: Verifier<B>, Trans
 	if let Some(keys) = maybe_keys {
 		cache.extend(keys.into_iter());
 	}
-	import_block.allow_missing_state = block.allow_missing_state;
+	import_block.import_existing = block.import_existing;
+	let mut import_block = import_block.clear_storage_changes_and_mutate();
+	if let Some(state) = block.state {
+		import_block.state_action = StateAction::ApplyChanges(crate::StorageChanges::Import(state));
+	} else if block.skip_execution {
+		import_block.state_action = StateAction::Skip;
+	} else if block.allow_missing_state {
+		import_block.state_action = StateAction::ExecuteIfPossible;
+	}
 
-	let imported = import_handle.import_block(import_block.convert_transaction(), cache).await;
+	let imported = import_handle.import_block(import_block, cache).await;
 	if let Some(metrics) = metrics.as_ref() {
 		metrics.report_verification_and_import(started.elapsed());
 	}

substrate/primitives/consensus/common/src/import_queue/basic_queue.rs

@@ -564,6 +564,8 @@ mod tests {
 					origin: None,
 					allow_missing_state: false,
 					import_existing: false,
+					state: None,
+					skip_execution: false,
 				}],
 			)))
 			.unwrap();

substrate/primitives/consensus/common/src/lib.rs

@@ -50,7 +50,8 @@ mod metrics;
 pub use self::error::Error;
 pub use block_import::{
 	BlockCheckParams, BlockImport, BlockImportParams, BlockOrigin, ForkChoiceStrategy,
-	ImportResult, ImportedAux, JustificationImport, JustificationSyncLink,
+	ImportResult, ImportedAux, ImportedState, JustificationImport, JustificationSyncLink,
+	StateAction, StorageChanges,
 };
 pub use select_chain::SelectChain;
 pub use sp_state_machine::Backend as StateBackend;