Allow updating configuration of changes tries (#3201)

* DigestItem::ChangesTrieSignal * introduce changes_trie::State * introduce config activation block * ChangesTrieSignal::as_new_configuration * moved well_known_cache_keys to client * extracted DbChangesTrieStorage to separate file * change meaning of none in blockchain cache * changes trie config (FULL) cache draft * eliminating const ChangesTrieConfiguration * delay pruning * continue elimination * do not prune CT config from cache * removed redundant code * fix some TODOs * introduce ConfigurationRange * use Configuration range in build * build skewed digest * remove debug print * extracted surface iterator * key_changes works with skewed digests * fix client build * add test for NeverPrune * fix TODO * fixed some TODOs * more tests * fixing TODOs * fixed compilation * update runtime version * git rid of large tuple * too long lines * config_activation_block -> zero * obsolete TODO * removed unjustified expect * update TODOs with issue number * new CT pruning algorithm fixed cache + multiple blocks finalization track CT configuraiton on light clients support CT configuration change revert revert CT config test new CT pruning algorithm fixed cache + multiple blocks finalization track CT configuraiton on light clients support CT configuration change revert revert CT config test * BlockIdOrHeader isn't really required * removed debug leftovers + some docs * more docs * more post-merge fixes * more post-merge fixes * revertes some unnecessary changes * reverted unnecessary changes * fix compilation + unnecessary changes * (restart CI) * fix cache update when finalizing multiple blocks * fixed tests * collect_extrinsics -> set_collect_extrinsics * restore lost test * do not calculate block number twice * Update primitives/blockchain/src/error.rs Co-Authored-By: cheme <emericchevalier.pro@gmail.com> * map_err -> unwrap_or * document get_at Result * delete abandoned file * added weight for set_changes_trie_config * prefer_configs -> fail_if_disabled * Update client/api/src/backend.rs Co-Authored-By: cheme <emericchevalier.pro@gmail.com> * Update client/db/src/changes_tries_storage.rs Co-Authored-By: cheme <emericchevalier.pro@gmail.com> * CommitOperation+merge -> CommitOperations * fixed test compilation * merged two different CTRange structs * lost file * uggrade db from v0 to v1 (init CT cache + add column) * fix after merge Co-authored-by: cheme <emericchevalier.pro@gmail.com> Co-authored-by: Gavin Wood <github@gavwood.com>
2026-06-13 08:11:04 +00:00 · 2020-01-16 19:38:24 +03:00
parent 45fbf09dac
commit febf29390a
48 changed files with 2743 additions and 1548 deletions
@@ -23,8 +23,8 @@ use log::{warn, trace};
 use hash_db::Hasher;
 use codec::{Decode, Encode, Codec};
 use sp_core::{
-	storage::{well_known_keys, ChildInfo}, NativeOrEncoded, NeverNativeValue,
-	traits::{CodeExecutor, CallInWasmExt}, hexdisplay::HexDisplay
+	storage::ChildInfo, NativeOrEncoded, NeverNativeValue,
+	traits::{CodeExecutor, CallInWasmExt}, hexdisplay::HexDisplay,
 };
 use overlayed_changes::OverlayedChangeSet;
 use sp_externalities::Extensions;
@@ -49,15 +49,17 @@ pub use ext::Ext;
 pub use backend::Backend;
 pub use changes_trie::{
 	AnchorBlockId as ChangesTrieAnchorBlockId,
+	State as ChangesTrieState,
 	Storage as ChangesTrieStorage,
 	RootsStorage as ChangesTrieRootsStorage,
 	InMemoryStorage as InMemoryChangesTrieStorage,
 	BuildCache as ChangesTrieBuildCache,
 	CacheAction as ChangesTrieCacheAction,
 	ConfigurationRange as ChangesTrieConfigurationRange,
-	key_changes, key_changes_proof, key_changes_proof_check,
+	key_changes, key_changes_proof,
+	key_changes_proof_check, key_changes_proof_check_with_db,
 	prune as prune_changes_tries,
-	oldest_non_pruned_trie as oldest_non_pruned_changes_trie,
+	disabled_state as disabled_changes_trie_state,
 	BlockNumber as ChangesTrieBlockNumber,
 };
 pub use overlayed_changes::{OverlayedChanges, StorageChanges, StorageTransactionCache};
@@ -171,7 +173,7 @@ fn always_untrusted_wasm<E, R: Decode>() -> ExecutionManager<DefaultHandler<R, E
 }

 /// The substrate state machine.
-pub struct StateMachine<'a, B, H, N, T, Exec>
+pub struct StateMachine<'a, B, H, N, Exec>
 	where
 		H: Hasher,
 		B: Backend<H>,
@@ -183,23 +185,22 @@ pub struct StateMachine<'a, B, H, N, T, Exec>
 	call_data: &'a [u8],
 	overlay: &'a mut OverlayedChanges,
 	extensions: Extensions,
-	changes_trie_storage: Option<&'a T>,
+	changes_trie_state: Option<ChangesTrieState<'a, H, N>>,
 	_marker: PhantomData<(H, N)>,
 	storage_transaction_cache: Option<&'a mut StorageTransactionCache<B::Transaction, H, N>>,
 }

-impl<'a, B, H, N, T, Exec> StateMachine<'a, B, H, N, T, Exec> where
+impl<'a, B, H, N, Exec> StateMachine<'a, B, H, N, Exec> where
 	H: Hasher,
 	H::Out: Ord + 'static + codec::Codec,
 	Exec: CodeExecutor + Clone + 'static,
 	B: Backend<H>,
-	T: ChangesTrieStorage<H, N>,
 	N: crate::changes_trie::BlockNumber,
 {
 	/// Creates new substrate state machine.
 	pub fn new(
 		backend: &'a B,
-		changes_trie_storage: Option<&'a T>,
+		changes_trie_state: Option<ChangesTrieState<'a, H, N>>,
 		overlay: &'a mut OverlayedChanges,
 		exec: &'a Exec,
 		method: &'a str,
@@ -215,7 +216,7 @@ impl<'a, B, H, N, T, Exec> StateMachine<'a, B, H, N, T, Exec> where
 			call_data,
 			extensions,
 			overlay,
-			changes_trie_storage,
+			changes_trie_state,
 			_marker: PhantomData,
 			storage_transaction_cache: None,
 		}
@@ -273,7 +274,7 @@ impl<'a, B, H, N, T, Exec> StateMachine<'a, B, H, N, T, Exec> where
 			self.overlay,
 			cache,
 			self.backend,
-			self.changes_trie_storage.clone(),
+			self.changes_trie_state.clone(),
 			Some(&mut self.extensions),
 		);

@@ -388,20 +389,8 @@ impl<'a, B, H, N, T, Exec> StateMachine<'a, B, H, N, T, Exec> where
 				CallResult<R, Exec::Error>,
 			) -> CallResult<R, Exec::Error>
 	{
-		// read changes trie configuration. The reason why we're doing it here instead of the
-		// `OverlayedChanges` constructor is that we need proofs for this read as a part of
-		// proof-of-execution on light clients. And the proof is recorded by the backend which
-		// is created after OverlayedChanges
-
-		let init_overlay = |overlay: &mut OverlayedChanges, final_check: bool, backend: &B| {
-			let changes_trie_config = try_read_overlay_value(
-				overlay,
-				backend,
-				well_known_keys::CHANGES_TRIE_CONFIG
-			)?;
-			set_changes_trie_config(overlay, changes_trie_config, final_check)
-		};
-		init_overlay(self.overlay, false, &self.backend)?;
+		let changes_tries_enabled = self.changes_trie_state.is_some();
+		self.overlay.set_collect_extrinsics(changes_tries_enabled);

 		let result = {
 			let orig_prospective = self.overlay.prospective.clone();
@@ -433,16 +422,12 @@ impl<'a, B, H, N, T, Exec> StateMachine<'a, B, H, N, T, Exec> where
 			}
 		};

-		if result.is_ok() {
-			init_overlay(self.overlay, true, self.backend)?;
-		}
-
 		result.map_err(|e| Box::new(e) as _)
 	}
 }

 /// Prove execution using the given state backend, overlayed changes, and call executor.
-pub fn prove_execution<B, H, Exec>(
+pub fn prove_execution<B, H, N, Exec>(
 	mut backend: B,
 	overlay: &mut OverlayedChanges,
 	exec: &Exec,
@@ -454,10 +439,11 @@ where
 	H: Hasher,
 	H::Out: Ord + 'static + codec::Codec,
 	Exec: CodeExecutor + Clone + 'static,
+	N: crate::changes_trie::BlockNumber,
 {
 	let trie_backend = backend.as_trie_backend()
 		.ok_or_else(|| Box::new(ExecutionError::UnableToGenerateProof) as Box<dyn Error>)?;
-	prove_execution_on_trie_backend(trie_backend, overlay, exec, method, call_data)
+	prove_execution_on_trie_backend::<_, _, N, _>(trie_backend, overlay, exec, method, call_data)
 }

 /// Prove execution using the given trie backend, overlayed changes, and call executor.
@@ -469,7 +455,7 @@ where
 ///
 /// Note: changes to code will be in place if this call is made again. For running partial
 /// blocks (e.g. a transaction at a time), ensure a different method is used.
-pub fn prove_execution_on_trie_backend<S, H, Exec>(
+pub fn prove_execution_on_trie_backend<S, H, N, Exec>(
 	trie_backend: &TrieBackend<S, H>,
 	overlay: &mut OverlayedChanges,
 	exec: &Exec,
@@ -481,9 +467,10 @@ where
 	H: Hasher,
 	H::Out: Ord + 'static + codec::Codec,
 	Exec: CodeExecutor + 'static + Clone,
+	N: crate::changes_trie::BlockNumber,
 {
 	let proving_backend = proving_backend::ProvingBackend::new(trie_backend);
-	let mut sm = StateMachine::<_, H, _, InMemoryChangesTrieStorage<H, u64>, Exec>::new(
+	let mut sm = StateMachine::<_, H, N, Exec>::new(
 		&proving_backend, None, overlay, exec, method, call_data, Extensions::default(),
 	);

@@ -496,7 +483,7 @@ where
 }

 /// Check execution proof, generated by `prove_execution` call.
-pub fn execution_proof_check<H, Exec>(
+pub fn execution_proof_check<H, N, Exec>(
 	root: H::Out,
 	proof: StorageProof,
 	overlay: &mut OverlayedChanges,
@@ -508,13 +495,14 @@ where
 	H: Hasher,
 	Exec: CodeExecutor + Clone + 'static,
 	H::Out: Ord + 'static + codec::Codec,
+	N: crate::changes_trie::BlockNumber,
 {
 	let trie_backend = create_proof_check_backend::<H>(root.into(), proof)?;
-	execution_proof_check_on_trie_backend(&trie_backend, overlay, exec, method, call_data)
+	execution_proof_check_on_trie_backend::<_, N, _>(&trie_backend, overlay, exec, method, call_data)
 }

 /// Check execution proof on proving backend, generated by `prove_execution` call.
-pub fn execution_proof_check_on_trie_backend<H, Exec>(
+pub fn execution_proof_check_on_trie_backend<H, N, Exec>(
 	trie_backend: &TrieBackend<MemoryDB<H>, H>,
 	overlay: &mut OverlayedChanges,
 	exec: &Exec,
@@ -525,8 +513,9 @@ where
 	H: Hasher,
 	H::Out: Ord + 'static + codec::Codec,
 	Exec: CodeExecutor + Clone + 'static,
+	N: crate::changes_trie::BlockNumber,
 {
-	let mut sm = StateMachine::<_, H, _, InMemoryChangesTrieStorage<H, u64>, Exec>::new(
+	let mut sm = StateMachine::<_, H, N, Exec>::new(
 		trie_backend, None, overlay, exec, method, call_data, Extensions::default(),
 	);

@@ -692,44 +681,6 @@ where
 		.map_err(|e| Box::new(e) as Box<dyn Error>)
 }

-/// Sets overlayed changes' changes trie configuration. Returns error if configuration
-/// differs from previous OR config decode has failed.
-fn set_changes_trie_config(
-	overlay: &mut OverlayedChanges,
-	config: Option<Vec<u8>>,
-	final_check: bool,
-) -> Result<(), Box<dyn Error>> {
-	let config = match config {
-		Some(v) => Some(Decode::decode(&mut &v[..])
-			.map_err(|_| Box::new("Failed to decode changes trie configuration".to_owned()) as Box<dyn Error>)?),
-		None => None,
-	};
-
-	if final_check && overlay.changes_trie_config.is_some() != config.is_some() {
-		return Err(Box::new("Changes trie configuration change is not supported".to_owned()));
-	}
-
-	if let Some(config) = config {
-		if !overlay.set_changes_trie_config(config) {
-			return Err(Box::new("Changes trie configuration change is not supported".to_owned()));
-		}
-	}
-	Ok(())
-}
-
-/// Reads storage value from overlay or from the backend.
-fn try_read_overlay_value<H, B>(
-	overlay: &OverlayedChanges,
-	backend: &B, key: &[u8],
-) -> Result<Option<Vec<u8>>, Box<dyn Error>> where H: Hasher, B: Backend<H> {
-	match overlay.storage(key).map(|x| x.map(|x| x.to_vec())) {
-		Some(value) => Ok(value),
-		None => backend
-			.storage(key)
-			.map_err(|err| Box::new(ExecutionError::Backend(format!("{}", err))) as Box<dyn Error>),
-	}
-}
-
 #[cfg(test)]
 mod tests {
 	use std::collections::BTreeMap;
@@ -737,10 +688,7 @@ mod tests {
 	use overlayed_changes::OverlayedValue;
 	use super::*;
 	use super::ext::Ext;
-	use super::changes_trie::{
-		InMemoryStorage as InMemoryChangesTrieStorage,
-		Configuration as ChangesTrieConfig,
-	};
+	use super::changes_trie::Configuration as ChangesTrieConfig;
 	use sp_core::{Blake2Hasher, map, traits::Externalities, storage::ChildStorageKey};

 	#[derive(Clone)]
@@ -770,7 +718,7 @@ mod tests {
 		) -> (CallResult<R, Self::Error>, bool) {
 			if self.change_changes_trie_config {
 				ext.place_storage(
-					well_known_keys::CHANGES_TRIE_CONFIG.to_vec(),
+					sp_core::storage::well_known_keys::CHANGES_TRIE_CONFIG.to_vec(),
 					Some(
 						ChangesTrieConfig {
 							digest_interval: 777,
@@ -816,11 +764,10 @@ mod tests {
 	fn execute_works() {
 		let backend = trie_backend::tests::test_trie();
 		let mut overlayed_changes = Default::default();
-		let changes_trie_storage = InMemoryChangesTrieStorage::<Blake2Hasher, u64>::new();

 		let mut state_machine = StateMachine::new(
 			&backend,
-			Some(&changes_trie_storage),
+			changes_trie::disabled_state::<_, u64>(),
 			&mut overlayed_changes,
 			&DummyCodeExecutor {
 				change_changes_trie_config: false,
@@ -844,11 +791,10 @@ mod tests {
 	fn execute_works_with_native_else_wasm() {
 		let backend = trie_backend::tests::test_trie();
 		let mut overlayed_changes = Default::default();
-		let changes_trie_storage = InMemoryChangesTrieStorage::<Blake2Hasher, u64>::new();

 		let mut state_machine = StateMachine::new(
 			&backend,
-			Some(&changes_trie_storage),
+			changes_trie::disabled_state::<_, u64>(),
 			&mut overlayed_changes,
 			&DummyCodeExecutor {
 				change_changes_trie_config: false,
@@ -869,11 +815,10 @@ mod tests {
 		let mut consensus_failed = false;
 		let backend = trie_backend::tests::test_trie();
 		let mut overlayed_changes = Default::default();
-		let changes_trie_storage = InMemoryChangesTrieStorage::<Blake2Hasher, u64>::new();

 		let mut state_machine = StateMachine::new(
 			&backend,
-			Some(&changes_trie_storage),
+			changes_trie::disabled_state::<_, u64>(),
 			&mut overlayed_changes,
 			&DummyCodeExecutor {
 				change_changes_trie_config: false,
@@ -910,7 +855,7 @@ mod tests {
 		// fetch execution proof from 'remote' full node
 		let remote_backend = trie_backend::tests::test_trie();
 		let remote_root = remote_backend.storage_root(std::iter::empty()).0;
-		let (remote_result, remote_proof) = prove_execution(
+		let (remote_result, remote_proof) = prove_execution::<_, _, u64, _>(
 			remote_backend,
 			&mut Default::default(),
 			&executor,
@@ -919,7 +864,7 @@ mod tests {
 		).unwrap();

 		// check proof locally
-		let local_result = execution_proof_check::<Blake2Hasher, _>(
+		let local_result = execution_proof_check::<Blake2Hasher, u64, _>(
 			remote_root,
 			remote_proof,
 			&mut Default::default(),
@@ -956,13 +901,12 @@ mod tests {
 		};

 		{
-			let changes_trie_storage = InMemoryChangesTrieStorage::<Blake2Hasher, u64>::new();
 			let mut cache = StorageTransactionCache::default();
 			let mut ext = Ext::new(
 				&mut overlay,
 				&mut cache,
 				backend,
-				Some(&changes_trie_storage),
+				changes_trie::disabled_state::<_, u64>(),
 				None,
 			);
 			ext.clear_prefix(b"ab");
@@ -987,14 +931,13 @@ mod tests {
 	fn set_child_storage_works() {
 		let mut state = InMemoryBackend::<Blake2Hasher>::default();
 		let backend = state.as_trie_backend().unwrap();
-		let changes_trie_storage = InMemoryChangesTrieStorage::<Blake2Hasher, u64>::new();
 		let mut overlay = OverlayedChanges::default();
 		let mut cache = StorageTransactionCache::default();
 		let mut ext = Ext::new(
 			&mut overlay,
 			&mut cache,
 			backend,
-			Some(&changes_trie_storage),
+			changes_trie::disabled_state::<_, u64>(),
 			None,
 		);

@@ -1080,30 +1023,6 @@ mod tests {
 		);
 	}

-	#[test]
-	fn cannot_change_changes_trie_config() {
-		let backend = trie_backend::tests::test_trie();
-		let mut overlayed_changes = Default::default();
-		let changes_trie_storage = InMemoryChangesTrieStorage::<Blake2Hasher, u64>::new();
-
-		let mut state_machine = StateMachine::new(
-			&backend,
-			Some(&changes_trie_storage),
-			&mut overlayed_changes,
-			&DummyCodeExecutor {
-				change_changes_trie_config: true,
-				native_available: false,
-				native_succeeds: true,
-				fallback_succeeds: true,
-			},
-			"test",
-			&[],
-			Default::default(),
-		);
-
-		assert!(state_machine.execute(ExecutionStrategy::NativeWhenPossible).is_err());
-	}
-
 	#[test]
 	fn child_storage_uuid() {
 		const CHILD_INFO_1: ChildInfo<'static> = ChildInfo::new_default(b"unique_id_1");
@@ -1115,13 +1034,12 @@ mod tests {
 		let subtrie2 = ChildStorageKey::from_slice(b":child_storage:default:sub_test2").unwrap();
 		let mut transaction = {
 			let backend = test_trie();
-			let changes_trie_storage = InMemoryChangesTrieStorage::<Blake2Hasher, u64>::new();
 			let mut cache = StorageTransactionCache::default();
 			let mut ext = Ext::new(
 				&mut overlay,
 				&mut cache,
 				&backend,
-				Some(&changes_trie_storage),
+				changes_trie::disabled_state::<_, u64>(),
 				None,
 			);
 			ext.set_child_storage(subtrie1, CHILD_INFO_1, b"abc".to_vec(), b"def".to_vec());
@@ -1139,28 +1057,4 @@ mod tests {
 		}
 		assert!(!duplicate);
 	}
-
-	#[test]
-	fn cannot_change_changes_trie_config_with_native_else_wasm() {
-		let backend = trie_backend::tests::test_trie();
-		let mut overlayed_changes = Default::default();
-		let changes_trie_storage = InMemoryChangesTrieStorage::<Blake2Hasher, u64>::new();
-
-		let mut state_machine = StateMachine::new(
-			&backend,
-			Some(&changes_trie_storage),
-			&mut overlayed_changes,
-			&DummyCodeExecutor {
-				change_changes_trie_config: true,
-				native_available: false,
-				native_succeeds: true,
-				fallback_succeeds: true,
-			},
-			"test",
-			&[],
-			Default::default(),
-		);
-
-		assert!(state_machine.execute(ExecutionStrategy::NativeElseWasm).is_err());
-	}
 }