Allow updating configuration of changes tries (#3201)

* DigestItem::ChangesTrieSignal

* introduce changes_trie::State

* introduce config activation block

* ChangesTrieSignal::as_new_configuration

* moved well_known_cache_keys to client

* extracted DbChangesTrieStorage to separate file

* change meaning of none in blockchain cache

* changes trie config (FULL) cache draft

* eliminating const ChangesTrieConfiguration

* delay pruning

* continue elimination

* do not prune CT config from cache

* removed redundant code

* fix some TODOs

* introduce ConfigurationRange

* use Configuration range in build

* build skewed digest

* remove debug print

* extracted surface iterator

* key_changes works with skewed digests

* fix client build

* add test for NeverPrune

* fix TODO

* fixed some TODOs

* more tests

* fixing TODOs

* fixed compilation

* update runtime version

* git rid of large tuple

* too long lines

* config_activation_block -> zero

* obsolete TODO

* removed unjustified expect

* update TODOs with issue number

* new CT pruning algorithm

fixed cache + multiple blocks finalization

track CT configuraiton on light clients

support CT configuration change revert

revert CT config test

new CT pruning algorithm

fixed cache + multiple blocks finalization

track CT configuraiton on light clients

support CT configuration change revert

revert CT config test

* BlockIdOrHeader isn't really required

* removed debug leftovers + some docs

* more docs

* more post-merge fixes

* more post-merge fixes

* revertes some unnecessary changes

* reverted unnecessary changes

* fix compilation + unnecessary changes

* (restart CI)

* fix cache update when finalizing multiple blocks

* fixed tests

* collect_extrinsics -> set_collect_extrinsics

* restore lost test

* do not calculate block number twice

* Update primitives/blockchain/src/error.rs

Co-Authored-By: cheme <emericchevalier.pro@gmail.com>

* map_err -> unwrap_or

* document get_at Result

* delete abandoned file

* added weight for set_changes_trie_config

* prefer_configs -> fail_if_disabled

* Update client/api/src/backend.rs

Co-Authored-By: cheme <emericchevalier.pro@gmail.com>

* Update client/db/src/changes_tries_storage.rs

Co-Authored-By: cheme <emericchevalier.pro@gmail.com>

* CommitOperation+merge -> CommitOperations

* fixed test compilation

* merged two different CTRange structs

* lost file

* uggrade db from v0 to v1 (init CT cache + add column)

* fix after merge

Co-authored-by: cheme <emericchevalier.pro@gmail.com>
Co-authored-by: Gavin Wood <github@gavwood.com>

substrate/primitives/state-machine/src/changes_trie/mod.rs

@@ -63,7 +63,7 @@ pub use self::changes_iterator::{
 	key_changes, key_changes_proof,
 	key_changes_proof_check, key_changes_proof_check_with_db,
 };
-pub use self::prune::{prune, oldest_non_pruned_trie};
+pub use self::prune::prune;
 
 use std::collections::{HashMap, HashSet};
 use std::convert::TryInto;
@@ -121,6 +121,18 @@ pub struct AnchorBlockId<Hash: std::fmt::Debug, Number: BlockNumber> {
 	pub number: Number,
 }
 
+/// Changes tries state at some block.
+pub struct State<'a, H, Number> {
+	/// Configuration that is active at given block.
+	pub config: Configuration,
+	/// Configuration activation block number. Zero if it is the first coonfiguration on the chain,
+	/// or number of the block that have emit NewConfiguration signal (thus activating configuration
+	/// starting from the **next** block).
+	pub zero: Number,
+	/// Underlying changes tries storage reference.
+	pub storage: &'a dyn Storage<H, Number>,
+}
+
 /// Changes trie storage. Provides access to trie roots and trie nodes.
 pub trait RootsStorage<H: Hasher, Number: BlockNumber>: Send + Sync {
 	/// Resolve hash of the block into anchor.
@@ -170,14 +182,43 @@ pub struct ConfigurationRange<'a, N> {
 	pub end: Option<N>,
 }
 
+impl<'a, H, Number> State<'a, H, Number> {
+	/// Create state with given config and storage.
+	pub fn new(
+		config: Configuration,
+		zero: Number,
+		storage: &'a dyn Storage<H, Number>,
+	) -> Self {
+		Self {
+			config,
+			zero,
+			storage,
+		}
+	}
+}
+
+impl<'a, H, Number: Clone> Clone for State<'a, H, Number> {
+	fn clone(&self) -> Self {
+		State {
+			config: self.config.clone(),
+			zero: self.zero.clone(),
+			storage: self.storage,
+		}
+	}
+}
+
+/// Create state where changes tries are disabled.
+pub fn disabled_state<'a, H, Number>() -> Option<State<'a, H, Number>> {
+	None
+}
+
 /// Compute the changes trie root and transaction for given block.
 /// Returns Err(()) if unknown `parent_hash` has been passed.
 /// Returns Ok(None) if there's no data to perform computation.
-/// Panics if background storage returns an error (and `panic_on_storage_error` is `true`) OR
-/// if insert to MemoryDB fails.
-pub fn build_changes_trie<'a, B: Backend<H>, S: Storage<H, Number>, H: Hasher, Number: BlockNumber>(
+/// Panics if background storage returns an error OR if insert to MemoryDB fails.
+pub fn build_changes_trie<'a, B: Backend<H>, H: Hasher, Number: BlockNumber>(
 	backend: &B,
-	storage: Option<&'a S>,
+	state: Option<&'a State<'a, H, Number>>,
 	changes: &OverlayedChanges,
 	parent_hash: H::Out,
 	panic_on_storage_error: bool,
@@ -185,11 +226,6 @@ pub fn build_changes_trie<'a, B: Backend<H>, S: Storage<H, Number>, H: Hasher, N
 	where
 		H::Out: Ord + 'static + Encode,
 {
-	let (storage, config) = match (storage, changes.changes_trie_config.as_ref()) {
-		(Some(storage), Some(config)) => (storage, config),
-		_ => return Ok(None),
-	};
-
 	/// Panics when `res.is_err() && panic`, otherwise it returns `Err(())` on an error.
 	fn maybe_panic<R, E: std::fmt::Debug>(
 		res: std::result::Result<R, E>,
@@ -203,23 +239,35 @@ pub fn build_changes_trie<'a, B: Backend<H>, S: Storage<H, Number>, H: Hasher, N
 			})
 	}
 
-	// FIXME: remove this in https://github.com/paritytech/substrate/pull/3201
-	let config = ConfigurationRange {
-		config,
-		zero: Zero::zero(),
-		end: None,
+	// when storage isn't provided, changes tries aren't created
+	let state = match state {
+		Some(state) => state,
+		None => return Ok(None),
 	};
 
 	// build_anchor error should not be considered fatal
-	let parent = storage.build_anchor(parent_hash).map_err(|_| ())?;
+	let parent = state.storage.build_anchor(parent_hash).map_err(|_| ())?;
 	let block = parent.number.clone() + One::one();
 
+	// prepare configuration range - we already know zero block. Current block may be the end block if configuration
+	// has been changed in this block
+	let is_config_changed = match changes.storage(sp_core::storage::well_known_keys::CHANGES_TRIE_CONFIG) {
+		Some(Some(new_config)) => new_config != &state.config.encode()[..],
+		Some(None) => true,
+		None => false,
+	};
+	let config_range = ConfigurationRange {
+		config: &state.config,
+		zero: state.zero.clone(),
+		end: if is_config_changed { Some(block.clone()) } else { None },
+	};
+
 	// storage errors are considered fatal (similar to situations when runtime fetches values from storage)
 	let (input_pairs, child_input_pairs, digest_input_blocks) = maybe_panic(
 		prepare_input::<B, H, Number>(
 			backend,
-			storage,
-			config.clone(),
+			state.storage,
+			config_range.clone(),
 			changes,
 			&parent,
 		),
@@ -227,7 +275,7 @@ pub fn build_changes_trie<'a, B: Backend<H>, S: Storage<H, Number>, H: Hasher, N
 	)?;
 
 	// prepare cached data
-	let mut cache_action = prepare_cached_build_data(config, block.clone());
+	let mut cache_action = prepare_cached_build_data(config_range, block.clone());
 	let needs_changed_keys = cache_action.collects_changed_keys();
 	cache_action = cache_action.set_digest_input_blocks(digest_input_blocks);