Reorganising the repository - external renames and moves (#4074)

* Adding first rough ouline of the repository structure

* Remove old CI stuff

* add title

* formatting fixes

* move node-exits job's script to scripts dir

* Move docs into subdir

* move to bin

* move maintainence scripts, configs and helpers into its own dir

* add .local to ignore

* move core->client

* start up 'test' area

* move test client

* move test runtime

* make test move compile

* Add dependencies rule enforcement.

* Fix indexing.

* Update docs to reflect latest changes

* Moving /srml->/paint

* update docs

* move client/sr-* -> primitives/

* clean old readme

* remove old broken code in rhd

* update lock

* Step 1.

* starting to untangle client

* Fix after merge.

* start splitting out client interfaces

* move children and blockchain interfaces

* Move trie and state-machine to primitives.

* Fix WASM builds.

* fixing broken imports

* more interface moves

* move backend and light to interfaces

* move CallExecutor

* move cli off client

* moving around more interfaces

* re-add consensus crates into the mix

* fix subkey path

* relieve client from executor

* starting to pull out client from grandpa

* move is_decendent_of out of client

* grandpa still depends on client directly

* lemme tests pass

* rename srml->paint

* Make it compile.

* rename interfaces->client-api

* Move keyring to primitives.

* fixup libp2p dep

* fix broken use

* allow dependency enforcement to fail

* move fork-tree

* Moving wasm-builder

* make env

* move build-script-utils

* fixup broken crate depdencies and names

* fix imports for authority discovery

* fix typo

* update cargo.lock

* fixing imports

* Fix paths and add missing crates

* re-add missing crates

substrate/primitives/state-machine/src/changes_trie/build.rs

@@ -0,0 +1,762 @@
+// Copyright 2017-2019 Parity Technologies (UK) Ltd.
+// This file is part of Substrate.
+
+// Substrate is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// Substrate is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with Substrate.  If not, see <http://www.gnu.org/licenses/>.
+
+//! Structures and functions required to build changes trie for given block.
+
+use std::collections::{BTreeMap, BTreeSet};
+use std::collections::btree_map::Entry;
+use codec::Decode;
+use hash_db::Hasher;
+use num_traits::One;
+use crate::backend::Backend;
+use crate::overlayed_changes::OverlayedChanges;
+use crate::trie_backend_essence::TrieBackendEssence;
+use crate::changes_trie::build_iterator::digest_build_iterator;
+use crate::changes_trie::input::{InputKey, InputPair, DigestIndex, ExtrinsicIndex};
+use crate::changes_trie::{AnchorBlockId, ConfigurationRange, Storage, BlockNumber};
+use crate::changes_trie::input::ChildIndex;
+
+/// Prepare input pairs for building a changes trie of given block.
+///
+/// Returns Err if storage error has occurred OR if storage haven't returned
+/// required data.
+pub(crate) fn prepare_input<'a, B, H, Number>(
+	backend: &'a B,
+	storage: &'a dyn Storage<H, Number>,
+	config: ConfigurationRange<'a, Number>,
+	changes: &'a OverlayedChanges,
+	parent: &'a AnchorBlockId<H::Out, Number>,
+) -> Result<(
+		impl Iterator<Item=InputPair<Number>> + 'a,
+		Vec<(ChildIndex<Number>, impl Iterator<Item=InputPair<Number>> + 'a)>,
+		Vec<Number>,
+	), String>
+	where
+		B: Backend<H>,
+		H: Hasher + 'a,
+		Number: BlockNumber,
+{
+	let number = parent.number.clone() + One::one();
+	let (extrinsics_input, children_extrinsics_input) = prepare_extrinsics_input(
+		backend,
+		&number,
+		changes,
+	)?;
+	let (digest_input, mut children_digest_input, digest_input_blocks) = prepare_digest_input::<H, Number>(
+		parent,
+		config,
+		number,
+		storage,
+	)?;
+
+	let mut children_digest = Vec::with_capacity(children_extrinsics_input.len());
+	for (child_index, ext_iter) in children_extrinsics_input.into_iter() {
+		let dig_iter = children_digest_input.remove(&child_index);
+		children_digest.push((
+			child_index,
+			Some(ext_iter).into_iter().flatten()
+				.chain(dig_iter.into_iter().flatten()),
+		));
+	}
+	for (child_index, dig_iter) in children_digest_input.into_iter() {
+		children_digest.push((
+			child_index,
+			None.into_iter().flatten()
+				.chain(Some(dig_iter).into_iter().flatten()),
+		));
+	}
+
+	Ok((
+		extrinsics_input.chain(digest_input),
+		children_digest,
+		digest_input_blocks,
+	))
+}
+/// Prepare ExtrinsicIndex input pairs.
+fn prepare_extrinsics_input<'a, B, H, Number>(
+	backend: &'a B,
+	block: &Number,
+	changes: &'a OverlayedChanges,
+) -> Result<(
+		impl Iterator<Item=InputPair<Number>> + 'a,
+		BTreeMap<ChildIndex<Number>, impl Iterator<Item=InputPair<Number>> + 'a>,
+	), String>
+	where
+		B: Backend<H>,
+		H: Hasher + 'a,
+		Number: BlockNumber,
+{
+
+	let mut children_keys = BTreeSet::<Vec<u8>>::new();
+	let mut children_result = BTreeMap::new();
+	for (storage_key, _) in changes.prospective.children.iter()
+		.chain(changes.committed.children.iter()) {
+		children_keys.insert(storage_key.clone());
+	}
+	for storage_key in children_keys {
+		let child_index = ChildIndex::<Number> {
+			block: block.clone(),
+			storage_key: storage_key.clone(),
+		};
+
+		let iter = prepare_extrinsics_input_inner(backend, block, changes, Some(storage_key))?;
+		children_result.insert(child_index, iter);
+	}
+
+	let top = prepare_extrinsics_input_inner(backend, block, changes, None)?;
+
+	Ok((top, children_result))
+}
+
+fn prepare_extrinsics_input_inner<'a, B, H, Number>(
+	backend: &'a B,
+	block: &Number,
+	changes: &'a OverlayedChanges,
+	storage_key: Option<Vec<u8>>,
+) -> Result<impl Iterator<Item=InputPair<Number>> + 'a, String>
+	where
+		B: Backend<H>,
+		H: Hasher,
+		Number: BlockNumber,
+{
+	let (committed, prospective) = if let Some(sk) = storage_key.as_ref() {
+		(changes.committed.children.get(sk), changes.prospective.children.get(sk))
+	} else {
+		(Some(&changes.committed.top), Some(&changes.prospective.top))
+	};
+	committed.iter().flat_map(|c| c.iter())
+		.chain(prospective.iter().flat_map(|c| c.iter()))
+		.filter(|( _, v)| v.extrinsics.is_some())
+		.try_fold(BTreeMap::new(), |mut map: BTreeMap<&[u8], (ExtrinsicIndex<Number>, Vec<u32>)>, (k, v)| {
+			match map.entry(k) {
+				Entry::Vacant(entry) => {
+					// ignore temporary values (values that have null value at the end of operation
+					// AND are not in storage at the beginning of operation
+					if let Some(sk) = storage_key.as_ref() {
+						if !changes.child_storage(sk, k).map(|v| v.is_some()).unwrap_or_default() {
+							if !backend.exists_child_storage(sk, k).map_err(|e| format!("{}", e))? {
+								return Ok(map);
+							}
+						}
+					} else {
+						if !changes.storage(k).map(|v| v.is_some()).unwrap_or_default() {
+							if !backend.exists_storage(k).map_err(|e| format!("{}", e))? {
+								return Ok(map);
+							}
+						}
+					};
+
+					let extrinsics = v.extrinsics.as_ref()
+						.expect("filtered by filter() call above; qed")
+						.iter().cloned().collect();
+					entry.insert((ExtrinsicIndex {
+						block: block.clone(),
+						key: k.to_vec(),
+					}, extrinsics));
+				},
+				Entry::Occupied(mut entry) => {
+					// we do not need to check for temporary values here, because entry is Occupied
+					// AND we are checking it before insertion
+					let extrinsics = &mut entry.get_mut().1;
+					extrinsics.extend(
+						v.extrinsics.as_ref()
+							.expect("filtered by filter() call above; qed")
+							.iter()
+							.cloned()
+					);
+					extrinsics.sort_unstable();
+				},
+			}
+
+			Ok(map)
+		})
+		.map(|pairs| pairs.into_iter().map(|(_, (k, v))| InputPair::ExtrinsicIndex(k, v)))
+}
+
+
+/// Prepare DigestIndex input pairs.
+fn prepare_digest_input<'a, H, Number>(
+	parent: &'a AnchorBlockId<H::Out, Number>,
+	config: ConfigurationRange<Number>,
+	block: Number,
+	storage: &'a dyn Storage<H, Number>,
+) -> Result<(
+		impl Iterator<Item=InputPair<Number>> + 'a,
+		BTreeMap<ChildIndex<Number>, impl Iterator<Item=InputPair<Number>> + 'a>,
+		Vec<Number>,
+	), String>
+	where
+		H: Hasher,
+		H::Out: 'a,
+		Number: BlockNumber,
+{
+	let build_skewed_digest = config.end.as_ref() == Some(&block);
+	let block_for_digest = if build_skewed_digest {
+		config.config.next_max_level_digest_range(config.zero.clone(), block.clone())
+			.map(|(_, end)| end)
+			.unwrap_or_else(|| block.clone())
+	} else {
+		block.clone()
+	};
+
+	let digest_input_blocks = digest_build_iterator(config, block_for_digest).collect::<Vec<_>>();
+	digest_input_blocks.clone().into_iter()
+		.try_fold(
+			(BTreeMap::new(), BTreeMap::new()), move |(mut map, mut child_map), digest_build_block| {
+			let extrinsic_prefix = ExtrinsicIndex::key_neutral_prefix(digest_build_block.clone());
+			let digest_prefix = DigestIndex::key_neutral_prefix(digest_build_block.clone());
+			let child_prefix = ChildIndex::key_neutral_prefix(digest_build_block.clone());
+			let trie_root = storage.root(parent, digest_build_block.clone())?;
+			let trie_root = trie_root.ok_or_else(|| format!("No changes trie root for block {}", digest_build_block.clone()))?;
+
+			let insert_to_map = |map: &mut BTreeMap<_,_>, key: Vec<u8>| {
+				match map.entry(key.clone()) {
+					Entry::Vacant(entry) => {
+						entry.insert((DigestIndex {
+							block: block.clone(),
+							key,
+						}, vec![digest_build_block.clone()]));
+					},
+					Entry::Occupied(mut entry) => {
+						// DigestIndexValue must be sorted. Here we are relying on the fact that digest_build_iterator()
+						// returns blocks in ascending order => we only need to check for duplicates
+						//
+						// is_dup_block could be true when key has been changed in both digest block
+						// AND other blocks that it covers
+						let is_dup_block = entry.get().1.last() == Some(&digest_build_block);
+						if !is_dup_block {
+							entry.get_mut().1.push(digest_build_block.clone());
+						}
+					},
+				}
+			};
+
+			// try to get all updated keys from cache
+			let populated_from_cache = storage.with_cached_changed_keys(
+				&trie_root,
+				&mut |changed_keys| {
+					for (storage_key, changed_keys) in changed_keys {
+						let map = match storage_key {
+							Some(storage_key) => child_map
+								.entry(ChildIndex::<Number> {
+									block: block.clone(),
+									storage_key: storage_key.clone(),
+								})
+								.or_default(),
+							None => &mut map,
+						};
+						for changed_key in changed_keys.iter().cloned() {
+							insert_to_map(map, changed_key);
+						}
+					}
+				}
+			);
+			if populated_from_cache {
+				return Ok((map, child_map));
+			}
+
+			let mut children_roots = BTreeMap::<Vec<u8>, _>::new();
+			{
+				let trie_storage = TrieBackendEssence::<_, H>::new(
+					crate::changes_trie::TrieBackendStorageAdapter(storage),
+					trie_root,
+				);
+
+				trie_storage.for_key_values_with_prefix(&child_prefix, |key, value|
+					if let Ok(InputKey::ChildIndex::<Number>(trie_key)) = Decode::decode(&mut &key[..]) {
+						if let Ok(value) = <Vec<u8>>::decode(&mut &value[..]) {
+							let mut trie_root = <H as Hasher>::Out::default();
+							trie_root.as_mut().copy_from_slice(&value[..]);
+							children_roots.insert(trie_key.storage_key, trie_root);
+						}
+					});
+
+				trie_storage.for_keys_with_prefix(&extrinsic_prefix, |key|
+					if let Ok(InputKey::ExtrinsicIndex::<Number>(trie_key)) = Decode::decode(&mut &key[..]) {
+						insert_to_map(&mut map, trie_key.key);
+					});
+
+				trie_storage.for_keys_with_prefix(&digest_prefix, |key|
+					if let Ok(InputKey::DigestIndex::<Number>(trie_key)) = Decode::decode(&mut &key[..]) {
+						insert_to_map(&mut map, trie_key.key);
+					});
+			}
+
+			for (storage_key, trie_root) in children_roots.into_iter() {
+				let child_index = ChildIndex::<Number> {
+					block: block.clone(),
+					storage_key,
+				};
+
+				let mut map = child_map.entry(child_index).or_default();
+				let trie_storage = TrieBackendEssence::<_, H>::new(
+					crate::changes_trie::TrieBackendStorageAdapter(storage),
+					trie_root,
+				);
+				trie_storage.for_keys_with_prefix(&extrinsic_prefix, |key|
+					if let Ok(InputKey::ExtrinsicIndex::<Number>(trie_key)) = Decode::decode(&mut &key[..]) {
+						insert_to_map(&mut map, trie_key.key);
+					});
+
+				trie_storage.for_keys_with_prefix(&digest_prefix, |key|
+					if let Ok(InputKey::DigestIndex::<Number>(trie_key)) = Decode::decode(&mut &key[..]) {
+						insert_to_map(&mut map, trie_key.key);
+					});
+			}
+			Ok((map, child_map))
+		})
+		.map(|(pairs, child_pairs)| (
+			pairs.into_iter().map(|(_, (k, v))| InputPair::DigestIndex(k, v)),
+			child_pairs.into_iter().map(|(sk, pairs)|
+				(sk, pairs.into_iter().map(|(_, (k, v))| InputPair::DigestIndex(k, v)))).collect(),
+			digest_input_blocks,
+		))
+}
+
+#[cfg(test)]
+mod test {
+	use codec::Encode;
+	use primitives::Blake2Hasher;
+	use primitives::storage::well_known_keys::{EXTRINSIC_INDEX};
+	use crate::backend::InMemory;
+	use crate::changes_trie::{RootsStorage, Configuration, storage::InMemoryStorage};
+	use crate::changes_trie::build_cache::{IncompleteCacheAction, IncompleteCachedBuildData};
+	use crate::overlayed_changes::{OverlayedValue, OverlayedChangeSet};
+	use super::*;
+
+	fn prepare_for_build(zero: u64) -> (
+		InMemory<Blake2Hasher>,
+		InMemoryStorage<Blake2Hasher, u64>,
+		OverlayedChanges,
+		Configuration,
+	) {
+		let config = Configuration { digest_interval: 4, digest_levels: 2 };
+		let backend: InMemory<_> = vec![
+			(vec![100], vec![255]),
+			(vec![101], vec![255]),
+			(vec![102], vec![255]),
+			(vec![103], vec![255]),
+			(vec![104], vec![255]),
+			(vec![105], vec![255]),
+		].into_iter().collect::<::std::collections::HashMap<_, _>>().into();
+		let child_trie_key1 = b"1".to_vec();
+		let child_trie_key2 = b"2".to_vec();
+		let storage = InMemoryStorage::with_inputs(vec![
+			(zero + 1, vec![
+				InputPair::ExtrinsicIndex(ExtrinsicIndex { block: zero + 1, key: vec![100] }, vec![1, 3]),
+				InputPair::ExtrinsicIndex(ExtrinsicIndex { block: zero + 1, key: vec![101] }, vec![0, 2]),
+				InputPair::ExtrinsicIndex(ExtrinsicIndex { block: zero + 1, key: vec![105] }, vec![0, 2, 4]),
+			]),
+			(zero + 2, vec![
+				InputPair::ExtrinsicIndex(ExtrinsicIndex { block: zero + 2, key: vec![102] }, vec![0]),
+			]),
+			(zero + 3, vec![
+				InputPair::ExtrinsicIndex(ExtrinsicIndex { block: zero + 3, key: vec![100] }, vec![0]),
+				InputPair::ExtrinsicIndex(ExtrinsicIndex { block: zero + 3, key: vec![105] }, vec![1]),
+			]),
+			(zero + 4, vec![
+				InputPair::ExtrinsicIndex(ExtrinsicIndex { block: zero + 4, key: vec![100] }, vec![0, 2, 3]),
+				InputPair::ExtrinsicIndex(ExtrinsicIndex { block: zero + 4, key: vec![101] }, vec![1]),
+				InputPair::ExtrinsicIndex(ExtrinsicIndex { block: zero + 4, key: vec![103] }, vec![0, 1]),
+
+				InputPair::DigestIndex(DigestIndex { block: zero + 4, key: vec![100] }, vec![zero + 1, zero + 3]),
+				InputPair::DigestIndex(DigestIndex { block: zero + 4, key: vec![101] }, vec![zero + 1]),
+				InputPair::DigestIndex(DigestIndex { block: zero + 4, key: vec![102] }, vec![zero + 2]),
+				InputPair::DigestIndex(DigestIndex { block: zero + 4, key: vec![105] }, vec![zero + 1, zero + 3]),
+			]),
+			(zero + 5, Vec::new()),
+			(zero + 6, vec![
+				InputPair::ExtrinsicIndex(ExtrinsicIndex { block: zero + 6, key: vec![105] }, vec![2]),
+			]),
+			(zero + 7, Vec::new()),
+			(zero + 8, vec![
+				InputPair::DigestIndex(DigestIndex { block: zero + 8, key: vec![105] }, vec![zero + 6]),
+			]),
+			(zero + 9, Vec::new()), (zero + 10, Vec::new()), (zero + 11, Vec::new()), (zero + 12, Vec::new()),
+			(zero + 13, Vec::new()), (zero + 14, Vec::new()), (zero + 15, Vec::new()),
+		], vec![(child_trie_key1.clone(), vec![
+				(zero + 1, vec![
+					InputPair::ExtrinsicIndex(ExtrinsicIndex { block: zero + 1, key: vec![100] }, vec![1, 3]),
+					InputPair::ExtrinsicIndex(ExtrinsicIndex { block: zero + 1, key: vec![101] }, vec![0, 2]),
+					InputPair::ExtrinsicIndex(ExtrinsicIndex { block: zero + 1, key: vec![105] }, vec![0, 2, 4]),
+				]),
+				(zero + 2, vec![
+					InputPair::ExtrinsicIndex(ExtrinsicIndex { block: zero + 2, key: vec![102] }, vec![0]),
+				]),
+				(zero + 4, vec![
+					InputPair::ExtrinsicIndex(ExtrinsicIndex { block: zero + 2, key: vec![102] }, vec![0, 3]),
+
+					InputPair::DigestIndex(DigestIndex { block: zero + 4, key: vec![102] }, vec![zero + 2]),
+				]),
+			]),
+		]);
+		let changes = OverlayedChanges {
+			prospective: OverlayedChangeSet { top: vec![
+				(vec![100], OverlayedValue {
+					value: Some(vec![200]),
+					extrinsics: Some(vec![0, 2].into_iter().collect())
+				}),
+				(vec![103], OverlayedValue {
+					value: None,
+					extrinsics: Some(vec![0, 1].into_iter().collect())
+				}),
+			].into_iter().collect(),
+				children: vec![
+					(child_trie_key1.clone(), vec![
+						(vec![100], OverlayedValue {
+							value: Some(vec![200]),
+							extrinsics: Some(vec![0, 2].into_iter().collect())
+						})
+					].into_iter().collect()),
+					(child_trie_key2, vec![
+						(vec![100], OverlayedValue {
+							value: Some(vec![200]),
+							extrinsics: Some(vec![0, 2].into_iter().collect())
+						})
+					].into_iter().collect()),
+				].into_iter().collect()
+			},
+			committed: OverlayedChangeSet { top: vec![
+				(EXTRINSIC_INDEX.to_vec(), OverlayedValue {
+					value: Some(3u32.encode()),
+					extrinsics: None,
+				}),
+				(vec![100], OverlayedValue {
+					value: Some(vec![202]),
+					extrinsics: Some(vec![3].into_iter().collect())
+				}),
+				(vec![101], OverlayedValue {
+					value: Some(vec![203]),
+					extrinsics: Some(vec![1].into_iter().collect())
+				}),
+			].into_iter().collect(),
+				children: vec![
+					(child_trie_key1, vec![
+						(vec![100], OverlayedValue {
+							value: Some(vec![202]),
+							extrinsics: Some(vec![3].into_iter().collect())
+						})
+					].into_iter().collect()),
+				].into_iter().collect(),
+			},
+			changes_trie_config: Some(config.clone()),
+		};
+
+		(backend, storage, changes, config)
+	}
+
+	fn configuration_range<'a>(config: &'a Configuration, zero: u64) -> ConfigurationRange<'a, u64> {
+		ConfigurationRange {
+			config,
+			zero,
+			end: None,
+		}
+	}
+
+	#[test]
+	fn build_changes_trie_nodes_on_non_digest_block() {
+		fn test_with_zero(zero: u64) {
+			let (backend, storage, changes, config) = prepare_for_build(zero);
+			let parent = AnchorBlockId { hash: Default::default(), number: zero + 4 };
+			let changes_trie_nodes = prepare_input(
+				&backend,
+				&storage,
+				configuration_range(&config, zero),
+				&changes,
+				&parent,
+			).unwrap();
+			assert_eq!(changes_trie_nodes.0.collect::<Vec<InputPair<u64>>>(), vec![
+				InputPair::ExtrinsicIndex(ExtrinsicIndex { block: zero + 5, key: vec![100] }, vec![0, 2, 3]),
+				InputPair::ExtrinsicIndex(ExtrinsicIndex { block: zero + 5, key: vec![101] }, vec![1]),
+				InputPair::ExtrinsicIndex(ExtrinsicIndex { block: zero + 5, key: vec![103] }, vec![0, 1]),
+			]);
+			assert_eq!(changes_trie_nodes.1.into_iter()
+				.map(|(k,v)| (k, v.collect::<Vec<_>>())).collect::<Vec<_>>(), vec![
+				(ChildIndex { block: zero + 5u64, storage_key: b"1".to_vec() },
+					vec![
+						InputPair::ExtrinsicIndex(ExtrinsicIndex { block: zero + 5u64, key: vec![100] }, vec![0, 2, 3]),
+					]),
+				(ChildIndex { block: zero + 5, storage_key: b"2".to_vec() },
+					vec![
+						InputPair::ExtrinsicIndex(ExtrinsicIndex { block: zero + 5, key: vec![100] }, vec![0, 2]),
+					]),
+			]);
+
+		}
+
+		test_with_zero(0);
+		test_with_zero(16);
+		test_with_zero(17);
+	}
+
+	#[test]
+	fn build_changes_trie_nodes_on_digest_block_l1() {
+		fn test_with_zero(zero: u64) {
+			let (backend, storage, changes, config) = prepare_for_build(zero);
+			let parent = AnchorBlockId { hash: Default::default(), number: zero + 3 };
+			let changes_trie_nodes = prepare_input(
+				&backend,
+				&storage,
+				configuration_range(&config, zero),
+				&changes,
+				&parent,
+			).unwrap();
+			assert_eq!(changes_trie_nodes.0.collect::<Vec<InputPair<u64>>>(), vec![
+				InputPair::ExtrinsicIndex(ExtrinsicIndex { block: zero + 4, key: vec![100] }, vec![0, 2, 3]),
+				InputPair::ExtrinsicIndex(ExtrinsicIndex { block: zero + 4, key: vec![101] }, vec![1]),
+				InputPair::ExtrinsicIndex(ExtrinsicIndex { block: zero + 4, key: vec![103] }, vec![0, 1]),
+
+				InputPair::DigestIndex(DigestIndex { block: zero + 4, key: vec![100] }, vec![zero + 1, zero + 3]),
+				InputPair::DigestIndex(DigestIndex { block: zero + 4, key: vec![101] }, vec![zero + 1]),
+				InputPair::DigestIndex(DigestIndex { block: zero + 4, key: vec![102] }, vec![zero + 2]),
+				InputPair::DigestIndex(DigestIndex { block: zero + 4, key: vec![105] }, vec![zero + 1, zero + 3]),
+			]);
+			assert_eq!(changes_trie_nodes.1.into_iter()
+				.map(|(k,v)| (k, v.collect::<Vec<_>>())).collect::<Vec<_>>(), vec![
+				(ChildIndex { block: zero + 4u64, storage_key: b"1".to_vec() },
+					vec![
+						InputPair::ExtrinsicIndex(ExtrinsicIndex { block: zero + 4u64, key: vec![100] }, vec![0, 2, 3]),
+
+						InputPair::DigestIndex(DigestIndex { block: zero + 4, key: vec![100] }, vec![zero + 1]),
+						InputPair::DigestIndex(DigestIndex { block: zero + 4, key: vec![101] }, vec![zero + 1]),
+						InputPair::DigestIndex(DigestIndex { block: zero + 4, key: vec![102] }, vec![zero + 2]),
+						InputPair::DigestIndex(DigestIndex { block: zero + 4, key: vec![105] }, vec![zero + 1]),
+					]),
+				(ChildIndex { block: zero + 4, storage_key: b"2".to_vec() },
+					vec![
+						InputPair::ExtrinsicIndex(ExtrinsicIndex { block: zero + 4, key: vec![100] }, vec![0, 2]),
+					]),
+			]);
+
+		}
+
+		test_with_zero(0);
+		test_with_zero(16);
+		test_with_zero(17);
+	}
+
+	#[test]
+	fn build_changes_trie_nodes_on_digest_block_l2() {
+		fn test_with_zero(zero: u64) {
+			let (backend, storage, changes, config) = prepare_for_build(zero);
+			let parent = AnchorBlockId { hash: Default::default(), number: zero + 15 };
+			let changes_trie_nodes = prepare_input(
+				&backend,
+				&storage,
+				configuration_range(&config, zero),
+				&changes,
+				&parent,
+			).unwrap();
+			assert_eq!(changes_trie_nodes.0.collect::<Vec<InputPair<u64>>>(), vec![
+				InputPair::ExtrinsicIndex(ExtrinsicIndex { block: zero + 16, key: vec![100] }, vec![0, 2, 3]),
+				InputPair::ExtrinsicIndex(ExtrinsicIndex { block: zero + 16, key: vec![101] }, vec![1]),
+				InputPair::ExtrinsicIndex(ExtrinsicIndex { block: zero + 16, key: vec![103] }, vec![0, 1]),
+
+				InputPair::DigestIndex(DigestIndex { block: zero + 16, key: vec![100] }, vec![zero + 4]),
+				InputPair::DigestIndex(DigestIndex { block: zero + 16, key: vec![101] }, vec![zero + 4]),
+				InputPair::DigestIndex(DigestIndex { block: zero + 16, key: vec![102] }, vec![zero + 4]),
+				InputPair::DigestIndex(DigestIndex { block: zero + 16, key: vec![103] }, vec![zero + 4]),
+				InputPair::DigestIndex(DigestIndex { block: zero + 16, key: vec![105] }, vec![zero + 4, zero + 8]),
+			]);
+			assert_eq!(changes_trie_nodes.1.into_iter()
+				.map(|(k,v)| (k, v.collect::<Vec<_>>())).collect::<Vec<_>>(), vec![
+				(ChildIndex { block: zero + 16u64, storage_key: b"1".to_vec() },
+					vec![
+						InputPair::ExtrinsicIndex(ExtrinsicIndex { block: zero + 16u64, key: vec![100] }, vec![0, 2, 3]),
+
+						InputPair::DigestIndex(DigestIndex { block: zero + 16, key: vec![102] }, vec![zero + 4]),
+					]),
+				(ChildIndex { block: zero + 16, storage_key: b"2".to_vec() },
+					vec![
+						InputPair::ExtrinsicIndex(ExtrinsicIndex { block: zero + 16, key: vec![100] }, vec![0, 2]),
+					]),
+			]);
+
+		}
+
+		test_with_zero(0);
+		test_with_zero(16);
+		test_with_zero(17);
+	}
+
+	#[test]
+	fn build_changes_trie_nodes_on_skewed_digest_block() {
+		fn test_with_zero(zero: u64) {
+			let (backend, storage, changes, config) = prepare_for_build(zero);
+			let parent = AnchorBlockId { hash: Default::default(), number: zero + 10 };
+
+			let mut configuration_range = configuration_range(&config, zero);
+			let changes_trie_nodes = prepare_input(
+				&backend,
+				&storage,
+				configuration_range.clone(),
+				&changes,
+				&parent,
+			).unwrap();
+			assert_eq!(changes_trie_nodes.0.collect::<Vec<InputPair<u64>>>(), vec![
+				InputPair::ExtrinsicIndex(ExtrinsicIndex { block: zero + 11, key: vec![100] }, vec![0, 2, 3]),
+				InputPair::ExtrinsicIndex(ExtrinsicIndex { block: zero + 11, key: vec![101] }, vec![1]),
+				InputPair::ExtrinsicIndex(ExtrinsicIndex { block: zero + 11, key: vec![103] }, vec![0, 1]),
+			]);
+
+			configuration_range.end = Some(zero + 11);
+			let changes_trie_nodes = prepare_input(
+				&backend,
+				&storage,
+				configuration_range,
+				&changes,
+				&parent,
+			).unwrap();
+			assert_eq!(changes_trie_nodes.0.collect::<Vec<InputPair<u64>>>(), vec![
+				InputPair::ExtrinsicIndex(ExtrinsicIndex { block: zero + 11, key: vec![100] }, vec![0, 2, 3]),
+				InputPair::ExtrinsicIndex(ExtrinsicIndex { block: zero + 11, key: vec![101] }, vec![1]),
+				InputPair::ExtrinsicIndex(ExtrinsicIndex { block: zero + 11, key: vec![103] }, vec![0, 1]),
+
+				InputPair::DigestIndex(DigestIndex { block: zero + 11, key: vec![100] }, vec![zero + 4]),
+				InputPair::DigestIndex(DigestIndex { block: zero + 11, key: vec![101] }, vec![zero + 4]),
+				InputPair::DigestIndex(DigestIndex { block: zero + 11, key: vec![102] }, vec![zero + 4]),
+				InputPair::DigestIndex(DigestIndex { block: zero + 11, key: vec![103] }, vec![zero + 4]),
+				InputPair::DigestIndex(DigestIndex { block: zero + 11, key: vec![105] }, vec![zero + 4, zero + 8]),
+			]);
+		}
+
+		test_with_zero(0);
+		test_with_zero(16);
+		test_with_zero(17);
+	}
+
+	#[test]
+	fn build_changes_trie_nodes_ignores_temporary_storage_values() {
+		fn test_with_zero(zero: u64) {
+			let (backend, storage, mut changes, config) = prepare_for_build(zero);
+
+			// 110: missing from backend, set to None in overlay
+			changes.prospective.top.insert(vec![110], OverlayedValue {
+				value: None,
+				extrinsics: Some(vec![1].into_iter().collect())
+			});
+
+			let parent = AnchorBlockId { hash: Default::default(), number: zero + 3 };
+			let changes_trie_nodes = prepare_input(
+				&backend,
+				&storage,
+				configuration_range(&config, zero),
+				&changes,
+				&parent,
+			).unwrap();
+			assert_eq!(changes_trie_nodes.0.collect::<Vec<InputPair<u64>>>(), vec![
+				InputPair::ExtrinsicIndex(ExtrinsicIndex { block: zero + 4, key: vec![100] }, vec![0, 2, 3]),
+				InputPair::ExtrinsicIndex(ExtrinsicIndex { block: zero + 4, key: vec![101] }, vec![1]),
+				InputPair::ExtrinsicIndex(ExtrinsicIndex { block: zero + 4, key: vec![103] }, vec![0, 1]),
+
+				InputPair::DigestIndex(DigestIndex { block: zero + 4, key: vec![100] }, vec![zero + 1, zero + 3]),
+				InputPair::DigestIndex(DigestIndex { block: zero + 4, key: vec![101] }, vec![zero + 1]),
+				InputPair::DigestIndex(DigestIndex { block: zero + 4, key: vec![102] }, vec![zero + 2]),
+				InputPair::DigestIndex(DigestIndex { block: zero + 4, key: vec![105] }, vec![zero + 1, zero + 3]),
+			]);
+			assert_eq!(changes_trie_nodes.1.into_iter()
+				.map(|(k,v)| (k, v.collect::<Vec<_>>())).collect::<Vec<_>>(), vec![
+				(ChildIndex { block: zero + 4u64, storage_key: b"1".to_vec() },
+					vec![
+						InputPair::ExtrinsicIndex(ExtrinsicIndex { block: zero + 4u64, key: vec![100] }, vec![0, 2, 3]),
+
+						InputPair::DigestIndex(DigestIndex { block: zero + 4, key: vec![100] }, vec![zero + 1]),
+						InputPair::DigestIndex(DigestIndex { block: zero + 4, key: vec![101] }, vec![zero + 1]),
+						InputPair::DigestIndex(DigestIndex { block: zero + 4, key: vec![102] }, vec![zero + 2]),
+						InputPair::DigestIndex(DigestIndex { block: zero + 4, key: vec![105] }, vec![zero + 1]),
+					]),
+				(ChildIndex { block: zero + 4, storage_key: b"2".to_vec() },
+					vec![
+						InputPair::ExtrinsicIndex(ExtrinsicIndex { block: zero + 4, key: vec![100] }, vec![0, 2]),
+					]),
+			]);
+
+		}
+
+		test_with_zero(0);
+		test_with_zero(16);
+		test_with_zero(17);
+	}
+
+	#[test]
+	fn cache_is_used_when_changes_trie_is_built() {
+		let (backend, mut storage, changes, _) = prepare_for_build(0);
+		let config = changes.changes_trie_config.as_ref().unwrap();
+		let parent = AnchorBlockId { hash: Default::default(), number: 15 };
+
+		// override some actual values from storage with values from the cache
+		//
+		// top-level storage:
+		// (keys 100, 101, 103, 105 are now missing from block#4 => they do not appear
+		// in l2 digest at block 16)
+		//
+		// "1" child storage:
+		// key 102 is now missing from block#4 => it doesn't appear in l2 digest at block 16
+		// (keys 103, 104) are now added to block#4 => they appear in l2 digest at block 16
+		//
+		// "2" child storage:
+		// (keys 105, 106) are now added to block#4 => they appear in l2 digest at block 16
+		let trie_root4 = storage.root(&parent, 4).unwrap().unwrap();
+		let cached_data4 = IncompleteCacheAction::CacheBuildData(IncompleteCachedBuildData::new())
+			.set_digest_input_blocks(vec![1, 2, 3])
+			.insert(None, vec![vec![100], vec![102]].into_iter().collect())
+			.insert(Some(b"1".to_vec()), vec![vec![103], vec![104]].into_iter().collect())
+			.insert(Some(b"2".to_vec()), vec![vec![105], vec![106]].into_iter().collect())
+			.complete(4, &trie_root4);
+		storage.cache_mut().perform(cached_data4);
+
+		let (root_changes_trie_nodes, child_changes_tries_nodes, _) = prepare_input(
+			&backend,
+			&storage,
+			configuration_range(&config, 0),
+			&changes,
+			&parent,
+		).unwrap();
+		assert_eq!(root_changes_trie_nodes.collect::<Vec<InputPair<u64>>>(), vec![
+			InputPair::ExtrinsicIndex(ExtrinsicIndex { block: 16, key: vec![100] }, vec![0, 2, 3]),
+			InputPair::ExtrinsicIndex(ExtrinsicIndex { block: 16, key: vec![101] }, vec![1]),
+			InputPair::ExtrinsicIndex(ExtrinsicIndex { block: 16, key: vec![103] }, vec![0, 1]),
+
+			InputPair::DigestIndex(DigestIndex { block: 16, key: vec![100] }, vec![4]),
+			InputPair::DigestIndex(DigestIndex { block: 16, key: vec![102] }, vec![4]),
+			InputPair::DigestIndex(DigestIndex { block: 16, key: vec![105] }, vec![8]),
+		]);
+
+		let child_changes_tries_nodes = child_changes_tries_nodes
+			.into_iter()
+			.map(|(k, i)| (k, i.collect::<Vec<_>>()))
+			.collect::<BTreeMap<_, _>>();
+		assert_eq!(
+			child_changes_tries_nodes.get(&ChildIndex { block: 16u64, storage_key: b"1".to_vec() }).unwrap(),
+			&vec![
+				InputPair::ExtrinsicIndex(ExtrinsicIndex { block: 16u64, key: vec![100] }, vec![0, 2, 3]),
+
+				InputPair::DigestIndex(DigestIndex { block: 16u64, key: vec![103] }, vec![4]),
+				InputPair::DigestIndex(DigestIndex { block: 16u64, key: vec![104] }, vec![4]),
+			],
+		);
+		assert_eq!(
+			child_changes_tries_nodes.get(&ChildIndex { block: 16u64, storage_key: b"2".to_vec() }).unwrap(),
+			&vec![
+				InputPair::ExtrinsicIndex(ExtrinsicIndex { block: 16u64, key: vec![100] }, vec![0, 2]),
+
+				InputPair::DigestIndex(DigestIndex { block: 16u64, key: vec![105] }, vec![4]),
+				InputPair::DigestIndex(DigestIndex { block: 16u64, key: vec![106] }, vec![4]),
+			],
+		);
+
+	}
+}

substrate/primitives/state-machine/src/changes_trie/build_cache.rs

@@ -0,0 +1,262 @@
+// Copyright 2019 Parity Technologies (UK) Ltd.
+// This file is part of Substrate.
+
+// Substrate is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// Substrate is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with Substrate.  If not, see <http://www.gnu.org/licenses/>.
+
+//! Changes tries build cache.
+
+use std::collections::{HashMap, HashSet};
+
+/// Changes trie build cache.
+///
+/// Helps to avoid read of changes tries from the database when digest trie
+/// is built. It holds changed keys for every block (indexed by changes trie
+/// root) that could be referenced by future digest items. For digest entries
+/// it also holds keys covered by this digest. Entries for top level digests
+/// are never created, because they'll never be used to build other digests.
+///
+/// Entries are pruned from the cache once digest block that is using this entry
+/// is inserted (because digest block will includes all keys from this entry).
+/// When there's a fork, entries are pruned when first changes trie is inserted.
+pub struct BuildCache<H, N> {
+	/// Map of block (implies changes true) number => changes trie root.
+	roots_by_number: HashMap<N, H>,
+	/// Map of changes trie root => set of storage keys that are in this trie.
+	/// The `Option<Vec<u8>>` in inner `HashMap` stands for the child storage key.
+	/// If it is `None`, then the `HashSet` contains keys changed in top-level storage.
+	/// If it is `Some`, then the `HashSet` contains keys changed in child storage, identified by the key.
+	changed_keys: HashMap<H, HashMap<Option<Vec<u8>>, HashSet<Vec<u8>>>>,
+}
+
+/// The action to perform when block-with-changes-trie is imported.
+#[derive(Debug, PartialEq)]
+pub enum CacheAction<H, N> {
+	/// Cache data that has been collected when CT has been built.
+	CacheBuildData(CachedBuildData<H, N>),
+	/// Clear cache from all existing entries.
+	Clear,
+}
+
+/// The data that has been cached during changes trie building.
+#[derive(Debug, PartialEq)]
+pub struct CachedBuildData<H, N> {
+	block: N,
+	trie_root: H,
+	digest_input_blocks: Vec<N>,
+	changed_keys: HashMap<Option<Vec<u8>>, HashSet<Vec<u8>>>,
+}
+
+/// The action to perform when block-with-changes-trie is imported.
+#[derive(Debug, PartialEq)]
+pub(crate) enum IncompleteCacheAction<N> {
+	/// Cache data that has been collected when CT has been built.
+	CacheBuildData(IncompleteCachedBuildData<N>),
+	/// Clear cache from all existing entries.
+	Clear,
+}
+
+/// The data (without changes trie root) that has been cached during changes trie building.
+#[derive(Debug, PartialEq)]
+pub(crate) struct IncompleteCachedBuildData<N> {
+	digest_input_blocks: Vec<N>,
+	changed_keys: HashMap<Option<Vec<u8>>, HashSet<Vec<u8>>>,
+}
+
+impl<H, N> BuildCache<H, N>
+	where
+		N: Eq + ::std::hash::Hash,
+		H: Eq + ::std::hash::Hash + Clone,
+{
+	/// Create new changes trie build cache.
+	pub fn new() -> Self {
+		BuildCache {
+			roots_by_number: HashMap::new(),
+			changed_keys: HashMap::new(),
+		}
+	}
+
+	/// Get cached changed keys for changes trie with given root.
+	pub fn get(&self, root: &H) -> Option<&HashMap<Option<Vec<u8>>, HashSet<Vec<u8>>>> {
+		self.changed_keys.get(&root)
+	}
+
+	/// Execute given functor with cached entry for given block.
+	/// Returns true if the functor has been called and false otherwise.
+	pub fn with_changed_keys(
+		&self,
+		root: &H,
+		functor: &mut dyn FnMut(&HashMap<Option<Vec<u8>>, HashSet<Vec<u8>>>),
+	) -> bool {
+		match self.changed_keys.get(&root) {
+			Some(changed_keys) => {
+				functor(changed_keys);
+				true
+			},
+			None => false,
+		}
+	}
+
+	/// Insert data into cache.
+	pub fn perform(&mut self, action: CacheAction<H, N>) {
+		match action {
+			CacheAction::CacheBuildData(data) => {
+				self.roots_by_number.insert(data.block, data.trie_root.clone());
+				self.changed_keys.insert(data.trie_root, data.changed_keys);
+
+				for digest_input_block in data.digest_input_blocks {
+					let digest_input_block_hash = self.roots_by_number.remove(&digest_input_block);
+					if let Some(digest_input_block_hash) = digest_input_block_hash {
+						self.changed_keys.remove(&digest_input_block_hash);
+					}
+				}
+			},
+			CacheAction::Clear => {
+				self.roots_by_number.clear();
+				self.changed_keys.clear();
+			},
+		}
+	}
+}
+
+impl<N> IncompleteCacheAction<N> {
+	/// Returns true if we need to collect changed keys for this action.
+	pub fn collects_changed_keys(&self) -> bool {
+		match *self {
+			IncompleteCacheAction::CacheBuildData(_) => true,
+			IncompleteCacheAction::Clear => false,
+		}
+	}
+
+	/// Complete cache action with computed changes trie root.
+	pub(crate) fn complete<H: Clone>(self, block: N, trie_root: &H) -> CacheAction<H, N> {
+		match self {
+			IncompleteCacheAction::CacheBuildData(build_data) =>
+				CacheAction::CacheBuildData(build_data.complete(block, trie_root.clone())),
+			IncompleteCacheAction::Clear => CacheAction::Clear,
+		}
+	}
+
+	/// Set numbers of blocks that are superseded by this new entry.
+	///
+	/// If/when this build data is committed to the cache, entries for these blocks
+	/// will be removed from the cache.
+	pub(crate) fn set_digest_input_blocks(self, digest_input_blocks: Vec<N>) -> Self {
+		match self {
+			IncompleteCacheAction::CacheBuildData(build_data) =>
+				IncompleteCacheAction::CacheBuildData(build_data.set_digest_input_blocks(digest_input_blocks)),
+			IncompleteCacheAction::Clear => IncompleteCacheAction::Clear,
+		}
+	}
+
+	/// Insert changed keys of given storage into cached data.
+	pub(crate) fn insert(
+		self,
+		storage_key: Option<Vec<u8>>,
+		changed_keys: HashSet<Vec<u8>>,
+	) -> Self {
+		match self {
+			IncompleteCacheAction::CacheBuildData(build_data) =>
+				IncompleteCacheAction::CacheBuildData(build_data.insert(storage_key, changed_keys)),
+			IncompleteCacheAction::Clear => IncompleteCacheAction::Clear,
+		}
+	}
+}
+
+impl<N> IncompleteCachedBuildData<N> {
+	/// Create new cached data.
+	pub(crate) fn new() -> Self {
+		IncompleteCachedBuildData {
+			digest_input_blocks: Vec::new(),
+			changed_keys: HashMap::new(),
+		}
+	}
+
+	fn complete<H>(self, block: N, trie_root: H) -> CachedBuildData<H, N> {
+		CachedBuildData {
+			block,
+			trie_root,
+			digest_input_blocks: self.digest_input_blocks,
+			changed_keys: self.changed_keys,
+		}
+	}
+
+	fn set_digest_input_blocks(mut self, digest_input_blocks: Vec<N>) -> Self {
+		self.digest_input_blocks = digest_input_blocks;
+		self
+	}
+
+	fn insert(
+		mut self,
+		storage_key: Option<Vec<u8>>,
+		changed_keys: HashSet<Vec<u8>>,
+	) -> Self {
+		self.changed_keys.insert(storage_key, changed_keys);
+		self
+	}
+}
+
+#[cfg(test)]
+mod tests {
+	use super::*;
+
+	#[test]
+	fn updated_keys_are_stored_when_non_top_level_digest_is_built() {
+		let mut data = IncompleteCachedBuildData::<u32>::new();
+		data = data.insert(None, vec![vec![1]].into_iter().collect());
+		assert_eq!(data.changed_keys.len(), 1);
+
+		let mut cache = BuildCache::new();
+		cache.perform(CacheAction::CacheBuildData(data.complete(1, 1)));
+		assert_eq!(cache.changed_keys.len(), 1);
+		assert_eq!(
+			cache.get(&1).unwrap().clone(),
+			vec![(None, vec![vec![1]].into_iter().collect())].into_iter().collect(),
+		);
+	}
+
+	#[test]
+	fn obsolete_entries_are_purged_when_new_ct_is_built() {
+		let mut cache = BuildCache::<u32, u32>::new();
+		cache.perform(CacheAction::CacheBuildData(IncompleteCachedBuildData::new()
+			.insert(None, vec![vec![1]].into_iter().collect())
+			.complete(1, 1)));
+		cache.perform(CacheAction::CacheBuildData(IncompleteCachedBuildData::new()
+			.insert(None, vec![vec![2]].into_iter().collect())
+			.complete(2, 2)));
+		cache.perform(CacheAction::CacheBuildData(IncompleteCachedBuildData::new()
+			.insert(None, vec![vec![3]].into_iter().collect())
+			.complete(3, 3)));
+
+		assert_eq!(cache.changed_keys.len(), 3);
+
+		cache.perform(CacheAction::CacheBuildData(IncompleteCachedBuildData::new()
+			.set_digest_input_blocks(vec![1, 2, 3])
+			.complete(4, 4)));
+
+		assert_eq!(cache.changed_keys.len(), 1);
+
+		cache.perform(CacheAction::CacheBuildData(IncompleteCachedBuildData::new()
+			.insert(None, vec![vec![8]].into_iter().collect())
+			.complete(8, 8)));
+		cache.perform(CacheAction::CacheBuildData(IncompleteCachedBuildData::new()
+			.insert(None, vec![vec![12]].into_iter().collect())
+			.complete(12, 12)));
+
+		assert_eq!(cache.changed_keys.len(), 3);
+
+		cache.perform(CacheAction::Clear);
+
+		assert_eq!(cache.changed_keys.len(), 0);
+	}
+}

substrate/primitives/state-machine/src/changes_trie/build_iterator.rs

@@ -0,0 +1,431 @@
+// Copyright 2017-2019 Parity Technologies (UK) Ltd.
+// This file is part of Substrate.
+
+// Substrate is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// Substrate is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with Substrate.  If not, see <http://www.gnu.org/licenses/>.
+
+//! Structures and functions to return blocks whose changes are to be included
+//! in given block's changes trie.
+
+use num_traits::Zero;
+use crate::changes_trie::{ConfigurationRange, BlockNumber};
+
+/// Returns iterator of OTHER blocks that are required for inclusion into
+/// changes trie of given block. Blocks are guaranteed to be returned in
+/// ascending order.
+///
+/// Skewed digest is built IF block >= config.end.
+pub fn digest_build_iterator<'a, Number: BlockNumber>(
+	config: ConfigurationRange<'a, Number>,
+	block: Number,
+) -> DigestBuildIterator<Number> {
+	// prepare digest build parameters
+	let (_, _, digest_step) = match config.config.digest_level_at_block(config.zero, block.clone()) {
+		Some((current_level, digest_interval, digest_step)) =>
+			(current_level, digest_interval, digest_step),
+		None => return DigestBuildIterator::empty(),
+	};
+
+	DigestBuildIterator::new(block.clone(), config.end.unwrap_or(block), config.config.digest_interval, digest_step)
+}
+
+/// Changes trie build iterator that returns numbers of OTHER blocks that are
+/// required for inclusion into changes trie of given block.
+#[derive(Debug)]
+pub struct DigestBuildIterator<Number: BlockNumber> {
+	/// Block we're building changes trie for. It could (logically) be a post-end block if we are creating
+	/// skewed digest.
+	block: Number,
+	/// Block that is a last block where current configuration is active. We have never yet created anything
+	/// after this block => digest that we're creating can't reference any blocks that are >= end.
+	end: Number,
+	/// Interval of L1 digest blocks.
+	digest_interval: u32,
+	/// Max step that could be used when digest is created.
+	max_step: u32,
+
+	// Mutable data below:
+
+	/// Step of current blocks range.
+	current_step: u32,
+	/// Reverse step of current blocks range.
+	current_step_reverse: u32,
+	/// Current blocks range.
+	current_range: Option<BlocksRange<Number>>,
+	/// Last block that we have returned.
+	last_block: Option<Number>,
+}
+
+impl<Number: BlockNumber> DigestBuildIterator<Number> {
+	/// Create new digest build iterator.
+	pub fn new(block: Number, end: Number, digest_interval: u32, max_step: u32) -> Self {
+		DigestBuildIterator {
+			block,
+			end,
+			digest_interval,
+			max_step,
+			current_step: max_step,
+			current_step_reverse: 0,
+			current_range: None,
+			last_block: None,
+		}
+	}
+
+	/// Create empty digest build iterator.
+	pub fn empty() -> Self {
+		Self::new(Zero::zero(), Zero::zero(), 0, 0)
+	}
+}
+
+impl<Number: BlockNumber> Iterator for DigestBuildIterator<Number> {
+	type Item = Number;
+
+	fn next(&mut self) -> Option<Self::Item> {
+		// when we're building skewed digest, we might want to skip some blocks if
+		// they're not covered by current configuration
+		loop {
+			if let Some(next) = self.current_range.as_mut().and_then(|iter| iter.next()) {
+				if next < self.end {
+					self.last_block = Some(next.clone());
+					return Some(next);
+				}
+			}
+
+			// we are safe to use non-checking mul/sub versions here because:
+			// DigestBuildIterator is created only by internal function that is checking
+			// that all multiplications/subtractions are safe within max_step limit
+
+			let next_step_reverse = if self.current_step_reverse == 0 {
+				1
+			} else {
+				self.current_step_reverse * self.digest_interval
+			};
+			if next_step_reverse > self.max_step {
+				return None;
+			}
+
+			self.current_step_reverse = next_step_reverse;
+			self.current_range = Some(BlocksRange::new(
+				match self.last_block.clone() {
+					Some(last_block) => last_block + self.current_step.into(),
+					None => self.block.clone() - (self.current_step * self.digest_interval - self.current_step).into(),
+				},
+				self.block.clone(),
+				self.current_step.into(),
+			));
+
+			self.current_step = self.current_step / self.digest_interval;
+			if self.current_step == 0 {
+				self.current_step = 1;
+			}
+		}
+	}
+}
+
+/// Blocks range iterator with builtin step_by support.
+#[derive(Debug)]
+struct BlocksRange<Number: BlockNumber> {
+	current: Number,
+	end: Number,
+	step: Number,
+}
+
+impl<Number: BlockNumber> BlocksRange<Number> {
+	pub fn new(begin: Number, end: Number, step: Number) -> Self {
+		BlocksRange {
+			current: begin,
+			end,
+			step,
+		}
+	}
+}
+
+impl<Number: BlockNumber> Iterator for BlocksRange<Number> {
+	type Item = Number;
+
+	fn next(&mut self) -> Option<Self::Item> {
+		if self.current >= self.end {
+			return None;
+		}
+
+		let current = Some(self.current.clone());
+		self.current += self.step.clone();
+		current
+	}
+}
+
+#[cfg(test)]
+mod tests {
+	use crate::changes_trie::Configuration;
+	use super::*;
+
+	fn digest_build_iterator(
+		digest_interval: u32,
+		digest_levels: u32,
+		zero: u64,
+		block: u64,
+		end: Option<u64>,
+	) -> DigestBuildIterator<u64> {
+		super::digest_build_iterator(
+			ConfigurationRange {
+				config: &Configuration {
+					digest_interval,
+					digest_levels,
+				},
+				zero,
+				end,
+			},
+			block,
+		)
+	}
+
+	fn digest_build_iterator_basic(
+		digest_interval: u32,
+		digest_levels: u32,
+		zero: u64,
+		block: u64,
+	) -> (u64, u32, u32) {
+		let iter = digest_build_iterator(digest_interval, digest_levels, zero, block, None);
+		(iter.block, iter.digest_interval, iter.max_step)
+	}
+
+	fn digest_build_iterator_blocks(
+		digest_interval: u32,
+		digest_levels: u32,
+		zero: u64,
+		block: u64,
+		end: Option<u64>,
+	) -> Vec<u64> {
+		digest_build_iterator(digest_interval, digest_levels, zero, block, end).collect()
+	}
+
+	#[test]
+	fn suggest_digest_inclusion_returns_empty_iterator() {
+		fn test_with_zero(zero: u64) {
+			let empty = (0, 0, 0);
+			assert_eq!(digest_build_iterator_basic(4, 16, zero, zero + 0), empty, "block is 0");
+			assert_eq!(digest_build_iterator_basic(0, 16, zero, zero + 64), empty, "digest_interval is 0");
+			assert_eq!(digest_build_iterator_basic(1, 16, zero, zero + 64), empty, "digest_interval is 1");
+			assert_eq!(digest_build_iterator_basic(4, 0, zero, zero + 64), empty, "digest_levels is 0");
+			assert_eq!(
+				digest_build_iterator_basic(4, 16, zero, zero + 1),
+				empty,
+				"digest is not required for this block",
+			);
+			assert_eq!(
+				digest_build_iterator_basic(4, 16, zero, zero + 2),
+				empty,
+				"digest is not required for this block",
+			);
+			assert_eq!(
+				digest_build_iterator_basic(4, 16, zero, zero + 15),
+				empty,
+				"digest is not required for this block",
+			);
+			assert_eq!(
+				digest_build_iterator_basic(4, 16, zero, zero + 17),
+				empty,
+				"digest is not required for this block",
+			);
+			assert_eq!(digest_build_iterator_basic(
+				::std::u32::MAX / 2 + 1,
+				16,
+				zero,
+				::std::u64::MAX,
+			), empty, "digest_interval * 2 is greater than u64::MAX");
+		}
+
+		test_with_zero(0);
+		test_with_zero(1);
+		test_with_zero(2);
+		test_with_zero(4);
+		test_with_zero(17);
+	}
+
+	#[test]
+	fn suggest_digest_inclusion_returns_level1_iterator() {
+		fn test_with_zero(zero: u64) {
+			assert_eq!(
+				digest_build_iterator_basic(16, 1, zero, zero + 16),
+				(zero + 16, 16, 1),
+				"!(block % interval) && first digest level == block",
+			);
+			assert_eq!(
+				digest_build_iterator_basic(16, 1, zero, zero + 256),
+				(zero + 256, 16, 1),
+				"!(block % interval^2), but there's only 1 digest level",
+			);
+			assert_eq!(
+				digest_build_iterator_basic(16, 2, zero, zero + 32),
+				(zero + 32, 16, 1),
+				"second level digest is not required for this block",
+			);
+			assert_eq!(
+				digest_build_iterator_basic(16, 3, zero, zero + 4080),
+				(zero + 4080, 16, 1),
+				"second && third level digest are not required for this block",
+			);
+		}
+
+		test_with_zero(0);
+		test_with_zero(16);
+		test_with_zero(17);
+	}
+
+	#[test]
+	fn suggest_digest_inclusion_returns_level2_iterator() {
+		fn test_with_zero(zero: u64) {
+			assert_eq!(
+				digest_build_iterator_basic(16, 2, zero, zero + 256),
+				(zero + 256, 16, 16),
+				"second level digest",
+			);
+			assert_eq!(
+				digest_build_iterator_basic(16, 2, zero, zero + 4096),
+				(zero + 4096, 16, 16),
+				"!(block % interval^3), but there's only 2 digest levels",
+			);
+		}
+
+		test_with_zero(0);
+		test_with_zero(16);
+		test_with_zero(17);
+	}
+
+	#[test]
+	fn suggest_digest_inclusion_returns_level3_iterator() {
+		fn test_with_zero(zero: u64) {
+			assert_eq!(
+				digest_build_iterator_basic(16, 3, zero, zero + 4096),
+				(zero + 4096, 16, 256),
+				"third level digest: beginning",
+			);
+			assert_eq!(
+				digest_build_iterator_basic(16, 3, zero, zero + 8192),
+				(zero + 8192, 16, 256),
+				"third level digest: next",
+			);
+		}
+
+		test_with_zero(0);
+		test_with_zero(16);
+		test_with_zero(17);
+	}
+
+	#[test]
+	fn digest_iterator_returns_level1_blocks() {
+		fn test_with_zero(zero: u64) {
+			assert_eq!(digest_build_iterator_blocks(16, 1, zero, zero + 16, None),
+				[1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15]
+					.iter().map(|item| zero + item).collect::<Vec<_>>());
+			assert_eq!(digest_build_iterator_blocks(16, 1, zero, zero + 256, None),
+				[241, 242, 243, 244, 245, 246, 247, 248, 249, 250, 251, 252, 253, 254, 255]
+					.iter().map(|item| zero + item).collect::<Vec<_>>());
+			assert_eq!(digest_build_iterator_blocks(16, 2, zero, zero + 32, None),
+				[17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31]
+					.iter().map(|item| zero + item).collect::<Vec<_>>());
+			assert_eq!(digest_build_iterator_blocks(16, 3, zero, zero + 4080, None),
+				[4065, 4066, 4067, 4068, 4069, 4070, 4071, 4072, 4073, 4074, 4075, 4076, 4077, 4078, 4079]
+					.iter().map(|item| zero + item).collect::<Vec<_>>());
+		}
+
+		test_with_zero(0);
+		test_with_zero(16);
+		test_with_zero(17);
+	}
+
+	#[test]
+	fn digest_iterator_returns_level1_and_level2_blocks() {
+		fn test_with_zero(zero: u64) {
+			assert_eq!(digest_build_iterator_blocks(16, 2, zero, zero + 256, None),
+				[
+					// level2 points to previous 16-1 level1 digests:
+					16, 32, 48, 64, 80, 96, 112, 128, 144, 160, 176, 192, 208, 224, 240,
+					// level2 is a level1 digest of 16-1 previous blocks:
+					241, 242, 243, 244, 245, 246, 247, 248, 249, 250, 251, 252, 253, 254, 255,
+				].iter().map(|item| zero + item).collect::<Vec<_>>(),
+			);
+			assert_eq!(digest_build_iterator_blocks(16, 2, zero, zero + 4096, None),
+				[
+					// level2 points to previous 16-1 level1 digests:
+					3856, 3872, 3888, 3904, 3920, 3936, 3952, 3968, 3984, 4000, 4016, 4032, 4048, 4064, 4080,
+					// level2 is a level1 digest of 16-1 previous blocks:
+					4081, 4082, 4083, 4084, 4085, 4086, 4087, 4088, 4089, 4090, 4091, 4092, 4093, 4094, 4095,
+				].iter().map(|item| zero + item).collect::<Vec<_>>(),
+			);
+		}
+
+		test_with_zero(0);
+		test_with_zero(16);
+		test_with_zero(17);
+	}
+
+	#[test]
+	fn digest_iterator_returns_level1_and_level2_and_level3_blocks() {
+		fn test_with_zero(zero: u64) {
+			assert_eq!(digest_build_iterator_blocks(16, 3, zero, zero + 4096, None),
+				[
+					// level3 points to previous 16-1 level2 digests:
+					256, 512, 768, 1024, 1280, 1536, 1792, 2048, 2304, 2560, 2816, 3072, 3328, 3584, 3840,
+					// level3 points to previous 16-1 level1 digests:
+					3856, 3872, 3888, 3904, 3920, 3936, 3952, 3968, 3984, 4000, 4016, 4032, 4048, 4064, 4080,
+					// level3 is a level1 digest of 16-1 previous blocks:
+					4081, 4082, 4083, 4084, 4085, 4086, 4087, 4088, 4089, 4090, 4091, 4092, 4093, 4094, 4095,
+				].iter().map(|item| zero + item).collect::<Vec<_>>(),
+			);
+		}
+
+		test_with_zero(0);
+		test_with_zero(16);
+		test_with_zero(17);
+	}
+
+	#[test]
+	fn digest_iterator_returns_skewed_digest_blocks() {
+		fn test_with_zero(zero: u64) {
+			assert_eq!(digest_build_iterator_blocks(16, 3, zero, zero + 4096, Some(zero + 1338)),
+				[
+					// level3 MUST point to previous 16-1 level2 digests, BUT there are only 5:
+					256, 512, 768, 1024, 1280,
+					// level3 MUST point to previous 16-1 level1 digests, BUT there are only 3:
+					1296, 1312, 1328,
+					// level3 MUST be a level1 digest of 16-1 previous blocks, BUT there are only 9:
+					1329, 1330, 1331, 1332, 1333, 1334, 1335, 1336, 1337,
+				].iter().map(|item| zero + item).collect::<Vec<_>>(),
+			);
+		}
+
+		test_with_zero(0);
+		test_with_zero(16);
+		test_with_zero(17);
+	}
+
+	#[test]
+	fn digest_iterator_returns_skewed_digest_blocks_skipping_level() {
+		fn test_with_zero(zero: u64) {
+			assert_eq!(digest_build_iterator_blocks(16, 3, zero, zero + 4096, Some(zero + 1284)),
+				[
+					// level3 MUST point to previous 16-1 level2 digests, BUT there are only 5:
+					256, 512, 768, 1024, 1280,
+					// level3 MUST point to previous 16-1 level1 digests, BUT there are NO ANY L1-digests:
+					// level3 MUST be a level1 digest of 16-1 previous blocks, BUT there are only 3:
+					1281, 1282, 1283,
+				].iter().map(|item| zero + item).collect::<Vec<_>>(),
+			);
+		}
+
+		test_with_zero(0);
+		test_with_zero(16);
+		test_with_zero(17);
+	}
+}

substrate/primitives/state-machine/src/changes_trie/changes_iterator.rs

@@ -0,0 +1,635 @@
+// Copyright 2017-2019 Parity Technologies (UK) Ltd.
+// This file is part of Substrate.
+
+// Substrate is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// Substrate is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with Substrate.  If not, see <http://www.gnu.org/licenses/>.
+
+//! Functions + iterator that traverses changes tries and returns all
+//! (block, extrinsic) pairs where given key has been changed.
+
+use std::cell::RefCell;
+use std::collections::VecDeque;
+use codec::{Decode, Encode};
+use hash_db::Hasher;
+use num_traits::Zero;
+use trie::Recorder;
+use crate::changes_trie::{AnchorBlockId, ConfigurationRange, RootsStorage, Storage, BlockNumber};
+use crate::changes_trie::input::{DigestIndex, ExtrinsicIndex, DigestIndexValue, ExtrinsicIndexValue};
+use crate::changes_trie::storage::{TrieBackendAdapter, InMemoryStorage};
+use crate::changes_trie::input::ChildIndex;
+use crate::changes_trie::surface_iterator::{surface_iterator, SurfaceIterator};
+use crate::proving_backend::ProvingBackendRecorder;
+use crate::trie_backend_essence::{TrieBackendEssence};
+
+/// Return changes of given key at given blocks range.
+/// `max` is the number of best known block.
+/// Changes are returned in descending order (i.e. last block comes first).
+pub fn key_changes<'a, H: Hasher, Number: BlockNumber>(
+	config: ConfigurationRange<'a, Number>,
+	storage: &'a dyn Storage<H, Number>,
+	begin: Number,
+	end: &'a AnchorBlockId<H::Out, Number>,
+	max: Number,
+	storage_key: Option<&'a [u8]>,
+	key: &'a [u8],
+) -> Result<DrilldownIterator<'a, H, Number>, String> {
+	// we can't query any roots before root
+	let max = ::std::cmp::min(max.clone(), end.number.clone());
+
+	Ok(DrilldownIterator {
+		essence: DrilldownIteratorEssence {
+			storage_key,
+			key,
+			roots_storage: storage.as_roots_storage(),
+			storage,
+			begin: begin.clone(),
+			end,
+			config: config.clone(),
+			surface: surface_iterator(
+				config,
+				max,
+				begin,
+				end.number.clone(),
+			)?,
+
+			extrinsics: Default::default(),
+			blocks: Default::default(),
+
+			_hasher: ::std::marker::PhantomData::<H>::default(),
+		},
+	})
+}
+
+
+/// Returns proof of changes of given key at given blocks range.
+/// `max` is the number of best known block.
+pub fn key_changes_proof<'a, H: Hasher, Number: BlockNumber>(
+	config: ConfigurationRange<'a, Number>,
+	storage: &dyn Storage<H, Number>,
+	begin: Number,
+	end: &AnchorBlockId<H::Out, Number>,
+	max: Number,
+	storage_key: Option<&[u8]>,
+	key: &[u8],
+) -> Result<Vec<Vec<u8>>, String> {
+	// we can't query any roots before root
+	let max = ::std::cmp::min(max.clone(), end.number.clone());
+
+	let mut iter = ProvingDrilldownIterator {
+		essence: DrilldownIteratorEssence {
+			storage_key,
+			key,
+			roots_storage: storage.as_roots_storage(),
+			storage,
+			begin: begin.clone(),
+			end,
+			config: config.clone(),
+			surface: surface_iterator(
+				config,
+				max,
+				begin,
+				end.number.clone(),
+			)?,
+
+			extrinsics: Default::default(),
+			blocks: Default::default(),
+
+			_hasher: ::std::marker::PhantomData::<H>::default(),
+		},
+		proof_recorder: Default::default(),
+	};
+
+	// iterate to collect proof
+	while let Some(item) = iter.next() {
+		item?;
+	}
+
+	Ok(iter.extract_proof())
+}
+
+/// Check key changes proof and return changes of the key at given blocks range.
+/// `max` is the number of best known block.
+/// Changes are returned in descending order (i.e. last block comes first).
+pub fn key_changes_proof_check<'a, H: Hasher, Number: BlockNumber>(
+	config: ConfigurationRange<'a, Number>,
+	roots_storage: &dyn RootsStorage<H, Number>,
+	proof: Vec<Vec<u8>>,
+	begin: Number,
+	end: &AnchorBlockId<H::Out, Number>,
+	max: Number,
+	storage_key: Option<&[u8]>,
+	key: &[u8]
+) -> Result<Vec<(Number, u32)>, String> {
+	key_changes_proof_check_with_db(
+		config,
+		roots_storage,
+		&InMemoryStorage::with_proof(proof),
+		begin,
+		end,
+		max,
+		storage_key,
+		key,
+	)
+}
+
+/// Similar to the `key_changes_proof_check` function, but works with prepared proof storage.
+pub fn key_changes_proof_check_with_db<'a, H: Hasher, Number: BlockNumber>(
+	config: ConfigurationRange<'a, Number>,
+	roots_storage: &dyn RootsStorage<H, Number>,
+	proof_db: &InMemoryStorage<H, Number>,
+	begin: Number,
+	end: &AnchorBlockId<H::Out, Number>,
+	max: Number,
+	storage_key: Option<&[u8]>,
+	key: &[u8]
+) -> Result<Vec<(Number, u32)>, String> {
+	// we can't query any roots before root
+	let max = ::std::cmp::min(max.clone(), end.number.clone());
+
+	DrilldownIterator {
+		essence: DrilldownIteratorEssence {
+			storage_key,
+			key,
+			roots_storage,
+			storage: proof_db,
+			begin: begin.clone(),
+			end,
+			config: config.clone(),
+			surface: surface_iterator(
+				config,
+				max,
+				begin,
+				end.number.clone(),
+			)?,
+
+			extrinsics: Default::default(),
+			blocks: Default::default(),
+
+			_hasher: ::std::marker::PhantomData::<H>::default(),
+		},
+	}.collect()
+}
+
+/// Drilldown iterator - receives 'digest points' from surface iterator and explores
+/// every point until extrinsic is found.
+pub struct DrilldownIteratorEssence<'a, H, Number>
+	where
+		H: Hasher,
+		Number: BlockNumber,
+		H::Out: 'a,
+{
+	storage_key: Option<&'a [u8]>,
+	key: &'a [u8],
+	roots_storage: &'a dyn RootsStorage<H, Number>,
+	storage: &'a dyn Storage<H, Number>,
+	begin: Number,
+	end: &'a AnchorBlockId<H::Out, Number>,
+	config: ConfigurationRange<'a, Number>,
+	surface: SurfaceIterator<'a, Number>,
+
+	extrinsics: VecDeque<(Number, u32)>,
+	blocks: VecDeque<(Number, Option<u32>)>,
+
+	_hasher: ::std::marker::PhantomData<H>,
+}
+
+impl<'a, H, Number> DrilldownIteratorEssence<'a, H, Number>
+	where
+		H: Hasher,
+		Number: BlockNumber,
+		H::Out: 'a,
+{
+	pub fn next<F>(&mut self, trie_reader: F) -> Option<Result<(Number, u32), String>>
+		where
+			F: FnMut(&dyn Storage<H, Number>, H::Out, &[u8]) -> Result<Option<Vec<u8>>, String>,
+	{
+		match self.do_next(trie_reader) {
+			Ok(Some(res)) => Some(Ok(res)),
+			Ok(None) => None,
+			Err(err) => Some(Err(err)),
+		}
+	}
+
+	fn do_next<F>(&mut self, mut trie_reader: F) -> Result<Option<(Number, u32)>, String>
+		where
+			F: FnMut(&dyn Storage<H, Number>, H::Out, &[u8]) -> Result<Option<Vec<u8>>, String>,
+	{
+		loop {
+			if let Some((block, extrinsic)) = self.extrinsics.pop_front() {
+				return Ok(Some((block, extrinsic)));
+			}
+
+			if let Some((block, level)) = self.blocks.pop_front() {
+				// not having a changes trie root is an error because:
+				// we never query roots for future blocks
+				// AND trie roots for old blocks are known (both on full + light node)
+				let trie_root = self.roots_storage.root(&self.end, block.clone())?
+					.ok_or_else(|| format!("Changes trie root for block {} is not found", block.clone()))?;
+				let trie_root = if let Some(storage_key) = self.storage_key {
+					let child_key = ChildIndex {
+						block: block.clone(),
+						storage_key: storage_key.to_vec(),
+					}.encode();
+					if let Some(trie_root) = trie_reader(self.storage, trie_root, &child_key)?
+						.and_then(|v| <Vec<u8>>::decode(&mut &v[..]).ok())
+						.map(|v| {
+							let mut hash = H::Out::default();
+							hash.as_mut().copy_from_slice(&v[..]);
+							hash
+						}) {
+						trie_root
+					} else {
+						continue;
+					}
+				} else {
+					trie_root
+				};
+
+				// only return extrinsics for blocks before self.max
+				// most of blocks will be filtered out before pushing to `self.blocks`
+				// here we just throwing away changes at digest blocks we're processing
+				debug_assert!(block >= self.begin, "We shall not touch digests earlier than a range' begin");
+				if block <= self.end.number {
+					let extrinsics_key = ExtrinsicIndex { block: block.clone(), key: self.key.to_vec() }.encode();
+					let extrinsics = trie_reader(self.storage, trie_root, &extrinsics_key);
+					if let Some(extrinsics) = extrinsics? {
+						if let Ok(extrinsics) = ExtrinsicIndexValue::decode(&mut &extrinsics[..]) {
+							self.extrinsics.extend(extrinsics.into_iter().rev().map(|e| (block.clone(), e)));
+						}
+					}
+				}
+
+				let blocks_key = DigestIndex { block: block.clone(), key: self.key.to_vec() }.encode();
+				let blocks = trie_reader(self.storage, trie_root, &blocks_key);
+				if let Some(blocks) = blocks? {
+					if let Ok(blocks) = <DigestIndexValue<Number>>::decode(&mut &blocks[..]) {
+						// filter level0 blocks here because we tend to use digest blocks,
+						// AND digest block changes could also include changes for out-of-range blocks
+						let begin = self.begin.clone();
+						let end = self.end.number.clone();
+						let config = self.config.clone();
+						self.blocks.extend(blocks.into_iter()
+							.rev()
+							.filter(|b| level.map(|level| level > 1).unwrap_or(true) || (*b >= begin && *b <= end))
+							.map(|b| {
+								let prev_level = level
+									.map(|level| Some(level - 1))
+									.unwrap_or_else(||
+										Some(config.config.digest_level_at_block(config.zero.clone(), b.clone())
+											.map(|(level, _, _)| level)
+											.unwrap_or_else(|| Zero::zero())));
+								(b, prev_level)
+							})
+						);
+					}
+				}
+
+				continue;
+			}
+
+			match self.surface.next() {
+				Some(Ok(block)) => self.blocks.push_back(block),
+				Some(Err(err)) => return Err(err),
+				None => return Ok(None),
+			}
+		}
+	}
+}
+
+/// Exploring drilldown operator.
+pub struct DrilldownIterator<'a, H, Number>
+	where
+		Number: BlockNumber,
+		H: Hasher,
+		H::Out: 'a,
+{
+	essence: DrilldownIteratorEssence<'a, H, Number>,
+}
+
+impl<'a, H: Hasher, Number: BlockNumber> Iterator
+	for DrilldownIterator<'a, H, Number>
+{
+	type Item = Result<(Number, u32), String>;
+
+	fn next(&mut self) -> Option<Self::Item> {
+		self.essence.next(|storage, root, key|
+			TrieBackendEssence::<_, H>::new(TrieBackendAdapter::new(storage), root).storage(key))
+	}
+}
+
+/// Proving drilldown iterator.
+struct ProvingDrilldownIterator<'a, H, Number>
+	where
+		Number: BlockNumber,
+		H: Hasher,
+		H::Out: 'a,
+{
+	essence: DrilldownIteratorEssence<'a, H, Number>,
+	proof_recorder: RefCell<Recorder<H::Out>>,
+}
+
+impl<'a, H, Number> ProvingDrilldownIterator<'a, H, Number>
+	where
+		Number: BlockNumber,
+		H: Hasher,
+		H::Out: 'a,
+{
+	/// Consume the iterator, extracting the gathered proof in lexicographical order
+	/// by value.
+	pub fn extract_proof(self) -> Vec<Vec<u8>> {
+		self.proof_recorder.into_inner().drain()
+			.into_iter()
+			.map(|n| n.data.to_vec())
+			.collect()
+	}
+}
+
+impl<'a, H, Number> Iterator for ProvingDrilldownIterator<'a, H, Number>
+	where
+		Number: BlockNumber,
+		H: Hasher,
+		H::Out: 'a,
+{
+	type Item = Result<(Number, u32), String>;
+
+	fn next(&mut self) -> Option<Self::Item> {
+		let proof_recorder = &mut *self.proof_recorder.try_borrow_mut()
+			.expect("only fails when already borrowed; storage() is non-reentrant; qed");
+		self.essence.next(|storage, root, key|
+			ProvingBackendRecorder::<_, H> {
+				backend: &TrieBackendEssence::new(TrieBackendAdapter::new(storage), root),
+				proof_recorder,
+			}.storage(key))
+	}
+}
+
+#[cfg(test)]
+mod tests {
+	use std::iter::FromIterator;
+	use primitives::Blake2Hasher;
+	use crate::changes_trie::Configuration;
+	use crate::changes_trie::input::InputPair;
+	use crate::changes_trie::storage::InMemoryStorage;
+	use super::*;
+
+	fn prepare_for_drilldown() -> (Configuration, InMemoryStorage<Blake2Hasher, u64>) {
+		let config = Configuration { digest_interval: 4, digest_levels: 2 };
+		let backend = InMemoryStorage::with_inputs(vec![
+			// digest: 1..4 => [(3, 0)]
+			(1, vec![
+			]),
+			(2, vec![
+			]),
+			(3, vec![
+				InputPair::ExtrinsicIndex(ExtrinsicIndex { block: 3, key: vec![42] }, vec![0]),
+			]),
+			(4, vec![
+				InputPair::DigestIndex(DigestIndex { block: 4, key: vec![42] }, vec![3]),
+			]),
+			// digest: 5..8 => [(6, 3), (8, 1+2)]
+			(5, vec![]),
+			(6, vec![
+				InputPair::ExtrinsicIndex(ExtrinsicIndex { block: 6, key: vec![42] }, vec![3]),
+			]),
+			(7, vec![]),
+			(8, vec![
+				InputPair::ExtrinsicIndex(ExtrinsicIndex { block: 8, key: vec![42] }, vec![1, 2]),
+				InputPair::DigestIndex(DigestIndex { block: 8, key: vec![42] }, vec![6]),
+			]),
+			// digest: 9..12 => []
+			(9, vec![]),
+			(10, vec![]),
+			(11, vec![]),
+			(12, vec![]),
+			// digest: 0..16 => [4, 8]
+			(13, vec![]),
+			(14, vec![]),
+			(15, vec![]),
+			(16, vec![
+				InputPair::DigestIndex(DigestIndex { block: 16, key: vec![42] }, vec![4, 8]),
+			]),
+		], vec![(b"1".to_vec(), vec![
+				(1, vec![
+					InputPair::ExtrinsicIndex(ExtrinsicIndex { block: 1, key: vec![42] }, vec![0]),
+				]),
+				(2, vec![
+					InputPair::ExtrinsicIndex(ExtrinsicIndex { block: 2, key: vec![42] }, vec![3]),
+				]),
+				(16, vec![
+					InputPair::ExtrinsicIndex(ExtrinsicIndex { block: 16, key: vec![42] }, vec![5]),
+
+					InputPair::DigestIndex(DigestIndex { block: 16, key: vec![42] }, vec![2]),
+				]),
+			]),
+		]);
+
+		(config, backend)
+	}
+
+	fn configuration_range<'a>(config: &'a Configuration, zero: u64) -> ConfigurationRange<'a, u64> {
+		ConfigurationRange {
+			config,
+			zero,
+			end: None,
+		}
+	}
+
+	#[test]
+	fn drilldown_iterator_works() {
+		let (config, storage) = prepare_for_drilldown();
+		let drilldown_result = key_changes::<Blake2Hasher, u64>(
+			configuration_range(&config, 0),
+			&storage,
+			1,
+			&AnchorBlockId { hash: Default::default(), number: 16 },
+			16,
+			None,
+			&[42],
+		).and_then(Result::from_iter);
+		assert_eq!(drilldown_result, Ok(vec![(8, 2), (8, 1), (6, 3), (3, 0)]));
+
+		let drilldown_result = key_changes::<Blake2Hasher, u64>(
+			configuration_range(&config, 0),
+			&storage,
+			1,
+			&AnchorBlockId { hash: Default::default(), number: 2 },
+			4,
+			None,
+			&[42],
+		).and_then(Result::from_iter);
+		assert_eq!(drilldown_result, Ok(vec![]));
+
+		let drilldown_result = key_changes::<Blake2Hasher, u64>(
+			configuration_range(&config, 0),
+			&storage,
+			1,
+			&AnchorBlockId { hash: Default::default(), number: 3 },
+			4,
+			None,
+			&[42],
+		).and_then(Result::from_iter);
+		assert_eq!(drilldown_result, Ok(vec![(3, 0)]));
+
+		let drilldown_result = key_changes::<Blake2Hasher, u64>(
+			configuration_range(&config, 0),
+			&storage,
+			1,
+			&AnchorBlockId { hash: Default::default(), number: 7 },
+			7,
+			None,
+			&[42],
+		).and_then(Result::from_iter);
+		assert_eq!(drilldown_result, Ok(vec![(6, 3), (3, 0)]));
+
+		let drilldown_result = key_changes::<Blake2Hasher, u64>(
+			configuration_range(&config, 0),
+			&storage,
+			7,
+			&AnchorBlockId { hash: Default::default(), number: 8 },
+			8,
+			None,
+			&[42],
+		).and_then(Result::from_iter);
+		assert_eq!(drilldown_result, Ok(vec![(8, 2), (8, 1)]));
+
+		let drilldown_result = key_changes::<Blake2Hasher, u64>(
+			configuration_range(&config, 0),
+			&storage,
+			5,
+			&AnchorBlockId { hash: Default::default(), number: 7 },
+			8,
+			None,
+			&[42],
+		).and_then(Result::from_iter);
+		assert_eq!(drilldown_result, Ok(vec![(6, 3)]));
+	}
+
+	#[test]
+	fn drilldown_iterator_fails_when_storage_fails() {
+		let (config, storage) = prepare_for_drilldown();
+		storage.clear_storage();
+
+		assert!(key_changes::<Blake2Hasher, u64>(
+			configuration_range(&config, 0),
+			&storage,
+			1,
+			&AnchorBlockId { hash: Default::default(), number: 100 },
+			1000,
+			None,
+			&[42],
+		).and_then(|i| i.collect::<Result<Vec<_>, _>>()).is_err());
+
+		assert!(key_changes::<Blake2Hasher, u64>(
+			configuration_range(&config, 0),
+			&storage,
+			1,
+			&AnchorBlockId { hash: Default::default(), number: 100 },
+			1000,
+			Some(&b"1"[..]),
+			&[42],
+		).and_then(|i| i.collect::<Result<Vec<_>, _>>()).is_err());
+	}
+
+	#[test]
+	fn drilldown_iterator_fails_when_range_is_invalid() {
+		let (config, storage) = prepare_for_drilldown();
+		assert!(key_changes::<Blake2Hasher, u64>(
+			configuration_range(&config, 0),
+			&storage,
+			1,
+			&AnchorBlockId { hash: Default::default(), number: 100 },
+			50,
+			None,
+			&[42],
+		).is_err());
+		assert!(key_changes::<Blake2Hasher, u64>(
+			configuration_range(&config, 0),
+			&storage,
+			20,
+			&AnchorBlockId { hash: Default::default(), number: 10 },
+			100,
+			None,
+			&[42],
+		).is_err());
+	}
+
+
+	#[test]
+	fn proving_drilldown_iterator_works() {
+		// happens on remote full node:
+
+		// create drilldown iterator that records all trie nodes during drilldown
+		let (remote_config, remote_storage) = prepare_for_drilldown();
+		let remote_proof = key_changes_proof::<Blake2Hasher, u64>(
+			configuration_range(&remote_config, 0), &remote_storage, 1,
+			&AnchorBlockId { hash: Default::default(), number: 16 }, 16, None, &[42]).unwrap();
+
+		let (remote_config, remote_storage) = prepare_for_drilldown();
+		let remote_proof_child = key_changes_proof::<Blake2Hasher, u64>(
+			configuration_range(&remote_config, 0), &remote_storage, 1,
+			&AnchorBlockId { hash: Default::default(), number: 16 }, 16, Some(&b"1"[..]), &[42]).unwrap();
+
+		// happens on local light node:
+
+		// create drilldown iterator that works the same, but only depends on trie
+		let (local_config, local_storage) = prepare_for_drilldown();
+		local_storage.clear_storage();
+		let local_result = key_changes_proof_check::<Blake2Hasher, u64>(
+			configuration_range(&local_config, 0), &local_storage, remote_proof, 1,
+			&AnchorBlockId { hash: Default::default(), number: 16 }, 16, None, &[42]);
+
+		let (local_config, local_storage) = prepare_for_drilldown();
+		local_storage.clear_storage();
+		let local_result_child = key_changes_proof_check::<Blake2Hasher, u64>(
+			configuration_range(&local_config, 0), &local_storage, remote_proof_child, 1,
+			&AnchorBlockId { hash: Default::default(), number: 16 }, 16, Some(&b"1"[..]), &[42]);
+
+		// check that drilldown result is the same as if it was happening at the full node
+		assert_eq!(local_result, Ok(vec![(8, 2), (8, 1), (6, 3), (3, 0)]));
+		assert_eq!(local_result_child, Ok(vec![(16, 5), (2, 3)]));
+	}
+
+	#[test]
+	fn drilldown_iterator_works_with_skewed_digest() {
+		let config = Configuration { digest_interval: 4, digest_levels: 3 };
+		let mut config_range = configuration_range(&config, 0);
+		config_range.end = Some(91);
+
+		// when 4^3 deactivates at block 91:
+		// last L3 digest has been created at block#64
+		// skewed digest covers:
+		// L2 digests at blocks: 80
+		// L1 digests at blocks: 84, 88
+		// regular blocks: 89, 90, 91
+		let mut input = (1u64..92u64).map(|b| (b, vec![])).collect::<Vec<_>>();
+		// changed at block#63 and covered by L3 digest at block#64
+		input[63 - 1].1.push(InputPair::ExtrinsicIndex(ExtrinsicIndex { block: 63, key: vec![42] }, vec![0]));
+		input[64 - 1].1.push(InputPair::DigestIndex(DigestIndex { block: 64, key: vec![42] }, vec![63]));
+		// changed at block#79 and covered by L2 digest at block#80 + skewed digest at block#91
+		input[79 - 1].1.push(InputPair::ExtrinsicIndex(ExtrinsicIndex { block: 79, key: vec![42] }, vec![1]));
+		input[80 - 1].1.push(InputPair::DigestIndex(DigestIndex { block: 80, key: vec![42] }, vec![79]));
+		input[91 - 1].1.push(InputPair::DigestIndex(DigestIndex { block: 91, key: vec![42] }, vec![80]));
+		let storage = InMemoryStorage::with_inputs(input, vec![]);
+
+		let drilldown_result = key_changes::<Blake2Hasher, u64>(
+			config_range,
+			&storage,
+			1,
+			&AnchorBlockId { hash: Default::default(), number: 91 },
+			100_000u64,
+			None,
+			&[42],
+		).and_then(Result::from_iter);
+		assert_eq!(drilldown_result, Ok(vec![(79, 1), (63, 0)]));
+	}
+}

substrate/primitives/state-machine/src/changes_trie/input.rs

@@ -0,0 +1,204 @@
+// Copyright 2017-2019 Parity Technologies (UK) Ltd.
+// This file is part of Substrate.
+
+// Substrate is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// Substrate is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with Substrate.  If not, see <http://www.gnu.org/licenses/>.
+
+//! Different types of changes trie input pairs.
+
+use codec::{Decode, Encode, Input, Output, Error};
+use crate::changes_trie::BlockNumber;
+
+/// Key of { changed key => set of extrinsic indices } mapping.
+#[derive(Clone, Debug, PartialEq, Eq)]
+pub struct ExtrinsicIndex<Number: BlockNumber> {
+	/// Block at which this key has been inserted in the trie.
+	pub block: Number,
+	/// Storage key this node is responsible for.
+	pub key: Vec<u8>,
+}
+
+/// Value of { changed key => set of extrinsic indices } mapping.
+pub type ExtrinsicIndexValue = Vec<u32>;
+
+/// Key of { changed key => block/digest block numbers } mapping.
+#[derive(Clone, Debug, PartialEq, Eq)]
+pub struct DigestIndex<Number: BlockNumber> {
+	/// Block at which this key has been inserted in the trie.
+	pub block: Number,
+	/// Storage key this node is responsible for.
+	pub key: Vec<u8>,
+}
+
+/// Key of { childtrie key => Childchange trie } mapping.
+#[derive(Clone, Debug, PartialEq, Eq, PartialOrd, Ord)]
+pub struct ChildIndex<Number: BlockNumber> {
+	/// Block at which this key has been inserted in the trie.
+	pub block: Number,
+	/// Storage key this node is responsible for.
+	pub storage_key: Vec<u8>,
+}
+
+/// Value of { changed key => block/digest block numbers } mapping.
+pub type DigestIndexValue<Number> = Vec<Number>;
+
+/// Value of { changed key => block/digest block numbers } mapping.
+/// That is the root of the child change trie.
+pub type ChildIndexValue = Vec<u8>;
+
+/// Single input pair of changes trie.
+#[derive(Clone, Debug, PartialEq, Eq)]
+pub enum InputPair<Number: BlockNumber> {
+	/// Element of { key => set of extrinsics where key has been changed } element mapping.
+	ExtrinsicIndex(ExtrinsicIndex<Number>, ExtrinsicIndexValue),
+	/// Element of { key => set of blocks/digest blocks where key has been changed } element mapping.
+	DigestIndex(DigestIndex<Number>, DigestIndexValue<Number>),
+	/// Element of { childtrie key => Childchange trie } where key has been changed } element mapping.
+	ChildIndex(ChildIndex<Number>, ChildIndexValue),
+}
+
+/// Single input key of changes trie.
+#[derive(Clone, Debug, PartialEq, Eq)]
+pub enum InputKey<Number: BlockNumber> {
+	/// Key of { key => set of extrinsics where key has been changed } element mapping.
+	ExtrinsicIndex(ExtrinsicIndex<Number>),
+	/// Key of { key => set of blocks/digest blocks where key has been changed } element mapping.
+	DigestIndex(DigestIndex<Number>),
+	/// Key of { childtrie key => Childchange trie } where key has been changed } element mapping.
+	ChildIndex(ChildIndex<Number>),
+}
+
+impl<Number: BlockNumber> InputPair<Number> {
+	/// Extract storage key that this pair corresponds to.
+	pub fn key(&self) -> Option<&[u8]> {
+		match *self {
+			InputPair::ExtrinsicIndex(ref key, _) => Some(&key.key),
+			InputPair::DigestIndex(ref key, _) => Some(&key.key),
+			InputPair::ChildIndex(_, _) => None,
+		}
+	}
+}
+
+impl<Number: BlockNumber> Into<(Vec<u8>, Vec<u8>)> for InputPair<Number> {
+	fn into(self) -> (Vec<u8>, Vec<u8>) {
+		match self {
+			InputPair::ExtrinsicIndex(key, value) => (key.encode(), value.encode()),
+			InputPair::DigestIndex(key, value) => (key.encode(), value.encode()),
+			InputPair::ChildIndex(key, value) => (key.encode(), value.encode()),
+		}
+	}
+}
+
+impl<Number: BlockNumber> Into<InputKey<Number>> for InputPair<Number> {
+	fn into(self) -> InputKey<Number> {
+		match self {
+			InputPair::ExtrinsicIndex(key, _) => InputKey::ExtrinsicIndex(key),
+			InputPair::DigestIndex(key, _) => InputKey::DigestIndex(key),
+			InputPair::ChildIndex(key, _) => InputKey::ChildIndex(key),
+		}
+	}
+}
+
+impl<Number: BlockNumber> ExtrinsicIndex<Number> {
+	pub fn key_neutral_prefix(block: Number) -> Vec<u8> {
+		let mut prefix = vec![1];
+		prefix.extend(block.encode());
+		prefix
+	}
+}
+
+impl<Number: BlockNumber> Encode for ExtrinsicIndex<Number> {
+	fn encode_to<W: Output>(&self, dest: &mut W) {
+		dest.push_byte(1);
+		self.block.encode_to(dest);
+		self.key.encode_to(dest);
+	}
+}
+
+impl<Number: BlockNumber> codec::EncodeLike for ExtrinsicIndex<Number> {}
+
+impl<Number: BlockNumber> DigestIndex<Number> {
+	pub fn key_neutral_prefix(block: Number) -> Vec<u8> {
+		let mut prefix = vec![2];
+		prefix.extend(block.encode());
+		prefix
+	}
+}
+
+
+impl<Number: BlockNumber> Encode for DigestIndex<Number> {
+	fn encode_to<W: Output>(&self, dest: &mut W) {
+		dest.push_byte(2);
+		self.block.encode_to(dest);
+		self.key.encode_to(dest);
+	}
+}
+
+impl<Number: BlockNumber> ChildIndex<Number> {
+	pub fn key_neutral_prefix(block: Number) -> Vec<u8> {
+		let mut prefix = vec![3];
+		prefix.extend(block.encode());
+		prefix
+	}
+}
+
+impl<Number: BlockNumber> Encode for ChildIndex<Number> {
+	fn encode_to<W: Output>(&self, dest: &mut W) {
+		dest.push_byte(3);
+		self.block.encode_to(dest);
+		self.storage_key.encode_to(dest);
+	}
+}
+
+impl<Number: BlockNumber> codec::EncodeLike for DigestIndex<Number> {}
+
+impl<Number: BlockNumber> Decode for InputKey<Number> {
+	fn decode<I: Input>(input: &mut I) -> Result<Self, Error> {
+		match input.read_byte()? {
+			1 => Ok(InputKey::ExtrinsicIndex(ExtrinsicIndex {
+				block: Decode::decode(input)?,
+				key: Decode::decode(input)?,
+			})),
+			2 => Ok(InputKey::DigestIndex(DigestIndex {
+				block: Decode::decode(input)?,
+				key: Decode::decode(input)?,
+			})),
+			3 => Ok(InputKey::ChildIndex(ChildIndex {
+				block: Decode::decode(input)?,
+				storage_key: Decode::decode(input)?,
+			})),
+			_ => Err("Invalid input key variant".into()),
+		}
+	}
+}
+
+#[cfg(test)]
+mod tests {
+	use super::*;
+
+	#[test]
+	fn extrinsic_index_serialized_and_deserialized() {
+		let original = ExtrinsicIndex { block: 777u64, key: vec![42] };
+		let serialized = original.encode();
+		let deserialized: InputKey<u64> = Decode::decode(&mut &serialized[..]).unwrap();
+		assert_eq!(InputKey::ExtrinsicIndex(original), deserialized);
+	}
+
+	#[test]
+	fn digest_index_serialized_and_deserialized() {
+		let original = DigestIndex { block: 777u64, key: vec![42] };
+		let serialized = original.encode();
+		let deserialized: InputKey<u64> = Decode::decode(&mut &serialized[..]).unwrap();
+		assert_eq!(InputKey::DigestIndex(original), deserialized);
+	}
+}

substrate/primitives/state-machine/src/changes_trie/mod.rs

@@ -0,0 +1,336 @@
+// Copyright 2017-2019 Parity Technologies (UK) Ltd.
+// This file is part of Substrate.
+
+// Substrate is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// Substrate is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with Substrate.  If not, see <http://www.gnu.org/licenses/>.
+
+//! Changes trie related structures and functions.
+//!
+//! Changes trie is a trie built of { storage key => extrinsiscs } pairs
+//! at the end of each block. For every changed storage key it contains
+//! a pair, mapping key to the set of extrinsics where it has been changed.
+//!
+//! Optionally, every N blocks, additional level1-digest nodes are appended
+//! to the changes trie, containing pairs { storage key => blocks }. For every
+//! storage key that has been changed in PREVIOUS N-1 blocks (except for genesis
+//! block) it contains a pair, mapping this key to the set of blocks where it
+//! has been changed.
+//!
+//! Optionally, every N^digest_level (where digest_level > 1) blocks, additional
+//! digest_level digest is created. It is built out of pairs { storage key => digest
+//! block }, containing entries for every storage key that has been changed in
+//! the last N*digest_level-1 blocks (except for genesis block), mapping these keys
+//! to the set of lower-level digest blocks.
+//!
+//! Changes trie configuration could change within a time. The range of blocks, where
+//! configuration has been active, is given by two blocks: zero and end. Zero block is
+//! the block where configuration has been set. But the first changes trie that uses
+//! this configuration will be built at the block zero+1. If configuration deactivates
+//! at some block, this will be the end block of the configuration. It is also the
+//! zero block of the next configuration.
+//!
+//! If configuration has the end block, it also means that 'skewed digest' has/should
+//! been built at that block. If this is the block where max-level digest should have
+//! been created, than it is simply max-level digest of this configuration. Otherwise,
+//! it is the digest that covers all blocks since last max-level digest block was
+//! created.
+//!
+//! Changes trie only contains the top level storage changes. Sub-level changes
+//! are propagated through its storage root on the top level storage.
+
+mod build;
+mod build_cache;
+mod build_iterator;
+mod changes_iterator;
+mod input;
+mod prune;
+mod storage;
+mod surface_iterator;
+
+pub use self::build_cache::{BuildCache, CachedBuildData, CacheAction};
+pub use self::storage::InMemoryStorage;
+pub use self::changes_iterator::{
+	key_changes, key_changes_proof,
+	key_changes_proof_check, key_changes_proof_check_with_db,
+};
+pub use self::prune::{prune, oldest_non_pruned_trie};
+
+use std::collections::{HashMap, HashSet};
+use std::convert::TryInto;
+use hash_db::{Hasher, Prefix};
+use crate::backend::Backend;
+use num_traits::{One, Zero};
+use codec::{Decode, Encode};
+use primitives;
+use crate::changes_trie::build::prepare_input;
+use crate::changes_trie::build_cache::{IncompleteCachedBuildData, IncompleteCacheAction};
+use crate::overlayed_changes::OverlayedChanges;
+use trie::{MemoryDB, DBValue, TrieMut};
+use trie::trie_types::TrieDBMut;
+
+/// Changes that are made outside of extrinsics are marked with this index;
+pub const NO_EXTRINSIC_INDEX: u32 = 0xffffffff;
+
+/// Requirements for block number that can be used with changes tries.
+pub trait BlockNumber:
+	Send + Sync + 'static +
+	::std::fmt::Display +
+	Clone +
+	From<u32> + TryInto<u32> + One + Zero +
+	PartialEq + Ord +
+	::std::hash::Hash +
+	::std::ops::Add<Self, Output=Self> + ::std::ops::Sub<Self, Output=Self> +
+	::std::ops::Mul<Self, Output=Self> + ::std::ops::Div<Self, Output=Self> +
+	::std::ops::Rem<Self, Output=Self> +
+	::std::ops::AddAssign<Self> +
+	num_traits::CheckedMul + num_traits::CheckedSub +
+	Decode + Encode
+{}
+
+impl<T> BlockNumber for T where T:
+	Send + Sync + 'static +
+	::std::fmt::Display +
+	Clone +
+	From<u32> + TryInto<u32> + One + Zero +
+	PartialEq + Ord +
+	::std::hash::Hash +
+	::std::ops::Add<Self, Output=Self> + ::std::ops::Sub<Self, Output=Self> +
+	::std::ops::Mul<Self, Output=Self> + ::std::ops::Div<Self, Output=Self> +
+	::std::ops::Rem<Self, Output=Self> +
+	::std::ops::AddAssign<Self> +
+	num_traits::CheckedMul + num_traits::CheckedSub +
+	Decode + Encode,
+{}
+
+/// Block identifier that could be used to determine fork of this block.
+#[derive(Debug)]
+pub struct AnchorBlockId<Hash: ::std::fmt::Debug, Number: BlockNumber> {
+	/// Hash of this block.
+	pub hash: Hash,
+	/// Number of this block.
+	pub number: Number,
+}
+
+/// Changes trie storage. Provides access to trie roots and trie nodes.
+pub trait RootsStorage<H: Hasher, Number: BlockNumber>: Send + Sync {
+	/// Resolve hash of the block into anchor.
+	fn build_anchor(&self, hash: H::Out) -> Result<AnchorBlockId<H::Out, Number>, String>;
+	/// Get changes trie root for the block with given number which is an ancestor (or the block
+	/// itself) of the anchor_block (i.e. anchor_block.number >= block).
+	fn root(&self, anchor: &AnchorBlockId<H::Out, Number>, block: Number) -> Result<Option<H::Out>, String>;
+}
+
+/// Changes trie storage. Provides access to trie roots and trie nodes.
+pub trait Storage<H: Hasher, Number: BlockNumber>: RootsStorage<H, Number> {
+	/// Casts from self reference to RootsStorage reference.
+	fn as_roots_storage(&self) -> &dyn RootsStorage<H, Number>;
+	/// Execute given functor with cached entry for given trie root.
+	/// Returns true if the functor has been called (cache entry exists) and false otherwise.
+	fn with_cached_changed_keys(
+		&self,
+		root: &H::Out,
+		functor: &mut dyn FnMut(&HashMap<Option<Vec<u8>>, HashSet<Vec<u8>>>),
+	) -> bool;
+	/// Get a trie node.
+	fn get(&self, key: &H::Out, prefix: Prefix) -> Result<Option<DBValue>, String>;
+}
+
+/// Changes trie storage -> trie backend essence adapter.
+pub struct TrieBackendStorageAdapter<'a, H: Hasher, Number: BlockNumber>(pub &'a dyn Storage<H, Number>);
+
+impl<'a, H: Hasher, N: BlockNumber> crate::TrieBackendStorage<H> for TrieBackendStorageAdapter<'a, H, N> {
+	type Overlay = trie::MemoryDB<H>;
+
+	fn get(&self, key: &H::Out, prefix: Prefix) -> Result<Option<DBValue>, String> {
+		self.0.get(key, prefix)
+	}
+}
+
+/// Changes trie configuration.
+pub type Configuration = primitives::ChangesTrieConfiguration;
+
+/// Blocks range where configuration has been constant.
+#[derive(Clone)]
+pub struct ConfigurationRange<'a, N> {
+	/// Active configuration.
+	pub config: &'a Configuration,
+	/// Zero block of this configuration. The configuration is active starting from the next block.
+	pub zero: N,
+	/// End block of this configuration. It is the last block where configuration has been active.
+	pub end: Option<N>,
+}
+
+/// Compute the changes trie root and transaction for given block.
+/// Returns Err(()) if unknown `parent_hash` has been passed.
+/// Returns Ok(None) if there's no data to perform computation.
+/// Panics if background storage returns an error OR if insert to MemoryDB fails.
+pub fn build_changes_trie<'a, B: Backend<H>, S: Storage<H, Number>, H: Hasher, Number: BlockNumber>(
+	backend: &B,
+	storage: Option<&'a S>,
+	changes: &OverlayedChanges,
+	parent_hash: H::Out,
+) -> Result<Option<(MemoryDB<H>, H::Out, CacheAction<H::Out, Number>)>, ()>
+	where
+		H::Out: Ord + 'static,
+{
+	let (storage, config) = match (storage, changes.changes_trie_config.as_ref()) {
+		(Some(storage), Some(config)) => (storage, config),
+		_ => return Ok(None),
+	};
+
+	// FIXME: remove this in https://github.com/paritytech/substrate/pull/3201
+	let config = ConfigurationRange {
+		config,
+		zero: Zero::zero(),
+		end: None,
+	};
+
+	// build_anchor error should not be considered fatal
+	let parent = storage.build_anchor(parent_hash).map_err(|_| ())?;
+	let block = parent.number.clone() + One::one();
+
+	// storage errors are considered fatal (similar to situations when runtime fetches values from storage)
+	let (input_pairs, child_input_pairs, digest_input_blocks) = prepare_input::<B, H, Number>(
+		backend,
+		storage,
+		config.clone(),
+		changes,
+		&parent,
+	).expect("changes trie: storage access is not allowed to fail within runtime");
+
+	// prepare cached data
+	let mut cache_action = prepare_cached_build_data(config, block.clone());
+	let needs_changed_keys = cache_action.collects_changed_keys();
+	cache_action = cache_action.set_digest_input_blocks(digest_input_blocks);
+
+	let mut mdb = MemoryDB::default();
+	let mut child_roots = Vec::with_capacity(child_input_pairs.len());
+	for (child_index, input_pairs) in child_input_pairs {
+		let mut not_empty = false;
+		let mut root = Default::default();
+		{
+			let mut trie = TrieDBMut::<H>::new(&mut mdb, &mut root);
+			let mut storage_changed_keys = HashSet::new();
+			for input_pair in input_pairs {
+				if needs_changed_keys {
+					if let Some(key) = input_pair.key() {
+						storage_changed_keys.insert(key.to_vec());
+					}
+				}
+
+				let (key, value) = input_pair.into();
+				not_empty = true;
+				trie.insert(&key, &value)
+					.expect("changes trie: insertion to trie is not allowed to fail within runtime");
+			}
+
+			cache_action = cache_action.insert(
+				Some(child_index.storage_key.clone()),
+				storage_changed_keys,
+			);
+		}
+		if not_empty {
+			child_roots.push(input::InputPair::ChildIndex(child_index, root.as_ref().to_vec()));
+		}
+	}
+	let mut root = Default::default();
+	{
+		let mut trie = TrieDBMut::<H>::new(&mut mdb, &mut root);
+		for (key, value) in child_roots.into_iter().map(Into::into) {
+			trie.insert(&key, &value)
+				.expect("changes trie: insertion to trie is not allowed to fail within runtime");
+		}
+
+		let mut storage_changed_keys = HashSet::new();
+		for input_pair in input_pairs {
+			if needs_changed_keys {
+				if let Some(key) = input_pair.key() {
+					storage_changed_keys.insert(key.to_vec());
+				}
+			}
+
+			let (key, value) = input_pair.into();
+			trie.insert(&key, &value)
+				.expect("changes trie: insertion to trie is not allowed to fail within runtime");
+		}
+		cache_action = cache_action.insert(
+			None,
+			storage_changed_keys,
+		);
+	}
+
+	let cache_action = cache_action.complete(block, &root);
+	Ok(Some((mdb, root, cache_action)))
+}
+
+/// Prepare empty cached build data for given block.
+fn prepare_cached_build_data<Number: BlockNumber>(
+	config: ConfigurationRange<Number>,
+	block: Number,
+) -> IncompleteCacheAction<Number> {
+	// when digests are not enabled in configuration, we do not need to cache anything
+	// because it'll never be used again for building other tries
+	// => let's clear the cache
+	if !config.config.is_digest_build_enabled() {
+		return IncompleteCacheAction::Clear;
+	}
+
+	// when this is the last block where current configuration is active
+	// => let's clear the cache
+	if config.end.as_ref() == Some(&block) {
+		return IncompleteCacheAction::Clear;
+	}
+
+	// we do not need to cache anything when top-level digest trie is created, because
+	// it'll never be used again for building other tries
+	// => let's clear the cache
+	match config.config.digest_level_at_block(config.zero.clone(), block) {
+		Some((digest_level, _, _)) if digest_level == config.config.digest_levels => IncompleteCacheAction::Clear,
+		_ => IncompleteCacheAction::CacheBuildData(IncompleteCachedBuildData::new()),
+	}
+}
+
+#[cfg(test)]
+mod tests {
+	use super::*;
+
+	#[test]
+	fn cache_is_cleared_when_digests_are_disabled() {
+		let config = Configuration { digest_interval: 0, digest_levels: 0 };
+		let config_range = ConfigurationRange { zero: 0, end: None, config: &config };
+		assert_eq!(prepare_cached_build_data(config_range, 8u32), IncompleteCacheAction::Clear);
+	}
+
+	#[test]
+	fn build_data_is_cached_when_digests_are_enabled() {
+		let config = Configuration { digest_interval: 8, digest_levels: 2 };
+		let config_range = ConfigurationRange { zero: 0, end: None, config: &config };
+		assert!(prepare_cached_build_data(config_range.clone(), 4u32).collects_changed_keys());
+		assert!(prepare_cached_build_data(config_range.clone(), 7u32).collects_changed_keys());
+		assert!(prepare_cached_build_data(config_range, 8u32).collects_changed_keys());
+	}
+
+	#[test]
+	fn cache_is_cleared_when_digests_are_enabled_and_top_level_digest_is_built() {
+		let config = Configuration { digest_interval: 8, digest_levels: 2 };
+		let config_range = ConfigurationRange { zero: 0, end: None, config: &config };
+		assert_eq!(prepare_cached_build_data(config_range, 64u32), IncompleteCacheAction::Clear);
+	}
+
+	#[test]
+	fn cache_is_cleared_when_end_block_of_configuration_is_built() {
+		let config = Configuration { digest_interval: 8, digest_levels: 2 };
+		let config_range = ConfigurationRange { zero: 0, end: Some(4u32), config: &config };
+		assert_eq!(prepare_cached_build_data(config_range.clone(), 4u32), IncompleteCacheAction::Clear);
+	}
+}

substrate/primitives/state-machine/src/changes_trie/prune.rs

@@ -0,0 +1,370 @@
+// Copyright 2017-2019 Parity Technologies (UK) Ltd.
+// This file is part of Substrate.
+
+// Substrate is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// Substrate is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with Substrate.  If not, see <http://www.gnu.org/licenses/>.
+
+//! Changes trie pruning-related functions.
+
+use hash_db::Hasher;
+use trie::Recorder;
+use log::warn;
+use num_traits::{One, Zero};
+use crate::proving_backend::ProvingBackendRecorder;
+use crate::trie_backend_essence::TrieBackendEssence;
+use crate::changes_trie::{AnchorBlockId, Configuration, Storage, BlockNumber};
+use crate::changes_trie::storage::TrieBackendAdapter;
+use crate::changes_trie::input::{ChildIndex, InputKey};
+use codec::Decode;
+
+/// Get number of oldest block for which changes trie is not pruned
+/// given changes trie configuration, pruning parameter and number of
+/// best finalized block.
+pub fn oldest_non_pruned_trie<Number: BlockNumber>(
+	config: &Configuration,
+	min_blocks_to_keep: Number,
+	best_finalized_block: Number,
+) -> Number {
+	let max_digest_interval = config.max_digest_interval();
+	let best_finalized_block_rem = best_finalized_block.clone() % max_digest_interval.into();
+	let max_digest_block = best_finalized_block - best_finalized_block_rem;
+	match pruning_range(config, min_blocks_to_keep, max_digest_block) {
+		Some((_, last_pruned_block)) => last_pruned_block + One::one(),
+		None => One::one(),
+	}
+}
+
+/// Prune obsolete changes tries. Pruning happens at the same block, where highest
+/// level digest is created. Pruning guarantees to save changes tries for last
+/// `min_blocks_to_keep` blocks. We only prune changes tries at `max_digest_interval`
+/// ranges.
+/// Returns MemoryDB that contains all deleted changes tries nodes.
+pub fn prune<S: Storage<H, Number>, H: Hasher, Number: BlockNumber, F: FnMut(H::Out)>(
+	config: &Configuration,
+	storage: &S,
+	min_blocks_to_keep: Number,
+	current_block: &AnchorBlockId<H::Out, Number>,
+	mut remove_trie_node: F,
+) {
+
+	// select range for pruning
+	let (first, last) = match pruning_range(config, min_blocks_to_keep, current_block.number.clone()) {
+		Some((first, last)) => (first, last),
+		None => return,
+	};
+
+	// delete changes trie for every block in range
+	// FIXME: limit `max_digest_interval` so that this cycle won't involve huge ranges
+	let mut block = first;
+	loop {
+		if block >= last.clone() + One::one() {
+			break;
+		}
+
+		let prev_block = block.clone();
+		block += One::one();
+
+		let block = prev_block;
+		let root = match storage.root(current_block, block.clone()) {
+			Ok(Some(root)) => root,
+			Ok(None) => continue,
+			Err(error) => {
+				// try to delete other tries
+				warn!(target: "trie", "Failed to read changes trie root from DB: {}", error);
+				continue;
+			},
+		};
+		let children_roots = {
+			let trie_storage = TrieBackendEssence::<_, H>::new(
+				crate::changes_trie::TrieBackendStorageAdapter(storage),
+				root,
+			);
+			let child_prefix = ChildIndex::key_neutral_prefix(block.clone());
+			let mut children_roots = Vec::new();
+			trie_storage.for_key_values_with_prefix(&child_prefix, |key, value| {
+				if let Ok(InputKey::ChildIndex::<Number>(_trie_key)) = Decode::decode(&mut &key[..]) {
+					if let Ok(value) = <Vec<u8>>::decode(&mut &value[..]) {
+						let mut trie_root = <H as Hasher>::Out::default();
+						trie_root.as_mut().copy_from_slice(&value[..]);
+						children_roots.push(trie_root);
+					}
+				}
+			});
+
+			children_roots
+		};
+		for root in children_roots.into_iter() {
+			prune_trie(storage, root, &mut remove_trie_node);
+		}
+
+		prune_trie(storage, root, &mut remove_trie_node);
+	}
+}
+
+// Prune a trie.
+fn prune_trie<S: Storage<H, Number>, H: Hasher, Number: BlockNumber, F: FnMut(H::Out)>(
+	storage: &S,
+	root: H::Out,
+	remove_trie_node: &mut F,
+) {
+
+	// enumerate all changes trie' keys, recording all nodes that have been 'touched'
+	// (effectively - all changes trie nodes)
+	let mut proof_recorder: Recorder<H::Out> = Default::default();
+	{
+		let mut trie = ProvingBackendRecorder::<_, H> {
+			backend: &TrieBackendEssence::new(TrieBackendAdapter::new(storage), root),
+			proof_recorder: &mut proof_recorder,
+		};
+		trie.record_all_keys();
+	}
+
+	// all nodes of this changes trie should be pruned
+	remove_trie_node(root);
+	for node in proof_recorder.drain().into_iter().map(|n| n.hash) {
+		remove_trie_node(node);
+	}
+}
+
+/// Select blocks range (inclusive from both ends) for pruning changes tries in.
+fn pruning_range<Number: BlockNumber>(
+	config: &Configuration,
+	min_blocks_to_keep: Number,
+	block: Number,
+) -> Option<(Number, Number)> {
+	// compute number of changes tries we actually want to keep
+	let (prune_interval, blocks_to_keep) = if config.is_digest_build_enabled() {
+		// we only CAN prune at block where max-level-digest is created
+		let max_digest_interval = match config.digest_level_at_block(Zero::zero(), block.clone()) {
+			Some((digest_level, digest_interval, _)) if digest_level == config.digest_levels =>
+				digest_interval,
+			_ => return None,
+		};
+
+		// compute maximal number of high-level digests to keep
+		let max_digest_intervals_to_keep = max_digest_intervals_to_keep(min_blocks_to_keep, max_digest_interval);
+
+		// number of blocks BEFORE current block where changes tries are not pruned
+		(
+			max_digest_interval,
+			max_digest_intervals_to_keep.checked_mul(&max_digest_interval.into())
+		)
+	} else {
+		(
+			1,
+			Some(min_blocks_to_keep)
+		)
+	};
+
+	// last block for which changes trie is pruned
+	let last_block_to_prune = blocks_to_keep.and_then(|b| block.checked_sub(&b));
+	let first_block_to_prune = last_block_to_prune
+		.clone()
+		.and_then(|b| b.checked_sub(&prune_interval.into()));
+
+	last_block_to_prune
+		.and_then(|last| first_block_to_prune.map(|first| (first + One::one(), last)))
+}
+
+/// Select pruning delay for the changes tries. To make sure we could build a changes
+/// trie at block B, we need an access to previous:
+/// max_digest_interval = config.digest_interval ^ config.digest_levels
+/// blocks. So we can only prune blocks that are earlier than B - max_digest_interval.
+/// The pruning_delay stands for number of max_digest_interval-s that we want to keep:
+/// 0 or 1: means that only last changes trie is guaranteed to exists;
+/// 2: the last changes trie + previous changes trie
+/// ...
+fn max_digest_intervals_to_keep<Number: BlockNumber>(
+	min_blocks_to_keep: Number,
+	max_digest_interval: u32,
+) -> Number {
+	// config.digest_level_at_block ensures that it is not zero
+	debug_assert!(max_digest_interval != 0);
+
+	let max_digest_intervals_to_keep = min_blocks_to_keep / max_digest_interval.into();
+	if max_digest_intervals_to_keep.is_zero() {
+		One::one()
+	} else {
+		max_digest_intervals_to_keep
+	}
+}
+
+#[cfg(test)]
+mod tests {
+	use std::collections::HashSet;
+	use trie::MemoryDB;
+	use primitives::Blake2Hasher;
+	use crate::backend::insert_into_memory_db;
+	use crate::changes_trie::storage::InMemoryStorage;
+	use codec::Encode;
+	use super::*;
+
+	fn config(interval: u32, levels: u32) -> Configuration {
+		Configuration {
+			digest_interval: interval,
+			digest_levels: levels,
+		}
+	}
+
+	fn prune_by_collect<S: Storage<H, u64>, H: Hasher>(
+		config: &Configuration,
+		storage: &S,
+		min_blocks_to_keep: u64,
+		current_block: u64,
+	) -> HashSet<H::Out> {
+		let mut pruned_trie_nodes = HashSet::new();
+		prune(config,
+			storage,
+			min_blocks_to_keep,
+			&AnchorBlockId { hash: Default::default(), number: current_block },
+			|node| { pruned_trie_nodes.insert(node); });
+		pruned_trie_nodes
+	}
+
+	#[test]
+	fn prune_works() {
+		fn prepare_storage() -> InMemoryStorage<Blake2Hasher, u64> {
+
+			let child_key = ChildIndex { block: 67u64, storage_key: b"1".to_vec() }.encode();
+			let mut mdb1 = MemoryDB::<Blake2Hasher>::default();
+			let root1 = insert_into_memory_db::<Blake2Hasher, _>(
+				&mut mdb1, vec![(vec![10], vec![20])]).unwrap();
+			let mut mdb2 = MemoryDB::<Blake2Hasher>::default();
+			let root2 = insert_into_memory_db::<Blake2Hasher, _>(
+				&mut mdb2, vec![(vec![11], vec![21]), (vec![12], vec![22])]).unwrap();
+			let mut mdb3 = MemoryDB::<Blake2Hasher>::default();
+			let ch_root3 = insert_into_memory_db::<Blake2Hasher, _>(
+				&mut mdb3, vec![(vec![110], vec![120])]).unwrap();
+			let root3 = insert_into_memory_db::<Blake2Hasher, _>(&mut mdb3, vec![
+				(vec![13], vec![23]),
+				(vec![14], vec![24]),
+				(child_key, ch_root3.as_ref().encode()),
+			]).unwrap();
+			let mut mdb4 = MemoryDB::<Blake2Hasher>::default();
+			let root4 = insert_into_memory_db::<Blake2Hasher, _>(
+				&mut mdb4, vec![(vec![15], vec![25])]).unwrap();
+			let storage = InMemoryStorage::new();
+			storage.insert(65, root1, mdb1);
+			storage.insert(66, root2, mdb2);
+			storage.insert(67, root3, mdb3);
+			storage.insert(68, root4, mdb4);
+
+			storage
+		}
+
+		// l1-digest is created every 2 blocks
+		// l2-digest is created every 4 blocks
+		// we do not want to keep any additional changes tries
+		// => only one l2-digest is saved AND it is pruned once next is created
+		let config = Configuration { digest_interval: 2, digest_levels: 2 };
+		let storage = prepare_storage();
+		assert!(prune_by_collect(&config, &storage, 0, 69).is_empty());
+		assert!(prune_by_collect(&config, &storage, 0, 70).is_empty());
+		assert!(prune_by_collect(&config, &storage, 0, 71).is_empty());
+		let non_empty = prune_by_collect(&config, &storage, 0, 72);
+		assert!(!non_empty.is_empty());
+		storage.remove_from_storage(&non_empty);
+		assert!(storage.into_mdb().drain().is_empty());
+
+		// l1-digest is created every 2 blocks
+		// l2-digest is created every 4 blocks
+		// we want keep 1 additional changes tries
+		let config = Configuration { digest_interval: 2, digest_levels: 2 };
+		let storage = prepare_storage();
+		assert!(prune_by_collect(&config, &storage, 8, 69).is_empty());
+		assert!(prune_by_collect(&config, &storage, 8, 70).is_empty());
+		assert!(prune_by_collect(&config, &storage, 8, 71).is_empty());
+		assert!(prune_by_collect(&config, &storage, 8, 72).is_empty());
+		assert!(prune_by_collect(&config, &storage, 8, 73).is_empty());
+		assert!(prune_by_collect(&config, &storage, 8, 74).is_empty());
+		assert!(prune_by_collect(&config, &storage, 8, 75).is_empty());
+		let non_empty = prune_by_collect(&config, &storage, 8, 76);
+		assert!(!non_empty.is_empty());
+		storage.remove_from_storage(&non_empty);
+		assert!(storage.into_mdb().drain().is_empty());
+
+		// l1-digest is created every 2 blocks
+		// we want keep 2 additional changes tries
+		let config = Configuration { digest_interval: 2, digest_levels: 1 };
+		let storage = prepare_storage();
+		assert!(prune_by_collect(&config, &storage, 4, 69).is_empty());
+		let non_empty = prune_by_collect(&config, &storage, 4, 70);
+		assert!(!non_empty.is_empty());
+		storage.remove_from_storage(&non_empty);
+		assert!(prune_by_collect(&config, &storage, 4, 71).is_empty());
+		let non_empty = prune_by_collect(&config, &storage, 4, 72);
+		assert!(!non_empty.is_empty());
+		storage.remove_from_storage(&non_empty);
+		assert!(storage.into_mdb().drain().is_empty());
+	}
+
+	#[test]
+	fn pruning_range_works() {
+		// DIGESTS ARE NOT CREATED + NO TRIES ARE PRUNED
+		assert_eq!(pruning_range(&config(10, 0), 2u64, 2u64), None);
+
+		// DIGESTS ARE NOT CREATED + SOME TRIES ARE PRUNED
+		assert_eq!(pruning_range(&config(10, 0), 100u64, 110u64), Some((10, 10)));
+		assert_eq!(pruning_range(&config(10, 0), 100u64, 210u64), Some((110, 110)));
+
+		// DIGESTS ARE CREATED + NO TRIES ARE PRUNED
+
+		assert_eq!(pruning_range(&config(10, 2), 2u64, 0u64), None);
+		assert_eq!(pruning_range(&config(10, 2), 30u64, 100u64), None);
+		assert_eq!(pruning_range(&config(::std::u32::MAX, 2), 1u64, 1024u64), None);
+		assert_eq!(pruning_range(&config(::std::u32::MAX, 2), ::std::u64::MAX, 1024u64), None);
+		assert_eq!(pruning_range(&config(32, 2), 2048u64, 512u64), None);
+		assert_eq!(pruning_range(&config(32, 2), 2048u64, 1024u64), None);
+
+		// DIGESTS ARE CREATED + SOME TRIES ARE PRUNED
+
+		// when we do not want to keep any highest-level-digests
+		// (system forces to keep at least one)
+		assert_eq!(pruning_range(&config(4, 2), 0u64, 32u64), Some((1, 16)));
+		assert_eq!(pruning_range(&config(4, 2), 0u64, 64u64), Some((33, 48)));
+		// when we want to keep 1 (last) highest-level-digest
+		assert_eq!(pruning_range(&config(4, 2), 16u64, 32u64), Some((1, 16)));
+		assert_eq!(pruning_range(&config(4, 2), 16u64, 64u64), Some((33, 48)));
+		// when we want to keep 1 (last) + 1 additional level digests
+		assert_eq!(pruning_range(&config(32, 2), 4096u64, 5120u64), Some((1, 1024)));
+		assert_eq!(pruning_range(&config(32, 2), 4096u64, 6144u64), Some((1025, 2048)));
+	}
+
+	#[test]
+	fn max_digest_intervals_to_keep_works() {
+		assert_eq!(max_digest_intervals_to_keep(1024u64, 1025), 1u64);
+		assert_eq!(max_digest_intervals_to_keep(1024u64, 1023), 1u64);
+		assert_eq!(max_digest_intervals_to_keep(1024u64, 512), 2u64);
+		assert_eq!(max_digest_intervals_to_keep(1024u64, 511), 2u64);
+		assert_eq!(max_digest_intervals_to_keep(1024u64, 100), 10u64);
+	}
+
+	#[test]
+	fn oldest_non_pruned_trie_works() {
+		// when digests are not created at all
+		assert_eq!(oldest_non_pruned_trie(&config(0, 0), 100u64, 10u64), 1);
+		assert_eq!(oldest_non_pruned_trie(&config(0, 0), 100u64, 110u64), 11);
+
+		// when only l1 digests are created
+		assert_eq!(oldest_non_pruned_trie(&config(100, 1), 100u64, 50u64), 1);
+		assert_eq!(oldest_non_pruned_trie(&config(100, 1), 100u64, 110u64), 1);
+		assert_eq!(oldest_non_pruned_trie(&config(100, 1), 100u64, 210u64), 101);
+
+		// when l2 digests are created
+		assert_eq!(oldest_non_pruned_trie(&config(100, 2), 100u64, 50u64), 1);
+		assert_eq!(oldest_non_pruned_trie(&config(100, 2), 100u64, 110u64), 1);
+		assert_eq!(oldest_non_pruned_trie(&config(100, 2), 100u64, 210u64), 1);
+		assert_eq!(oldest_non_pruned_trie(&config(100, 2), 100u64, 10110u64), 1);
+		assert_eq!(oldest_non_pruned_trie(&config(100, 2), 100u64, 20110u64), 10001);
+	}
+}

substrate/primitives/state-machine/src/changes_trie/storage.rs

@@ -0,0 +1,208 @@
+// Copyright 2017-2019 Parity Technologies (UK) Ltd.
+// This file is part of Substrate.
+
+// Substrate is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// Substrate is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with Substrate.  If not, see <http://www.gnu.org/licenses/>.
+
+//! Changes trie storage utilities.
+
+use std::collections::{BTreeMap, HashSet, HashMap};
+use hash_db::{Hasher, Prefix, EMPTY_PREFIX};
+use trie::DBValue;
+use trie::MemoryDB;
+use parking_lot::RwLock;
+use crate::changes_trie::{BuildCache, RootsStorage, Storage, AnchorBlockId, BlockNumber};
+use crate::trie_backend_essence::TrieBackendStorage;
+
+#[cfg(test)]
+use crate::backend::insert_into_memory_db;
+#[cfg(test)]
+use crate::changes_trie::input::{InputPair, ChildIndex};
+
+/// In-memory implementation of changes trie storage.
+pub struct InMemoryStorage<H: Hasher, Number: BlockNumber> {
+	data: RwLock<InMemoryStorageData<H, Number>>,
+	cache: BuildCache<H::Out, Number>,
+}
+
+/// Adapter for using changes trie storage as a TrieBackendEssence' storage.
+pub struct TrieBackendAdapter<'a, H: Hasher, Number: BlockNumber> {
+	storage: &'a dyn Storage<H, Number>,
+	_hasher: ::std::marker::PhantomData<(H, Number)>,
+}
+
+struct InMemoryStorageData<H: Hasher, Number: BlockNumber> {
+	roots: BTreeMap<Number, H::Out>,
+	mdb: MemoryDB<H>,
+}
+
+impl<H: Hasher, Number: BlockNumber> InMemoryStorage<H, Number> {
+	/// Creates storage from given in-memory database.
+	pub fn with_db(mdb: MemoryDB<H>) -> Self {
+		Self {
+			data: RwLock::new(InMemoryStorageData {
+				roots: BTreeMap::new(),
+				mdb,
+			}),
+			cache: BuildCache::new(),
+		}
+	}
+
+	/// Creates storage with empty database.
+	pub fn new() -> Self {
+		Self::with_db(Default::default())
+	}
+
+	/// Creates storage with given proof.
+	pub fn with_proof(proof: Vec<Vec<u8>>) -> Self {
+		use hash_db::HashDB;
+
+ 		let mut proof_db = MemoryDB::<H>::default();
+		for item in proof {
+			proof_db.insert(EMPTY_PREFIX, &item);
+		}
+		Self::with_db(proof_db)
+	}
+
+	/// Get mutable cache reference.
+	pub fn cache_mut(&mut self) -> &mut BuildCache<H::Out, Number> {
+		&mut self.cache
+	}
+
+	/// Create the storage with given blocks.
+	pub fn with_blocks(blocks: Vec<(Number, H::Out)>) -> Self {
+		Self {
+			data: RwLock::new(InMemoryStorageData {
+				roots: blocks.into_iter().collect(),
+				mdb: MemoryDB::default(),
+			}),
+			cache: BuildCache::new(),
+		}
+	}
+
+	#[cfg(test)]
+	pub fn with_inputs(
+		mut top_inputs: Vec<(Number, Vec<InputPair<Number>>)>,
+		children_inputs: Vec<(Vec<u8>, Vec<(Number, Vec<InputPair<Number>>)>)>,
+	) -> Self {
+		let mut mdb = MemoryDB::default();
+		let mut roots = BTreeMap::new();
+		for (storage_key, child_input) in children_inputs {
+			for (block, pairs) in child_input {
+				let root = insert_into_memory_db::<H, _>(&mut mdb, pairs.into_iter().map(Into::into));
+
+				if let Some(root) = root {
+					let ix = if let Some(ix) = top_inputs.iter().position(|v| v.0 == block) {
+						ix
+					} else {
+						top_inputs.push((block.clone(), Default::default()));
+						top_inputs.len() - 1
+					};
+					top_inputs[ix].1.push(InputPair::ChildIndex(
+						ChildIndex { block: block.clone(), storage_key: storage_key.clone() },
+						root.as_ref().to_vec(),
+					));
+				}
+			}
+		}
+
+		for (block, pairs) in top_inputs {
+			let root = insert_into_memory_db::<H, _>(&mut mdb, pairs.into_iter().map(Into::into));
+			if let Some(root) = root {
+				roots.insert(block, root);
+			}
+		}
+
+		InMemoryStorage {
+			data: RwLock::new(InMemoryStorageData {
+				roots,
+				mdb,
+			}),
+			cache: BuildCache::new(),
+		}
+	}
+
+	#[cfg(test)]
+	pub fn clear_storage(&self) {
+		self.data.write().mdb = MemoryDB::default();	// use new to be more correct
+	}
+
+	#[cfg(test)]
+	pub fn remove_from_storage(&self, keys: &HashSet<H::Out>) {
+		let mut data = self.data.write();
+		for key in keys {
+			data.mdb.remove_and_purge(key, hash_db::EMPTY_PREFIX);
+		}
+	}
+
+	#[cfg(test)]
+	pub fn into_mdb(self) -> MemoryDB<H> {
+		self.data.into_inner().mdb
+	}
+
+	/// Insert changes trie for given block.
+	pub fn insert(&self, block: Number, changes_trie_root: H::Out, trie: MemoryDB<H>) {
+		let mut data = self.data.write();
+		data.roots.insert(block, changes_trie_root);
+		data.mdb.consolidate(trie);
+	}
+}
+
+impl<H: Hasher, Number: BlockNumber> RootsStorage<H, Number> for InMemoryStorage<H, Number> {
+	fn build_anchor(&self, parent_hash: H::Out) -> Result<AnchorBlockId<H::Out, Number>, String> {
+		self.data.read().roots.iter()
+			.find(|(_, v)| **v == parent_hash)
+			.map(|(k, _)| AnchorBlockId { hash: parent_hash, number: k.clone() })
+			.ok_or_else(|| format!("Can't find associated number for block {:?}", parent_hash))
+	}
+
+	fn root(&self, _anchor_block: &AnchorBlockId<H::Out, Number>, block: Number) -> Result<Option<H::Out>, String> {
+		Ok(self.data.read().roots.get(&block).cloned())
+	}
+}
+
+impl<H: Hasher, Number: BlockNumber> Storage<H, Number> for InMemoryStorage<H, Number> {
+	fn as_roots_storage(&self) -> &dyn RootsStorage<H, Number> {
+		self
+	}
+
+	fn with_cached_changed_keys(
+		&self,
+		root: &H::Out,
+		functor: &mut dyn FnMut(&HashMap<Option<Vec<u8>>, HashSet<Vec<u8>>>),
+	) -> bool {
+		self.cache.with_changed_keys(root, functor)
+	}
+
+	fn get(&self, key: &H::Out, prefix: Prefix) -> Result<Option<DBValue>, String> {
+		MemoryDB::<H>::get(&self.data.read().mdb, key, prefix)
+	}
+}
+
+impl<'a, H: Hasher, Number: BlockNumber> TrieBackendAdapter<'a, H, Number> {
+	pub fn new(storage: &'a dyn Storage<H, Number>) -> Self {
+		Self { storage, _hasher: Default::default() }
+	}
+}
+
+impl<'a, H, Number> TrieBackendStorage<H> for TrieBackendAdapter<'a, H, Number>
+	where
+		Number: BlockNumber,
+		H: Hasher,
+{
+	type Overlay = MemoryDB<H>;
+
+	fn get(&self, key: &H::Out, prefix: Prefix) -> Result<Option<DBValue>, String> {
+		self.storage.get(key, prefix)
+	}
+}

substrate/primitives/state-machine/src/changes_trie/surface_iterator.rs

@@ -0,0 +1,285 @@
+// Copyright 2017-2019 Parity Technologies (UK) Ltd.
+// This file is part of Substrate.
+
+// Substrate is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// Substrate is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with Substrate.  If not, see <http://www.gnu.org/licenses/>.
+
+//! The best way to understand how this iterator works is to imagine some 2D terrain that have some mountains
+//! (digest changes tries) and valleys (changes tries for regular blocks). There are gems (blocks) beneath the
+//! terrain. Given the request to find all gems in the range [X1; X2] this iterator will return **minimal set**
+//! of points at the terrain (mountains and valleys) inside this range that have to be drilled down to
+//! search for gems.
+
+use num_traits::One;
+use crate::changes_trie::{ConfigurationRange, BlockNumber};
+
+/// Returns surface iterator for given range of blocks.
+///
+/// `max` is the number of best block, known to caller. We can't access any changes tries
+/// that are built after this block, even though we may have them built already.
+pub fn surface_iterator<'a, Number: BlockNumber>(
+	config: ConfigurationRange<'a, Number>,
+	max: Number,
+	begin: Number,
+	end: Number,
+) -> Result<SurfaceIterator<'a, Number>, String> {
+	let (current, current_begin, digest_step, digest_level) = lower_bound_max_digest(
+		config.clone(),
+		max.clone(),
+		begin.clone(),
+		end,
+	)?;
+	Ok(SurfaceIterator {
+		config,
+		begin,
+		max,
+		current: Some(current),
+		current_begin,
+		digest_step,
+		digest_level,
+	})
+}
+
+/// Surface iterator - only traverses top-level digests from given range and tries to find
+/// all valid digest changes.
+///
+/// Iterator item is the tuple of (last block of the current point + digest level of the current point).
+/// Digest level is Some(0) when it is regular block, is Some(non-zero) when it is digest block and None
+/// if it is skewed digest block.
+pub struct SurfaceIterator<'a, Number: BlockNumber> {
+	config: ConfigurationRange<'a, Number>,
+	begin: Number,
+	max: Number,
+	current: Option<Number>,
+	current_begin: Number,
+	digest_step: u32,
+	digest_level: Option<u32>,
+}
+
+impl<'a, Number: BlockNumber> Iterator for SurfaceIterator<'a, Number> {
+	type Item = Result<(Number, Option<u32>), String>;
+
+	fn next(&mut self) -> Option<Self::Item> {
+		let current = self.current.clone()?;
+		let digest_level = self.digest_level;
+
+		if current < self.digest_step.into() {
+			self.current = None;
+		} else {
+			let next = current.clone() - self.digest_step.into();
+			if next.is_zero() || next < self.begin {
+				self.current = None;
+			} else if next > self.current_begin {
+				self.current = Some(next);
+			} else {
+				let max_digest_interval = lower_bound_max_digest(
+					self.config.clone(),
+					self.max.clone(),
+					self.begin.clone(),
+					next,
+				);
+				let (current, current_begin, digest_step, digest_level) = match max_digest_interval {
+					Err(err) => return Some(Err(err)),
+					Ok(range) => range,
+				};
+
+				self.current = Some(current);
+				self.current_begin = current_begin;
+				self.digest_step = digest_step;
+				self.digest_level = digest_level;
+			}
+		}
+
+		Some(Ok((current, digest_level)))
+	}
+}
+
+/// Returns parameters of highest level digest block that includes the end of given range
+/// and tends to include the whole range.
+fn lower_bound_max_digest<'a, Number: BlockNumber>(
+	config: ConfigurationRange<'a, Number>,
+	max: Number,
+	begin: Number,
+	end: Number,
+) -> Result<(Number, Number, u32, Option<u32>), String> {
+	if end > max || begin > end {
+		return Err(format!("invalid changes range: {}..{}/{}", begin, end, max));
+	}
+	if begin <= config.zero || config.end.as_ref().map(|config_end| end > *config_end).unwrap_or(false) {
+		return Err(format!("changes trie range is not covered by configuration: {}..{}/{}..{}",
+			begin, end, config.zero, match config.end.as_ref() {
+				Some(config_end) => format!("{}", config_end),
+				None => "None".into(),
+			}));
+	}
+
+	let mut digest_level = 0u32;
+	let mut digest_step = 1u32;
+	let mut digest_interval = 0u32;
+	let mut current = end.clone();
+	let mut current_begin = begin.clone();
+	if current_begin != current {
+		while digest_level != config.config.digest_levels {
+			// try to use next level digest
+			let new_digest_level = digest_level + 1;
+			let new_digest_step = digest_step * config.config.digest_interval;
+			let new_digest_interval = config.config.digest_interval * {
+				if digest_interval == 0 { 1 } else { digest_interval }
+			};
+			let new_digest_begin = config.zero.clone() + ((current.clone() - One::one() - config.zero.clone())
+				/ new_digest_interval.into()) * new_digest_interval.into();
+			let new_digest_end = new_digest_begin.clone() + new_digest_interval.into();
+			let new_current = new_digest_begin.clone() + new_digest_interval.into();
+
+			// check if we met skewed digest
+			if let Some(skewed_digest_end) = config.end.as_ref() {
+				if new_digest_end > *skewed_digest_end {
+					let skewed_digest_start = config.config.prev_max_level_digest_block(
+						config.zero.clone(),
+						skewed_digest_end.clone(),
+					);
+					if let Some(skewed_digest_start) = skewed_digest_start {
+						let skewed_digest_range = (skewed_digest_end.clone() - skewed_digest_start.clone())
+							.try_into().ok()
+							.expect("skewed digest range is always <= max level digest range;\
+								max level digest range always fits u32; qed");
+						return Ok((
+							skewed_digest_end.clone(),
+							skewed_digest_start,
+							skewed_digest_range,
+							None,
+						));
+					}
+				}
+			}
+
+			// we can't use next level digest if it touches any unknown (> max) blocks
+			if new_digest_end > max {
+				if begin < new_digest_begin {
+					current_begin = new_digest_begin;
+				}
+				break;
+			}
+
+			// we can (and will) use this digest
+			digest_level = new_digest_level;
+			digest_step = new_digest_step;
+			digest_interval = new_digest_interval;
+			current = new_current;
+			current_begin = new_digest_begin;
+
+			// if current digest covers the whole range => no need to use next level digest
+			if current_begin <= begin && new_digest_end >= end {
+				break;
+			}
+		}
+	}
+
+	Ok((
+		current,
+		current_begin,
+		digest_step,
+		Some(digest_level),
+	))
+}
+
+#[cfg(test)]
+mod tests {
+	use crate::changes_trie::{Configuration};
+	use super::*;
+
+	fn configuration_range<'a>(config: &'a Configuration, zero: u64) -> ConfigurationRange<'a, u64> {
+		ConfigurationRange {
+			config,
+			zero,
+			end: None,
+		}
+	}
+
+	#[test]
+	fn lower_bound_max_digest_works() {
+		let config = Configuration { digest_interval: 4, digest_levels: 2 };
+
+		// when config activates at 0
+		assert_eq!(
+			lower_bound_max_digest(configuration_range(&config, 0u64), 100_000u64, 20u64, 180u64).unwrap(),
+			(192, 176, 16, Some(2)),
+		);
+
+		// when config activates at 30
+		assert_eq!(
+			lower_bound_max_digest(configuration_range(&config, 30u64), 100_000u64, 50u64, 210u64).unwrap(),
+			(222, 206, 16, Some(2)),
+		);
+	}
+
+	#[test]
+	fn surface_iterator_works() {
+		let config = Configuration { digest_interval: 4, digest_levels: 2 };
+
+		// when config activates at 0
+		assert_eq!(
+			surface_iterator(
+				configuration_range(&config, 0u64),
+				100_000u64,
+				40u64,
+				180u64,
+			).unwrap().collect::<Vec<_>>(),
+			vec![
+				Ok((192, Some(2))), Ok((176, Some(2))), Ok((160, Some(2))), Ok((144, Some(2))),
+				Ok((128, Some(2))), Ok((112, Some(2))), Ok((96, Some(2))), Ok((80, Some(2))),
+				Ok((64, Some(2))), Ok((48, Some(2))),
+			],
+		);
+
+		// when config activates at 30
+		assert_eq!(
+			surface_iterator(
+				configuration_range(&config, 30u64),
+				100_000u64,
+				40u64,
+				180u64,
+			).unwrap().collect::<Vec<_>>(),
+			vec![
+				Ok((190, Some(2))), Ok((174, Some(2))), Ok((158, Some(2))), Ok((142, Some(2))), Ok((126, Some(2))),
+				Ok((110, Some(2))), Ok((94, Some(2))), Ok((78, Some(2))), Ok((62, Some(2))), Ok((46, Some(2))),
+			],
+		);
+
+		// when config activates at 0 AND max block is before next digest
+		assert_eq!(
+			surface_iterator(configuration_range(&config, 0u64), 183u64, 40u64, 183u64).unwrap().collect::<Vec<_>>(),
+			vec![
+				Ok((183, Some(0))), Ok((182, Some(0))), Ok((181, Some(0))), Ok((180, Some(1))),
+				Ok((176, Some(2))), Ok((160, Some(2))), Ok((144, Some(2))), Ok((128, Some(2))), Ok((112, Some(2))),
+				Ok((96, Some(2))), Ok((80, Some(2))), Ok((64, Some(2))), Ok((48, Some(2))),
+			],
+		);
+	}
+
+	#[test]
+	fn surface_iterator_works_with_skewed_digest() {
+		let config = Configuration { digest_interval: 4, digest_levels: 2 };
+		let mut config_range = configuration_range(&config, 0u64);
+
+		// when config activates at 0 AND ends at 170
+		config_range.end = Some(170);
+		assert_eq!(
+			surface_iterator(config_range, 100_000u64, 40u64, 170u64).unwrap().collect::<Vec<_>>(),
+			vec![
+				Ok((170, None)), Ok((160, Some(2))), Ok((144, Some(2))), Ok((128, Some(2))), Ok((112, Some(2))),
+				Ok((96, Some(2))), Ok((80, Some(2))), Ok((64, Some(2))), Ok((48, Some(2))),
+			],
+		);
+	}
+}