Light friendly storage tracking: changes trie + extending over ranges (#628)

* changes_trie

* changs_trie: continue

* changes_trie: adding tests

* fixed TODO

* removed obsolete ExtrinsicChanges

* encodable ChangesTrieConfiguration

* removed polkadot fle

* fixed grumbles

* ext_storage_changes_root returns u32

* moved changes trie root to digest

* removed commented code

* read storage values from native code

* fixed grumbles

* fixed grumbles

* missing comma

substrate/core/state-machine/src/backend.rs

@@ -20,11 +20,11 @@ use std::{error, fmt};
 use std::cmp::Ord;
 use std::collections::HashMap;
 use std::marker::PhantomData;
-use std::sync::Arc;
-
 use hashdb::Hasher;
+use memorydb::MemoryDB;
 use rlp::Encodable;
-use trie_backend::{TryIntoTrieBackend, TrieBackend};
+use trie_backend::TrieBackend;
+use trie_backend_essence::TrieBackendStorage;
 use patricia_trie::{TrieDBMut, TrieMut, NodeCodec};
 use heapsize::HeapSizeOf;
 
@@ -32,13 +32,16 @@ use heapsize::HeapSizeOf;
 /// to it.
 ///
 /// The clone operation (if implemented) should be cheap.
-pub trait Backend<H: Hasher, C: NodeCodec<H>>: TryIntoTrieBackend<H, C> {
+pub trait Backend<H: Hasher, C: NodeCodec<H>> {
 	/// An error type when fetching data is not possible.
 	type Error: super::Error;
 
-	/// Changes to be applied if committing
+	/// Storage changes to be applied if committing
 	type Transaction;
 
+	/// Type of trie backend storage.
+	type TrieBackendStorage: TrieBackendStorage<H>;
+
 	/// Get keyed storage associated with specific address, or None if there is nothing associated.
 	fn storage(&self, key: &[u8]) -> Result<Option<Vec<u8>>, Self::Error>;
 
@@ -60,6 +63,9 @@ pub trait Backend<H: Hasher, C: NodeCodec<H>>: TryIntoTrieBackend<H, C> {
 
 	/// Get all key/value pairs into a Vec.
 	fn pairs(&self) -> Vec<(Vec<u8>, Vec<u8>)>;
+
+	/// Try convert into trie backend.
+	fn try_into_trie_backend(self) -> Option<TrieBackend<Self::TrieBackendStorage, H, C>>;
 }
 
 /// Error impossible.
@@ -81,7 +87,7 @@ impl error::Error for Void {
 /// tests.
 #[derive(Eq)]
 pub struct InMemory<H, C> {
-	inner: Arc<HashMap<Vec<u8>, Vec<u8>>>,
+	inner: HashMap<Vec<u8>, Vec<u8>>,
 	_hasher: PhantomData<H>,
 	_codec: PhantomData<C>,
 }
@@ -89,7 +95,7 @@ pub struct InMemory<H, C> {
 impl<H, C> Default for InMemory<H, C> {
 	fn default() -> Self {
 		InMemory {
-			inner: Arc::new(Default::default()),
+			inner: Default::default(),
 			_hasher: PhantomData,
 			_codec: PhantomData,
 		}
@@ -111,9 +117,17 @@ impl<H, C> PartialEq for InMemory<H, C> {
 }
 
 impl<H: Hasher, C: NodeCodec<H>> InMemory<H, C> where H::Out: HeapSizeOf {
+	/// Try convert into trie backend.
+	pub fn try_into_trie_backend(self) -> Option<TrieBackend<MemoryDB<H>, H, C>> {
+		let mut mdb = MemoryDB::default();
+		let root = insert_into_memory_db::<H, C, _>(&mut mdb, self.inner.into_iter())?;
+
+		Some(TrieBackend::new(mdb, root))
+	}
+
 	/// Copy the state, with applied updates
 	pub fn update(&self, changes: <Self as Backend<H, C>>::Transaction) -> Self {
-		let mut inner: HashMap<_, _> = (&*self.inner).clone();
+		let mut inner: HashMap<_, _> = self.inner.clone();
 		for (key, val) in changes {
 			match val {
 				Some(v) => { inner.insert(key, v); },
@@ -128,7 +142,7 @@ impl<H: Hasher, C: NodeCodec<H>> InMemory<H, C> where H::Out: HeapSizeOf {
 impl<H, C> From<HashMap<Vec<u8>, Vec<u8>>> for InMemory<H, C> {
 	fn from(inner: HashMap<Vec<u8>, Vec<u8>>) -> Self {
 		InMemory {
-			inner: Arc::new(inner), _hasher: PhantomData, _codec: PhantomData
+			inner: inner, _hasher: PhantomData, _codec: PhantomData
 		}
 	}
 }
@@ -138,6 +152,7 @@ impl super::Error for Void {}
 impl<H: Hasher, C: NodeCodec<H>> Backend<H, C> for InMemory<H, C> where H::Out: HeapSizeOf {
 	type Error = Void;
 	type Transaction = Vec<(Vec<u8>, Option<Vec<u8>>)>;
+	type TrieBackendStorage = MemoryDB<H>;
 
 	fn storage(&self, key: &[u8]) -> Result<Option<Vec<u8>>, Self::Error> {
 		Ok(self.inner.get(key).map(Clone::clone))
@@ -171,23 +186,32 @@ impl<H: Hasher, C: NodeCodec<H>> Backend<H, C> for InMemory<H, C> where H::Out:
 	fn pairs(&self) -> Vec<(Vec<u8>, Vec<u8>)> {
 		self.inner.iter().map(|(k, v)| (k.clone(), v.clone())).collect()
 	}
-}
 
-impl<H: Hasher, C: NodeCodec<H>> TryIntoTrieBackend<H, C> for InMemory<H, C> where H::Out: HeapSizeOf {
-	fn try_into_trie_backend(self) -> Option<TrieBackend<H, C>> {
-		use memorydb::MemoryDB;
-		let mut root = <H as Hasher>::Out::default();
+	fn try_into_trie_backend(self) -> Option<TrieBackend<Self::TrieBackendStorage, H, C>> {
 		let mut mdb = MemoryDB::new();
-		{
-			let mut trie = TrieDBMut::<H, C>::new(&mut mdb, &mut root);
-			for (key, value) in self.inner.iter() {
-				if let Err(e) = trie.insert(&key, &value) {
-					warn!(target: "trie", "Failed to write to trie: {}", e);
-					return None;
-				}
-			}
-		}
-
-		Some(TrieBackend::with_memorydb(mdb, root))
+		let root = insert_into_memory_db::<H, C, _>(&mut mdb, self.inner.clone().into_iter())?;
+		Some(TrieBackend::new(mdb, root))
 	}
 }
+
+/// Insert input pairs into memory db.
+pub(crate) fn insert_into_memory_db<H, C, I>(mdb: &mut MemoryDB<H>, input: I) -> Option<H::Out>
+	where
+		H: Hasher,
+		H::Out: HeapSizeOf,
+		C: NodeCodec<H>,
+		I: Iterator<Item=(Vec<u8>, Vec<u8>)>,
+{
+	let mut root = <H as Hasher>::Out::default();
+	{
+		let mut trie = TrieDBMut::<H, C>::new(mdb, &mut root);
+		for (key, value) in input {
+			if let Err(e) = trie.insert(&key, &value) {
+				warn!(target: "trie", "Failed to write to trie: {}", e);
+				return None;
+			}
+		}
+	}
+
+	Some(root)
+}

substrate/core/state-machine/src/changes_trie/build.rs

@@ -0,0 +1,296 @@
+// Copyright 2017 Parity Technologies (UK) Ltd.
+// This file is part of Substrate.
+
+// Substrate is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// Substrate is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with Substrate.  If not, see <http://www.gnu.org/licenses/>.
+
+//! Structures and functions required to build changes trie for given block.
+
+use std::collections::{BTreeMap, BTreeSet};
+use codec::Decode;
+use hashdb::Hasher;
+use heapsize::HeapSizeOf;
+use patricia_trie::NodeCodec;
+use backend::Backend;
+use overlayed_changes::OverlayedChanges;
+use trie_backend_essence::{TrieBackendStorage, TrieBackendEssence};
+use changes_trie::build_iterator::digest_build_iterator;
+use changes_trie::input::{InputKey, InputPair, DigestIndex, ExtrinsicIndex};
+use changes_trie::{Configuration, Storage};
+
+/// Prepare input pairs for building a changes trie of given block.
+///
+/// Returns Err if storage error has occured OR if storage haven't returned
+/// required data.
+/// Returns Ok(None) data required to prepare input pairs is not collected
+/// or storage is not provided.
+pub fn prepare_input<'a, B, S, H, C>(
+	backend: &B,
+	storage: Option<&'a S>,
+	changes: &OverlayedChanges,
+	block: u64,
+) -> Result<Option<Vec<InputPair>>, String>
+	where
+		B: Backend<H, C>,
+		S: Storage<H>,
+		&'a S: TrieBackendStorage<H>,
+		H: Hasher,
+		H::Out: HeapSizeOf,
+		C: NodeCodec<H>,
+{
+	let (storage, config) = match (storage, changes.changes_trie_config.as_ref()) {
+		(Some(storage), Some(config)) => (storage, config),
+		_ => return Ok(None),
+	};
+
+	let mut input = Vec::new();
+	input.extend(prepare_extrinsics_input(
+		backend,
+		block,
+		changes)?);
+	input.extend(prepare_digest_input::<_, H, C>(
+		block,
+		config,
+		storage)?);
+
+	Ok(Some(input))
+}
+
+/// Prepare ExtrinsicIndex input pairs.
+fn prepare_extrinsics_input<B, H, C>(
+	backend: &B,
+	block: u64,
+	changes: &OverlayedChanges,
+) -> Result<impl Iterator<Item=InputPair>, String>
+	where
+		B: Backend<H, C>,
+		H: Hasher,
+		C: NodeCodec<H>,
+{
+	let mut extrinsic_map = BTreeMap::<Vec<u8>, BTreeSet<u32>>::new();
+	for (key, val) in changes.prospective.iter().chain(changes.committed.iter()) {
+		let extrinsics = match val.extrinsics {
+			Some(ref extrinsics) => extrinsics,
+			None => continue,
+		};
+
+		// ignore values that have null value at the end of operation AND are not in storage
+		// at the beginning of operation
+		if !changes.storage(key).map(|v| v.is_some()).unwrap_or_default() {
+			if !backend.exists_storage(key).map_err(|e| format!("{}", e))? {
+				continue;
+			}
+		}
+
+		extrinsic_map.entry(key.clone()).or_default()
+			.extend(extrinsics.iter().cloned());
+	}
+
+	Ok(extrinsic_map.into_iter()
+		.map(move |(key, extrinsics)| InputPair::ExtrinsicIndex(ExtrinsicIndex {
+			block,
+			key,
+		}, extrinsics.iter().cloned().collect())))
+}
+
+/// Prepare DigestIndex input pairs.
+fn prepare_digest_input<'a, S, H, C>(
+	block: u64,
+	config: &Configuration,
+	storage: &'a S
+) -> Result<impl Iterator<Item=InputPair>, String>
+	where
+		S: Storage<H>,
+		&'a S: TrieBackendStorage<H>,
+		H: Hasher,
+		H::Out: HeapSizeOf,
+		C: NodeCodec<H>,
+{
+	let mut digest_map = BTreeMap::<Vec<u8>, BTreeSet<u64>>::new();
+	for digest_build_block in digest_build_iterator(config, block) {
+		let trie_root = storage.root(digest_build_block)?;
+		let trie_root = trie_root.ok_or_else(|| format!("No changes trie root for block {}", digest_build_block))?;
+		let trie_storage = TrieBackendEssence::<_, H, C>::new(storage, trie_root);
+
+		let extrinsic_prefix = ExtrinsicIndex::key_neutral_prefix(digest_build_block);
+		trie_storage.for_keys_with_prefix(&extrinsic_prefix, |key|
+			if let Some(InputKey::ExtrinsicIndex(trie_key)) = Decode::decode(&mut &key[..]) {
+				digest_map.entry(trie_key.key).or_default()
+					.insert(digest_build_block);
+			});
+
+		let digest_prefix = DigestIndex::key_neutral_prefix(digest_build_block);
+		trie_storage.for_keys_with_prefix(&digest_prefix, |key|
+			if let Some(InputKey::DigestIndex(trie_key)) = Decode::decode(&mut &key[..]) {
+				digest_map.entry(trie_key.key).or_default()
+					.insert(digest_build_block);
+			});
+	}
+
+	Ok(digest_map.into_iter()
+		.map(move |(key, set)| InputPair::DigestIndex(DigestIndex {
+			block,
+			key
+		}, set.into_iter().collect())))
+}
+
+#[cfg(test)]
+mod test {
+	use codec::Encode;
+	use primitives::{Blake2Hasher, RlpCodec};
+	use backend::InMemory;
+	use changes_trie::storage::InMemoryStorage;
+	use overlayed_changes::OverlayedValue;
+	use super::*;
+
+	fn prepare_for_build() -> (InMemory<Blake2Hasher, RlpCodec>, InMemoryStorage<Blake2Hasher>, OverlayedChanges) {
+		let backend: InMemory<_, _> = vec![
+			(vec![100], vec![255]),
+			(vec![101], vec![255]),
+			(vec![102], vec![255]),
+			(vec![103], vec![255]),
+			(vec![104], vec![255]),
+			(vec![105], vec![255]),
+		].into_iter().collect::<::std::collections::HashMap<_, _>>().into();
+		let storage = InMemoryStorage::with_inputs::<RlpCodec>(vec![
+			(1, vec![
+				InputPair::ExtrinsicIndex(ExtrinsicIndex { block: 1, key: vec![100] }, vec![1, 3]),
+				InputPair::ExtrinsicIndex(ExtrinsicIndex { block: 1, key: vec![101] }, vec![0, 2]),
+				InputPair::ExtrinsicIndex(ExtrinsicIndex { block: 1, key: vec![105] }, vec![0, 2, 4]),
+			]),
+			(2, vec![
+				InputPair::ExtrinsicIndex(ExtrinsicIndex { block: 2, key: vec![102] }, vec![0]),
+			]),
+			(3, vec![
+				InputPair::ExtrinsicIndex(ExtrinsicIndex { block: 3, key: vec![100] }, vec![0]),
+				InputPair::ExtrinsicIndex(ExtrinsicIndex { block: 3, key: vec![105] }, vec![1]),
+			]),
+			(4, vec![
+				InputPair::ExtrinsicIndex(ExtrinsicIndex { block: 4, key: vec![100] }, vec![0, 2, 3]),
+				InputPair::ExtrinsicIndex(ExtrinsicIndex { block: 4, key: vec![101] }, vec![1]),
+				InputPair::ExtrinsicIndex(ExtrinsicIndex { block: 4, key: vec![103] }, vec![0, 1]),
+
+				InputPair::DigestIndex(DigestIndex { block: 4, key: vec![100] }, vec![1, 3]),
+				InputPair::DigestIndex(DigestIndex { block: 4, key: vec![101] }, vec![1]),
+				InputPair::DigestIndex(DigestIndex { block: 4, key: vec![102] }, vec![2]),
+				InputPair::DigestIndex(DigestIndex { block: 4, key: vec![105] }, vec![1, 3]),
+			]),
+			(5, Vec::new()),
+			(6, vec![
+				InputPair::ExtrinsicIndex(ExtrinsicIndex { block: 6, key: vec![105] }, vec![2]),
+			]),
+			(7, Vec::new()),
+			(8, vec![
+				InputPair::DigestIndex(DigestIndex { block: 8, key: vec![105] }, vec![6]),
+			]),
+			(9, Vec::new()), (10, Vec::new()), (11, Vec::new()), (12, Vec::new()), (13, Vec::new()),
+			(14, Vec::new()), (15, Vec::new()),
+		]);
+		let changes = OverlayedChanges {
+			prospective: vec![
+				(vec![100], OverlayedValue {
+					value: Some(vec![200]),
+					extrinsics: Some(vec![0, 2].into_iter().collect())
+				}),
+				(vec![103], OverlayedValue {
+					value: None,
+					extrinsics: Some(vec![0, 1].into_iter().collect())
+				}),
+			].into_iter().collect(),
+			committed: vec![
+				(b":extrinsic_index".to_vec(), OverlayedValue {
+					value: Some(3u32.encode()),
+					extrinsics: None,
+				}),
+				(vec![100], OverlayedValue {
+					value: Some(vec![202]),
+					extrinsics: Some(vec![3].into_iter().collect())
+				}),
+				(vec![101], OverlayedValue {
+					value: Some(vec![203]),
+					extrinsics: Some(vec![1].into_iter().collect())
+				}),
+			].into_iter().collect(),
+			changes_trie_config: Some(Configuration { digest_interval: 4, digest_levels: 2 }),
+		};
+
+		(backend, storage, changes)
+	}
+
+	#[test]
+	fn build_changes_trie_nodes_on_non_digest_block() {
+		let (backend, storage, changes) = prepare_for_build();
+		let changes_trie_nodes = prepare_input::<_, _, _, RlpCodec>(&backend, Some(&storage), &changes, 5).unwrap();
+		assert_eq!(changes_trie_nodes, Some(vec![
+			InputPair::ExtrinsicIndex(ExtrinsicIndex { block: 5, key: vec![100] }, vec![0, 2, 3]),
+			InputPair::ExtrinsicIndex(ExtrinsicIndex { block: 5, key: vec![101] }, vec![1]),
+			InputPair::ExtrinsicIndex(ExtrinsicIndex { block: 5, key: vec![103] }, vec![0, 1]),
+		]));
+	}
+
+	#[test]
+	fn build_changes_trie_nodes_on_digest_block_l1() {
+		let (backend, storage, changes) = prepare_for_build();
+		let changes_trie_nodes = prepare_input::<_, _, _, RlpCodec>(&backend, Some(&storage), &changes, 4).unwrap();
+		assert_eq!(changes_trie_nodes, Some(vec![
+			InputPair::ExtrinsicIndex(ExtrinsicIndex { block: 4, key: vec![100] }, vec![0, 2, 3]),
+			InputPair::ExtrinsicIndex(ExtrinsicIndex { block: 4, key: vec![101] }, vec![1]),
+			InputPair::ExtrinsicIndex(ExtrinsicIndex { block: 4, key: vec![103] }, vec![0, 1]),
+
+			InputPair::DigestIndex(DigestIndex { block: 4, key: vec![100] }, vec![1, 3]),
+			InputPair::DigestIndex(DigestIndex { block: 4, key: vec![101] }, vec![1]),
+			InputPair::DigestIndex(DigestIndex { block: 4, key: vec![102] }, vec![2]),
+			InputPair::DigestIndex(DigestIndex { block: 4, key: vec![105] }, vec![1, 3]),
+		]));
+	}
+
+	#[test]
+	fn build_changes_trie_nodes_on_digest_block_l2() {
+		let (backend, storage, changes) = prepare_for_build();
+		let changes_trie_nodes = prepare_input::<_, _, _, RlpCodec>(&backend, Some(&storage), &changes, 16).unwrap();
+		assert_eq!(changes_trie_nodes, Some(vec![
+			InputPair::ExtrinsicIndex(ExtrinsicIndex { block: 16, key: vec![100] }, vec![0, 2, 3]),
+			InputPair::ExtrinsicIndex(ExtrinsicIndex { block: 16, key: vec![101] }, vec![1]),
+			InputPair::ExtrinsicIndex(ExtrinsicIndex { block: 16, key: vec![103] }, vec![0, 1]),
+
+			InputPair::DigestIndex(DigestIndex { block: 16, key: vec![100] }, vec![4]),
+			InputPair::DigestIndex(DigestIndex { block: 16, key: vec![101] }, vec![4]),
+			InputPair::DigestIndex(DigestIndex { block: 16, key: vec![102] }, vec![4]),
+			InputPair::DigestIndex(DigestIndex { block: 16, key: vec![103] }, vec![4]),
+			InputPair::DigestIndex(DigestIndex { block: 16, key: vec![105] }, vec![4, 8]),
+		]));
+	}
+
+	#[test]
+	fn build_changes_trie_nodes_ignores_temporary_storage_values() {
+		let (backend, storage, mut changes) = prepare_for_build();
+
+		// 110: missing from backend, set to None in overlay
+		changes.prospective.insert(vec![110], OverlayedValue {
+			value: None,
+			extrinsics: Some(vec![1].into_iter().collect())
+		});
+
+		let changes_trie_nodes = prepare_input::<_, _, _, RlpCodec>(&backend, Some(&storage), &changes, 4).unwrap();
+		assert_eq!(changes_trie_nodes, Some(vec![
+			InputPair::ExtrinsicIndex(ExtrinsicIndex { block: 4, key: vec![100] }, vec![0, 2, 3]),
+			InputPair::ExtrinsicIndex(ExtrinsicIndex { block: 4, key: vec![101] }, vec![1]),
+			InputPair::ExtrinsicIndex(ExtrinsicIndex { block: 4, key: vec![103] }, vec![0, 1]),
+
+			InputPair::DigestIndex(DigestIndex { block: 4, key: vec![100] }, vec![1, 3]),
+			InputPair::DigestIndex(DigestIndex { block: 4, key: vec![101] }, vec![1]),
+			InputPair::DigestIndex(DigestIndex { block: 4, key: vec![102] }, vec![2]),
+			InputPair::DigestIndex(DigestIndex { block: 4, key: vec![105] }, vec![1, 3]),
+		]));
+	}
+}

substrate/core/state-machine/src/changes_trie/build_iterator.rs

@@ -0,0 +1,212 @@
+// Copyright 2017 Parity Technologies (UK) Ltd.
+// This file is part of Substrate.
+
+// Substrate is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// Substrate is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with Substrate.  If not, see <http://www.gnu.org/licenses/>.
+
+//! Structures and functions to return blocks whose changes are to be included
+//! in given block' changes trie.
+
+use changes_trie::Configuration;
+
+/// Returns iterator of OTHER blocks that are required for inclusion into
+/// changes trie of given block.
+pub fn digest_build_iterator(config: &Configuration, block: u64) -> DigestBuildIterator {
+	// digest is never built in these cases
+	if block == 0 || config.digest_interval <= 1 || config.digest_levels == 0 {
+		return DigestBuildIterator::empty();
+	}
+
+	// digest is built every digest_multiplier blocks
+	let mut digest_interval = config.digest_interval;
+	if block % digest_interval != 0 {
+		return DigestBuildIterator::empty();
+	}
+
+	// we have checked that the block is at least level1-digest
+	// => try to find highest digest level for inclusion
+	let mut current_level = 1u32;
+	let mut digest_step = 1u64;
+	while current_level < config.digest_levels {
+		let new_digest_interval = match digest_interval.checked_mul(config.digest_interval) {
+			Some(new_digest_interval) if block % new_digest_interval == 0 => new_digest_interval,
+			_ => break,
+		};
+
+		digest_step = digest_interval;
+		digest_interval = new_digest_interval;
+		current_level = current_level + 1;
+	}
+
+	DigestBuildIterator::new(block, config.digest_interval, digest_step)
+}
+
+/// Changes trie build iterator that returns numbers of OTHER blocks that are
+/// required for inclusion into changes trie of given block.
+#[derive(Debug)]
+pub struct DigestBuildIterator {
+	/// Block we're building changes trie for.
+	block: u64,
+	/// Interval for creation digest blocks.
+	digest_interval: u64,
+	/// Step of current blocks range.
+	current_step: u64,
+	/// Current blocks range.
+	current_range: Option<::std::iter::StepBy<::std::ops::Range<u64>>>,
+	/// Max step of blocks range.
+	max_step: u64,
+}
+
+impl DigestBuildIterator {
+	/// Create new digest build iterator.
+	pub fn new(block: u64, digest_interval: u64, max_step: u64) -> Self {
+		DigestBuildIterator {
+			block, digest_interval, max_step,
+			current_step: 0,
+			current_range: None,
+		}
+	}
+
+	/// Create empty digest build iterator.
+	pub fn empty() -> Self {
+		Self::new(0, 0, 0)
+	}
+}
+
+impl Iterator for DigestBuildIterator {
+	type Item = u64;
+
+	fn next(&mut self) -> Option<Self::Item> {
+		if let Some(next) = self.current_range.as_mut().and_then(|iter| iter.next()) {
+			return Some(next);
+		}
+
+		// we are safe to use non-checking mul/sub versions here because:
+		// DigestBuildIterator is created only by internal function that is checking
+		// that all multiplications/subtractions are safe within max_step limit
+
+		let next_step = if self.current_step == 0 { 1 } else { self.current_step * self.digest_interval };
+		if next_step > self.max_step {
+			return None;
+		}
+
+		self.current_step = next_step;
+		self.current_range = Some(
+			((self.block - self.current_step * self.digest_interval + self.current_step)..self.block)
+				.step_by(self.current_step as usize)
+		);
+
+		Some(self.current_range.as_mut()
+			.expect("assigned one line above; qed")
+			.next()
+			.expect("X - I^(N+1) + I^N > X when X,I,N are > 1; qed"))
+	}
+}
+
+#[cfg(test)]
+mod tests {
+	use super::*;
+
+	fn digest_build_iterator(digest_interval: u64, digest_levels: u32, block: u64) -> DigestBuildIterator {
+		super::digest_build_iterator(&Configuration { digest_interval, digest_levels }, block)
+	}
+
+	fn digest_build_iterator_basic(digest_interval: u64, digest_levels: u32, block: u64) -> (u64, u64, u64) {
+		let iter = digest_build_iterator(digest_interval, digest_levels, block);
+		(iter.block, iter.digest_interval, iter.max_step)
+	}
+
+	fn digest_build_iterator_blocks(digest_interval: u64, digest_levels: u32, block: u64) -> Vec<u64> {
+		digest_build_iterator(digest_interval, digest_levels, block).collect()
+	}
+
+	#[test]
+	fn suggest_digest_inclusion_returns_empty_iterator() {
+		let empty = (0, 0, 0);
+		assert_eq!(digest_build_iterator_basic(4, 16, 0), empty, "block is 0");
+		assert_eq!(digest_build_iterator_basic(0, 16, 64), empty, "digest_interval is 0");
+		assert_eq!(digest_build_iterator_basic(1, 16, 64), empty, "digest_interval is 1");
+		assert_eq!(digest_build_iterator_basic(4, 0, 64), empty, "digest_levels is 0");
+		assert_eq!(digest_build_iterator_basic(4, 16, 1), empty, "digest is not required for this block");
+		assert_eq!(digest_build_iterator_basic(4, 16, 2), empty, "digest is not required for this block");
+		assert_eq!(digest_build_iterator_basic(4, 16, 15), empty, "digest is not required for this block");
+		assert_eq!(digest_build_iterator_basic(4, 16, 17), empty, "digest is not required for this block");
+		assert_eq!(digest_build_iterator_basic(::std::u64::MAX / 2 + 1, 16, ::std::u64::MAX), empty, "digest_interval * 2 is greater than u64::MAX");
+	}
+
+	#[test]
+	fn suggest_digest_inclusion_returns_level1_iterator() {
+		assert_eq!(digest_build_iterator_basic(16, 1, 16), (16, 16, 1), "!(block % interval) && first digest level == block");
+		assert_eq!(digest_build_iterator_basic(16, 1, 256), (256, 16, 1), "!(block % interval^2), but there's only 1 digest level");
+		assert_eq!(digest_build_iterator_basic(16, 2, 32), (32, 16, 1), "second level digest is not required for this block");
+		assert_eq!(digest_build_iterator_basic(16, 3, 4080), (4080, 16, 1), "second && third level digest are not required for this block");
+	}
+
+	#[test]
+	fn suggest_digest_inclusion_returns_level2_iterator() {
+		assert_eq!(digest_build_iterator_basic(16, 2, 256), (256, 16, 16), "second level digest");
+		assert_eq!(digest_build_iterator_basic(16, 2, 4096), (4096, 16, 16), "!(block % interval^3), but there's only 2 digest levels");
+	}
+
+	#[test]
+	fn suggest_digest_inclusion_returns_level3_iterator() {
+		assert_eq!(digest_build_iterator_basic(16, 3, 4096), (4096, 16, 256), "third level digest: beginning");
+		assert_eq!(digest_build_iterator_basic(16, 3, 8192), (8192, 16, 256), "third level digest: next");
+	}
+
+	#[test]
+	fn digest_iterator_returns_level1_blocks() {
+		assert_eq!(digest_build_iterator_blocks(16, 1, 16),
+			vec![1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15]);
+		assert_eq!(digest_build_iterator_blocks(16, 1, 256),
+			vec![241, 242, 243, 244, 245, 246, 247, 248, 249, 250, 251, 252, 253, 254, 255]);
+		assert_eq!(digest_build_iterator_blocks(16, 2, 32),
+			vec![17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31]);
+		assert_eq!(digest_build_iterator_blocks(16, 3, 4080),
+			vec![4065, 4066, 4067, 4068, 4069, 4070, 4071, 4072, 4073, 4074, 4075, 4076, 4077, 4078, 4079]);
+	}
+
+	#[test]
+	fn digest_iterator_returns_level1_and_level2_blocks() {
+		assert_eq!(digest_build_iterator_blocks(16, 2, 256),
+			vec![
+				// level2 is a level1 digest of 16-1 previous blocks:
+				241, 242, 243, 244, 245, 246, 247, 248, 249, 250, 251, 252, 253, 254, 255,
+				// level2 points to previous 16-1 level1 digests:
+				16, 32, 48, 64, 80, 96, 112, 128, 144, 160, 176, 192, 208, 224, 240,
+			],
+		);
+		assert_eq!(digest_build_iterator_blocks(16, 2, 4096),
+			vec![
+				// level2 is a level1 digest of 16-1 previous blocks:
+				4081, 4082, 4083, 4084, 4085, 4086, 4087, 4088, 4089, 4090, 4091, 4092, 4093, 4094, 4095,
+				// level2 points to previous 16-1 level1 digests:
+				3856, 3872, 3888, 3904, 3920, 3936, 3952, 3968, 3984, 4000, 4016, 4032, 4048, 4064, 4080,
+			],
+		);
+	}
+
+	#[test]
+	fn digest_iterator_returns_level1_and_level2_and_level3_blocks() {
+		assert_eq!(digest_build_iterator_blocks(16, 3, 4096),
+			vec![
+				// level3 is a level1 digest of 16-1 previous blocks:
+				4081, 4082, 4083, 4084, 4085, 4086, 4087, 4088, 4089, 4090, 4091, 4092, 4093, 4094, 4095,
+				// level3 points to previous 16-1 level1 digests:
+				3856, 3872, 3888, 3904, 3920, 3936, 3952, 3968, 3984, 4000, 4016, 4032, 4048, 4064, 4080,
+				// level3 points to previous 16-1 level2 digests:
+				256, 512, 768, 1024, 1280, 1536, 1792, 2048, 2304, 2560, 2816, 3072, 3328, 3584, 3840,
+			],
+		);
+	}
+}

substrate/core/state-machine/src/changes_trie/changes_iterator.rs

@@ -0,0 +1,453 @@
+// Copyright 2017 Parity Technologies (UK) Ltd.
+// This file is part of Substrate.
+
+// Substrate is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// Substrate is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with Substrate.  If not, see <http://www.gnu.org/licenses/>.
+
+//! Functions + iterator that traverses changes tries and returns all
+//! (block, extrinsic) pairs where given key has been changed.
+
+use std::cell::RefCell;
+use std::collections::VecDeque;
+use codec::{Decode, Encode};
+use hashdb::{HashDB, Hasher};
+use heapsize::HeapSizeOf;
+use memorydb::MemoryDB;
+use patricia_trie::{NodeCodec, Recorder};
+use changes_trie::{Configuration, Storage};
+use changes_trie::input::{DigestIndex, ExtrinsicIndex, DigestIndexValue, ExtrinsicIndexValue};
+use changes_trie::storage::{TrieBackendAdapter, InMemoryStorage};
+use proving_backend::ProvingBackendEssence;
+use trie_backend_essence::{TrieBackendEssence};
+
+/// Return changes of given key at given blocks range.
+/// `max` is the number of best known block.
+pub fn key_changes<S: Storage<H>, H: Hasher, C: NodeCodec<H>>(
+	config: &Configuration,
+	storage: &S,
+	begin: u64,
+	end: u64,
+	max: u64,
+	key: &[u8],
+) -> Result<Vec<(u64, u32)>, String> where H::Out: HeapSizeOf {
+	DrilldownIterator {
+		essence: DrilldownIteratorEssence {
+			key,
+			roots_storage: storage,
+			storage,
+			surface: surface_iterator(config, max, begin, end)?,
+
+			extrinsics: Default::default(),
+			blocks: Default::default(),
+
+			_hasher: ::std::marker::PhantomData::<H>::default(),
+		},
+		_codec: ::std::marker::PhantomData::<C>::default(),
+	}.collect()
+}
+
+/// Returns proof of changes of given key at given blocks range.
+/// `max` is the number of best known block.
+pub fn key_changes_proof<S: Storage<H>, H: Hasher, C: NodeCodec<H>>(
+	config: &Configuration,
+	storage: &S,
+	begin: u64,
+	end: u64,
+	max: u64,
+	key: &[u8],
+) -> Result<Vec<Vec<u8>>, String> where H::Out: HeapSizeOf {
+	let mut iter = ProvingDrilldownIterator {
+		essence: DrilldownIteratorEssence {
+			key,
+			roots_storage: storage.clone(),
+			storage,
+			surface: surface_iterator(config, max, begin, end)?,
+
+			extrinsics: Default::default(),
+			blocks: Default::default(),
+
+			_hasher: ::std::marker::PhantomData::<H>::default(),
+		},
+		proof_recorder: Default::default(),
+		_codec: ::std::marker::PhantomData::<C>::default(),
+	};
+
+	// iterate to collect proof
+	while let Some(item) = iter.next() {
+		item?;
+	}
+
+	Ok(iter.extract_proof())
+}
+
+/// Check key changes proog and return changes of the key at given blocks range.
+/// `max` is the number of best known block.
+pub fn key_changes_proof_check<S: Storage<H>, H: Hasher, C: NodeCodec<H>>(
+	config: &Configuration,
+	roots_storage: &S, // TODO: use RootsStorage is only used to gather root
+	proof: Vec<Vec<u8>>,
+	begin: u64,
+	end: u64,
+	max: u64,
+	key: &[u8]
+) -> Result<Vec<(u64, u32)>, String> where H::Out: HeapSizeOf {
+	let mut proof_db = MemoryDB::<H>::new();
+	for item in proof {
+		proof_db.insert(&item);
+	}
+
+	let proof_db = InMemoryStorage::with_db(proof_db);
+	DrilldownIterator {
+		essence: DrilldownIteratorEssence {
+			key,
+			roots_storage,
+			storage: &proof_db,
+			surface: surface_iterator(config, max, begin, end)?,
+
+			extrinsics: Default::default(),
+			blocks: Default::default(),
+
+			_hasher: ::std::marker::PhantomData::<H>::default(),
+		},
+		_codec: ::std::marker::PhantomData::<C>::default(),
+	}.collect()
+}
+
+/// Surface iterator - only traverses top-level digests from given range and tries to find
+/// all digest changes for the key.
+pub struct SurfaceIterator<'a> {
+	config: &'a Configuration,
+	begin: u64,
+	max: u64,
+	current: Option<u64>,
+	current_begin: u64,
+	digest_step: u64,
+	digest_level: u32,
+}
+
+impl<'a> Iterator for SurfaceIterator<'a> {
+	type Item = Result<(u64, u32), String>;
+
+	fn next(&mut self) -> Option<Self::Item> {
+		let current = self.current?;
+		let digest_level = self.digest_level;
+
+		if current < self.digest_step {
+			self.current = None;
+		}
+		else {
+			let next = current - self.digest_step;
+			if next == 0 || next < self.begin {
+				self.current = None;
+			}
+			else if next > self.current_begin {
+				self.current = Some(next);
+			} else {
+				let (current, current_begin, digest_step, digest_level) = match
+					lower_bound_max_digest(self.config, self.max, self.begin, next) {
+					Err(err) => return Some(Err(err)),
+					Ok(range) => range,
+				};
+
+				self.current = Some(current);
+				self.current_begin = current_begin;
+				self.digest_step = digest_step;
+				self.digest_level = digest_level;
+			}
+		}
+
+		Some(Ok((current, digest_level)))
+	}
+}
+
+/// Drilldown iterator - receives 'digest points' from surface iterator and explores
+/// every point until extrinsic is found.
+pub struct DrilldownIteratorEssence<'a, RS: 'a + Storage<H>, S: 'a + Storage<H>, H: Hasher> {
+	key: &'a [u8],
+	roots_storage: &'a RS,
+	storage: &'a S,
+	surface: SurfaceIterator<'a>,
+
+	extrinsics: VecDeque<(u64, u32)>,
+	blocks: VecDeque<(u64, u32)>,
+
+	_hasher: ::std::marker::PhantomData<H>,
+}
+
+impl<'a, RS: 'a + Storage<H>, S: Storage<H>, H: Hasher> DrilldownIteratorEssence<'a, RS, S, H> {
+	pub fn next<F>(&mut self, trie_reader: F) -> Option<Result<(u64, u32), String>>
+		where
+			F: FnMut(&S, H::Out, &[u8]) -> Result<Option<Vec<u8>>, String>,
+	{
+		match self.do_next(trie_reader) {
+			Ok(Some(res)) => Some(Ok(res)),
+			Ok(None) => None,
+			Err(err) => Some(Err(err)),
+		}
+	}
+
+	fn do_next<F>(&mut self, mut trie_reader: F) -> Result<Option<(u64, u32)>, String>
+		where
+			F: FnMut(&S, H::Out, &[u8]) -> Result<Option<Vec<u8>>, String>,
+	{
+		loop {
+			if let Some((block, extrinsic)) = self.extrinsics.pop_front() {
+				return Ok(Some((block, extrinsic)));
+			}
+
+			if let Some((block, level)) = self.blocks.pop_front() {
+				if let Some(trie_root) = self.roots_storage.root(block)? {
+					let extrinsics_key = ExtrinsicIndex { block, key: self.key.to_vec() }.encode();
+					let extrinsics = trie_reader(&self.storage, trie_root, &extrinsics_key);
+					if let Some(extrinsics) = extrinsics? {
+						let extrinsics: Option<ExtrinsicIndexValue> = Decode::decode(&mut &extrinsics[..]);
+						if let Some(extrinsics) = extrinsics {
+							self.extrinsics.extend(extrinsics.into_iter().rev().map(|e| (block, e)));
+						}
+					}
+
+					let blocks_key = DigestIndex { block, key: self.key.to_vec() }.encode();
+					let blocks = trie_reader(&self.storage, trie_root, &blocks_key);
+					if let Some(blocks) = blocks? {
+						let blocks: Option<DigestIndexValue> = Decode::decode(&mut &blocks[..]);
+						if let Some(blocks) = blocks {
+							self.blocks.extend(blocks.into_iter().rev().map(|b| (b, level - 1)));
+						}
+					}
+				}
+
+				continue;
+			}
+
+			match self.surface.next() {
+				Some(Ok(block)) => self.blocks.push_back(block),
+				Some(Err(err)) => return Err(err),
+				None => return Ok(None),
+			}
+		}
+	}
+}
+
+/// Exploring drilldown operator.
+struct DrilldownIterator<'a, RS: 'a + Storage<H>, S: 'a + Storage<H>, H: Hasher, C: NodeCodec<H>> {
+	essence: DrilldownIteratorEssence<'a, RS, S, H>,
+	_codec: ::std::marker::PhantomData<C>,
+}
+
+impl<'a, RS: 'a + Storage<H>, S: Storage<H>, H: Hasher, C: NodeCodec<H>> Iterator for DrilldownIterator<'a, RS, S, H, C> where H::Out: HeapSizeOf {
+	type Item = Result<(u64, u32), String>;
+
+	fn next(&mut self) -> Option<Self::Item> {
+		self.essence.next(|storage, root, key|
+			TrieBackendEssence::<_, H, C>::new(TrieBackendAdapter::new(storage), root).storage(key))
+	}
+}
+
+/// Proving drilldown iterator.
+struct ProvingDrilldownIterator<'a, RS: 'a + Storage<H>, S: 'a + Storage<H>, H: Hasher, C: NodeCodec<H>> {
+	essence: DrilldownIteratorEssence<'a, RS, S, H>,
+	proof_recorder: RefCell<Recorder<H::Out>>,
+	_codec: ::std::marker::PhantomData<C>,
+}
+
+impl<'a, RS: 'a + Storage<H>, S: Storage<H>, H: Hasher, C: NodeCodec<H>> ProvingDrilldownIterator<'a, RS, S, H, C> {
+	/// Consume the iterator, extracting the gathered proof in lexicographical order
+	/// by value.
+	pub fn extract_proof(self) -> Vec<Vec<u8>> {
+		self.proof_recorder.into_inner().drain()
+			.into_iter()
+			.map(|n| n.data.to_vec())
+			.collect()
+	}
+}
+
+impl<'a, RS: 'a + Storage<H>, S: Storage<H>, H: Hasher, C: NodeCodec<H>> Iterator for ProvingDrilldownIterator<'a, RS, S, H, C> where H::Out: HeapSizeOf {
+	type Item = Result<(u64, u32), String>;
+
+	fn next(&mut self) -> Option<Self::Item> {
+		let proof_recorder = &mut *self.proof_recorder.try_borrow_mut()
+			.expect("only fails when already borrowed; storage() is non-reentrant; qed");
+		self.essence.next(|storage, root, key|
+			ProvingBackendEssence::<_, H, C> {
+				backend: &TrieBackendEssence::new(TrieBackendAdapter::new(storage), root),
+				proof_recorder,
+			}.storage(key))
+	}
+}
+
+/// Returns surface iterator for given range of blocks.
+fn surface_iterator<'a>(config: &'a Configuration, max: u64, begin: u64, end: u64) -> Result<SurfaceIterator<'a>, String> {
+	let (current, current_begin, digest_step, digest_level) = lower_bound_max_digest(config, max, begin, end)?;
+	Ok(SurfaceIterator {
+		config,
+		begin,
+		max,
+		current: Some(current),
+		current_begin,
+		digest_step,
+		digest_level,
+	})
+}
+
+/// Returns parameters of highest level digest block that includes the end of given range
+/// and tends to include the whole range.
+fn lower_bound_max_digest(
+	config: &Configuration,
+	max: u64,
+	begin: u64,
+	end: u64,
+) -> Result<(u64, u64, u64, u32), String> {
+	if end > max || begin > end {
+		return Err("invalid changes range".into());
+	}
+
+	let mut digest_level = 0u32;
+	let mut digest_step = 1u64;
+	let mut digest_interval = 0u64;
+	let mut current = end;
+	let mut current_begin = begin;
+	if begin != end {
+		while digest_level != config.digest_levels {
+			let new_digest_level = digest_level + 1;
+			let new_digest_step = digest_step * config.digest_interval;
+			let new_digest_interval = config.digest_interval * {
+				if digest_interval == 0 { 1 } else { digest_interval }
+			};
+			let new_digest_begin = ((current - 1) / new_digest_interval) * new_digest_interval;
+			let new_digest_end = new_digest_begin + new_digest_interval;
+			let new_current = new_digest_begin + new_digest_interval;
+
+			if new_digest_end > max {
+				if begin < new_digest_begin {
+					current_begin = new_digest_begin;
+				}
+				break;
+			}
+
+			digest_level = new_digest_level;
+			digest_step = new_digest_step;
+			digest_interval = new_digest_interval;
+			current = new_current;
+			current_begin = new_digest_begin;
+
+			if new_digest_begin <= begin && new_digest_end >= end {
+				break;
+			}
+		}
+	}
+
+	Ok((
+		current,
+		current_begin,
+		digest_step,
+		digest_level,
+	))
+}
+
+#[cfg(test)]
+mod tests {
+	use primitives::{Blake2Hasher, RlpCodec};
+	use changes_trie::input::InputPair;
+	use changes_trie::storage::InMemoryStorage;
+	use super::*;
+
+	fn prepare_for_drilldown() -> (Configuration, InMemoryStorage<Blake2Hasher>) {
+		let config = Configuration { digest_interval: 4, digest_levels: 2 };
+		let backend = InMemoryStorage::with_inputs::<RlpCodec>(vec![
+			// digest: 1..4 => [(3, 0)]
+			(1, vec![]),
+			(2, vec![]),
+			(3, vec![
+				InputPair::ExtrinsicIndex(ExtrinsicIndex { block: 3, key: vec![42] }, vec![0]),
+			]),
+			(4, vec![
+				InputPair::DigestIndex(DigestIndex { block: 4, key: vec![42] }, vec![3]),
+			]),
+			// digest: 5..8 => [(6, 3), (8, 1+2)]
+			(5, vec![]),
+			(6, vec![
+				InputPair::ExtrinsicIndex(ExtrinsicIndex { block: 6, key: vec![42] }, vec![3]),
+			]),
+			(7, vec![]),
+			(8, vec![
+				InputPair::ExtrinsicIndex(ExtrinsicIndex { block: 8, key: vec![42] }, vec![1, 2]),
+				InputPair::DigestIndex(DigestIndex { block: 8, key: vec![42] }, vec![6]),
+			]),
+			// digest: 9..12 => []
+			(9, vec![]),
+			(10, vec![]),
+			(11, vec![]),
+			(12, vec![]),
+			// digest: 0..16 => [4, 8]
+			(13, vec![]),
+			(14, vec![]),
+			(15, vec![]),
+			(16, vec![
+				InputPair::DigestIndex(DigestIndex { block: 16, key: vec![42] }, vec![4, 8]),
+			]),
+		]);
+
+		(config, backend)
+	}
+
+	#[test]
+	fn drilldown_iterator_works() {
+		let (config, storage) = prepare_for_drilldown();
+		let drilldown_result = key_changes::<InMemoryStorage<Blake2Hasher>, Blake2Hasher, RlpCodec>(
+			&config, &storage, 0, 100, 1000, &[42]);
+
+		assert_eq!(drilldown_result, Ok(vec![(8, 2), (8, 1), (6, 3), (3, 0)]));
+	}
+
+	#[test]
+	fn drilldown_iterator_fails_when_storage_fails() {
+		let (config, storage) = prepare_for_drilldown();
+		storage.clear_storage();
+
+		assert!(key_changes::<InMemoryStorage<Blake2Hasher>, Blake2Hasher, RlpCodec>(
+			&config, &storage, 0, 100, 1000, &[42]).is_err());
+	}
+
+	#[test]
+	fn drilldown_iterator_fails_when_range_is_invalid() {
+		let (config, storage) = prepare_for_drilldown();
+		assert!(key_changes::<InMemoryStorage<Blake2Hasher>, Blake2Hasher, RlpCodec>(
+			&config, &storage, 0, 100, 50, &[42]).is_err());
+		assert!(key_changes::<InMemoryStorage<Blake2Hasher>, Blake2Hasher, RlpCodec>(
+			&config, &storage, 20, 10, 100, &[42]).is_err());
+	}
+
+
+	#[test]
+	fn proving_drilldown_iterator_works() {
+		// happens on remote full node:
+
+		// create drilldown iterator that records all trie nodes during drilldown
+		let (remote_config, remote_storage) = prepare_for_drilldown();
+		let remote_proof = key_changes_proof::<InMemoryStorage<Blake2Hasher>, Blake2Hasher, RlpCodec>(
+			&remote_config, &remote_storage,
+			0, 100, 1000, &[42]).unwrap();
+
+		// happens on local light node:
+
+		// create drilldown iterator that works the same, but only depends on trie
+		let (local_config, local_storage) = prepare_for_drilldown();
+		local_storage.clear_storage();
+		let local_result = key_changes_proof_check::<InMemoryStorage<Blake2Hasher>, Blake2Hasher, RlpCodec>(
+			&local_config, &local_storage, remote_proof,
+			0, 100, 1000, &[42]);
+
+		// check that drilldown result is the same as if it was happening at the full node
+		assert_eq!(local_result, Ok(vec![(8, 2), (8, 1), (6, 3), (3, 0)]));
+	}
+}

substrate/core/state-machine/src/changes_trie/input.rs

@@ -0,0 +1,149 @@
+// Copyright 2017 Parity Technologies (UK) Ltd.
+// This file is part of Substrate.
+
+// Substrate is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// Substrate is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with Substrate.  If not, see <http://www.gnu.org/licenses/>.
+
+//! Different types of changes trie input pairs.
+
+use codec::{Decode, Encode, Input, Output};
+
+/// Key of { changed key => set of extrinsic indices } mapping.
+#[derive(Clone, Debug, PartialEq, Eq)]
+pub struct ExtrinsicIndex {
+	/// Block at which this key has been inserted in the trie.
+	pub block: u64,
+	/// Storage key this node is responsible for.
+	pub key: Vec<u8>,
+}
+
+/// Value of { changed key => set of extrinsic indices } mapping.
+pub type ExtrinsicIndexValue = Vec<u32>;
+
+/// Key of { changed key => block/digest block numbers } mapping.
+#[derive(Clone, Debug, PartialEq, Eq)]
+pub struct DigestIndex {
+	/// Block at which this key has been inserted in the trie.
+	pub block: u64,
+	/// Storage key this node is responsible for.
+	pub key: Vec<u8>,
+}
+
+/// Value of { changed key => block/digest block numbers } mapping.
+pub type DigestIndexValue = Vec<u64>;
+
+/// Single input pair of changes trie.
+#[derive(Clone, Debug, PartialEq, Eq)]
+pub enum InputPair {
+	/// Element of { key => set of extrinsics where key has been changed } element mapping.
+	ExtrinsicIndex(ExtrinsicIndex, ExtrinsicIndexValue),
+	/// Element of { key => set of blocks/digest blocks where key has been changed } element mapping.
+	DigestIndex(DigestIndex, DigestIndexValue),
+}
+
+/// Single input key of changes trie.
+#[derive(Clone, Debug, PartialEq, Eq)]
+pub enum InputKey {
+	/// Key of { key => set of extrinsics where key has been changed } element mapping.
+	ExtrinsicIndex(ExtrinsicIndex),
+	/// Key of { key => set of blocks/digest blocks where key has been changed } element mapping.
+	DigestIndex(DigestIndex),
+}
+
+impl Into<(Vec<u8>, Vec<u8>)> for InputPair {
+	fn into(self) -> (Vec<u8>, Vec<u8>) {
+		match self {
+			InputPair::ExtrinsicIndex(key, value) => (key.encode(), value.encode()),
+			InputPair::DigestIndex(key, value) => (key.encode(), value.encode()),
+		}
+	}
+}
+
+impl Into<InputKey> for InputPair {
+	fn into(self) -> InputKey {
+		match self {
+			InputPair::ExtrinsicIndex(key, _) => InputKey::ExtrinsicIndex(key),
+			InputPair::DigestIndex(key, _) => InputKey::DigestIndex(key),
+		}
+	}
+}
+
+impl ExtrinsicIndex {
+	pub fn key_neutral_prefix(block: u64) -> Vec<u8> {
+		let mut prefix = vec![1];
+		prefix.extend(block.encode());
+		prefix
+	}
+}
+
+impl Encode for ExtrinsicIndex {
+	fn encode_to<W: Output>(&self, dest: &mut W) {
+		dest.push_byte(1);
+		self.block.encode_to(dest);
+		self.key.encode_to(dest);
+	}
+}
+
+impl DigestIndex {
+	pub fn key_neutral_prefix(block: u64) -> Vec<u8> {
+		let mut prefix = vec![2];
+		prefix.extend(block.encode());
+		prefix
+	}
+}
+
+
+impl Encode for DigestIndex {
+	fn encode_to<W: Output>(&self, dest: &mut W) {
+		dest.push_byte(2);
+		self.block.encode_to(dest);
+		self.key.encode_to(dest);
+	}
+}
+
+impl Decode for InputKey {
+	fn decode<I: Input>(input: &mut I) -> Option<Self> {
+		match input.read_byte()? {
+			1 => Some(InputKey::ExtrinsicIndex(ExtrinsicIndex {
+				block: Decode::decode(input)?,
+				key: Decode::decode(input)?,
+			})),
+			2 => Some(InputKey::DigestIndex(DigestIndex {
+				block: Decode::decode(input)?,
+				key: Decode::decode(input)?,
+			})),
+			_ => None,
+		}
+	}
+}
+
+#[cfg(test)]
+mod tests {
+	use super::*;
+
+	#[test]
+	fn extrinsic_index_serialized_and_deserialized() {
+		let original = ExtrinsicIndex { block: 777, key: vec![42] };
+		let serialized = original.encode();
+		let deserialized: InputKey = Decode::decode(&mut &serialized[..]).unwrap();
+		assert_eq!(InputKey::ExtrinsicIndex(original), deserialized);
+	}
+
+	#[test]
+	fn digest_index_serialized_and_deserialized() {
+		let original = DigestIndex { block: 777, key: vec![42] };
+		let serialized = original.encode();
+		let deserialized: InputKey = Decode::decode(&mut &serialized[..]).unwrap();
+		assert_eq!(InputKey::DigestIndex(original), deserialized);
+	}
+}

substrate/core/state-machine/src/changes_trie/mod.rs

@@ -0,0 +1,89 @@
+// Copyright 2017 Parity Technologies (UK) Ltd.
+// This file is part of Substrate.
+
+// Substrate is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// Substrate is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with Substrate.  If not, see <http://www.gnu.org/licenses/>.
+
+//! Changes trie related structures and functions.
+//!
+//! Changes trie is a trie built of { storage key => extrinsiscs } pairs
+//! at the end of each block. For every changed storage key it contains
+//! a pair, mapping key to the set of extrinsics where it has been changed.
+//!
+//! Optionally, every N blocks, additional level1-digest nodes are appended
+//! to the changes trie, containing pairs { storage key => blocks }. For every
+//! storage key that has been changed in PREVIOUS N-1 blocks (except for genesis
+//! block) it contains a pair, mapping this key to the set of blocks where it
+//! has been changed.
+//!
+//! Optionally, every N^digest_level (where digest_level > 1) blocks, additional
+//! digest_level digest is created. It is built out of pairs { storage key => digest
+//! block }, containing entries for every storage key that has been changed in
+//! the last N*digest_level-1 blocks (except for genesis block), mapping these keys
+//! to the set of lower-level digest blocks.
+
+mod build;
+mod build_iterator;
+mod changes_iterator;
+mod input;
+mod storage;
+
+pub use self::storage::InMemoryStorage;
+pub use self::changes_iterator::{key_changes, key_changes_proof, key_changes_proof_check};
+
+use hashdb::{DBValue, Hasher};
+use heapsize::HeapSizeOf;
+use patricia_trie::NodeCodec;
+use rlp::Encodable;
+use backend::Backend;
+use primitives;
+use changes_trie::build::prepare_input;
+use overlayed_changes::OverlayedChanges;
+use trie_backend_essence::TrieBackendStorage;
+
+/// Changes that are made outside of extrinsics are marked with this index;
+pub const NO_EXTRINSIC_INDEX: u32 = 0xffffffff;
+
+/// Changes trie storage. Provides access to trie roots and trie nodes.
+pub trait Storage<H: Hasher>: Send + Sync {
+	/// Get changes trie root for given block.
+	fn root(&self, block: u64) -> Result<Option<H::Out>, String>;
+
+	/// Get a trie node.
+	fn get(&self, key: &H::Out) -> Result<Option<DBValue>, String>;
+}
+
+/// Changes trie configuration.
+pub type Configuration = primitives::ChangesTrieConfiguration;
+
+/// Compute the changes trie root and transaction for given block.
+/// Returns None if there's no data to perform computation.
+pub fn compute_changes_trie_root<'a, B: Backend<H, C>, S: Storage<H>, H: Hasher, C: NodeCodec<H>>(
+	backend: &B,
+	storage: Option<&'a S>,
+	changes: &OverlayedChanges,
+	block: u64,
+) -> Option<(H::Out, Vec<(Vec<u8>, Vec<u8>)>)>
+	where
+		&'a S: TrieBackendStorage<H>,
+		H::Out: Ord + Encodable + HeapSizeOf,
+{
+	let input_pairs = prepare_input::<B, S, H, C>(backend, storage, changes, block)
+		.expect("storage is not allowed to fail within runtime")?;
+	let transaction = input_pairs.into_iter()
+		.map(Into::into)
+		.collect::<Vec<_>>();
+	let root = ::triehash::trie_root::<H, _, _, _>(transaction.iter().map(|(k, v)| (&*k, &*v)));
+
+	Some((root, transaction))
+}

substrate/core/state-machine/src/changes_trie/storage.rs

@@ -0,0 +1,118 @@
+// Copyright 2017 Parity Technologies (UK) Ltd.
+// This file is part of Substrate.
+
+// Substrate is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// Substrate is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with Substrate.  If not, see <http://www.gnu.org/licenses/>.
+
+//! Changes trie storage utilities.
+
+use std::collections::HashMap;
+use hashdb::{Hasher, HashDB, DBValue};
+use heapsize::HeapSizeOf;
+use memorydb::MemoryDB;
+use parking_lot::RwLock;
+use changes_trie::Storage;
+use trie_backend_essence::TrieBackendStorage;
+
+#[cfg(test)]
+use backend::insert_into_memory_db;
+#[cfg(test)]
+use patricia_trie::NodeCodec;
+#[cfg(test)]
+use changes_trie::input::InputPair;
+
+/// In-memory implementation of changes trie storage.
+pub struct InMemoryStorage<H: Hasher> where H::Out: HeapSizeOf {
+	data: RwLock<InMemoryStorageData<H>>,
+}
+
+/// Adapter for using changes trie storage as a TrieBackendEssence' storage.
+pub struct TrieBackendAdapter<'a, H: Hasher, S: 'a + Storage<H>> {
+	storage: &'a S,
+	_hasher: ::std::marker::PhantomData<H>,
+}
+
+struct InMemoryStorageData<H: Hasher> where H::Out: HeapSizeOf {
+	roots: HashMap<u64, H::Out>,
+	mdb: MemoryDB<H>,
+}
+
+impl<H: Hasher> InMemoryStorage<H> where H::Out: HeapSizeOf {
+	/// Create the storage from given in-memory database.
+	pub fn with_db(mdb: MemoryDB<H>) -> Self {
+		Self {
+			data: RwLock::new(InMemoryStorageData {
+				roots: HashMap::new(),
+				mdb,
+			}),
+		}
+	}
+
+	/// Create the storage with empty database.
+	pub fn new() -> Self {
+		Self::with_db(Default::default())
+	}
+
+	#[cfg(test)]
+	pub fn with_inputs<C: NodeCodec<H>>(inputs: Vec<(u64, Vec<InputPair>)>) -> Self {
+		let mut mdb = MemoryDB::default();
+		let mut roots = HashMap::new();
+		for (block, pairs) in inputs {
+			let root = insert_into_memory_db::<H, C, _>(&mut mdb, pairs.into_iter().map(Into::into));
+			if let Some(root) = root {
+				roots.insert(block, root);
+			}
+		}
+
+		InMemoryStorage {
+			data: RwLock::new(InMemoryStorageData {
+				roots,
+				mdb,
+			}),
+		}
+	}
+
+	#[cfg(test)]
+	pub fn clear_storage(&self) {
+		self.data.write().mdb = MemoryDB::new();
+	}
+
+	/// Insert changes trie for given block.
+	pub fn insert(&self, block: u64, changes_trie_root: H::Out, trie: MemoryDB<H>) {
+		let mut data = self.data.write();
+		data.roots.insert(block, changes_trie_root);
+		data.mdb.consolidate(trie);
+	}
+}
+
+impl<H: Hasher> Storage<H> for InMemoryStorage<H> where H::Out: HeapSizeOf {
+	fn root(&self, block: u64) -> Result<Option<H::Out>, String> {
+		Ok(self.data.read().roots.get(&block).cloned())
+	}
+
+	fn get(&self, key: &H::Out) -> Result<Option<DBValue>, String> {
+		Ok(HashDB::<H>::get(&self.data.read().mdb, key))
+	}
+}
+
+impl<'a, H: Hasher, S: 'a + Storage<H>> TrieBackendAdapter<'a, H, S> {
+	pub fn new(storage: &'a S) -> Self {
+		Self { storage, _hasher: Default::default() }
+	}
+}
+
+impl<'a, H: Hasher, S: 'a + Storage<H>> TrieBackendStorage<H> for TrieBackendAdapter<'a, H, S> {
+	fn get(&self, key: &H::Out) -> Result<Option<DBValue>, String> {
+		self.storage.get(key)
+	}
+}

substrate/core/state-machine/src/ext.rs

@@ -18,10 +18,15 @@
 
 use std::{error, fmt, cmp::Ord};
 use backend::Backend;
+use changes_trie::{Storage as ChangesTrieStorage, compute_changes_trie_root};
 use {Externalities, OverlayedChanges};
 use hashdb::Hasher;
+use memorydb::MemoryDB;
 use rlp::Encodable;
-use patricia_trie::NodeCodec;
+use patricia_trie::{NodeCodec, TrieDBMut, TrieMut};
+use heapsize::HeapSizeOf;
+
+const EXT_NOT_ALLOWED_TO_FAIL: &'static str = "Externalities not allowed to fail within runtime";
 
 /// Errors that can occur when interacting with the externalities.
 #[derive(Debug, Copy, Clone)]
@@ -53,64 +58,90 @@ impl<B: error::Error, E: error::Error> error::Error for Error<B, E> {
 }
 
 /// Wraps a read-only backend, call executor, and current overlayed changes.
-pub struct Ext<'a, H, C, B>
+pub struct Ext<'a, H, C, B, T>
 where
 	H: Hasher,
 	C: NodeCodec<H>,
 	B: 'a + Backend<H, C>,
+	T: 'a + ChangesTrieStorage<H>,
 {
-	// The overlayed changes to write to.
+	/// The overlayed changes to write to.
 	overlay: &'a mut OverlayedChanges,
-	// The storage backend to read from.
+	/// The storage backend to read from.
 	backend: &'a B,
-	// The transaction necessary to commit to the backend.
-	transaction: Option<(B::Transaction, H::Out)>,
+	/// The storage transaction necessary to commit to the backend. Is cached when
+	/// `storage_root` is called and the cache is cleared on every subsequent change.
+	storage_transaction: Option<(B::Transaction, H::Out)>,
+	/// Changes trie storage to read from.
+	changes_trie_storage: Option<&'a T>,
+	/// The changes trie transaction necessary to commit to the changes trie backend.
+	/// Set to Some when `storage_changes_root` is called. Could be replaced later
+	/// by calling `storage_changes_root` again => never used as cache.
+	/// This differs from `storage_transaction` behavior, because the moment when
+	/// `storage_changes_root` is called matters + we need to remember additional
+	/// data at this moment (block number).
+	changes_trie_transaction: Option<(u64, MemoryDB<H>, H::Out)>,
 }
 
-impl<'a, H, C, B> Ext<'a, H, C, B>
+impl<'a, H, C, B, T> Ext<'a, H, C, B, T>
 where
 	H: Hasher,
 	C: NodeCodec<H>,
 	B: 'a + Backend<H, C>,
-	H::Out: Ord + Encodable
+	T: 'a + ChangesTrieStorage<H>,
+	H::Out: Ord + Encodable + HeapSizeOf,
 {
 	/// Create a new `Ext` from overlayed changes and read-only backend
-	pub fn new(overlay: &'a mut OverlayedChanges, backend: &'a B) -> Self {
+	pub fn new(overlay: &'a mut OverlayedChanges, backend: &'a B, changes_trie_storage: Option<&'a T>) -> Self {
 		Ext {
 			overlay,
 			backend,
-			transaction: None,
+			storage_transaction: None,
+			changes_trie_storage,
+			changes_trie_transaction: None,
 		}
 	}
 
 	/// Get the transaction necessary to update the backend.
-	pub fn transaction(mut self) -> B::Transaction {
+	pub fn transaction(mut self) -> (B::Transaction, Option<MemoryDB<H>>) {
 		let _ = self.storage_root();
-		self.transaction.expect("transaction always set after calling storage root; qed").0
+
+		let (storage_transaction, changes_trie_transaction) = (
+			self.storage_transaction
+				.expect("storage_transaction always set after calling storage root; qed"),
+			self.changes_trie_transaction
+				.map(|(_, tx, _)| tx),
+		);
+
+		(
+			storage_transaction.0,
+			changes_trie_transaction,
+		)
 	}
 
 	/// Invalidates the currently cached storage root and the db transaction.
 	///
 	/// Called when there are changes that likely will invalidate the storage root.
 	fn mark_dirty(&mut self) {
-		self.transaction = None;
+		self.storage_transaction = None;
 	}
 }
 
 #[cfg(test)]
-impl<'a, H, C, B> Ext<'a, H, C, B>
+impl<'a, H, C, B, T> Ext<'a, H, C, B, T>
 where
 	H: Hasher,
 	C: NodeCodec<H>,
 	B: 'a + Backend<H,C>,
+	T: 'a + ChangesTrieStorage<H>,
 {
 	pub fn storage_pairs(&self) -> Vec<(Vec<u8>, Vec<u8>)> {
 		use std::collections::HashMap;
 
 		self.backend.pairs().iter()
 			.map(|&(ref k, ref v)| (k.to_vec(), Some(v.to_vec())))
-			.chain(self.overlay.committed.clone().into_iter())
-			.chain(self.overlay.prospective.clone().into_iter())
+			.chain(self.overlay.committed.clone().into_iter().map(|(k, v)| (k, v.value)))
+			.chain(self.overlay.prospective.clone().into_iter().map(|(k, v)| (k, v.value)))
 			.collect::<HashMap<_, _>>()
 			.into_iter()
 			.filter_map(|(k, maybe_val)| maybe_val.map(|val| (k, val)))
@@ -118,22 +149,23 @@ where
 	}
 }
 
-impl<'a, B: 'a, H, C> Externalities<H> for Ext<'a, H, C, B>
+impl<'a, B: 'a, T: 'a, H, C> Externalities<H> for Ext<'a, H, C, B, T>
 where
 	H: Hasher,
 	C: NodeCodec<H>,
 	B: 'a + Backend<H, C>,
-	H::Out: Ord + Encodable
+	T: 'a + ChangesTrieStorage<H>,
+	H::Out: Ord + Encodable + HeapSizeOf,
 {
 	fn storage(&self, key: &[u8]) -> Option<Vec<u8>> {
 		self.overlay.storage(key).map(|x| x.map(|x| x.to_vec())).unwrap_or_else(||
-			self.backend.storage(key).expect("Externalities not allowed to fail within runtime"))
+			self.backend.storage(key).expect(EXT_NOT_ALLOWED_TO_FAIL))
 	}
 
 	fn exists_storage(&self, key: &[u8]) -> bool {
 		match self.overlay.storage(key) {
 			Some(x) => x.is_some(),
-			_ => self.backend.exists_storage(key).expect("Externalities not allowed to fail within runtime"),
+			_ => self.backend.exists_storage(key).expect(EXT_NOT_ALLOWED_TO_FAIL),
 		}
 	}
 
@@ -155,17 +187,114 @@ where
 	}
 
 	fn storage_root(&mut self) -> H::Out {
-		if let Some((_, ref root)) = self.transaction {
+		if let Some((_, ref root)) =  self.storage_transaction {
 			return root.clone();
 		}
 
 		// compute and memoize
-		let delta = self.overlay.committed.iter()
-			.chain(self.overlay.prospective.iter())
-			.map(|(k, v)| (k.clone(), v.clone()));
+		let delta = self.overlay.committed.iter().map(|(k, v)| (k.clone(), v.value.clone()))
+			.chain(self.overlay.prospective.iter().map(|(k, v)| (k.clone(), v.value.clone())));
 
 		let (root, transaction) = self.backend.storage_root(delta);
-		self.transaction = Some((transaction, root));
+		self.storage_transaction = Some((transaction, root));
+		root
+	}
+
+	fn storage_changes_root(&mut self, block: u64) -> Option<H::Out> {
+		let root_and_tx = compute_changes_trie_root::<_, T, H, C>(
+			self.backend,
+			self.changes_trie_storage.clone(),
+			self.overlay,
+			block,
+		);
+		let root_and_tx = root_and_tx.map(|(root, changes)| {
+			let mut calculated_root = Default::default();
+			let mut mdb = MemoryDB::new();
+			{
+				let mut trie = TrieDBMut::<H, C>::new(&mut mdb, &mut calculated_root);
+				for (key, value) in changes {
+					trie.insert(&key, &value).expect(EXT_NOT_ALLOWED_TO_FAIL);
+				}
+			}
+
+			(block, mdb, root)
+		});
+		let root = root_and_tx.as_ref().map(|(_, _, root)| root.clone());
+		self.changes_trie_transaction = root_and_tx;
 		root
 	}
 }
+
+#[cfg(test)]
+mod tests {
+	use codec::Encode;
+	use primitives::{Blake2Hasher, RlpCodec};
+	use backend::InMemory;
+	use changes_trie::{Configuration as ChangesTrieConfiguration,
+		InMemoryStorage as InMemoryChangesTrieStorage};
+	use overlayed_changes::OverlayedValue;
+	use super::*;
+
+	type TestBackend = InMemory<Blake2Hasher, RlpCodec>;
+	type TestChangesTrieStorage = InMemoryChangesTrieStorage<Blake2Hasher>;
+	type TestExt<'a> = Ext<'a, Blake2Hasher, RlpCodec, TestBackend, TestChangesTrieStorage>;
+
+	fn prepare_overlay_with_changes() -> OverlayedChanges {
+		OverlayedChanges {
+			prospective: vec![
+				(b":extrinsic_index".to_vec(), OverlayedValue {
+					value: Some(3u32.encode()),
+					extrinsics: Some(vec![1].into_iter().collect())
+				}),
+				(vec![1], OverlayedValue {
+					value: Some(vec![100].into_iter().collect()),
+					extrinsics: Some(vec![1].into_iter().collect())
+				}),
+			].into_iter().collect(),
+			committed: Default::default(),
+			changes_trie_config: Some(ChangesTrieConfiguration {
+				digest_interval: 0,
+				digest_levels: 0,
+			}),
+		}
+	}
+
+	#[test]
+	fn storage_changes_root_is_none_when_storage_is_not_provided() {
+		let mut overlay = prepare_overlay_with_changes();
+		let backend = TestBackend::default();
+		let mut ext = TestExt::new(&mut overlay, &backend, None);
+		assert_eq!(ext.storage_changes_root(100), None);
+	}
+
+	#[test]
+	fn storage_changes_root_is_none_when_extrinsic_changes_are_none() {
+		let mut overlay = prepare_overlay_with_changes();
+		overlay.changes_trie_config = None;
+		let storage = TestChangesTrieStorage::new();
+		let backend = TestBackend::default();
+		let mut ext = TestExt::new(&mut overlay, &backend, Some(&storage));
+		assert_eq!(ext.storage_changes_root(100), None);
+	}
+
+	#[test]
+	fn storage_changes_root_is_some_when_extrinsic_changes_are_non_empty() {
+		let mut overlay = prepare_overlay_with_changes();
+		let storage = TestChangesTrieStorage::new();
+		let backend = TestBackend::default();
+		let mut ext = TestExt::new(&mut overlay, &backend, Some(&storage));
+		assert_eq!(ext.storage_changes_root(100),
+			Some(hex!("b2ecc5ca20de9f8a2d82482fcaa0fdfcca2fb76bf3d89860edf422bd15d075ec").into()));
+	}
+
+	#[test]
+	fn storage_changes_root_is_some_when_extrinsic_changes_are_empty() {
+		let mut overlay = prepare_overlay_with_changes();
+		overlay.prospective.get_mut(&vec![1]).unwrap().value = None;
+		let storage = TestChangesTrieStorage::new();
+		let backend = TestBackend::default();
+		let mut ext = TestExt::new(&mut overlay, &backend, Some(&storage));
+		assert_eq!(ext.storage_changes_root(100),
+			Some(hex!("8c12eccf80c166aefc23af540649979581cb404d95af25b0ed38dc6949ba2453").into()));
+	}
+}

substrate/core/state-machine/src/lib.rs

@@ -30,15 +30,14 @@ extern crate hashdb;
 extern crate memorydb;
 extern crate triehash;
 extern crate patricia_trie;
+
 extern crate byteorder;
 extern crate parking_lot;
 extern crate rlp;
 extern crate heapsize;
-#[cfg(test)]
 extern crate substrate_primitives as primitives;
 extern crate parity_codec as codec;
 
-use std::collections::HashMap;
 use std::fmt;
 use hashdb::Hasher;
 use patricia_trie::NodeCodec;
@@ -47,99 +46,24 @@ use heapsize::HeapSizeOf;
 use codec::Decode;
 
 pub mod backend;
+mod changes_trie;
 mod ext;
 mod testing;
+mod overlayed_changes;
 mod proving_backend;
 mod trie_backend;
+mod trie_backend_essence;
 
+pub use patricia_trie::{TrieMut, TrieDBMut};
 pub use testing::TestExternalities;
 pub use ext::Ext;
 pub use backend::Backend;
-pub use trie_backend::{TryIntoTrieBackend, TrieBackend, Storage, DBValue};
-
-/// The overlayed changes to state to be queried on top of the backend.
-///
-/// A transaction shares all prospective changes within an inner overlay
-/// that can be cleared.
-#[derive(Debug, Default, Clone)]
-pub struct OverlayedChanges {
-	prospective: HashMap<Vec<u8>, Option<Vec<u8>>>,
-	committed: HashMap<Vec<u8>, Option<Vec<u8>>>,
-}
-
-impl OverlayedChanges {
-	/// Returns a double-Option: None if the key is unknown (i.e. and the query should be refered
-	/// to the backend); Some(None) if the key has been deleted. Some(Some(...)) for a key whose
-	/// value has been set.
-	pub fn storage(&self, key: &[u8]) -> Option<Option<&[u8]>> {
-		self.prospective.get(key)
-			.or_else(|| self.committed.get(key))
-			.map(|x| x.as_ref().map(AsRef::as_ref))
-	}
-
-	/// Inserts the given key-value pair into the prospective change set.
-	///
-	/// `None` can be used to delete a value specified by the given key.
-	fn set_storage(&mut self, key: Vec<u8>, val: Option<Vec<u8>>) {
-		self.prospective.insert(key, val);
-	}
-
-	/// Removes all key-value pairs which keys share the given prefix.
-	///
-	/// NOTE that this doesn't take place immediately but written into the prospective
-	/// change set, and still can be reverted by [`discard_prospective`].
-	///
-	/// [`discard_prospective`]: #method.discard_prospective
-	fn clear_prefix(&mut self, prefix: &[u8]) {
-		// Iterate over all prospective and mark all keys that share
-		// the given prefix as removed (None).
-		for (key, value) in self.prospective.iter_mut() {
-			if key.starts_with(prefix) {
-				*value = None;
-			}
-		}
-
-		// Then do the same with keys from commited changes.
-		// NOTE that we are making changes in the prospective change set.
-		for key in self.committed.keys() {
-			if key.starts_with(prefix) {
-				self.prospective.insert(key.to_owned(), None);
-			}
-		}
-	}
-
-	/// Discard prospective changes to state.
-	pub fn discard_prospective(&mut self) {
-		self.prospective.clear();
-	}
-
-	/// Commit prospective changes to state.
-	pub fn commit_prospective(&mut self) {
-		if self.committed.is_empty() {
-			::std::mem::swap(&mut self.prospective, &mut self.committed);
-		} else {
-			self.committed.extend(self.prospective.drain());
-		}
-	}
-
-	/// Drain committed changes to an iterator.
-	///
-	/// Panics:
-	/// Will panic if there are any uncommitted prospective changes.
-	pub fn drain<'a>(&'a mut self) -> impl Iterator<Item=(Vec<u8>, Option<Vec<u8>>)> + 'a {
-		assert!(self.prospective.is_empty());
-		self.committed.drain()
-	}
-
-	/// Consume `OverlayedChanges` and take committed set.
-	///
-	/// Panics:
-	/// Will panic if there are any uncommitted prospective changes.
-	pub fn into_committed(self) -> impl Iterator<Item=(Vec<u8>, Option<Vec<u8>>)> {
-		assert!(self.prospective.is_empty());
-		self.committed.into_iter()
-	}
-}
+pub use changes_trie::{Storage as ChangesTrieStorage,
+	InMemoryStorage as InMemoryChangesTrieStorage,
+	key_changes, key_changes_proof, key_changes_proof_check};
+pub use overlayed_changes::OverlayedChanges;
+pub use trie_backend_essence::Storage;
+pub use trie_backend::{TrieBackend, DBValue};
 
 /// State Machine Error bound.
 ///
@@ -155,6 +79,8 @@ impl Error for ExecutionError {}
 /// and as a transition away from the pre-existing framework.
 #[derive(Debug, Eq, PartialEq)]
 pub enum ExecutionError {
+	/// Backend error.
+	Backend(String),
 	/// The entry `:code` doesn't exist in storage so there's no way we can execute anything.
 	CodeEntryDoesNotExist,
 	/// Backend is incompatible with execution proof generation process.
@@ -198,6 +124,9 @@ pub trait Externalities<H: Hasher> {
 
 	/// Get the trie root of the current storage map.
 	fn storage_root(&mut self) -> H::Out where H::Out: Ord + Encodable;
+
+	/// Get the change trie root of the current storage overlay at given block.
+	fn storage_changes_root(&mut self, block: u64) -> Option<H::Out> where H::Out: Ord + Encodable;
 }
 
 /// Code execution engine.
@@ -267,23 +196,26 @@ pub fn always_wasm<E>() -> ExecutionManager<fn(Result<Vec<u8>, E>, Result<Vec<u8
 ///
 /// Note: changes to code will be in place if this call is made again. For running partial
 /// blocks (e.g. a transaction at a time), ensure a different method is used.
-pub fn execute<H, C, B, Exec>(
+pub fn execute<H, C, B, T, Exec>(
 	backend: &B,
+	changes_trie_storage: Option<&T>,
 	overlay: &mut OverlayedChanges,
 	exec: &Exec,
 	method: &str,
 	call_data: &[u8],
 	strategy: ExecutionStrategy,
-) -> Result<(Vec<u8>, B::Transaction), Box<Error>>
+) -> Result<(Vec<u8>, B::Transaction, Option<memorydb::MemoryDB<H>>), Box<Error>>
 where
 	H: Hasher,
 	C: NodeCodec<H>,
 	Exec: CodeExecutor<H>,
 	B: Backend<H, C>,
-	H::Out: Ord + Encodable
+	T: ChangesTrieStorage<H>,
+	H::Out: Ord + Encodable + HeapSizeOf,
 {
 	execute_using_consensus_failure_handler(
 		backend,
+		changes_trie_storage,
 		overlay,
 		exec,
 		method,
@@ -307,38 +239,48 @@ where
 ///
 /// Note: changes to code will be in place if this call is made again. For running partial
 /// blocks (e.g. a transaction at a time), ensure a different method is used.
-pub fn execute_using_consensus_failure_handler<H, C, B, Exec, Handler>(
+pub fn execute_using_consensus_failure_handler<H, C, B, T, Exec, Handler>(
 	backend: &B,
+	changes_trie_storage: Option<&T>,
 	overlay: &mut OverlayedChanges,
 	exec: &Exec,
 	method: &str,
 	call_data: &[u8],
 	manager: ExecutionManager<Handler>,
-) -> Result<(Vec<u8>, B::Transaction), Box<Error>>
+) -> Result<(Vec<u8>, B::Transaction, Option<memorydb::MemoryDB<H>>), Box<Error>>
 where
 	H: Hasher,
 	C: NodeCodec<H>,
 	Exec: CodeExecutor<H>,
 	B: Backend<H, C>,
-	H::Out: Ord + Encodable,
+	T: ChangesTrieStorage<H>,
+	H::Out: Ord + Encodable + HeapSizeOf,
 	Handler: FnOnce(Result<Vec<u8>, Exec::Error>, Result<Vec<u8>, Exec::Error>) -> Result<Vec<u8>, Exec::Error>
 {
 	let strategy: ExecutionStrategy = (&manager).into();
 
 	// make a copy.
-	let code = ext::Ext::new(overlay, backend).storage(b":code")
+	let code = try_read_overlay_value(overlay, backend, b":code")?
 		.ok_or_else(|| Box::new(ExecutionError::CodeEntryDoesNotExist) as Box<Error>)?
 		.to_vec();
 
-	let heap_pages = ext::Ext::new(overlay, backend).storage(b":heappages")
+	let heap_pages = try_read_overlay_value(overlay, backend, b":heappages")?
 		.and_then(|v| u64::decode(&mut &v[..])).unwrap_or(8) as usize;
 
+	// read changes trie configuration. The reason why we're doing it here instead of the
+	// `OverlayedChanges` constructor is that we need proofs for this read as a part of
+	// proof-of-execution on light clients. And the proof is recorded by the backend which
+	// is created after OverlayedChanges
+
+	let changes_trie_config = try_read_overlay_value(overlay, backend, b":changes_trie")?;
+	set_changes_trie_config(overlay, changes_trie_config)?;
+
 	let result = {
 		let mut orig_prospective = overlay.prospective.clone();
 
-		let (result, was_native, delta) = {
-			let ((result, was_native), delta) = {
-				let mut externalities = ext::Ext::new(overlay, backend);
+		let (result, was_native, storage_delta, changes_delta) = {
+			let ((result, was_native), (storage_delta, changes_delta)) = {
+				let mut externalities = ext::Ext::new(overlay, backend, changes_trie_storage);
 				(
 					exec.call(
 						&mut externalities,
@@ -352,18 +294,18 @@ where
 					externalities.transaction()
 				)
 			};
-			(result, was_native, delta)
+			(result, was_native, storage_delta, changes_delta)
 		};
 
 		// run wasm separately if we did run native the first time and we're meant to run both
-		let (result, delta) = if let (true, ExecutionManager::Both(on_consensus_failure)) =
+		let (result, storage_delta, changes_delta) = if let (true, ExecutionManager::Both(on_consensus_failure)) =
 			(was_native, manager)
 		{
 			overlay.prospective = orig_prospective.clone();
 
-			let (wasm_result, wasm_delta) = {
-				let ((result, _), delta) = {
-					let mut externalities = ext::Ext::new(overlay, backend);
+			let (wasm_result, wasm_storage_delta, wasm_changes_delta) = {
+				let ((result, _), (storage_delta, changes_delta)) = {
+					let mut externalities = ext::Ext::new(overlay, backend, changes_trie_storage);
 					(
 						exec.call(
 							&mut externalities,
@@ -376,21 +318,21 @@ where
 						externalities.transaction()
 					)
 				};
-				(result, delta)
+				(result, storage_delta, changes_delta)
 			};
 
 			if (result.is_ok() && wasm_result.is_ok() && result.as_ref().unwrap() == wasm_result.as_ref().unwrap()/* && delta == wasm_delta*/)
 				|| (result.is_err() && wasm_result.is_err())
 			{
-				(result, delta)
+				(result, storage_delta, changes_delta)
 			} else {
 				// Consensus error.
-				(on_consensus_failure(wasm_result, result), wasm_delta)
+				(on_consensus_failure(wasm_result, result), wasm_storage_delta, wasm_changes_delta)
 			}
 		} else {
-			(result, delta)
+			(result, storage_delta, changes_delta)
 		};
-		result.map(move |out| (out, delta))
+		result.map(move |out| (out, storage_delta, changes_delta))
 	};
 
 	result.map_err(|e| Box::new(e) as _)
@@ -405,26 +347,34 @@ where
 ///
 /// Note: changes to code will be in place if this call is made again. For running partial
 /// blocks (e.g. a transaction at a time), ensure a different method is used.
-pub fn prove_execution<H, C, B, Exec>(
+pub fn prove_execution<B, H, C, Exec>(
 	backend: B,
 	overlay: &mut OverlayedChanges,
 	exec: &Exec,
 	method: &str,
 	call_data: &[u8],
-) -> Result<(Vec<u8>, Vec<Vec<u8>>, <TrieBackend<H, C> as Backend<H, C>>::Transaction), Box<Error>>
+) -> Result<(Vec<u8>, Vec<Vec<u8>>), Box<Error>>
 where
+	B: Backend<H, C>,
 	H: Hasher,
 	Exec: CodeExecutor<H>,
 	C: NodeCodec<H>,
-	B: TryIntoTrieBackend<H, C>,
 	H::Out: Ord + Encodable + HeapSizeOf,
 {
 	let trie_backend = backend.try_into_trie_backend()
 		.ok_or_else(|| Box::new(ExecutionError::UnableToGenerateProof) as Box<Error>)?;
 	let proving_backend = proving_backend::ProvingBackend::new(trie_backend);
-	let (result, transaction) = execute::<H, C, _, _>(&proving_backend, overlay, exec, method, call_data, ExecutionStrategy::NativeWhenPossible)?;
+	let (result, _, _) = execute::<H, C, _, changes_trie::InMemoryStorage<H>, _>(
+		&proving_backend,
+		None,
+		overlay,
+		exec,
+		method,
+		call_data,
+		ExecutionStrategy::NativeWhenPossible
+	)?;
 	let proof = proving_backend.extract_proof();
-	Ok((result, proof, transaction))
+	Ok((result, proof))
 }
 
 /// Check execution proof, generated by `prove_execution` call.
@@ -435,15 +385,16 @@ pub fn execution_proof_check<H, C, Exec>(
 	exec: &Exec,
 	method: &str,
 	call_data: &[u8],
-) -> Result<(Vec<u8>, memorydb::MemoryDB<H>), Box<Error>>
+) -> Result<Vec<u8>, Box<Error>>
 where
-H: Hasher,
-C: NodeCodec<H>,
-Exec: CodeExecutor<H>,
-H::Out: Ord + Encodable + HeapSizeOf,
+	H: Hasher,
+	C: NodeCodec<H>,
+	Exec: CodeExecutor<H>,
+	H::Out: Ord + Encodable + HeapSizeOf,
 {
 	let backend = proving_backend::create_proof_check_backend::<H, C>(root.into(), proof)?;
-	execute::<H, C, _, _>(&backend, overlay, exec, method, call_data, ExecutionStrategy::NativeWhenPossible)
+	execute::<H, C, _, changes_trie::InMemoryStorage<H>, _>(&backend, None, overlay, exec, method, call_data, ExecutionStrategy::NativeWhenPossible)
+		.map(|(result, _, _)| result)
 }
 
 /// Generate storage read proof.
@@ -452,14 +403,14 @@ pub fn prove_read<B, H, C>(
 	key: &[u8]
 ) -> Result<(Option<Vec<u8>>, Vec<Vec<u8>>), Box<Error>>
 where
-	B: TryIntoTrieBackend<H, C>,
+	B: Backend<H, C>,
 	H: Hasher,
 	C: NodeCodec<H>,
 	H::Out: Ord + Encodable + HeapSizeOf
 {
 	let trie_backend = backend.try_into_trie_backend()
 		.ok_or_else(|| Box::new(ExecutionError::UnableToGenerateProof) as Box<Error>)?;
-	let proving_backend = proving_backend::ProvingBackend::<H, C>::new(trie_backend);
+	let proving_backend = proving_backend::ProvingBackend::<_, H, C>::new(trie_backend);
 	let result = proving_backend.storage(key).map_err(|e| Box::new(e) as Box<Error>)?;
 	Ok((result, proving_backend.extract_proof()))
 }
@@ -479,12 +430,46 @@ where
 	backend.storage(key).map_err(|e| Box::new(e) as Box<Error>)
 }
 
+/// Sets overlayed changes' changes trie configuration. Returns error if configuration
+/// differs from previous OR config decode has failed.
+pub(crate) fn set_changes_trie_config(overlay: &mut OverlayedChanges, config: Option<Vec<u8>>) -> Result<(), Box<Error>> {
+	let config = match config {
+		Some(v) => Some(changes_trie::Configuration::decode(&mut &v[..])
+			.ok_or_else(|| Box::new("Failed to decode changes trie configuration".to_owned()) as Box<Error>)?),
+		None => None,
+	};
+	if let Some(config) = config {
+		if !overlay.set_changes_trie_config(config) {
+			return Err(Box::new("Changes trie configuration change is not supported".to_owned()));
+		}
+	}
+
+	Ok(())
+}
+
+/// Reads storage value from overlay or from the backend.
+fn try_read_overlay_value<H, C, B>(overlay: &OverlayedChanges, backend: &B, key: &[u8])
+	-> Result<Option<Vec<u8>>, Box<Error>>
+where
+	H: Hasher,
+	C: NodeCodec<H>,
+	B: Backend<H, C>,
+{
+	match overlay.storage(key).map(|x| x.map(|x| x.to_vec())) {
+		Some(value) => Ok(value),
+		None => backend.storage(key)
+			.map_err(|err| Box::new(ExecutionError::Backend(format!("{}", err))) as Box<Error>),
+	}
+}
+
 #[cfg(test)]
 mod tests {
+	use std::collections::HashMap;
 	use super::*;
 	use super::backend::InMemory;
 	use super::ext::Ext;
-	use primitives::{Blake2Hasher, RlpCodec, H256};
+	use super::changes_trie::InMemoryStorage as InMemoryChangesTrieStorage;
+	use primitives::{Blake2Hasher, RlpCodec};
 
 	struct DummyCodeExecutor {
 		native_available: bool,
@@ -515,69 +500,17 @@ mod tests {
 
 	impl Error for u8 {}
 
-	#[test]
-	fn overlayed_storage_works() {
-		let mut overlayed = OverlayedChanges::default();
-
-		let key = vec![42, 69, 169, 142];
-
-		assert!(overlayed.storage(&key).is_none());
-
-		overlayed.set_storage(key.clone(), Some(vec![1, 2, 3]));
-		assert_eq!(overlayed.storage(&key).unwrap(), Some(&[1, 2, 3][..]));
-
-		overlayed.commit_prospective();
-		assert_eq!(overlayed.storage(&key).unwrap(), Some(&[1, 2, 3][..]));
-
-		overlayed.set_storage(key.clone(), Some(vec![]));
-		assert_eq!(overlayed.storage(&key).unwrap(), Some(&[][..]));
-
-		overlayed.set_storage(key.clone(), None);
-		assert!(overlayed.storage(&key).unwrap().is_none());
-
-		overlayed.discard_prospective();
-		assert_eq!(overlayed.storage(&key).unwrap(), Some(&[1, 2, 3][..]));
-
-		overlayed.set_storage(key.clone(), None);
-		overlayed.commit_prospective();
-		assert!(overlayed.storage(&key).unwrap().is_none());
-	}
-
 	macro_rules! map {
 		($( $name:expr => $value:expr ),*) => (
 			vec![ $( ( $name, $value ) ),* ].into_iter().collect()
 		)
 	}
 
-	#[test]
-	fn overlayed_storage_root_works() {
-		let initial: HashMap<_, _> = map![
-			b"doe".to_vec() => b"reindeer".to_vec(),
-			b"dog".to_vec() => b"puppyXXX".to_vec(),
-			b"dogglesworth".to_vec() => b"catXXX".to_vec(),
-			b"doug".to_vec() => b"notadog".to_vec()
-		];
-		let backend = InMemory::<Blake2Hasher, RlpCodec>::from(initial);
-		let mut overlay = OverlayedChanges {
-			committed: map![
-				b"dog".to_vec() => Some(b"puppy".to_vec()),
-				b"dogglesworth".to_vec() => Some(b"catYYY".to_vec()),
-				b"doug".to_vec() => Some(vec![])
-			],
-			prospective: map![
-				b"dogglesworth".to_vec() => Some(b"cat".to_vec()),
-				b"doug".to_vec() => None
-			],
-		};
-		let mut ext = Ext::new(&mut overlay, &backend);
-		const ROOT: [u8; 32] = hex!("6ca394ff9b13d6690a51dea30b1b5c43108e52944d30b9095227c49bae03ff8b");
-		assert_eq!(ext.storage_root(), H256(ROOT));
-	}
-
 	#[test]
 	fn execute_works() {
 		assert_eq!(execute(
 			&trie_backend::tests::test_trie(),
+			Some(&InMemoryChangesTrieStorage::new()),
 			&mut Default::default(),
 			&DummyCodeExecutor {
 				native_available: true,
@@ -595,6 +528,7 @@ mod tests {
 		let mut consensus_failed = false;
 		assert!(execute_using_consensus_failure_handler(
 			&trie_backend::tests::test_trie(),
+			Some(&InMemoryChangesTrieStorage::new()),
 			&mut Default::default(),
 			&DummyCodeExecutor {
 				native_available: true,
@@ -623,11 +557,11 @@ mod tests {
 		// fetch execution proof from 'remote' full node
 		let remote_backend = trie_backend::tests::test_trie();
 		let remote_root = remote_backend.storage_root(::std::iter::empty()).0;
-		let (remote_result, remote_proof, _) = prove_execution(remote_backend,
+		let (remote_result, remote_proof) = prove_execution(remote_backend,
 			&mut Default::default(), &executor, "test", &[]).unwrap();
 
 		// check proof locally
-		let (local_result, _) = execution_proof_check::<Blake2Hasher, RlpCodec,_,>(remote_root, remote_proof,
+		let local_result = execution_proof_check::<Blake2Hasher, RlpCodec, _>(remote_root, remote_proof,
 			&mut Default::default(), &executor, "test", &[]).unwrap();
 
 		// check that both results are correct
@@ -646,17 +580,19 @@ mod tests {
 		let backend = InMemory::<Blake2Hasher, RlpCodec>::from(initial).try_into_trie_backend().unwrap();
 		let mut overlay = OverlayedChanges {
 			committed: map![
-				b"aba".to_vec() => Some(b"1312".to_vec()),
-				b"bab".to_vec() => Some(b"228".to_vec())
+				b"aba".to_vec() => Some(b"1312".to_vec()).into(),
+				b"bab".to_vec() => Some(b"228".to_vec()).into()
 			],
 			prospective: map![
-				b"abd".to_vec() => Some(b"69".to_vec()),
-				b"bbd".to_vec() => Some(b"42".to_vec())
+				b"abd".to_vec() => Some(b"69".to_vec()).into(),
+				b"bbd".to_vec() => Some(b"42".to_vec()).into()
 			],
+			..Default::default()
 		};
 
 		{
-			let mut ext = Ext::new(&mut overlay, &backend);
+			let changes_trie_storage = InMemoryChangesTrieStorage::new();
+			let mut ext = Ext::new(&mut overlay, &backend, Some(&changes_trie_storage));
 			ext.clear_prefix(b"ab");
 		}
 		overlay.commit_prospective();
@@ -664,13 +600,13 @@ mod tests {
 		assert_eq!(
 			overlay.committed,
 			map![
-				b"abb".to_vec() => None,
-				b"abc".to_vec() => None,
-				b"aba".to_vec() => None,
-				b"abd".to_vec() => None,
+				b"abc".to_vec() => None.into(),
+				b"abb".to_vec() => None.into(),
+				b"aba".to_vec() => None.into(),
+				b"abd".to_vec() => None.into(),
 
-				b"bab".to_vec() => Some(b"228".to_vec()),
-				b"bbd".to_vec() => Some(b"42".to_vec())
+				b"bab".to_vec() => Some(b"228".to_vec()).into(),
+				b"bbd".to_vec() => Some(b"42".to_vec()).into()
 			],
 		);
 	}

substrate/core/state-machine/src/overlayed_changes.rs

@@ -0,0 +1,371 @@
+// Copyright 2017 Parity Technologies (UK) Ltd.
+// This file is part of Substrate.
+
+// Substrate is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// Substrate is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with Substrate.  If not, see <http://www.gnu.org/licenses/>.
+
+//! The overlayed changes to state.
+
+use std::collections::{HashMap, HashSet};
+use codec::Decode;
+use changes_trie::{NO_EXTRINSIC_INDEX, Configuration as ChangesTrieConfig};
+
+/// The overlayed changes to state to be queried on top of the backend.
+///
+/// A transaction shares all prospective changes within an inner overlay
+/// that can be cleared.
+#[derive(Debug, Default, Clone)]
+pub struct OverlayedChanges {
+	/// Changes that are not yet committed.
+	pub(crate) prospective: HashMap<Vec<u8>, OverlayedValue>,
+	/// Committed changes.
+	pub(crate) committed: HashMap<Vec<u8>, OverlayedValue>,
+	/// Changes trie configuration. None by default, but could be installed by the
+	/// runtime if it supports change tries.
+	pub(crate) changes_trie_config: Option<ChangesTrieConfig>,
+}
+
+/// The storage value, used inside OverlayedChanges.
+#[derive(Debug, Default, Clone)]
+#[cfg_attr(test, derive(PartialEq))]
+pub struct OverlayedValue {
+	/// Current value. None if value has been deleted.
+	pub value: Option<Vec<u8>>,
+	/// The set of extinsic indices where the values has been changed.
+	/// Is filled only if runtime ahs announced changes trie support.
+	pub extrinsics: Option<HashSet<u32>>,
+}
+
+impl OverlayedChanges {
+	/// Sets the changes trie configuration.
+	///
+	/// Returns false if configuration has been set already and we now trying
+	/// to install different configuration. This isn't supported now.
+	#[must_use = "Result must be checked"]
+	pub(crate) fn set_changes_trie_config(&mut self, config: ChangesTrieConfig) -> bool {
+		if let Some(ref old_config) = self.changes_trie_config {
+			// we do not support changes trie configuration' change now
+			if *old_config != config {
+				return false;
+			}
+		}
+
+		self.changes_trie_config = Some(config);
+		true
+	}
+
+	/// Returns a double-Option: None if the key is unknown (i.e. and the query should be refered
+	/// to the backend); Some(None) if the key has been deleted. Some(Some(...)) for a key whose
+	/// value has been set.
+	pub fn storage(&self, key: &[u8]) -> Option<Option<&[u8]>> {
+		self.prospective.get(key)
+			.or_else(|| self.committed.get(key))
+			.map(|x| x.value.as_ref().map(AsRef::as_ref))
+	}
+
+	/// Inserts the given key-value pair into the prospective change set.
+	///
+	/// `None` can be used to delete a value specified by the given key.
+	pub(crate) fn set_storage(&mut self, key: Vec<u8>, val: Option<Vec<u8>>) {
+		let extrinsic_index = self.extrinsic_index();
+		let entry = self.prospective.entry(key).or_default();
+		entry.value = val;
+
+		if let Some(extrinsic) = extrinsic_index {
+			entry.extrinsics.get_or_insert_with(Default::default)
+				.insert(extrinsic);
+		}
+	}
+
+	/// Removes all key-value pairs which keys share the given prefix.
+	///
+	/// NOTE that this doesn't take place immediately but written into the prospective
+	/// change set, and still can be reverted by [`discard_prospective`].
+	///
+	/// [`discard_prospective`]: #method.discard_prospective
+	pub(crate) fn clear_prefix(&mut self, prefix: &[u8]) {
+		let extrinsic_index = self.extrinsic_index();
+
+		// Iterate over all prospective and mark all keys that share
+		// the given prefix as removed (None).
+		for (key, entry) in self.prospective.iter_mut() {
+			if key.starts_with(prefix) {
+				entry.value = None;
+
+				if let Some(extrinsic) = extrinsic_index {
+					entry.extrinsics.get_or_insert_with(Default::default)
+						.insert(extrinsic);
+				}
+			}
+		}
+
+		// Then do the same with keys from commited changes.
+		// NOTE that we are making changes in the prospective change set.
+		for key in self.committed.keys() {
+			if key.starts_with(prefix) {
+				let entry = self.prospective.entry(key.clone()).or_default();
+				entry.value = None;
+
+				if let Some(extrinsic) = extrinsic_index {
+					entry.extrinsics.get_or_insert_with(Default::default)
+						.insert(extrinsic);
+				}
+			}
+		}
+	}
+
+	/// Discard prospective changes to state.
+	pub fn discard_prospective(&mut self) {
+		self.prospective.clear();
+	}
+
+	/// Commit prospective changes to state.
+	pub fn commit_prospective(&mut self) {
+		if self.committed.is_empty() {
+			::std::mem::swap(&mut self.prospective, &mut self.committed);
+		} else {
+			for (key, val) in self.prospective.drain() {
+				let entry = self.committed.entry(key).or_default();
+				entry.value = val.value;
+
+				if let Some(prospective_extrinsics) = val.extrinsics {
+					entry.extrinsics.get_or_insert_with(Default::default)
+						.extend(prospective_extrinsics);
+				}
+			}
+		}
+	}
+
+	/// Drain committed changes to an iterator.
+	///
+	/// Panics:
+	/// Will panic if there are any uncommitted prospective changes.
+	pub fn drain<'a>(&'a mut self) -> impl Iterator<Item=(Vec<u8>, OverlayedValue)> + 'a {
+		assert!(self.prospective.is_empty());
+		self.committed.drain()
+	}
+
+	/// Consume `OverlayedChanges` and take committed set.
+	///
+	/// Panics:
+	/// Will panic if there are any uncommitted prospective changes.
+	pub fn into_committed(self) -> impl Iterator<Item=(Vec<u8>, Option<Vec<u8>>)> {
+		assert!(self.prospective.is_empty());
+		self.committed.into_iter().map(|(k, v)| (k, v.value))
+	}
+
+	/// Inserts storage entry responsible for current extrinsic index.
+	#[cfg(test)]
+	pub(crate) fn set_extrinsic_index(&mut self, extrinsic_index: u32) {
+		use codec::Encode;
+		self.prospective.insert(b":extrinsic_index".to_vec(), OverlayedValue {
+			value: Some(extrinsic_index.encode()),
+			extrinsics: None,
+		});
+	}
+
+	/// Returns current extrinsic index to use in changes trie construction.
+	/// None is returned if it is not set or changes trie config is not set.
+	/// Persistent value (from the backend) can be ignored because runtime must
+	/// set this index before first and unset after last extrinsic is executied.
+	/// Changes that are made outside of extrinsics, are marked with
+	/// `NO_EXTRINSIC_INDEX` index.
+	fn extrinsic_index(&self) -> Option<u32> {
+		match self.changes_trie_config.is_some() {
+			true => Some(
+				self.storage(b":extrinsic_index")
+					.and_then(|idx| idx.and_then(|idx| Decode::decode(&mut &*idx)))
+					.unwrap_or(NO_EXTRINSIC_INDEX)),
+			false => None,
+		}
+	}
+}
+
+#[cfg(test)]
+impl From<Option<Vec<u8>>> for OverlayedValue {
+	fn from(value: Option<Vec<u8>>) -> OverlayedValue {
+		OverlayedValue { value, ..Default::default() }
+	}
+}
+
+#[cfg(test)]
+mod tests {
+	use primitives::{Blake2Hasher, RlpCodec, H256};
+	use backend::InMemory;
+	use changes_trie::InMemoryStorage as InMemoryChangesTrieStorage;
+	use ext::Ext;
+	use {Externalities};
+	use super::*;
+
+	fn strip_extrinsic_index(map: &HashMap<Vec<u8>, OverlayedValue>) -> HashMap<Vec<u8>, OverlayedValue> {
+		let mut clone = map.clone();
+		clone.remove(&b":extrinsic_index".to_vec());
+		clone
+	}
+
+	#[test]
+	fn overlayed_storage_works() {
+		let mut overlayed = OverlayedChanges::default();
+
+		let key = vec![42, 69, 169, 142];
+
+		assert!(overlayed.storage(&key).is_none());
+
+		overlayed.set_storage(key.clone(), Some(vec![1, 2, 3]));
+		assert_eq!(overlayed.storage(&key).unwrap(), Some(&[1, 2, 3][..]));
+
+		overlayed.commit_prospective();
+		assert_eq!(overlayed.storage(&key).unwrap(), Some(&[1, 2, 3][..]));
+
+		overlayed.set_storage(key.clone(), Some(vec![]));
+		assert_eq!(overlayed.storage(&key).unwrap(), Some(&[][..]));
+
+		overlayed.set_storage(key.clone(), None);
+		assert!(overlayed.storage(&key).unwrap().is_none());
+
+		overlayed.discard_prospective();
+		assert_eq!(overlayed.storage(&key).unwrap(), Some(&[1, 2, 3][..]));
+
+		overlayed.set_storage(key.clone(), None);
+		overlayed.commit_prospective();
+		assert!(overlayed.storage(&key).unwrap().is_none());
+	}
+
+	#[test]
+	fn overlayed_storage_root_works() {
+		let initial: HashMap<_, _> = vec![
+			(b"doe".to_vec(), b"reindeer".to_vec()),
+			(b"dog".to_vec(), b"puppyXXX".to_vec()),
+			(b"dogglesworth".to_vec(), b"catXXX".to_vec()),
+			(b"doug".to_vec(), b"notadog".to_vec()),
+		].into_iter().collect();
+		let backend = InMemory::<Blake2Hasher, RlpCodec>::from(initial);
+		let mut overlay = OverlayedChanges {
+			committed: vec![
+				(b"dog".to_vec(), Some(b"puppy".to_vec()).into()),
+				(b"dogglesworth".to_vec(), Some(b"catYYY".to_vec()).into()),
+				(b"doug".to_vec(), Some(vec![]).into()),
+			].into_iter().collect(),
+			prospective: vec![
+				(b"dogglesworth".to_vec(), Some(b"cat".to_vec()).into()),
+				(b"doug".to_vec(), None.into()),
+			].into_iter().collect(),
+			..Default::default()
+		};
+
+		let changes_trie_storage = InMemoryChangesTrieStorage::new();
+		let mut ext = Ext::new(&mut overlay, &backend, Some(&changes_trie_storage));
+		const ROOT: [u8; 32] = hex!("6ca394ff9b13d6690a51dea30b1b5c43108e52944d30b9095227c49bae03ff8b");
+		assert_eq!(ext.storage_root(), H256(ROOT));
+	}
+
+	#[test]
+	fn changes_trie_configuration_is_saved() {
+		let mut overlay = OverlayedChanges::default();
+		assert!(overlay.changes_trie_config.is_none());
+		assert_eq!(overlay.set_changes_trie_config(ChangesTrieConfig {
+			digest_interval: 4, digest_levels: 1,
+		}), true);
+		assert!(overlay.changes_trie_config.is_some());
+	}
+
+	#[test]
+	fn changes_trie_configuration_is_saved_twice() {
+		let mut overlay = OverlayedChanges::default();
+		assert!(overlay.changes_trie_config.is_none());
+		assert_eq!(overlay.set_changes_trie_config(ChangesTrieConfig {
+			digest_interval: 4, digest_levels: 1,
+		}), true);
+		overlay.set_extrinsic_index(0);
+		overlay.set_storage(vec![1], Some(vec![2]));
+		assert_eq!(overlay.set_changes_trie_config(ChangesTrieConfig {
+			digest_interval: 4, digest_levels: 1,
+		}), true);
+		assert_eq!(
+			strip_extrinsic_index(&overlay.prospective),
+			vec![
+				(vec![1], OverlayedValue { value: Some(vec![2]), extrinsics: Some(vec![0].into_iter().collect()) }),
+			].into_iter().collect(),
+		);
+	}
+
+	#[test]
+	fn panics_when_trying_to_save_different_changes_trie_configuration() {
+		let mut overlay = OverlayedChanges::default();
+		assert_eq!(overlay.set_changes_trie_config(ChangesTrieConfig {
+			digest_interval: 4, digest_levels: 1,
+		}), true);
+		assert_eq!(overlay.set_changes_trie_config(ChangesTrieConfig {
+			digest_interval: 2, digest_levels: 1,
+		}), false);
+	}
+
+	#[test]
+	fn extrinsic_changes_are_collected() {
+		let mut overlay = OverlayedChanges::default();
+		let _ = overlay.set_changes_trie_config(ChangesTrieConfig {
+			digest_interval: 4, digest_levels: 1,
+		});
+
+		overlay.set_storage(vec![100], Some(vec![101]));
+
+		overlay.set_extrinsic_index(0);
+		overlay.set_storage(vec![1], Some(vec![2]));
+
+		overlay.set_extrinsic_index(1);
+		overlay.set_storage(vec![3], Some(vec![4]));
+
+		overlay.set_extrinsic_index(2);
+		overlay.set_storage(vec![1], Some(vec![6]));
+
+		assert_eq!(strip_extrinsic_index(&overlay.prospective),
+			vec![
+				(vec![1], OverlayedValue { value: Some(vec![6]), extrinsics: Some(vec![0, 2].into_iter().collect()) }),
+				(vec![3], OverlayedValue { value: Some(vec![4]), extrinsics: Some(vec![1].into_iter().collect()) }),
+				(vec![100], OverlayedValue { value: Some(vec![101]), extrinsics: Some(vec![NO_EXTRINSIC_INDEX].into_iter().collect()) }),
+			].into_iter().collect());
+
+		overlay.commit_prospective();
+
+		overlay.set_extrinsic_index(3);
+		overlay.set_storage(vec![3], Some(vec![7]));
+
+		overlay.set_extrinsic_index(4);
+		overlay.set_storage(vec![1], Some(vec![8]));
+
+		assert_eq!(strip_extrinsic_index(&overlay.committed),
+			vec![
+				(vec![1], OverlayedValue { value: Some(vec![6]), extrinsics: Some(vec![0, 2].into_iter().collect()) }),
+				(vec![3], OverlayedValue { value: Some(vec![4]), extrinsics: Some(vec![1].into_iter().collect()) }),
+				(vec![100], OverlayedValue { value: Some(vec![101]), extrinsics: Some(vec![NO_EXTRINSIC_INDEX].into_iter().collect()) }),
+			].into_iter().collect());
+
+		assert_eq!(strip_extrinsic_index(&overlay.prospective),
+			vec![
+				(vec![1], OverlayedValue { value: Some(vec![8]), extrinsics: Some(vec![4].into_iter().collect()) }),
+				(vec![3], OverlayedValue { value: Some(vec![7]), extrinsics: Some(vec![3].into_iter().collect()) }),
+			].into_iter().collect());
+
+		overlay.commit_prospective();
+
+		assert_eq!(strip_extrinsic_index(&overlay.committed),
+			vec![
+				(vec![1], OverlayedValue { value: Some(vec![8]), extrinsics: Some(vec![0, 2, 4].into_iter().collect()) }),
+				(vec![3], OverlayedValue { value: Some(vec![7]), extrinsics: Some(vec![1, 3].into_iter().collect()) }),
+				(vec![100], OverlayedValue { value: Some(vec![101]), extrinsics: Some(vec![NO_EXTRINSIC_INDEX].into_iter().collect()) }),
+			].into_iter().collect());
+
+		assert_eq!(overlay.prospective,
+			Default::default());
+	}
+}

substrate/core/state-machine/src/proving_backend.rs

@@ -18,23 +18,54 @@
 
 use std::cell::RefCell;
 use hashdb::{Hasher, HashDB};
+use heapsize::HeapSizeOf;
 use memorydb::MemoryDB;
 use patricia_trie::{TrieDB, Trie, Recorder, NodeCodec};
-use trie_backend::{TrieBackend, Ephemeral};
-use {Error, ExecutionError, Backend, TryIntoTrieBackend};
 use rlp::Encodable;
-use heapsize::HeapSizeOf;
+use trie_backend::TrieBackend;
+use trie_backend_essence::{Ephemeral, TrieBackendEssence, TrieBackendStorage};
+use {Error, ExecutionError, Backend};
+
+/// Patricia trie-based backend essence which also tracks all touched storage trie values.
+/// These can be sent to remote node and used as a proof of execution.
+pub struct ProvingBackendEssence<'a, S: 'a + TrieBackendStorage<H>, H: 'a + Hasher, C: 'a + NodeCodec<H>> {
+	pub(crate) backend: &'a TrieBackendEssence<S, H, C>,
+	pub(crate) proof_recorder: &'a mut Recorder<H::Out>,
+}
+
+impl<'a, S, H, C> ProvingBackendEssence<'a, S, H, C>
+	where
+		S: TrieBackendStorage<H>,
+		H: Hasher,
+		H::Out: HeapSizeOf,
+		C: NodeCodec<H>,
+{
+	pub fn storage(&mut self, key: &[u8]) -> Result<Option<Vec<u8>>, String> {
+		let mut read_overlay = MemoryDB::default();
+		let eph = Ephemeral::new(
+			self.backend.backend_storage(),
+			&mut read_overlay,
+		);
+
+		let map_e = |e| format!("Trie lookup error: {}", e);
+
+		TrieDB::<H, C>::new(&eph, self.backend.root()).map_err(map_e)?
+			.get_with(key, &mut *self.proof_recorder)
+			.map(|x| x.map(|val| val.to_vec()))
+			.map_err(map_e)
+	}
+}
 
 /// Patricia trie-based backend which also tracks all touched storage trie values.
 /// These can be sent to remote node and used as a proof of execution.
-pub struct ProvingBackend<H: Hasher, C: NodeCodec<H>> {
-	backend: TrieBackend<H, C>,
+pub struct ProvingBackend<S: TrieBackendStorage<H>, H: Hasher, C: NodeCodec<H>> {
+	backend: TrieBackend<S, H, C>,
 	proof_recorder: RefCell<Recorder<H::Out>>,
 }
 
-impl<H: Hasher, C: NodeCodec<H>> ProvingBackend<H, C> {
+impl<S: TrieBackendStorage<H>, H: Hasher, C: NodeCodec<H>> ProvingBackend<S, H, C> {
 	/// Create new proving backend.
-	pub fn new(backend: TrieBackend<H, C>) -> Self {
+	pub fn new(backend: TrieBackend<S, H, C>) -> Self {
 		ProvingBackend {
 			backend,
 			proof_recorder: RefCell::new(Recorder::new()),
@@ -51,27 +82,23 @@ impl<H: Hasher, C: NodeCodec<H>> ProvingBackend<H, C> {
 	}
 }
 
-impl<H, C> Backend<H, C> for ProvingBackend<H, C>
-where
-	H: Hasher,
-	C: NodeCodec<H>,
-	H::Out: Ord + Encodable + HeapSizeOf
+impl<S, H, C> Backend<H, C> for ProvingBackend<S, H, C>
+	where
+		S: TrieBackendStorage<H>,
+		H: Hasher,
+		C: NodeCodec<H>,
+		H::Out: Ord + Encodable + HeapSizeOf,
 {
 	type Error = String;
 	type Transaction = MemoryDB<H>;
+	type TrieBackendStorage = MemoryDB<H>;
 
 	fn storage(&self, key: &[u8]) -> Result<Option<Vec<u8>>, Self::Error> {
-		let mut read_overlay = MemoryDB::new();
-		let eph = Ephemeral::new(
-			self.backend.backend_storage(),
-			&mut read_overlay,
-		);
-		let map_e = |e| format!("Trie lookup error: {}", e);
-
-		let mut proof_recorder = self.proof_recorder.try_borrow_mut()
-			.expect("only fails when already borrowed; storage() is non-reentrant; qed");
-		TrieDB::<H, C>::new(&eph, &self.backend.root()).map_err(map_e)?
-			.get_with(key, &mut *proof_recorder).map(|x| x.map(|val| val.to_vec())).map_err(map_e)
+		ProvingBackendEssence {
+			backend: self.backend.essence(),
+			proof_recorder: &mut *self.proof_recorder.try_borrow_mut()
+				.expect("only fails when already borrowed; storage() is non-reentrant; qed"),
+		}.storage(key)
 	}
 
 	fn for_keys_with_prefix<F: FnMut(&[u8])>(&self, prefix: &[u8], f: F) {
@@ -87,10 +114,8 @@ where
 	{
 		self.backend.storage_root(delta)
 	}
-}
 
-impl<H: Hasher, C: NodeCodec<H>> TryIntoTrieBackend<H, C> for ProvingBackend<H, C> {
-	fn try_into_trie_backend(self) -> Option<TrieBackend<H, C>> {
+	fn try_into_trie_backend(self) -> Option<TrieBackend<Self::TrieBackendStorage, H, C>> {
 		None
 	}
 }
@@ -99,7 +124,7 @@ impl<H: Hasher, C: NodeCodec<H>> TryIntoTrieBackend<H, C> for ProvingBackend<H,
 pub fn create_proof_check_backend<H, C>(
 	root: H::Out,
 	proof: Vec<Vec<u8>>
-) -> Result<TrieBackend<H, C>, Box<Error>>
+) -> Result<TrieBackend<MemoryDB<H>, H, C>, Box<Error>>
 where
 	H: Hasher,
 	C: NodeCodec<H>,
@@ -114,8 +139,7 @@ where
 		return Err(Box::new(ExecutionError::InvalidProof) as Box<Error>);
 	}
 
-
-	Ok(TrieBackend::with_memorydb(db, root))
+	Ok(TrieBackend::new(db, root))
 }
 
 #[cfg(test)]
@@ -125,7 +149,7 @@ mod tests {
 	use super::*;
 	use primitives::{Blake2Hasher, RlpCodec};
 
-	fn test_proving() -> ProvingBackend<Blake2Hasher, RlpCodec> {
+	fn test_proving() -> ProvingBackend<MemoryDB<Blake2Hasher>, Blake2Hasher, RlpCodec> {
 		ProvingBackend::new(test_trie())
 	}
 

substrate/core/state-machine/src/testing.rs

@@ -17,41 +17,62 @@
 //! Test implementation for Externalities.
 
 use std::collections::HashMap;
-use std::cmp::Ord;
-use super::Externalities;
-use triehash::trie_root;
-use hashdb::Hasher;
-use rlp::Encodable;
-use std::marker::PhantomData;
 use std::iter::FromIterator;
+use hashdb::Hasher;
+use heapsize::HeapSizeOf;
+use patricia_trie::NodeCodec;
+use rlp::Encodable;
+use triehash::trie_root;
+use backend::InMemory;
+use changes_trie::{compute_changes_trie_root, InMemoryStorage as ChangesTrieInMemoryStorage};
+use super::{Externalities, OverlayedChanges};
 
 /// Simple HashMap-based Externalities impl.
-#[derive(Debug)]
-pub struct TestExternalities<H> {
+pub struct TestExternalities<H: Hasher, C: NodeCodec<H>> where H::Out: HeapSizeOf {
 	inner: HashMap<Vec<u8>, Vec<u8>>,
-	_hasher: PhantomData<H>,
+	changes_trie_storage: ChangesTrieInMemoryStorage<H>,
+	changes: OverlayedChanges,
+	_codec: ::std::marker::PhantomData<C>,
 }
 
-impl<H: Hasher> TestExternalities<H> {
+impl<H: Hasher, C: NodeCodec<H>> TestExternalities<H, C> where H::Out: HeapSizeOf {
 	/// Create a new instance of `TestExternalities`
-	pub fn new() -> Self {
-		TestExternalities {inner: HashMap::new(), _hasher: PhantomData}
+	pub fn new(inner: HashMap<Vec<u8>, Vec<u8>>) -> Self {
+		let mut overlay = OverlayedChanges::default();
+		super::set_changes_trie_config(
+			&mut overlay,
+			inner.get(&b":changes_trie".to_vec()).cloned())
+			.expect("changes trie configuration is correct in test env; qed");
+
+		TestExternalities {
+			inner,
+			changes_trie_storage: ChangesTrieInMemoryStorage::new(),
+			changes: overlay,
+			_codec: Default::default(),
+		}
 	}
+
 	/// Insert key/value
 	pub fn insert(&mut self, k: Vec<u8>, v: Vec<u8>) -> Option<Vec<u8>> {
 		self.inner.insert(k, v)
 	}
 }
 
-impl<H: Hasher> PartialEq for TestExternalities<H> {
-	fn eq(&self, other: &TestExternalities<H>) -> bool {
+impl<H: Hasher, C: NodeCodec<H>> ::std::fmt::Debug for TestExternalities<H, C> where H::Out: HeapSizeOf {
+	fn fmt(&self, f: &mut ::std::fmt::Formatter) -> ::std::fmt::Result {
+		write!(f, "{:?}", self.inner)
+	}
+}
+
+impl<H: Hasher, C: NodeCodec<H>> PartialEq for TestExternalities<H, C> where H::Out: HeapSizeOf {
+	fn eq(&self, other: &TestExternalities<H, C>) -> bool {
 		self.inner.eq(&other.inner)
 	}
 }
 
-impl<H: Hasher> FromIterator<(Vec<u8>, Vec<u8>)> for TestExternalities<H> {
+impl<H: Hasher, C: NodeCodec<H>> FromIterator<(Vec<u8>, Vec<u8>)> for TestExternalities<H, C> where H::Out: HeapSizeOf {
 	fn from_iter<I: IntoIterator<Item=(Vec<u8>, Vec<u8>)>>(iter: I) -> Self {
-		let mut t = Self::new();
+		let mut t = Self::new(Default::default());
 		for i in iter {
 			t.inner.insert(i.0, i.1);
 		}
@@ -59,29 +80,34 @@ impl<H: Hasher> FromIterator<(Vec<u8>, Vec<u8>)> for TestExternalities<H> {
 	}
 }
 
-impl<H: Hasher> Default for TestExternalities<H> {
-	fn default() -> Self { Self::new() }
+impl<H: Hasher, C: NodeCodec<H>> Default for TestExternalities<H, C> where H::Out: HeapSizeOf {
+	fn default() -> Self { Self::new(Default::default()) }
 }
 
-impl<H: Hasher> From<TestExternalities<H>> for HashMap<Vec<u8>, Vec<u8>> {
-	fn from(tex: TestExternalities<H>) -> Self {
+impl<H: Hasher, C: NodeCodec<H>> From<TestExternalities<H, C>> for HashMap<Vec<u8>, Vec<u8>> where H::Out: HeapSizeOf {
+	fn from(tex: TestExternalities<H, C>) -> Self {
 		tex.inner.into()
 	}
 }
 
-impl<H: Hasher> From< HashMap<Vec<u8>, Vec<u8>> > for TestExternalities<H> {
+impl<H: Hasher, C: NodeCodec<H>> From< HashMap<Vec<u8>, Vec<u8>> > for TestExternalities<H, C> where H::Out: HeapSizeOf {
 	fn from(hashmap: HashMap<Vec<u8>, Vec<u8>>) -> Self {
-		TestExternalities { inner: hashmap, _hasher: PhantomData }
+		TestExternalities {
+			inner: hashmap,
+			changes_trie_storage: ChangesTrieInMemoryStorage::new(),
+			changes: Default::default(),
+			_codec: ::std::marker::PhantomData::<C>::default(),
+		}
 	}
 }
 
-
-impl<H: Hasher> Externalities<H> for TestExternalities<H> where H::Out: Ord + Encodable {
+impl<H: Hasher, C: NodeCodec<H>> Externalities<H> for TestExternalities<H, C> where H::Out: Ord + Encodable + HeapSizeOf {
 	fn storage(&self, key: &[u8]) -> Option<Vec<u8>> {
 		self.inner.get(key).map(|x| x.to_vec())
 	}
 
 	fn place_storage(&mut self, key: Vec<u8>, maybe_value: Option<Vec<u8>>) {
+		self.changes.set_storage(key.clone(), maybe_value.clone());
 		match maybe_value {
 			Some(value) => { self.inner.insert(key, value); }
 			None => { self.inner.remove(&key); }
@@ -89,9 +115,8 @@ impl<H: Hasher> Externalities<H> for TestExternalities<H> where H::Out: Ord + En
 	}
 
 	fn clear_prefix(&mut self, prefix: &[u8]) {
-		self.inner.retain(|key, _|
-			!key.starts_with(prefix)
-		)
+		self.changes.clear_prefix(prefix);
+		self.inner.retain(|key, _| !key.starts_with(prefix));
 	}
 
 	fn chain_id(&self) -> u64 { 42 }
@@ -99,16 +124,25 @@ impl<H: Hasher> Externalities<H> for TestExternalities<H> where H::Out: Ord + En
 	fn storage_root(&mut self) -> H::Out {
 		trie_root::<H, _, _, _>(self.inner.clone())
 	}
+
+	fn storage_changes_root(&mut self, block: u64) -> Option<H::Out> {
+		compute_changes_trie_root::<_, _, H, C>(
+			&InMemory::default(),
+			Some(&self.changes_trie_storage),
+			&self.changes,
+			block,
+		).map(|(root, _)| root.clone())
+	}
 }
 
 #[cfg(test)]
 mod tests {
 	use super::*;
-	use primitives::{Blake2Hasher, H256};
+	use primitives::{Blake2Hasher, RlpCodec, H256};
 
 	#[test]
 	fn commit_should_work() {
-		let mut ext = TestExternalities::<Blake2Hasher>::new();
+		let mut ext = TestExternalities::<Blake2Hasher, RlpCodec>::default();
 		ext.set_storage(b"doe".to_vec(), b"reindeer".to_vec());
 		ext.set_storage(b"dog".to_vec(), b"puppy".to_vec());
 		ext.set_storage(b"dogglesworth".to_vec(), b"cat".to_vec());

substrate/core/state-machine/src/trie_backend.rs

@@ -15,135 +15,70 @@
 // along with Substrate.  If not, see <http://www.gnu.org/licenses/>.
 
 //! Trie-based state machine backend.
-use Backend;
-use hashdb::{Hasher, HashDB, AsHashDB};
-use memorydb::MemoryDB;
-use patricia_trie::{TrieDB, TrieDBMut, TrieError, Trie, TrieMut, NodeCodec};
-use std::collections::HashMap;
-use std::sync::Arc;
-use std::marker::PhantomData;
+
+use hashdb::Hasher;
 use heapsize::HeapSizeOf;
+use memorydb::MemoryDB;
+use rlp::Encodable;
+use patricia_trie::{TrieDB, TrieDBMut, TrieError, Trie, TrieMut, NodeCodec};
+use trie_backend_essence::{TrieBackendEssence, TrieBackendStorage, Ephemeral};
+use {Backend};
 
 pub use hashdb::DBValue;
 
-/// Backend trie storage trait.
-pub trait Storage<H: Hasher>: Send + Sync {
-	/// Get a trie node.
-	fn get(&self, key: &H::Out) -> Result<Option<DBValue>, String>;
-}
-
-/// Try convert into trie-based backend.
-pub trait TryIntoTrieBackend<H: Hasher, C: NodeCodec<H>> {
-	/// Try to convert self into trie backend.
-	fn try_into_trie_backend(self) -> Option<TrieBackend<H, C>>;
-}
-
 /// Patricia trie-based backend. Transaction type is an overlay of changes to commit.
-#[derive(Clone)]
-pub struct TrieBackend<H: Hasher, C: NodeCodec<H>> {
-	storage: TrieBackendStorage<H>,
-	root: H::Out,
-	_codec: PhantomData<C>
+pub struct TrieBackend<S: TrieBackendStorage<H>, H: Hasher, C: NodeCodec<H>> {
+	essence: TrieBackendEssence<S, H, C>,
 }
 
-impl<H: Hasher, C: NodeCodec<H>> TrieBackend<H, C> where H::Out: HeapSizeOf {
+impl<S: TrieBackendStorage<H>, H: Hasher, C: NodeCodec<H>> TrieBackend<S, H, C> where H::Out: HeapSizeOf {
 	/// Create new trie-based backend.
-	pub fn with_storage(db: Arc<Storage<H>>, root: H::Out) -> Self {
+	pub fn new(storage: S, root: H::Out) -> Self {
 		TrieBackend {
-			storage: TrieBackendStorage::Storage(db),
-			root,
-			_codec: PhantomData,
+			essence: TrieBackendEssence::new(storage, root),
 		}
 	}
 
-	/// Create new trie-based backend for genesis block.
-	pub fn with_storage_for_genesis(db: Arc<Storage<H>>) -> Self {
-		let mut root = <H as Hasher>::Out::default();
-		let mut mdb = MemoryDB::<H>::new();
-		TrieDBMut::<H, C>::new(&mut mdb, &mut root);
-
-		Self::with_storage(db, root)
-	}
-
-	/// Create new trie-based backend backed by MemoryDb storage.
-	pub fn with_memorydb(db: MemoryDB<H>, root: H::Out) -> Self {
-		// TODO: check that root is a part of db???
-		TrieBackend {
-			storage: TrieBackendStorage::MemoryDb(db),
-			root,
-			_codec: PhantomData,
-		}
+	/// Get backend essence reference.
+	pub fn essence(&self) -> &TrieBackendEssence<S, H, C> {
+		&self.essence
 	}
 
 	/// Get backend storage reference.
-	pub fn backend_storage(&self) -> &TrieBackendStorage<H> {
-		&self.storage
+	pub fn backend_storage(&self) -> &S {
+		self.essence.backend_storage()
 	}
 
 	/// Get trie root.
 	pub fn root(&self) -> &H::Out {
-		&self.root
+		self.essence.root()
 	}
 }
 
 impl super::Error for String {}
 
-impl<H: Hasher, C: NodeCodec<H>> Backend<H, C> for TrieBackend<H, C> where H::Out: HeapSizeOf {
+impl<S: TrieBackendStorage<H>, H: Hasher, C: NodeCodec<H>> Backend<H, C> for TrieBackend<S, H, C>
+	where
+		H::Out: Ord + Encodable + HeapSizeOf,
+{
 	type Error = String;
 	type Transaction = MemoryDB<H>;
+	type TrieBackendStorage = S;
 
 	fn storage(&self, key: &[u8]) -> Result<Option<Vec<u8>>, Self::Error> {
-		let mut read_overlay = MemoryDB::new();
-		let eph = Ephemeral {
-			storage: &self.storage,
-			overlay: &mut read_overlay,
-		};
-
-		let map_e = |e| format!("Trie lookup error: {}", e);
-		TrieDB::<H, C>::new(&eph, &self.root).map_err(map_e)?
-			.get(key).map(|x| x.map(|val| val.to_vec())).map_err(map_e)
+		self.essence.storage(key)
 	}
 
-	fn for_keys_with_prefix<F: FnMut(&[u8])>(&self, prefix: &[u8], mut f: F) {
-		let mut read_overlay = MemoryDB::new();
-		let eph = Ephemeral {
-			storage: &self.storage,
-			overlay: &mut read_overlay,
-		};
-
-		let mut iter = move || -> Result<(), Box<TrieError<H::Out, C::Error>>> {
-			let trie = TrieDB::<H, C>::new(&eph, &self.root)?;
-			let mut iter = trie.iter()?;
-
-			iter.seek(prefix)?;
-
-			for x in iter {
-				let (key, _) = x?;
-
-				if !key.starts_with(prefix) {
-					break;
-				}
-
-				f(&key);
-			}
-
-			Ok(())
-		};
-
-		if let Err(e) = iter() {
-			debug!(target: "trie", "Error while iterating by prefix: {}", e);
-		}
+	fn for_keys_with_prefix<F: FnMut(&[u8])>(&self, prefix: &[u8], f: F) {
+		self.essence.for_keys_with_prefix(prefix, f)
 	}
 
 	fn pairs(&self) -> Vec<(Vec<u8>, Vec<u8>)> {
 		let mut read_overlay = MemoryDB::new();
-		let eph = Ephemeral {
-			storage: &self.storage,
-			overlay: &mut read_overlay,
-		};
+		let eph = Ephemeral::new(self.essence.backend_storage(), &mut read_overlay);
 
 		let collect_all = || -> Result<_, Box<TrieError<H::Out, C::Error>>> {
-			let trie = TrieDB::<H, C>::new(&eph, &self.root)?;
+			let trie = TrieDB::<H, C>::new(&eph, self.essence.root())?;
 			let mut v = Vec::new();
 			for x in trie.iter()? {
 				let (key, value) = x?;
@@ -165,13 +100,13 @@ impl<H: Hasher, C: NodeCodec<H>> Backend<H, C> for TrieBackend<H, C> where H::Ou
 	fn storage_root<I>(&self, delta: I) -> (H::Out, MemoryDB<H>)
 		where I: IntoIterator<Item=(Vec<u8>, Option<Vec<u8>>)>
 	{
-		let mut write_overlay = MemoryDB::new();
-		let mut root = self.root;
+		let mut write_overlay = MemoryDB::default();
+		let mut root = *self.essence.root();
 		{
-			let mut eph = Ephemeral {
-				storage: &self.storage,
-				overlay: &mut write_overlay,
-			};
+			let mut eph = Ephemeral::new(
+				self.essence.backend_storage(),
+				&mut write_overlay,
+			);
 
 			let mut trie = TrieDBMut::<H, C>::from_existing(&mut eph, &mut root).expect("prior state root to exist"); // TODO: handle gracefully
 			for (key, change) in delta {
@@ -188,99 +123,17 @@ impl<H: Hasher, C: NodeCodec<H>> Backend<H, C> for TrieBackend<H, C> where H::Ou
 
 		(root, write_overlay)
 	}
-}
 
-impl<H: Hasher, C: NodeCodec<H>> TryIntoTrieBackend<H, C> for TrieBackend<H, C> {
-	fn try_into_trie_backend(self) -> Option<TrieBackend<H, C>> {
+	fn try_into_trie_backend(self) -> Option<TrieBackend<Self::TrieBackendStorage, H, C>> {
 		Some(self)
 	}
 }
 
-pub struct Ephemeral<'a, H: 'a + Hasher> {
-	storage: &'a TrieBackendStorage<H>,
-	overlay: &'a mut MemoryDB<H>,
-}
-
-impl<'a, H: Hasher> AsHashDB<H> for Ephemeral<'a, H> where H::Out: HeapSizeOf {
-	fn as_hashdb(&self) -> &HashDB<H> { self }
-	fn as_hashdb_mut(&mut self) -> &mut HashDB<H> { self }
-}
-
-impl<'a, H: Hasher> Ephemeral<'a, H> {
-	pub fn new(storage: &'a TrieBackendStorage<H>, overlay: &'a mut MemoryDB<H>) -> Self {
-		Ephemeral {
-			storage,
-			overlay,
-		}
-	}
-}
-
-impl<'a, H: Hasher> HashDB<H> for Ephemeral<'a, H> where H::Out: HeapSizeOf {
-	fn keys(&self) -> HashMap<H::Out, i32> {
-		self.overlay.keys() // TODO: iterate backing
-	}
-
-	fn get(&self, key: &H::Out) -> Option<DBValue> {
-		match self.overlay.raw(key) {
-			Some((val, i)) => {
-				if i <= 0 {
-					None
-				} else {
-					Some(val)
-				}
-			}
-			None => match self.storage.get(key) {
-				Ok(x) => x,
-				Err(e) => {
-					warn!(target: "trie", "Failed to read from DB: {}", e);
-					None
-				},
-			},
-		}
-	}
-
-	fn contains(&self, key: &H::Out) -> bool {
-		self.get(key).is_some()
-	}
-
-	fn insert(&mut self, value: &[u8]) -> H::Out {
-		self.overlay.insert(value)
-	}
-
-	fn emplace(&mut self, key: H::Out, value: DBValue) {
-		self.overlay.emplace(key, value)
-	}
-
-	fn remove(&mut self, key: &H::Out) {
-		self.overlay.remove(key)
-	}
-}
-
-#[derive(Clone)]
-pub enum TrieBackendStorage<H: Hasher> {
-	/// Key value db + storage column.
-	Storage(Arc<Storage<H>>),
-	/// Hash db.
-	MemoryDb(MemoryDB<H>),
-}
-
-impl<H: Hasher> TrieBackendStorage<H> {
-	pub fn get(&self, key: &H::Out) -> Result<Option<DBValue>, String> {
-		match *self {
-			TrieBackendStorage::Storage(ref db) =>
-				db.get(key)
-					.map_err(|e| format!("Trie lookup error: {}", e)),
-			TrieBackendStorage::MemoryDb(ref db) =>
-				Ok(db.get(key)),
-		}
-	}
-}
-
 #[cfg(test)]
 pub mod tests {
-	use super::*;
 	use std::collections::HashSet;
 	use primitives::{Blake2Hasher, RlpCodec, H256};
+	use super::*;
 
 	fn test_db() -> (MemoryDB<Blake2Hasher>, H256) {
 		let mut root = H256::default();
@@ -298,9 +151,9 @@ pub mod tests {
 		(mdb, root)
 	}
 
-	pub(crate) fn test_trie() -> TrieBackend<Blake2Hasher, RlpCodec> {
+	pub(crate) fn test_trie() -> TrieBackend<MemoryDB<Blake2Hasher>, Blake2Hasher, RlpCodec> {
 		let (mdb, root) = test_db();
-		TrieBackend::with_memorydb(mdb, root)
+		TrieBackend::new(mdb, root)
 	}
 
 	#[test]
@@ -320,11 +173,10 @@ pub mod tests {
 
 	#[test]
 	fn pairs_are_empty_on_empty_storage() {
-		let db = TrieBackend::<Blake2Hasher, RlpCodec>::with_memorydb(
+		assert!(TrieBackend::<MemoryDB<Blake2Hasher>, Blake2Hasher, RlpCodec>::new(
 			MemoryDB::new(),
-			Default::default()
-		);
-		assert!(db.pairs().is_empty());
+			Default::default(),
+		).pairs().is_empty());
 	}
 
 	#[test]

substrate/core/state-machine/src/trie_backend_essence.rs

@@ -0,0 +1,194 @@
+// Copyright 2017 Parity Technologies (UK) Ltd.
+// This file is part of Substrate.
+
+// Substrate is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// Substrate is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with Substrate.  If not, see <http://www.gnu.org/licenses/>.
+
+//! Trie-based state machine backend essence used to read values
+//! from storage.
+
+use std::collections::HashMap;
+use std::marker::PhantomData;
+use std::ops::Deref;
+use std::sync::Arc;
+use hashdb::{Hasher, DBValue, AsHashDB, HashDB};
+use heapsize::HeapSizeOf;
+use memorydb::MemoryDB;
+use patricia_trie::{TrieDB, TrieError, Trie, NodeCodec};
+use changes_trie::Storage as ChangesTrieStorage;
+
+/// Patricia trie-based storage trait.
+pub trait Storage<H: Hasher>: Send + Sync {
+	/// Get a trie node.
+	fn get(&self, key: &H::Out) -> Result<Option<DBValue>, String>;
+}
+
+/// Patricia trie-based pairs storage essence.
+pub struct TrieBackendEssence<S: TrieBackendStorage<H>, H: Hasher, C: NodeCodec<H>> {
+	storage: S,
+	root: H::Out,
+	_codec: PhantomData<C>,
+}
+
+impl<S: TrieBackendStorage<H>, H: Hasher, C: NodeCodec<H>> TrieBackendEssence<S, H, C> where H::Out: HeapSizeOf {
+	/// Create new trie-based backend.
+	pub fn new(storage: S, root: H::Out) -> Self {
+		TrieBackendEssence {
+			storage,
+			root,
+			_codec: Default::default(),
+		}
+	}
+
+	/// Get backend storage reference.
+	pub fn backend_storage(&self) -> &S {
+		&self.storage
+	}
+
+	/// Get trie root.
+	pub fn root(&self) -> &H::Out {
+		&self.root
+	}
+
+	/// Get the value of storage at given key.
+	pub fn storage(&self, key: &[u8]) -> Result<Option<Vec<u8>>, String> {
+		let mut read_overlay = MemoryDB::default();
+		let eph = Ephemeral {
+			storage: &self.storage,
+			overlay: &mut read_overlay,
+		};
+
+		let map_e = |e| format!("Trie lookup error: {}", e);
+
+		TrieDB::<H, C>::new(&eph, &self.root).map_err(map_e)?
+			.get(key).map(|x| x.map(|val| val.to_vec())).map_err(map_e)
+	}
+
+	/// Execute given closure for all keys starting with prefix.
+	pub fn for_keys_with_prefix<F: FnMut(&[u8])>(&self, prefix: &[u8], mut f: F) {
+		let mut read_overlay = MemoryDB::default();
+		let eph = Ephemeral {
+			storage: &self.storage,
+			overlay: &mut read_overlay,
+		};
+
+		let mut iter = move || -> Result<(), Box<TrieError<H::Out, C::Error>>> {
+			let trie = TrieDB::<H, C>::new(&eph, &self.root)?;
+			let mut iter = trie.iter()?;
+
+			iter.seek(prefix)?;
+
+			for x in iter {
+				let (key, _) = x?;
+
+				if !key.starts_with(prefix) {
+					break;
+				}
+
+				f(&key);
+			}
+
+			Ok(())
+		};
+
+		if let Err(e) = iter() {
+			debug!(target: "trie", "Error while iterating by prefix: {}", e);
+		}
+	}
+}
+
+pub(crate) struct Ephemeral<'a, S: 'a + TrieBackendStorage<H>, H: 'a + Hasher> {
+	storage: &'a S,
+	overlay: &'a mut MemoryDB<H>,
+}
+
+impl<'a, S: TrieBackendStorage<H>, H: Hasher> AsHashDB<H> for Ephemeral<'a, S, H> where H::Out: HeapSizeOf {
+	fn as_hashdb(&self) -> &HashDB<H> { self }
+	fn as_hashdb_mut(&mut self) -> &mut HashDB<H> { self }
+}
+
+impl<'a, S: 'a + TrieBackendStorage<H>, H: Hasher> Ephemeral<'a, S, H> {
+	pub fn new(storage: &'a S, overlay: &'a mut MemoryDB<H>) -> Self {
+		Ephemeral {
+			storage,
+			overlay,
+		}
+	}
+}
+
+impl<'a, S: TrieBackendStorage<H>, H: Hasher> HashDB<H> for Ephemeral<'a, S, H> where H::Out: HeapSizeOf {
+	fn keys(&self) -> HashMap<H::Out, i32> {
+		self.overlay.keys() // TODO: iterate backing
+	}
+
+	fn get(&self, key: &H::Out) -> Option<DBValue> {
+		match self.overlay.raw(key) {
+			Some((val, i)) => {
+				if i <= 0 {
+					None
+				} else {
+					Some(val)
+				}
+			}
+			None => match self.storage.get(&key) {
+				Ok(x) => x,
+				Err(e) => {
+					warn!(target: "trie", "Failed to read from DB: {}", e);
+					None
+				},
+			},
+		}
+	}
+
+	fn contains(&self, key: &H::Out) -> bool {
+		self.get(key).is_some()
+	}
+
+	fn insert(&mut self, value: &[u8]) -> H::Out {
+		self.overlay.insert(value)
+	}
+
+	fn emplace(&mut self, key: H::Out, value: DBValue) {
+		self.overlay.emplace(key, value)
+	}
+
+	fn remove(&mut self, key: &H::Out) {
+		self.overlay.remove(key)
+	}
+}
+
+/// Key-value pairs storage that is used by trie backend essence.
+pub trait TrieBackendStorage<H: Hasher>: Send + Sync {
+	fn get(&self, key: &H::Out) -> Result<Option<DBValue>, String>;
+}
+
+// This implementation is used by normal storage trie clients.
+impl<H: Hasher> TrieBackendStorage<H> for Arc<Storage<H>> {
+	fn get(&self, key: &H::Out) -> Result<Option<DBValue>, String> {
+		Storage::<H>::get(self.deref(), key)
+	}
+}
+
+// This implementation is used by test storage trie clients.
+impl<H: Hasher> TrieBackendStorage<H> for MemoryDB<H> {
+	fn get(&self, key: &H::Out) -> Result<Option<DBValue>, String> {
+		Ok(HashDB::<H>::get(self, key))
+	}
+}
+
+// This implementation is used by changes trie clients.
+impl<'a, S, H: Hasher> TrieBackendStorage<H> for &'a S where S: ChangesTrieStorage<H> {
+	fn get(&self, key: &H::Out) -> Result<Option<DBValue>, String> {
+		ChangesTrieStorage::<H>::get(*self, key)
+	}
+}