Phase 1 of repo reorg (#719)

* Remove unneeded script

* Rename Substrate Demo -> Substrate

* Rename demo -> node

* Build wasm from last rename.

* Merge ed25519 into substrate-primitives

* Minor tweak

* Rename substrate -> core

* Move substrate-runtime-support to core/runtime/support

* Rename/move substrate-runtime-version

* Move codec up a level

* Rename substrate-codec -> parity-codec

* Move environmental up a level

* Move pwasm-* up to top, ready for removal

* Remove requirement of s-r-support from s-r-primitives

* Move core/runtime/primitives into core/runtime-primitives

* Remove s-r-support dep from s-r-version

* Remove dep of s-r-support from bft

* Remove dep of s-r-support from node/consensus

* Sever all other core deps from s-r-support

* Forgot the no_std directive

* Rename non-SRML modules to sr-* to avoid match clashes

* Move runtime/* to srml/*

* Rename substrate-runtime-* -> srml-*

* Move srml to top-level

substrate/core/state-machine/src/backend.rs

@@ -0,0 +1,193 @@
+// Copyright 2017 Parity Technologies (UK) Ltd.
+// This file is part of Substrate.
+
+// Substrate is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// Substrate is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with Substrate.  If not, see <http://www.gnu.org/licenses/>.
+
+//! State machine backends. These manage the code and storage of contracts.
+
+use std::{error, fmt};
+use std::cmp::Ord;
+use std::collections::HashMap;
+use std::marker::PhantomData;
+use std::sync::Arc;
+
+use hashdb::Hasher;
+use rlp::Encodable;
+use trie_backend::{TryIntoTrieBackend, TrieBackend};
+use patricia_trie::{TrieDBMut, TrieMut, NodeCodec};
+use heapsize::HeapSizeOf;
+
+/// A state backend is used to read state data and can have changes committed
+/// to it.
+///
+/// The clone operation (if implemented) should be cheap.
+pub trait Backend<H: Hasher, C: NodeCodec<H>>: TryIntoTrieBackend<H, C> {
+	/// An error type when fetching data is not possible.
+	type Error: super::Error;
+
+	/// Changes to be applied if committing
+	type Transaction;
+
+	/// Get keyed storage associated with specific address, or None if there is nothing associated.
+	fn storage(&self, key: &[u8]) -> Result<Option<Vec<u8>>, Self::Error>;
+
+	/// true if a key exists in storage.
+	fn exists_storage(&self, key: &[u8]) -> Result<bool, Self::Error> {
+		Ok(self.storage(key)?.is_some())
+	}
+
+	/// Retrieve all entries keys of which start with the given prefix and
+	/// call `f` for each of those keys.
+	fn for_keys_with_prefix<F: FnMut(&[u8])>(&self, prefix: &[u8], f: F);
+
+	/// Calculate the storage root, with given delta over what is already stored in
+	/// the backend, and produce a "transaction" that can be used to commit.
+	fn storage_root<I>(&self, delta: I) -> (H::Out, Self::Transaction)
+	where
+		I: IntoIterator<Item=(Vec<u8>, Option<Vec<u8>>)>,
+		H::Out: Ord + Encodable;
+
+	/// Get all key/value pairs into a Vec.
+	fn pairs(&self) -> Vec<(Vec<u8>, Vec<u8>)>;
+}
+
+/// Error impossible.
+// TODO: use `!` type when stabilized.
+#[derive(Debug)]
+pub enum Void {}
+
+impl fmt::Display for Void {
+	fn fmt(&self, _: &mut fmt::Formatter) -> fmt::Result {
+		match *self {}
+	}
+}
+
+impl error::Error for Void {
+	fn description(&self) -> &str { "unreachable error" }
+}
+
+/// In-memory backend. Fully recomputes tries on each commit but useful for
+/// tests.
+#[derive(Eq)]
+pub struct InMemory<H, C> {
+	inner: Arc<HashMap<Vec<u8>, Vec<u8>>>,
+	_hasher: PhantomData<H>,
+	_codec: PhantomData<C>,
+}
+
+impl<H, C> Default for InMemory<H, C> {
+	fn default() -> Self {
+		InMemory {
+			inner: Arc::new(Default::default()),
+			_hasher: PhantomData,
+			_codec: PhantomData,
+		}
+	}
+}
+
+impl<H, C> Clone for InMemory<H, C> {
+	fn clone(&self) -> Self {
+		InMemory {
+			inner: self.inner.clone(), _hasher: PhantomData, _codec: PhantomData,
+		}
+	}
+}
+
+impl<H, C> PartialEq for InMemory<H, C> {
+	fn eq(&self, other: &Self) -> bool {
+		self.inner.eq(&other.inner)
+	}
+}
+
+impl<H: Hasher, C: NodeCodec<H>> InMemory<H, C> where H::Out: HeapSizeOf {
+	/// Copy the state, with applied updates
+	pub fn update(&self, changes: <Self as Backend<H, C>>::Transaction) -> Self {
+		let mut inner: HashMap<_, _> = (&*self.inner).clone();
+		for (key, val) in changes {
+			match val {
+				Some(v) => { inner.insert(key, v); },
+				None => { inner.remove(&key); },
+			}
+		}
+
+		inner.into()
+	}
+}
+
+impl<H, C> From<HashMap<Vec<u8>, Vec<u8>>> for InMemory<H, C> {
+	fn from(inner: HashMap<Vec<u8>, Vec<u8>>) -> Self {
+		InMemory {
+			inner: Arc::new(inner), _hasher: PhantomData, _codec: PhantomData
+		}
+	}
+}
+
+impl super::Error for Void {}
+
+impl<H: Hasher, C: NodeCodec<H>> Backend<H, C> for InMemory<H, C> where H::Out: HeapSizeOf {
+	type Error = Void;
+	type Transaction = Vec<(Vec<u8>, Option<Vec<u8>>)>;
+
+	fn storage(&self, key: &[u8]) -> Result<Option<Vec<u8>>, Self::Error> {
+		Ok(self.inner.get(key).map(Clone::clone))
+	}
+
+	fn exists_storage(&self, key: &[u8]) -> Result<bool, Self::Error> {
+		Ok(self.inner.get(key).is_some())
+	}
+
+	fn for_keys_with_prefix<F: FnMut(&[u8])>(&self, prefix: &[u8], f: F) {
+		self.inner.keys().filter(|key| key.starts_with(prefix)).map(|k| &**k).for_each(f);
+	}
+
+	fn storage_root<I>(&self, delta: I) -> (H::Out, Self::Transaction)
+	where
+		I: IntoIterator<Item=(Vec<u8>, Option<Vec<u8>>)>,
+		<H as Hasher>::Out: Ord + Encodable,
+	{
+		let existing_pairs = self.inner.iter().map(|(k, v)| (k.clone(), Some(v.clone())));
+
+		let transaction: Vec<_> = delta.into_iter().collect();
+		let root = ::triehash::trie_root::<H, _, _, _>(existing_pairs.chain(transaction.iter().cloned())
+			.collect::<HashMap<_, _>>()
+			.into_iter()
+			.filter_map(|(k, maybe_val)| maybe_val.map(|val| (k, val)))
+		);
+
+		(root, transaction)
+	}
+
+	fn pairs(&self) -> Vec<(Vec<u8>, Vec<u8>)> {
+		self.inner.iter().map(|(k, v)| (k.clone(), v.clone())).collect()
+	}
+}
+
+impl<H: Hasher, C: NodeCodec<H>> TryIntoTrieBackend<H, C> for InMemory<H, C> where H::Out: HeapSizeOf {
+	fn try_into_trie_backend(self) -> Option<TrieBackend<H, C>> {
+		use memorydb::MemoryDB;
+		let mut root = <H as Hasher>::Out::default();
+		let mut mdb = MemoryDB::new();
+		{
+			let mut trie = TrieDBMut::<H, C>::new(&mut mdb, &mut root);
+			for (key, value) in self.inner.iter() {
+				if let Err(e) = trie.insert(&key, &value) {
+					warn!(target: "trie", "Failed to write to trie: {}", e);
+					return None;
+				}
+			}
+		}
+
+		Some(TrieBackend::with_memorydb(mdb, root))
+	}
+}

substrate/core/state-machine/src/ext.rs

@@ -0,0 +1,171 @@
+// Copyright 2017 Parity Technologies (UK) Ltd.
+// This file is part of Substrate.
+
+// Substrate is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// Substrate is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with Substrate.  If not, see <http://www.gnu.org/licenses/>.
+
+//! Conrete externalities implementation.
+
+use std::{error, fmt, cmp::Ord};
+use backend::Backend;
+use {Externalities, OverlayedChanges};
+use hashdb::Hasher;
+use rlp::Encodable;
+use patricia_trie::NodeCodec;
+
+/// Errors that can occur when interacting with the externalities.
+#[derive(Debug, Copy, Clone)]
+pub enum Error<B, E> {
+	/// Failure to load state data from the backend.
+	#[allow(unused)]
+	Backend(B),
+	/// Failure to execute a function.
+	#[allow(unused)]
+	Executor(E),
+}
+
+impl<B: fmt::Display, E: fmt::Display> fmt::Display for Error<B, E> {
+	fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
+		match *self {
+			Error::Backend(ref e) => write!(f, "Storage backend error: {}", e),
+			Error::Executor(ref e) => write!(f, "Sub-call execution error: {}", e),
+		}
+	}
+}
+
+impl<B: error::Error, E: error::Error> error::Error for Error<B, E> {
+	fn description(&self) -> &str {
+		match *self {
+			Error::Backend(..) => "backend error",
+			Error::Executor(..) => "executor error",
+		}
+	}
+}
+
+/// Wraps a read-only backend, call executor, and current overlayed changes.
+pub struct Ext<'a, H, C, B>
+where
+	H: Hasher,
+	C: NodeCodec<H>,
+	B: 'a + Backend<H, C>,
+{
+	// The overlayed changes to write to.
+	overlay: &'a mut OverlayedChanges,
+	// The storage backend to read from.
+	backend: &'a B,
+	// The transaction necessary to commit to the backend.
+	transaction: Option<(B::Transaction, H::Out)>,
+}
+
+impl<'a, H, C, B> Ext<'a, H, C, B>
+where
+	H: Hasher,
+	C: NodeCodec<H>,
+	B: 'a + Backend<H, C>,
+	H::Out: Ord + Encodable
+{
+	/// Create a new `Ext` from overlayed changes and read-only backend
+	pub fn new(overlay: &'a mut OverlayedChanges, backend: &'a B) -> Self {
+		Ext {
+			overlay,
+			backend,
+			transaction: None,
+		}
+	}
+
+	/// Get the transaction necessary to update the backend.
+	pub fn transaction(mut self) -> B::Transaction {
+		let _ = self.storage_root();
+		self.transaction.expect("transaction always set after calling storage root; qed").0
+	}
+
+	/// Invalidates the currently cached storage root and the db transaction.
+	///
+	/// Called when there are changes that likely will invalidate the storage root.
+	fn mark_dirty(&mut self) {
+		self.transaction = None;
+	}
+}
+
+#[cfg(test)]
+impl<'a, H, C, B> Ext<'a, H, C, B>
+where
+	H: Hasher,
+	C: NodeCodec<H>,
+	B: 'a + Backend<H,C>,
+{
+	pub fn storage_pairs(&self) -> Vec<(Vec<u8>, Vec<u8>)> {
+		use std::collections::HashMap;
+
+		self.backend.pairs().iter()
+			.map(|&(ref k, ref v)| (k.to_vec(), Some(v.to_vec())))
+			.chain(self.overlay.committed.clone().into_iter())
+			.chain(self.overlay.prospective.clone().into_iter())
+			.collect::<HashMap<_, _>>()
+			.into_iter()
+			.filter_map(|(k, maybe_val)| maybe_val.map(|val| (k, val)))
+			.collect()
+	}
+}
+
+impl<'a, B: 'a, H, C> Externalities<H> for Ext<'a, H, C, B>
+where
+	H: Hasher,
+	C: NodeCodec<H>,
+	B: 'a + Backend<H, C>,
+	H::Out: Ord + Encodable
+{
+	fn storage(&self, key: &[u8]) -> Option<Vec<u8>> {
+		self.overlay.storage(key).map(|x| x.map(|x| x.to_vec())).unwrap_or_else(||
+			self.backend.storage(key).expect("Externalities not allowed to fail within runtime"))
+	}
+
+	fn exists_storage(&self, key: &[u8]) -> bool {
+		match self.overlay.storage(key) {
+			Some(x) => x.is_some(),
+			_ => self.backend.exists_storage(key).expect("Externalities not allowed to fail within runtime"),
+		}
+	}
+
+	fn place_storage(&mut self, key: Vec<u8>, value: Option<Vec<u8>>) {
+		self.mark_dirty();
+		self.overlay.set_storage(key, value);
+	}
+
+	fn clear_prefix(&mut self, prefix: &[u8]) {
+		self.mark_dirty();
+		self.overlay.clear_prefix(prefix);
+		self.backend.for_keys_with_prefix(prefix, |key| {
+			self.overlay.set_storage(key.to_vec(), None);
+		});
+	}
+
+	fn chain_id(&self) -> u64 {
+		42
+	}
+
+	fn storage_root(&mut self) -> H::Out {
+		if let Some((_, ref root)) = self.transaction {
+			return root.clone();
+		}
+
+		// compute and memoize
+		let delta = self.overlay.committed.iter()
+			.chain(self.overlay.prospective.iter())
+			.map(|(k, v)| (k.clone(), v.clone()));
+
+		let (root, transaction) = self.backend.storage_root(delta);
+		self.transaction = Some((transaction, root));
+		root
+	}
+}

substrate/core/state-machine/src/lib.rs

@@ -0,0 +1,691 @@
+// Copyright 2017 Parity Technologies (UK) Ltd.
+// This file is part of Substrate.
+
+// Substrate is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// Substrate is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with Substrate.  If not, see <http://www.gnu.org/licenses/>.
+
+// tag::description[]
+//! Substrate state machine implementation.
+// end::description[]
+
+#![warn(missing_docs)]
+
+#[cfg_attr(test, macro_use)]
+extern crate hex_literal;
+
+#[macro_use]
+extern crate log;
+
+extern crate hashdb;
+extern crate memorydb;
+extern crate triehash;
+extern crate patricia_trie;
+extern crate byteorder;
+extern crate parking_lot;
+extern crate rlp;
+extern crate heapsize;
+#[cfg(test)]
+extern crate substrate_primitives as primitives;
+extern crate parity_codec as codec;
+
+use std::collections::HashMap;
+use std::fmt;
+use hashdb::Hasher;
+use patricia_trie::NodeCodec;
+use rlp::Encodable;
+use heapsize::HeapSizeOf;
+use codec::Decode;
+
+pub mod backend;
+mod ext;
+mod testing;
+mod proving_backend;
+mod trie_backend;
+
+pub use testing::TestExternalities;
+pub use ext::Ext;
+pub use backend::Backend;
+pub use trie_backend::{TryIntoTrieBackend, TrieBackend, Storage, DBValue};
+
+/// The overlayed changes to state to be queried on top of the backend.
+///
+/// A transaction shares all prospective changes within an inner overlay
+/// that can be cleared.
+#[derive(Debug, Default, Clone)]
+pub struct OverlayedChanges {
+	prospective: HashMap<Vec<u8>, Option<Vec<u8>>>,
+	committed: HashMap<Vec<u8>, Option<Vec<u8>>>,
+}
+
+impl OverlayedChanges {
+	/// Returns a double-Option: None if the key is unknown (i.e. and the query should be refered
+	/// to the backend); Some(None) if the key has been deleted. Some(Some(...)) for a key whose
+	/// value has been set.
+	pub fn storage(&self, key: &[u8]) -> Option<Option<&[u8]>> {
+		self.prospective.get(key)
+			.or_else(|| self.committed.get(key))
+			.map(|x| x.as_ref().map(AsRef::as_ref))
+	}
+
+	/// Inserts the given key-value pair into the prospective change set.
+	///
+	/// `None` can be used to delete a value specified by the given key.
+	fn set_storage(&mut self, key: Vec<u8>, val: Option<Vec<u8>>) {
+		self.prospective.insert(key, val);
+	}
+
+	/// Removes all key-value pairs which keys share the given prefix.
+	///
+	/// NOTE that this doesn't take place immediately but written into the prospective
+	/// change set, and still can be reverted by [`discard_prospective`].
+	///
+	/// [`discard_prospective`]: #method.discard_prospective
+	fn clear_prefix(&mut self, prefix: &[u8]) {
+		// Iterate over all prospective and mark all keys that share
+		// the given prefix as removed (None).
+		for (key, value) in self.prospective.iter_mut() {
+			if key.starts_with(prefix) {
+				*value = None;
+			}
+		}
+
+		// Then do the same with keys from commited changes.
+		// NOTE that we are making changes in the prospective change set.
+		for key in self.committed.keys() {
+			if key.starts_with(prefix) {
+				self.prospective.insert(key.to_owned(), None);
+			}
+		}
+	}
+
+	/// Discard prospective changes to state.
+	pub fn discard_prospective(&mut self) {
+		self.prospective.clear();
+	}
+
+	/// Commit prospective changes to state.
+	pub fn commit_prospective(&mut self) {
+		if self.committed.is_empty() {
+			::std::mem::swap(&mut self.prospective, &mut self.committed);
+		} else {
+			self.committed.extend(self.prospective.drain());
+		}
+	}
+
+	/// Drain committed changes to an iterator.
+	///
+	/// Panics:
+	/// Will panic if there are any uncommitted prospective changes.
+	pub fn drain<'a>(&'a mut self) -> impl Iterator<Item=(Vec<u8>, Option<Vec<u8>>)> + 'a {
+		assert!(self.prospective.is_empty());
+		self.committed.drain()
+	}
+
+	/// Consume `OverlayedChanges` and take committed set.
+	///
+	/// Panics:
+	/// Will panic if there are any uncommitted prospective changes.
+	pub fn into_committed(self) -> impl Iterator<Item=(Vec<u8>, Option<Vec<u8>>)> {
+		assert!(self.prospective.is_empty());
+		self.committed.into_iter()
+	}
+}
+
+/// State Machine Error bound.
+///
+/// This should reflect WASM error type bound for future compatibility.
+pub trait Error: 'static + fmt::Debug + fmt::Display + Send {}
+
+impl Error for ExecutionError {}
+
+/// Externalities Error.
+///
+/// Externalities are not really allowed to have errors, since it's assumed that dependent code
+/// would not be executed unless externalities were available. This is included for completeness,
+/// and as a transition away from the pre-existing framework.
+#[derive(Debug, Eq, PartialEq)]
+pub enum ExecutionError {
+	/// The entry `:code` doesn't exist in storage so there's no way we can execute anything.
+	CodeEntryDoesNotExist,
+	/// Backend is incompatible with execution proof generation process.
+	UnableToGenerateProof,
+	/// Invalid execution proof.
+	InvalidProof,
+}
+
+impl fmt::Display for ExecutionError {
+	fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { write!(f, "Externalities Error") }
+}
+
+/// Externalities: pinned to specific active address.
+pub trait Externalities<H: Hasher> {
+	/// Read storage of current contract being called.
+	fn storage(&self, key: &[u8]) -> Option<Vec<u8>>;
+
+	/// Set storage entry `key` of current contract being called (effective immediately).
+	fn set_storage(&mut self, key: Vec<u8>, value: Vec<u8>) {
+		self.place_storage(key, Some(value));
+	}
+
+	/// Clear a storage entry (`key`) of current contract being called (effective immediately).
+	fn clear_storage(&mut self, key: &[u8]) {
+		self.place_storage(key.to_vec(), None);
+	}
+
+	/// Clear a storage entry (`key`) of current contract being called (effective immediately).
+	fn exists_storage(&self, key: &[u8]) -> bool {
+		self.storage(key).is_some()
+	}
+
+	/// Clear storage entries which keys are start with the given prefix.
+	fn clear_prefix(&mut self, prefix: &[u8]);
+
+	/// Set or clear a storage entry (`key`) of current contract being called (effective immediately).
+	fn place_storage(&mut self, key: Vec<u8>, value: Option<Vec<u8>>);
+
+	/// Get the identity of the chain.
+	fn chain_id(&self) -> u64;
+
+	/// Get the trie root of the current storage map.
+	fn storage_root(&mut self) -> H::Out where H::Out: Ord + Encodable;
+}
+
+/// Code execution engine.
+pub trait CodeExecutor<H: Hasher>: Sized + Send + Sync {
+	/// Externalities error type.
+	type Error: Error;
+
+	/// Call a given method in the runtime. Returns a tuple of the result (either the output data
+	/// or an execution error) together with a `bool`, which is true if native execution was used.
+	fn call<E: Externalities<H>>(
+		&self,
+		ext: &mut E,
+		heap_pages: usize,
+		code: &[u8],
+		method: &str,
+		data: &[u8],
+		use_native: bool
+	) -> (Result<Vec<u8>, Self::Error>, bool);
+}
+
+/// Strategy for executing a call into the runtime.
+#[derive(Copy, Clone, Eq, PartialEq, Debug)]
+pub enum ExecutionStrategy {
+	/// Execute with the native equivalent if it is compatible with the given wasm module; otherwise fall back to the wasm.
+	NativeWhenPossible,
+	/// Use the given wasm module.
+	AlwaysWasm,
+	/// Run with both the wasm and the native variant (if compatible). Report any discrepency as an error.
+	Both,
+}
+
+/// Like `ExecutionStrategy` only it also stores a handler in case of consensus failure.
+pub enum ExecutionManager<F> {
+	/// Execute with the native equivalent if it is compatible with the given wasm module; otherwise fall back to the wasm.
+	NativeWhenPossible,
+	/// Use the given wasm module.
+	AlwaysWasm,
+	/// Run with both the wasm and the native variant (if compatible). Call `F` in the case of any discrepency.
+	Both(F),
+}
+
+impl<'a, F> From<&'a ExecutionManager<F>> for ExecutionStrategy {
+	fn from(s: &'a ExecutionManager<F>) -> Self {
+		match *s {
+			ExecutionManager::NativeWhenPossible => ExecutionStrategy::NativeWhenPossible,
+			ExecutionManager::AlwaysWasm => ExecutionStrategy::AlwaysWasm,
+			ExecutionManager::Both(_) => ExecutionStrategy::Both,
+		}
+	}
+}
+
+/// Evaluate to ExecutionManager::NativeWhenPossible, without having to figure out the type.
+pub fn native_when_possible<E>() -> ExecutionManager<fn(Result<Vec<u8>, E>, Result<Vec<u8>, E>)->Result<Vec<u8>, E>> {
+	ExecutionManager::NativeWhenPossible
+}
+
+/// Evaluate to ExecutionManager::NativeWhenPossible, without having to figure out the type.
+pub fn always_wasm<E>() -> ExecutionManager<fn(Result<Vec<u8>, E>, Result<Vec<u8>, E>)->Result<Vec<u8>, E>> {
+	ExecutionManager::AlwaysWasm
+}
+
+/// Execute a call using the given state backend, overlayed changes, and call executor.
+/// Produces a state-backend-specific "transaction" which can be used to apply the changes
+/// to the backing store, such as the disk.
+///
+/// On an error, no prospective changes are written to the overlay.
+///
+/// Note: changes to code will be in place if this call is made again. For running partial
+/// blocks (e.g. a transaction at a time), ensure a different method is used.
+pub fn execute<H, C, B, Exec>(
+	backend: &B,
+	overlay: &mut OverlayedChanges,
+	exec: &Exec,
+	method: &str,
+	call_data: &[u8],
+	strategy: ExecutionStrategy,
+) -> Result<(Vec<u8>, B::Transaction), Box<Error>>
+where
+	H: Hasher,
+	C: NodeCodec<H>,
+	Exec: CodeExecutor<H>,
+	B: Backend<H, C>,
+	H::Out: Ord + Encodable
+{
+	execute_using_consensus_failure_handler(
+		backend,
+		overlay,
+		exec,
+		method,
+		call_data,
+		match strategy {
+			ExecutionStrategy::AlwaysWasm => ExecutionManager::AlwaysWasm,
+			ExecutionStrategy::NativeWhenPossible => ExecutionManager::NativeWhenPossible,
+			ExecutionStrategy::Both => ExecutionManager::Both(|wasm_result, native_result| {
+				warn!("Consensus error between wasm {:?} and native {:?}. Using wasm.", wasm_result, native_result);
+				wasm_result
+			}),
+		},
+	)
+}
+
+/// Execute a call using the given state backend, overlayed changes, and call executor.
+/// Produces a state-backend-specific "transaction" which can be used to apply the changes
+/// to the backing store, such as the disk.
+///
+/// On an error, no prospective changes are written to the overlay.
+///
+/// Note: changes to code will be in place if this call is made again. For running partial
+/// blocks (e.g. a transaction at a time), ensure a different method is used.
+pub fn execute_using_consensus_failure_handler<H, C, B, Exec, Handler>(
+	backend: &B,
+	overlay: &mut OverlayedChanges,
+	exec: &Exec,
+	method: &str,
+	call_data: &[u8],
+	manager: ExecutionManager<Handler>,
+) -> Result<(Vec<u8>, B::Transaction), Box<Error>>
+where
+	H: Hasher,
+	C: NodeCodec<H>,
+	Exec: CodeExecutor<H>,
+	B: Backend<H, C>,
+	H::Out: Ord + Encodable,
+	Handler: FnOnce(Result<Vec<u8>, Exec::Error>, Result<Vec<u8>, Exec::Error>) -> Result<Vec<u8>, Exec::Error>
+{
+	let strategy: ExecutionStrategy = (&manager).into();
+
+	// make a copy.
+	let code = ext::Ext::new(overlay, backend).storage(b":code")
+		.ok_or_else(|| Box::new(ExecutionError::CodeEntryDoesNotExist) as Box<Error>)?
+		.to_vec();
+
+	let heap_pages = ext::Ext::new(overlay, backend).storage(b":heappages")
+		.and_then(|v| u64::decode(&mut &v[..])).unwrap_or(8) as usize;
+
+	let result = {
+		let mut orig_prospective = overlay.prospective.clone();
+
+		let (result, was_native, delta) = {
+			let ((result, was_native), delta) = {
+				let mut externalities = ext::Ext::new(overlay, backend);
+				(
+					exec.call(
+						&mut externalities,
+						heap_pages,
+						&code,
+						method,
+						call_data,
+						// attempt to run native first, if we're not directed to run wasm only
+						strategy != ExecutionStrategy::AlwaysWasm,
+					),
+					externalities.transaction()
+				)
+			};
+			(result, was_native, delta)
+		};
+
+		// run wasm separately if we did run native the first time and we're meant to run both
+		let (result, delta) = if let (true, ExecutionManager::Both(on_consensus_failure)) =
+			(was_native, manager)
+		{
+			overlay.prospective = orig_prospective.clone();
+
+			let (wasm_result, wasm_delta) = {
+				let ((result, _), delta) = {
+					let mut externalities = ext::Ext::new(overlay, backend);
+					(
+						exec.call(
+							&mut externalities,
+							heap_pages,
+							&code,
+							method,
+							call_data,
+							false,
+						),
+						externalities.transaction()
+					)
+				};
+				(result, delta)
+			};
+
+			if (result.is_ok() && wasm_result.is_ok() && result.as_ref().unwrap() == wasm_result.as_ref().unwrap()/* && delta == wasm_delta*/)
+				|| (result.is_err() && wasm_result.is_err())
+			{
+				(result, delta)
+			} else {
+				// Consensus error.
+				(on_consensus_failure(wasm_result, result), wasm_delta)
+			}
+		} else {
+			(result, delta)
+		};
+		result.map(move |out| (out, delta))
+	};
+
+	result.map_err(|e| Box::new(e) as _)
+}
+
+/// Prove execution using the given state backend, overlayed changes, and call executor.
+/// Produces a state-backend-specific "transaction" which can be used to apply the changes
+/// to the backing store, such as the disk.
+/// Execution proof is the set of all 'touched' storage DBValues from the backend.
+///
+/// On an error, no prospective changes are written to the overlay.
+///
+/// Note: changes to code will be in place if this call is made again. For running partial
+/// blocks (e.g. a transaction at a time), ensure a different method is used.
+pub fn prove_execution<H, C, B, Exec>(
+	backend: B,
+	overlay: &mut OverlayedChanges,
+	exec: &Exec,
+	method: &str,
+	call_data: &[u8],
+) -> Result<(Vec<u8>, Vec<Vec<u8>>, <TrieBackend<H, C> as Backend<H, C>>::Transaction), Box<Error>>
+where
+	H: Hasher,
+	Exec: CodeExecutor<H>,
+	C: NodeCodec<H>,
+	B: TryIntoTrieBackend<H, C>,
+	H::Out: Ord + Encodable + HeapSizeOf,
+{
+	let trie_backend = backend.try_into_trie_backend()
+		.ok_or_else(|| Box::new(ExecutionError::UnableToGenerateProof) as Box<Error>)?;
+	let proving_backend = proving_backend::ProvingBackend::new(trie_backend);
+	let (result, transaction) = execute::<H, C, _, _>(&proving_backend, overlay, exec, method, call_data, ExecutionStrategy::NativeWhenPossible)?;
+	let proof = proving_backend.extract_proof();
+	Ok((result, proof, transaction))
+}
+
+/// Check execution proof, generated by `prove_execution` call.
+pub fn execution_proof_check<H, C, Exec>(
+	root: H::Out,
+	proof: Vec<Vec<u8>>,
+	overlay: &mut OverlayedChanges,
+	exec: &Exec,
+	method: &str,
+	call_data: &[u8],
+) -> Result<(Vec<u8>, memorydb::MemoryDB<H>), Box<Error>>
+where
+H: Hasher,
+C: NodeCodec<H>,
+Exec: CodeExecutor<H>,
+H::Out: Ord + Encodable + HeapSizeOf,
+{
+	let backend = proving_backend::create_proof_check_backend::<H, C>(root.into(), proof)?;
+	execute::<H, C, _, _>(&backend, overlay, exec, method, call_data, ExecutionStrategy::NativeWhenPossible)
+}
+
+/// Generate storage read proof.
+pub fn prove_read<B, H, C>(
+	backend: B,
+	key: &[u8]
+) -> Result<(Option<Vec<u8>>, Vec<Vec<u8>>), Box<Error>>
+where
+	B: TryIntoTrieBackend<H, C>,
+	H: Hasher,
+	C: NodeCodec<H>,
+	H::Out: Ord + Encodable + HeapSizeOf
+{
+	let trie_backend = backend.try_into_trie_backend()
+		.ok_or_else(|| Box::new(ExecutionError::UnableToGenerateProof) as Box<Error>)?;
+	let proving_backend = proving_backend::ProvingBackend::<H, C>::new(trie_backend);
+	let result = proving_backend.storage(key).map_err(|e| Box::new(e) as Box<Error>)?;
+	Ok((result, proving_backend.extract_proof()))
+}
+
+/// Check storage read proof, generated by `prove_read` call.
+pub fn read_proof_check<H, C>(
+	root: H::Out,
+	proof: Vec<Vec<u8>>,
+	key: &[u8],
+) -> Result<Option<Vec<u8>>, Box<Error>>
+where
+	H: Hasher,
+	C: NodeCodec<H>,
+	H::Out: Ord + Encodable + HeapSizeOf
+{
+	let backend = proving_backend::create_proof_check_backend::<H, C>(root, proof)?;
+	backend.storage(key).map_err(|e| Box::new(e) as Box<Error>)
+}
+
+#[cfg(test)]
+mod tests {
+	use super::*;
+	use super::backend::InMemory;
+	use super::ext::Ext;
+	use primitives::{Blake2Hasher, RlpCodec, H256};
+
+	struct DummyCodeExecutor {
+		native_available: bool,
+		native_succeeds: bool,
+		fallback_succeeds: bool,
+	}
+
+	impl<H: Hasher> CodeExecutor<H> for DummyCodeExecutor {
+		type Error = u8;
+
+		fn call<E: Externalities<H>>(
+			&self,
+			ext: &mut E,
+			_heap_pages: usize,
+			_code: &[u8],
+			_method: &str,
+			_data: &[u8],
+			use_native: bool
+		) -> (Result<Vec<u8>, Self::Error>, bool) {
+			let using_native = use_native && self.native_available;
+			match (using_native, self.native_succeeds, self.fallback_succeeds) {
+				(true, true, _) | (false, _, true) =>
+					(Ok(vec![ext.storage(b"value1").unwrap()[0] + ext.storage(b"value2").unwrap()[0]]), using_native),
+				_ => (Err(0), using_native),
+			}
+		}
+	}
+
+	impl Error for u8 {}
+
+	#[test]
+	fn overlayed_storage_works() {
+		let mut overlayed = OverlayedChanges::default();
+
+		let key = vec![42, 69, 169, 142];
+
+		assert!(overlayed.storage(&key).is_none());
+
+		overlayed.set_storage(key.clone(), Some(vec![1, 2, 3]));
+		assert_eq!(overlayed.storage(&key).unwrap(), Some(&[1, 2, 3][..]));
+
+		overlayed.commit_prospective();
+		assert_eq!(overlayed.storage(&key).unwrap(), Some(&[1, 2, 3][..]));
+
+		overlayed.set_storage(key.clone(), Some(vec![]));
+		assert_eq!(overlayed.storage(&key).unwrap(), Some(&[][..]));
+
+		overlayed.set_storage(key.clone(), None);
+		assert!(overlayed.storage(&key).unwrap().is_none());
+
+		overlayed.discard_prospective();
+		assert_eq!(overlayed.storage(&key).unwrap(), Some(&[1, 2, 3][..]));
+
+		overlayed.set_storage(key.clone(), None);
+		overlayed.commit_prospective();
+		assert!(overlayed.storage(&key).unwrap().is_none());
+	}
+
+	macro_rules! map {
+		($( $name:expr => $value:expr ),*) => (
+			vec![ $( ( $name, $value ) ),* ].into_iter().collect()
+		)
+	}
+
+	#[test]
+	fn overlayed_storage_root_works() {
+		let initial: HashMap<_, _> = map![
+			b"doe".to_vec() => b"reindeer".to_vec(),
+			b"dog".to_vec() => b"puppyXXX".to_vec(),
+			b"dogglesworth".to_vec() => b"catXXX".to_vec(),
+			b"doug".to_vec() => b"notadog".to_vec()
+		];
+		let backend = InMemory::<Blake2Hasher, RlpCodec>::from(initial);
+		let mut overlay = OverlayedChanges {
+			committed: map![
+				b"dog".to_vec() => Some(b"puppy".to_vec()),
+				b"dogglesworth".to_vec() => Some(b"catYYY".to_vec()),
+				b"doug".to_vec() => Some(vec![])
+			],
+			prospective: map![
+				b"dogglesworth".to_vec() => Some(b"cat".to_vec()),
+				b"doug".to_vec() => None
+			],
+		};
+		let mut ext = Ext::new(&mut overlay, &backend);
+		const ROOT: [u8; 32] = hex!("6ca394ff9b13d6690a51dea30b1b5c43108e52944d30b9095227c49bae03ff8b");
+		assert_eq!(ext.storage_root(), H256(ROOT));
+	}
+
+	#[test]
+	fn execute_works() {
+		assert_eq!(execute(
+			&trie_backend::tests::test_trie(),
+			&mut Default::default(),
+			&DummyCodeExecutor {
+				native_available: true,
+				native_succeeds: true,
+				fallback_succeeds: true,
+			},
+			"test",
+			&[],
+			ExecutionStrategy::NativeWhenPossible
+		).unwrap().0, vec![66]);
+	}
+
+	#[test]
+	fn dual_execution_strategy_detects_consensus_failure() {
+		let mut consensus_failed = false;
+		assert!(execute_using_consensus_failure_handler(
+			&trie_backend::tests::test_trie(),
+			&mut Default::default(),
+			&DummyCodeExecutor {
+				native_available: true,
+				native_succeeds: true,
+				fallback_succeeds: false,
+			},
+			"test",
+			&[],
+			ExecutionManager::Both(|we, _ne| {
+				consensus_failed = true;
+				println!("HELLO!");
+				we
+			}),
+		).is_err());
+		assert!(consensus_failed);
+	}
+
+	#[test]
+	fn prove_execution_and_proof_check_works() {
+		let executor = DummyCodeExecutor {
+			native_available: true,
+			native_succeeds: true,
+			fallback_succeeds: true,
+		};
+
+		// fetch execution proof from 'remote' full node
+		let remote_backend = trie_backend::tests::test_trie();
+		let remote_root = remote_backend.storage_root(::std::iter::empty()).0;
+		let (remote_result, remote_proof, _) = prove_execution(remote_backend,
+			&mut Default::default(), &executor, "test", &[]).unwrap();
+
+		// check proof locally
+		let (local_result, _) = execution_proof_check::<Blake2Hasher, RlpCodec,_,>(remote_root, remote_proof,
+			&mut Default::default(), &executor, "test", &[]).unwrap();
+
+		// check that both results are correct
+		assert_eq!(remote_result, vec![66]);
+		assert_eq!(remote_result, local_result);
+	}
+
+	#[test]
+	fn clear_prefix_in_ext_works() {
+		let initial: HashMap<_, _> = map![
+			b"aaa".to_vec() => b"0".to_vec(),
+			b"abb".to_vec() => b"1".to_vec(),
+			b"abc".to_vec() => b"2".to_vec(),
+			b"bbb".to_vec() => b"3".to_vec()
+		];
+		let backend = InMemory::<Blake2Hasher, RlpCodec>::from(initial).try_into_trie_backend().unwrap();
+		let mut overlay = OverlayedChanges {
+			committed: map![
+				b"aba".to_vec() => Some(b"1312".to_vec()),
+				b"bab".to_vec() => Some(b"228".to_vec())
+			],
+			prospective: map![
+				b"abd".to_vec() => Some(b"69".to_vec()),
+				b"bbd".to_vec() => Some(b"42".to_vec())
+			],
+		};
+
+		{
+			let mut ext = Ext::new(&mut overlay, &backend);
+			ext.clear_prefix(b"ab");
+		}
+		overlay.commit_prospective();
+
+		assert_eq!(
+			overlay.committed,
+			map![
+				b"abb".to_vec() => None,
+				b"abc".to_vec() => None,
+				b"aba".to_vec() => None,
+				b"abd".to_vec() => None,
+
+				b"bab".to_vec() => Some(b"228".to_vec()),
+				b"bbd".to_vec() => Some(b"42".to_vec())
+			],
+		);
+	}
+
+	#[test]
+	fn prove_read_and_proof_check_works() {
+		// fetch read proof from 'remote' full node
+		let remote_backend = trie_backend::tests::test_trie();
+		let remote_root = remote_backend.storage_root(::std::iter::empty()).0;
+		let remote_proof = prove_read(remote_backend, b"value2").unwrap().1;
+ 		// check proof locally
+		let local_result1 = read_proof_check::<Blake2Hasher, RlpCodec>(remote_root, remote_proof.clone(), b"value2").unwrap();
+		let local_result2 = read_proof_check::<Blake2Hasher, RlpCodec>(remote_root, remote_proof.clone(), &[0xff]).is_ok();
+ 		// check that results are correct
+		assert_eq!(local_result1, Some(vec![24]));
+		assert_eq!(local_result2, false);
+	}
+}

substrate/core/state-machine/src/proving_backend.rs

@@ -0,0 +1,183 @@
+// Copyright 2017 Parity Technologies (UK) Ltd.
+// This file is part of Substrate.
+
+// Substrate is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// Substrate is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with Substrate.  If not, see <http://www.gnu.org/licenses/>.
+
+//! Proving state machine backend.
+
+use std::cell::RefCell;
+use hashdb::{Hasher, HashDB};
+use memorydb::MemoryDB;
+use patricia_trie::{TrieDB, Trie, Recorder, NodeCodec};
+use trie_backend::{TrieBackend, Ephemeral};
+use {Error, ExecutionError, Backend, TryIntoTrieBackend};
+use rlp::Encodable;
+use heapsize::HeapSizeOf;
+
+/// Patricia trie-based backend which also tracks all touched storage trie values.
+/// These can be sent to remote node and used as a proof of execution.
+pub struct ProvingBackend<H: Hasher, C: NodeCodec<H>> {
+	backend: TrieBackend<H, C>,
+	proof_recorder: RefCell<Recorder<H::Out>>,
+}
+
+impl<H: Hasher, C: NodeCodec<H>> ProvingBackend<H, C> {
+	/// Create new proving backend.
+	pub fn new(backend: TrieBackend<H, C>) -> Self {
+		ProvingBackend {
+			backend,
+			proof_recorder: RefCell::new(Recorder::new()),
+		}
+	}
+
+	/// Consume the backend, extracting the gathered proof in lexicographical order
+	/// by value.
+	pub fn extract_proof(self) -> Vec<Vec<u8>> {
+		self.proof_recorder.into_inner().drain()
+			.into_iter()
+			.map(|n| n.data.to_vec())
+			.collect()
+	}
+}
+
+impl<H, C> Backend<H, C> for ProvingBackend<H, C>
+where
+	H: Hasher,
+	C: NodeCodec<H>,
+	H::Out: Ord + Encodable + HeapSizeOf
+{
+	type Error = String;
+	type Transaction = MemoryDB<H>;
+
+	fn storage(&self, key: &[u8]) -> Result<Option<Vec<u8>>, Self::Error> {
+		let mut read_overlay = MemoryDB::new();
+		let eph = Ephemeral::new(
+			self.backend.backend_storage(),
+			&mut read_overlay,
+		);
+		let map_e = |e| format!("Trie lookup error: {}", e);
+
+		let mut proof_recorder = self.proof_recorder.try_borrow_mut()
+			.expect("only fails when already borrowed; storage() is non-reentrant; qed");
+		TrieDB::<H, C>::new(&eph, &self.backend.root()).map_err(map_e)?
+			.get_with(key, &mut *proof_recorder).map(|x| x.map(|val| val.to_vec())).map_err(map_e)
+	}
+
+	fn for_keys_with_prefix<F: FnMut(&[u8])>(&self, prefix: &[u8], f: F) {
+		self.backend.for_keys_with_prefix(prefix, f)
+	}
+
+	fn pairs(&self) -> Vec<(Vec<u8>, Vec<u8>)> {
+		self.backend.pairs()
+	}
+
+	fn storage_root<I>(&self, delta: I) -> (H::Out, MemoryDB<H>)
+		where I: IntoIterator<Item=(Vec<u8>, Option<Vec<u8>>)>
+	{
+		self.backend.storage_root(delta)
+	}
+}
+
+impl<H: Hasher, C: NodeCodec<H>> TryIntoTrieBackend<H, C> for ProvingBackend<H, C> {
+	fn try_into_trie_backend(self) -> Option<TrieBackend<H, C>> {
+		None
+	}
+}
+
+/// Create proof check backend.
+pub fn create_proof_check_backend<H, C>(
+	root: H::Out,
+	proof: Vec<Vec<u8>>
+) -> Result<TrieBackend<H, C>, Box<Error>>
+where
+	H: Hasher,
+	C: NodeCodec<H>,
+	H::Out: HeapSizeOf,
+{
+	let mut db = MemoryDB::new();
+	for item in proof {
+		db.insert(&item);
+	}
+
+	if !db.contains(&root) {
+		return Err(Box::new(ExecutionError::InvalidProof) as Box<Error>);
+	}
+
+
+	Ok(TrieBackend::with_memorydb(db, root))
+}
+
+#[cfg(test)]
+mod tests {
+	use backend::{InMemory};
+	use trie_backend::tests::test_trie;
+	use super::*;
+	use primitives::{Blake2Hasher, RlpCodec};
+
+	fn test_proving() -> ProvingBackend<Blake2Hasher, RlpCodec> {
+		ProvingBackend::new(test_trie())
+	}
+
+	#[test]
+	fn proof_is_empty_until_value_is_read() {
+		assert!(test_proving().extract_proof().is_empty());
+	}
+
+	#[test]
+	fn proof_is_non_empty_after_value_is_read() {
+		let backend = test_proving();
+		assert_eq!(backend.storage(b"key").unwrap(), Some(b"value".to_vec()));
+		assert!(!backend.extract_proof().is_empty());
+	}
+
+	#[test]
+	fn proof_is_invalid_when_does_not_contains_root() {
+		assert!(create_proof_check_backend::<Blake2Hasher, RlpCodec>(1.into(), vec![]).is_err());
+	}
+
+	#[test]
+	fn passes_throgh_backend_calls() {
+		let trie_backend = test_trie();
+		let proving_backend = test_proving();
+		assert_eq!(trie_backend.storage(b"key").unwrap(), proving_backend.storage(b"key").unwrap());
+		assert_eq!(trie_backend.pairs(), proving_backend.pairs());
+
+		let (trie_root, mut trie_mdb) = trie_backend.storage_root(::std::iter::empty());
+		let (proving_root, mut proving_mdb) = proving_backend.storage_root(::std::iter::empty());
+		assert_eq!(trie_root, proving_root);
+		assert_eq!(trie_mdb.drain(), proving_mdb.drain());
+	}
+
+	#[test]
+	fn proof_recorded_and_checked() {
+		let contents = (0..64).map(|i| (vec![i], Some(vec![i]))).collect::<Vec<_>>();
+		let in_memory = InMemory::<Blake2Hasher, RlpCodec>::default();
+		let in_memory = in_memory.update(contents);
+		let in_memory_root = in_memory.storage_root(::std::iter::empty()).0;
+		(0..64).for_each(|i| assert_eq!(in_memory.storage(&[i]).unwrap().unwrap(), vec![i]));
+
+		let trie = in_memory.try_into_trie_backend().unwrap();
+		let trie_root = trie.storage_root(::std::iter::empty()).0;
+		assert_eq!(in_memory_root, trie_root);
+		(0..64).for_each(|i| assert_eq!(trie.storage(&[i]).unwrap().unwrap(), vec![i]));
+
+		let proving = ProvingBackend::new(trie);
+		assert_eq!(proving.storage(&[42]).unwrap().unwrap(), vec![42]);
+
+		let proof = proving.extract_proof();
+
+		let proof_check = create_proof_check_backend::<Blake2Hasher, RlpCodec>(in_memory_root.into(), proof).unwrap();
+		assert_eq!(proof_check.storage(&[42]).unwrap().unwrap(), vec![42]);
+	}
+}

substrate/core/state-machine/src/testing.rs

@@ -0,0 +1,118 @@
+// Copyright 2017 Parity Technologies (UK) Ltd.
+// This file is part of Substrate.
+
+// Substrate is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// Substrate is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with Substrate.  If not, see <http://www.gnu.org/licenses/>.
+
+//! Test implementation for Externalities.
+
+use std::collections::HashMap;
+use std::cmp::Ord;
+use super::Externalities;
+use triehash::trie_root;
+use hashdb::Hasher;
+use rlp::Encodable;
+use std::marker::PhantomData;
+use std::iter::FromIterator;
+
+/// Simple HashMap-based Externalities impl.
+#[derive(Debug)]
+pub struct TestExternalities<H> {
+	inner: HashMap<Vec<u8>, Vec<u8>>,
+	_hasher: PhantomData<H>,
+}
+
+impl<H: Hasher> TestExternalities<H> {
+	/// Create a new instance of `TestExternalities`
+	pub fn new() -> Self {
+		TestExternalities {inner: HashMap::new(), _hasher: PhantomData}
+	}
+	/// Insert key/value
+	pub fn insert(&mut self, k: Vec<u8>, v: Vec<u8>) -> Option<Vec<u8>> {
+		self.inner.insert(k, v)
+	}
+}
+
+impl<H: Hasher> PartialEq for TestExternalities<H> {
+	fn eq(&self, other: &TestExternalities<H>) -> bool {
+		self.inner.eq(&other.inner)
+	}
+}
+
+impl<H: Hasher> FromIterator<(Vec<u8>, Vec<u8>)> for TestExternalities<H> {
+	fn from_iter<I: IntoIterator<Item=(Vec<u8>, Vec<u8>)>>(iter: I) -> Self {
+		let mut t = Self::new();
+		for i in iter {
+			t.inner.insert(i.0, i.1);
+		}
+		t
+	}
+}
+
+impl<H: Hasher> Default for TestExternalities<H> {
+	fn default() -> Self { Self::new() }
+}
+
+impl<H: Hasher> From<TestExternalities<H>> for HashMap<Vec<u8>, Vec<u8>> {
+	fn from(tex: TestExternalities<H>) -> Self {
+		tex.inner.into()
+	}
+}
+
+impl<H: Hasher> From< HashMap<Vec<u8>, Vec<u8>> > for TestExternalities<H> {
+	fn from(hashmap: HashMap<Vec<u8>, Vec<u8>>) -> Self {
+		TestExternalities { inner: hashmap, _hasher: PhantomData }
+	}
+}
+
+
+impl<H: Hasher> Externalities<H> for TestExternalities<H> where H::Out: Ord + Encodable {
+	fn storage(&self, key: &[u8]) -> Option<Vec<u8>> {
+		self.inner.get(key).map(|x| x.to_vec())
+	}
+
+	fn place_storage(&mut self, key: Vec<u8>, maybe_value: Option<Vec<u8>>) {
+		match maybe_value {
+			Some(value) => { self.inner.insert(key, value); }
+			None => { self.inner.remove(&key); }
+		}
+	}
+
+	fn clear_prefix(&mut self, prefix: &[u8]) {
+		self.inner.retain(|key, _|
+			!key.starts_with(prefix)
+		)
+	}
+
+	fn chain_id(&self) -> u64 { 42 }
+
+	fn storage_root(&mut self) -> H::Out {
+		trie_root::<H, _, _, _>(self.inner.clone())
+	}
+}
+
+#[cfg(test)]
+mod tests {
+	use super::*;
+	use primitives::{Blake2Hasher, H256};
+
+	#[test]
+	fn commit_should_work() {
+		let mut ext = TestExternalities::<Blake2Hasher>::new();
+		ext.set_storage(b"doe".to_vec(), b"reindeer".to_vec());
+		ext.set_storage(b"dog".to_vec(), b"puppy".to_vec());
+		ext.set_storage(b"dogglesworth".to_vec(), b"cat".to_vec());
+		const ROOT: [u8; 32] = hex!("6ca394ff9b13d6690a51dea30b1b5c43108e52944d30b9095227c49bae03ff8b");
+		assert_eq!(ext.storage_root(), H256(ROOT));
+	}
+}

substrate/core/state-machine/src/trie_backend.rs

@@ -0,0 +1,362 @@
+// Copyright 2017 Parity Technologies (UK) Ltd.
+// This file is part of Substrate.
+
+// Substrate is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// Substrate is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with Substrate.  If not, see <http://www.gnu.org/licenses/>.
+
+//! Trie-based state machine backend.
+use Backend;
+use hashdb::{Hasher, HashDB, AsHashDB};
+use memorydb::MemoryDB;
+use patricia_trie::{TrieDB, TrieDBMut, TrieError, Trie, TrieMut, NodeCodec};
+use std::collections::HashMap;
+use std::sync::Arc;
+use std::marker::PhantomData;
+use heapsize::HeapSizeOf;
+
+pub use hashdb::DBValue;
+
+/// Backend trie storage trait.
+pub trait Storage<H: Hasher>: Send + Sync {
+	/// Get a trie node.
+	fn get(&self, key: &H::Out) -> Result<Option<DBValue>, String>;
+}
+
+/// Try convert into trie-based backend.
+pub trait TryIntoTrieBackend<H: Hasher, C: NodeCodec<H>> {
+	/// Try to convert self into trie backend.
+	fn try_into_trie_backend(self) -> Option<TrieBackend<H, C>>;
+}
+
+/// Patricia trie-based backend. Transaction type is an overlay of changes to commit.
+#[derive(Clone)]
+pub struct TrieBackend<H: Hasher, C: NodeCodec<H>> {
+	storage: TrieBackendStorage<H>,
+	root: H::Out,
+	_codec: PhantomData<C>
+}
+
+impl<H: Hasher, C: NodeCodec<H>> TrieBackend<H, C> where H::Out: HeapSizeOf {
+	/// Create new trie-based backend.
+	pub fn with_storage(db: Arc<Storage<H>>, root: H::Out) -> Self {
+		TrieBackend {
+			storage: TrieBackendStorage::Storage(db),
+			root,
+			_codec: PhantomData,
+		}
+	}
+
+	/// Create new trie-based backend for genesis block.
+	pub fn with_storage_for_genesis(db: Arc<Storage<H>>) -> Self {
+		let mut root = <H as Hasher>::Out::default();
+		let mut mdb = MemoryDB::<H>::new();
+		TrieDBMut::<H, C>::new(&mut mdb, &mut root);
+
+		Self::with_storage(db, root)
+	}
+
+	/// Create new trie-based backend backed by MemoryDb storage.
+	pub fn with_memorydb(db: MemoryDB<H>, root: H::Out) -> Self {
+		// TODO: check that root is a part of db???
+		TrieBackend {
+			storage: TrieBackendStorage::MemoryDb(db),
+			root,
+			_codec: PhantomData,
+		}
+	}
+
+	/// Get backend storage reference.
+	pub fn backend_storage(&self) -> &TrieBackendStorage<H> {
+		&self.storage
+	}
+
+	/// Get trie root.
+	pub fn root(&self) -> &H::Out {
+		&self.root
+	}
+}
+
+impl super::Error for String {}
+
+impl<H: Hasher, C: NodeCodec<H>> Backend<H, C> for TrieBackend<H, C> where H::Out: HeapSizeOf {
+	type Error = String;
+	type Transaction = MemoryDB<H>;
+
+	fn storage(&self, key: &[u8]) -> Result<Option<Vec<u8>>, Self::Error> {
+		let mut read_overlay = MemoryDB::new();
+		let eph = Ephemeral {
+			storage: &self.storage,
+			overlay: &mut read_overlay,
+		};
+
+		let map_e = |e| format!("Trie lookup error: {}", e);
+		TrieDB::<H, C>::new(&eph, &self.root).map_err(map_e)?
+			.get(key).map(|x| x.map(|val| val.to_vec())).map_err(map_e)
+	}
+
+	fn for_keys_with_prefix<F: FnMut(&[u8])>(&self, prefix: &[u8], mut f: F) {
+		let mut read_overlay = MemoryDB::new();
+		let eph = Ephemeral {
+			storage: &self.storage,
+			overlay: &mut read_overlay,
+		};
+
+		let mut iter = move || -> Result<(), Box<TrieError<H::Out, C::Error>>> {
+			let trie = TrieDB::<H, C>::new(&eph, &self.root)?;
+			let mut iter = trie.iter()?;
+
+			iter.seek(prefix)?;
+
+			for x in iter {
+				let (key, _) = x?;
+
+				if !key.starts_with(prefix) {
+					break;
+				}
+
+				f(&key);
+			}
+
+			Ok(())
+		};
+
+		if let Err(e) = iter() {
+			debug!(target: "trie", "Error while iterating by prefix: {}", e);
+		}
+	}
+
+	fn pairs(&self) -> Vec<(Vec<u8>, Vec<u8>)> {
+		let mut read_overlay = MemoryDB::new();
+		let eph = Ephemeral {
+			storage: &self.storage,
+			overlay: &mut read_overlay,
+		};
+
+		let collect_all = || -> Result<_, Box<TrieError<H::Out, C::Error>>> {
+			let trie = TrieDB::<H, C>::new(&eph, &self.root)?;
+			let mut v = Vec::new();
+			for x in trie.iter()? {
+				let (key, value) = x?;
+				v.push((key.to_vec(), value.to_vec()));
+			}
+
+			Ok(v)
+		};
+
+		match collect_all() {
+			Ok(v) => v,
+			Err(e) => {
+				debug!(target: "trie", "Error extracting trie values: {}", e);
+				Vec::new()
+			}
+		}
+	}
+
+	fn storage_root<I>(&self, delta: I) -> (H::Out, MemoryDB<H>)
+		where I: IntoIterator<Item=(Vec<u8>, Option<Vec<u8>>)>
+	{
+		let mut write_overlay = MemoryDB::new();
+		let mut root = self.root;
+		{
+			let mut eph = Ephemeral {
+				storage: &self.storage,
+				overlay: &mut write_overlay,
+			};
+
+			let mut trie = TrieDBMut::<H, C>::from_existing(&mut eph, &mut root).expect("prior state root to exist"); // TODO: handle gracefully
+			for (key, change) in delta {
+				let result = match change {
+					Some(val) => trie.insert(&key, &val),
+					None => trie.remove(&key), // TODO: archive mode
+				};
+
+				if let Err(e) = result {
+					warn!(target: "trie", "Failed to write to trie: {}", e);
+				}
+			}
+		}
+
+		(root, write_overlay)
+	}
+}
+
+impl<H: Hasher, C: NodeCodec<H>> TryIntoTrieBackend<H, C> for TrieBackend<H, C> {
+	fn try_into_trie_backend(self) -> Option<TrieBackend<H, C>> {
+		Some(self)
+	}
+}
+
+pub struct Ephemeral<'a, H: 'a + Hasher> {
+	storage: &'a TrieBackendStorage<H>,
+	overlay: &'a mut MemoryDB<H>,
+}
+
+impl<'a, H: Hasher> AsHashDB<H> for Ephemeral<'a, H> where H::Out: HeapSizeOf {
+	fn as_hashdb(&self) -> &HashDB<H> { self }
+	fn as_hashdb_mut(&mut self) -> &mut HashDB<H> { self }
+}
+
+impl<'a, H: Hasher> Ephemeral<'a, H> {
+	pub fn new(storage: &'a TrieBackendStorage<H>, overlay: &'a mut MemoryDB<H>) -> Self {
+		Ephemeral {
+			storage,
+			overlay,
+		}
+	}
+}
+
+impl<'a, H: Hasher> HashDB<H> for Ephemeral<'a, H> where H::Out: HeapSizeOf {
+	fn keys(&self) -> HashMap<H::Out, i32> {
+		self.overlay.keys() // TODO: iterate backing
+	}
+
+	fn get(&self, key: &H::Out) -> Option<DBValue> {
+		match self.overlay.raw(key) {
+			Some((val, i)) => {
+				if i <= 0 {
+					None
+				} else {
+					Some(val)
+				}
+			}
+			None => match self.storage.get(key) {
+				Ok(x) => x,
+				Err(e) => {
+					warn!(target: "trie", "Failed to read from DB: {}", e);
+					None
+				},
+			},
+		}
+	}
+
+	fn contains(&self, key: &H::Out) -> bool {
+		self.get(key).is_some()
+	}
+
+	fn insert(&mut self, value: &[u8]) -> H::Out {
+		self.overlay.insert(value)
+	}
+
+	fn emplace(&mut self, key: H::Out, value: DBValue) {
+		self.overlay.emplace(key, value)
+	}
+
+	fn remove(&mut self, key: &H::Out) {
+		self.overlay.remove(key)
+	}
+}
+
+#[derive(Clone)]
+pub enum TrieBackendStorage<H: Hasher> {
+	/// Key value db + storage column.
+	Storage(Arc<Storage<H>>),
+	/// Hash db.
+	MemoryDb(MemoryDB<H>),
+}
+
+impl<H: Hasher> TrieBackendStorage<H> {
+	pub fn get(&self, key: &H::Out) -> Result<Option<DBValue>, String> {
+		match *self {
+			TrieBackendStorage::Storage(ref db) =>
+				db.get(key)
+					.map_err(|e| format!("Trie lookup error: {}", e)),
+			TrieBackendStorage::MemoryDb(ref db) =>
+				Ok(db.get(key)),
+		}
+	}
+}
+
+#[cfg(test)]
+pub mod tests {
+	use super::*;
+	use std::collections::HashSet;
+	use primitives::{Blake2Hasher, RlpCodec, H256};
+
+	fn test_db() -> (MemoryDB<Blake2Hasher>, H256) {
+		let mut root = H256::default();
+		let mut mdb = MemoryDB::<Blake2Hasher>::new();
+		{
+			let mut trie = TrieDBMut::<_, RlpCodec>::new(&mut mdb, &mut root);
+			trie.insert(b"key", b"value").expect("insert failed");
+			trie.insert(b"value1", &[42]).expect("insert failed");
+			trie.insert(b"value2", &[24]).expect("insert failed");
+			trie.insert(b":code", b"return 42").expect("insert failed");
+			for i in 128u8..255u8 {
+				trie.insert(&[i], &[i]).unwrap();
+			}
+		}
+		(mdb, root)
+	}
+
+	pub(crate) fn test_trie() -> TrieBackend<Blake2Hasher, RlpCodec> {
+		let (mdb, root) = test_db();
+		TrieBackend::with_memorydb(mdb, root)
+	}
+
+	#[test]
+	fn read_from_storage_returns_some() {
+		assert_eq!(test_trie().storage(b"key").unwrap(), Some(b"value".to_vec()));
+	}
+
+	#[test]
+	fn read_from_storage_returns_none() {
+		assert_eq!(test_trie().storage(b"non-existing-key").unwrap(), None);
+	}
+
+	#[test]
+	fn pairs_are_not_empty_on_non_empty_storage() {
+		assert!(!test_trie().pairs().is_empty());
+	}
+
+	#[test]
+	fn pairs_are_empty_on_empty_storage() {
+		let db = TrieBackend::<Blake2Hasher, RlpCodec>::with_memorydb(
+			MemoryDB::new(),
+			Default::default()
+		);
+		assert!(db.pairs().is_empty());
+	}
+
+	#[test]
+	fn storage_root_is_non_default() {
+		assert!(test_trie().storage_root(::std::iter::empty()).0 != H256([0; 32]));
+	}
+
+	#[test]
+	fn storage_root_transaction_is_empty() {
+		assert!(test_trie().storage_root(::std::iter::empty()).1.drain().is_empty());
+	}
+
+	#[test]
+	fn storage_root_transaction_is_non_empty() {
+		let (new_root, mut tx) = test_trie().storage_root(vec![(b"new-key".to_vec(), Some(b"new-value".to_vec()))]);
+		assert!(!tx.drain().is_empty());
+		assert!(new_root != test_trie().storage_root(::std::iter::empty()).0);
+	}
+
+	#[test]
+	fn prefix_walking_works() {
+		let trie = test_trie();
+
+		let mut seen = HashSet::new();
+		trie.for_keys_with_prefix(b"value", |key| {
+			let for_first_time = seen.insert(key.to_vec());
+			assert!(for_first_time, "Seen key '{:?}' more than once", key);
+		});
+
+		let mut expected = HashSet::new();
+		expected.insert(b"value1".to_vec());
+		expected.insert(b"value2".to_vec());
+		assert_eq!(seen, expected);
+	}
+}