Trie-based execution proof (#177)

* TrieBasedBackend

* trie tests

* redunant return_value removed

* use Trie::get_with to record trie proofs

substrate/Cargo.lock

@@ -2509,7 +2509,14 @@ name = "substrate-state-machine"
 version = "0.1.0"
 dependencies = [
  "byteorder 1.2.3 (registry+https://github.com/rust-lang/crates.io-index)",
+ "ethereum-types 0.3.2 (registry+https://github.com/rust-lang/crates.io-index)",
+ "hashdb 0.1.1 (git+https://github.com/paritytech/parity.git)",
  "hex-literal 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)",
+ "kvdb 0.1.0 (git+https://github.com/paritytech/parity.git)",
+ "log 0.3.9 (registry+https://github.com/rust-lang/crates.io-index)",
+ "memorydb 0.1.1 (git+https://github.com/paritytech/parity.git)",
+ "parking_lot 0.4.8 (registry+https://github.com/rust-lang/crates.io-index)",
+ "patricia-trie 0.1.0 (git+https://github.com/paritytech/parity.git)",
  "substrate-primitives 0.1.0",
  "triehash 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)",
 ]

substrate/substrate/client/db/src/lib.rs

@@ -17,13 +17,11 @@
 //! Client backend that uses RocksDB database as storage. State is still kept in memory.
 
 extern crate substrate_client as client;
-extern crate ethereum_types;
 extern crate kvdb_rocksdb;
 extern crate kvdb;
 extern crate hashdb;
 extern crate memorydb;
 extern crate parking_lot;
-extern crate patricia_trie;
 extern crate substrate_state_machine as state_machine;
 extern crate substrate_primitives as primitives;
 extern crate substrate_runtime_support as runtime_support;
@@ -38,22 +36,22 @@ extern crate kvdb_memorydb;
 
 use std::sync::Arc;
 use std::path::PathBuf;
-use std::collections::HashMap;
 
 use codec::Slicable;
-use ethereum_types::H256 as TrieH256;
-use hashdb::{DBValue, HashDB};
+use hashdb::DBValue;
 use kvdb_rocksdb::{Database, DatabaseConfig};
 use kvdb::{KeyValueDB, DBTransaction};
 use memorydb::MemoryDB;
 use parking_lot::RwLock;
-use patricia_trie::{TrieDB, TrieDBMut, TrieError, Trie, TrieMut};
 use runtime_primitives::generic::BlockId;
 use runtime_primitives::bft::Justification;
 use runtime_primitives::traits::{Block as BlockT, Header as HeaderT, As, Hashing, HashingFor, Zero};
 use state_machine::backend::Backend as StateBackend;
 use state_machine::CodeExecutor;
 
+/// DB-backed patricia trie state, transaction type is an overlay of changes to commit.
+pub type DbState = state_machine::TrieBackend;
+
 /// Database settings.
 pub struct DatabaseSettings {
 	/// Cache size in bytes. If `None` default is used.
@@ -300,133 +298,6 @@ impl<Block: BlockT> client::backend::BlockImportOperation<Block> for BlockImport
 	}
 }
 
-struct Ephemeral<'a> {
-	backing: &'a KeyValueDB,
-	overlay: &'a mut MemoryDB,
-}
-
-impl<'a> HashDB for Ephemeral<'a> {
-	fn keys(&self) -> HashMap<TrieH256, i32> {
-		self.overlay.keys() // TODO: iterate backing
-	}
-
-	fn get(&self, key: &TrieH256) -> Option<DBValue> {
-		match self.overlay.raw(key) {
-			Some((val, i)) => {
-				if i <= 0 {
-					None
-				} else {
-					Some(val)
-				}
-			}
-			None => {
-				match self.backing.get(::columns::STATE, &key.0[..]) {
-					Ok(x) => x,
-					Err(e) => {
-						warn!("Failed to read from DB: {}", e);
-						None
-					}
-				}
-			}
-		}
-	}
-
-	fn contains(&self, key: &TrieH256) -> bool {
-		self.get(key).is_some()
-	}
-
-	fn insert(&mut self, value: &[u8]) -> TrieH256 {
-		self.overlay.insert(value)
-	}
-
-	fn emplace(&mut self, key: TrieH256, value: DBValue) {
-		self.overlay.emplace(key, value)
-	}
-
-	fn remove(&mut self, key: &TrieH256) {
-		self.overlay.remove(key)
-	}
-}
-
-/// DB-backed patricia trie state, transaction type is an overlay of changes to commit.
-#[derive(Clone)]
-pub struct DbState {
-	db: Arc<KeyValueDB>,
-	root: TrieH256,
-}
-
-impl state_machine::Backend for DbState {
-	type Error = client::error::Error;
-	type Transaction = MemoryDB;
-
-	fn storage(&self, key: &[u8]) -> Result<Option<Vec<u8>>, Self::Error> {
-		let mut read_overlay = MemoryDB::default();
-		let eph = Ephemeral {
-			backing: &*self.db,
-			overlay: &mut read_overlay,
-		};
-
-		let map_e = |e: Box<TrieError>| ::client::error::Error::from(format!("Trie lookup error: {}", e));
-
-		TrieDB::new(&eph, &self.root).map_err(map_e)?
-			.get(key).map(|x| x.map(|val| val.to_vec())).map_err(map_e)
-	}
-
-	fn pairs(&self) -> Vec<(Vec<u8>, Vec<u8>)> {
-		let mut read_overlay = MemoryDB::default();
-		let eph = Ephemeral {
-			backing: &*self.db,
-			overlay: &mut read_overlay,
-		};
-
-		let collect_all = || -> Result<_, Box<TrieError>> {
-			let trie = TrieDB::new(&eph, &self.root)?;
-			let mut v = Vec::new();
-			for x in trie.iter()? {
-				let (key, value) = x?;
-				v.push((key.to_vec(), value.to_vec()));
-			}
-
-			Ok(v)
-		};
-
-		match collect_all() {
-			Ok(v) => v,
-			Err(e) => {
-				debug!("Error extracting trie values: {}", e);
-				Vec::new()
-			}
-		}
-	}
-
-	fn storage_root<I>(&self, delta: I) -> ([u8; 32], MemoryDB)
-		where I: IntoIterator<Item=(Vec<u8>, Option<Vec<u8>>)>
-	{
-		let mut write_overlay = MemoryDB::default();
-		let mut root = self.root;
-		{
-			let mut eph = Ephemeral {
-				backing: &*self.db,
-				overlay: &mut write_overlay,
-			};
-
-			let mut trie = TrieDBMut::from_existing(&mut eph, &mut root).expect("prior state root to exist"); // TODO: handle gracefully
-			for (key, change) in delta {
-				let result = match change {
-					Some(val) => trie.insert(&key, &val),
-					None => trie.remove(&key), // TODO: archive mode
-				};
-
-				if let Err(e) = result {
-					warn!("Failed to write to trie: {}", e);
-				}
-			}
-		}
-
-		(root.0.into(), write_overlay)
-	}
-}
-
 /// Disk backend. Keeps data in a key-value store. In archive mode, trie nodes are kept from all blocks.
 /// Otherwise, trie nodes are kept only from the most recent block.
 pub struct Backend<Block: BlockT> {
@@ -522,25 +393,14 @@ impl<Block: BlockT> client::backend::Backend<Block> for Backend<Block> where
 
 		// special case for genesis initialization
 		match block {
-			BlockId::Hash(h) if h == Default::default() => {
-				let mut root = TrieH256::default();
-				let mut db = MemoryDB::default();
-				TrieDBMut::new(&mut db, &mut root);
-
-				return Ok(DbState {
-					db: self.db.clone(),
-					root,
-				})
-			}
+			BlockId::Hash(h) if h == Default::default() =>
+				return Ok(DbState::with_kvdb_for_genesis(self.db.clone(), ::columns::STATE)),
 			_ => {}
 		}
 
 		self.blockchain.header(block).and_then(|maybe_hdr| maybe_hdr.map(|hdr| {
 			let root: [u8; 32] = hdr.state_root().clone().into();
-			DbState {
-				db: self.db.clone(),
-				root: root.into(),
-			}
+			DbState::with_kvdb(self.db.clone(), ::columns::STATE, root.into())
 		}).ok_or_else(|| client::error::ErrorKind::UnknownBlock(format!("{:?}", block)).into()))
 	}
 }
@@ -552,6 +412,7 @@ impl<Block: BlockT> client::backend::LocalBackend<Block> for Backend<Block> wher
 
 #[cfg(test)]
 mod tests {
+	use hashdb::HashDB;
 	use super::*;
 	use client::backend::Backend as BTrait;
 	use client::backend::BlockImportOperation as Op;

substrate/substrate/client/src/call_executor.rs

@@ -17,9 +17,8 @@
 use std::sync::Arc;
 use futures::{IntoFuture, Future};
 use runtime_primitives::generic::BlockId;
-use runtime_primitives::traits::Block as BlockT;
+use runtime_primitives::traits::{Block as BlockT, Header as HeaderT};
 use state_machine::{self, OverlayedChanges, Backend as StateBackend, CodeExecutor};
-use state_machine::backend::InMemory as InMemoryStateBackend;
 
 use backend;
 use blockchain::Backend as ChainBackend;
@@ -49,6 +48,11 @@ pub trait CallExecutor<B: BlockT> {
 	///
 	/// No changes are made.
 	fn call_at_state<S: state_machine::Backend>(&self, state: &S, overlay: &mut OverlayedChanges, method: &str, call_data: &[u8]) -> Result<(Vec<u8>, S::Transaction), error::Error>;
+
+	/// Execute a call to a contract on top of given state, gathering execution proof.
+	///
+	/// No changes are made.
+	fn prove_at_state<S: state_machine::Backend>(&self, state: S, overlay: &mut OverlayedChanges, method: &str, call_data: &[u8]) -> Result<(Vec<u8>, Vec<Vec<u8>>), error::Error>;
 }
 
 /// Call executor that executes methods locally, querying all required
@@ -105,6 +109,18 @@ impl<B, E, Block> CallExecutor<Block> for LocalCallExecutor<B, E>
 			call_data,
 		).map_err(Into::into)
 	}
+
+	fn prove_at_state<S: state_machine::Backend>(&self, state: S, changes: &mut OverlayedChanges, method: &str, call_data: &[u8]) -> Result<(Vec<u8>, Vec<Vec<u8>>), error::Error> {
+		state_machine::prove(
+			state,
+			changes,
+			&self.executor,
+			method,
+			call_data,
+		)
+		.map(|(result, proof, _)| (result, proof))
+		.map_err(Into::into)
+	}
 }
 
 impl<B, F> RemoteCallExecutor<B, F> {
@@ -140,69 +156,70 @@ impl<B, F, Block> CallExecutor<Block> for RemoteCallExecutor<B, F>
 	fn call_at_state<S: state_machine::Backend>(&self, _state: &S, _changes: &mut OverlayedChanges, _method: &str, _call_data: &[u8]) -> error::Result<(Vec<u8>, S::Transaction)> {
 		Err(error::ErrorKind::NotAvailableOnLightClient.into())
 	}
+
+	fn prove_at_state<S: state_machine::Backend>(&self, _state: S, _changes: &mut OverlayedChanges, _method: &str, _call_data: &[u8]) -> Result<(Vec<u8>, Vec<Vec<u8>>), error::Error> {
+		Err(error::ErrorKind::NotAvailableOnLightClient.into())
+	}
 }
 
-/// Check remote execution proof.
-pub fn check_execution_proof<B, E, Block>(backend: &B, executor: &E, request: &RemoteCallRequest<Block::Hash>, remote_proof: (Vec<u8>, Vec<Vec<u8>>)) -> Result<CallResult, error::Error>
+/// Check remote execution proof using given backend.
+pub fn check_execution_proof<B, E, Block>(backend: &B, executor: &E, request: &RemoteCallRequest<Block::Hash>, remote_proof: Vec<Vec<u8>>) -> Result<CallResult, error::Error>
 	where
 		B: backend::RemoteBackend<Block>,
 		E: CodeExecutor,
 		Block: BlockT,
+		<<Block as BlockT>::Header as HeaderT>::Hash: Into<[u8; 32]>, // TODO: remove when patricia_trie generic.
 		error::Error: From<<<B as backend::Backend<Block>>::State as StateBackend>::Error>,
 {
-	use runtime_primitives::traits::{Header, Hashing, HashingFor};
-
-	let (remote_result, remote_proof) = remote_proof;
-
-	let remote_state = state_from_execution_proof(remote_proof);
-	let remote_state_root = HashingFor::<Block>::trie_root(remote_state.pairs().into_iter());
-
 	let local_header = backend.blockchain().header(BlockId::Hash(request.block))?;
-	let local_header = local_header.ok_or_else(|| error::ErrorKind::UnknownBlock(format!("{:?}", request.block)))?;
+	let local_header = local_header.ok_or_else(|| error::ErrorKind::UnknownBlock(format!("{}", request.block)))?;
 	let local_state_root = local_header.state_root().clone();
+	do_check_execution_proof(local_state_root, executor, request, remote_proof)
+}
 
-	if remote_state_root != local_state_root {
-		return Err(error::ErrorKind::InvalidExecutionProof.into());
-	}
-
+/// Check remote execution proof using given state root.
+fn do_check_execution_proof<E, H>(local_state_root: H, executor: &E, request: &RemoteCallRequest<H>, remote_proof: Vec<Vec<u8>>) -> Result<CallResult, error::Error>
+	where
+		E: CodeExecutor,
+		H: Into<[u8; 32]>, // TODO: remove when patricia_trie generic.
+{
 	let mut changes = OverlayedChanges::default();
-	let (local_result, _) = state_machine::execute(
-		&remote_state,
+	let (local_result, _) = state_machine::proof_check(
+		local_state_root.into(),
+		remote_proof,
 		&mut changes,
 		executor,
 		&request.method,
-		&request.call_data,
-	)?;
-
-	if local_result != remote_result {
-		return Err(error::ErrorKind::InvalidExecutionProof.into());
-	}
+		&request.call_data)?;
 
 	Ok(CallResult { return_data: local_result, changes })
 }
 
-/// Convert state to execution proof. Proof is simple the whole state (temporary).
-// TODO [light]: this method must be removed after trie-based proofs are landed.
-pub fn state_to_execution_proof<B: state_machine::Backend>(state: &B) -> Vec<Vec<u8>> {
-	state.pairs().into_iter()
-		.flat_map(|(k, v)| ::std::iter::once(k).chain(::std::iter::once(v)))
-		.collect()
-}
+#[cfg(test)]
+mod tests {
+	use runtime_primitives::generic::BlockId;
+	use state_machine::Backend;
+	use test_client;
+	use light::RemoteCallRequest;
+	use super::do_check_execution_proof;
 
-/// Convert execution proof to in-memory state for check. Reverse function for state_to_execution_proof.
-// TODO [light]: this method must be removed after trie-based proofs are landed.
-fn state_from_execution_proof(proof: Vec<Vec<u8>>) -> InMemoryStateBackend {
-	let mut changes = Vec::new();
-	let mut proof_iter = proof.into_iter();
-	loop {
-		let key = proof_iter.next();
-		let value = proof_iter.next();
-		if let (Some(key), Some(value)) = (key, value) {
-			changes.push((key, Some(value)));
-		} else {
-			break;
-		}
+	#[test]
+	fn execution_proof_is_generated_and_checked() {
+		// prepare remote client
+		let remote_client = test_client::new();
+		let remote_block_id = BlockId::Number(0);
+		let remote_block_storage_root = remote_client.state_at(&remote_block_id)
+			.unwrap().storage_root(::std::iter::empty()).0;
+
+		// 'fetch' execution proof from remote node
+		let remote_execution_proof = remote_client.execution_proof(&remote_block_id, "authorities", &[]).unwrap().1;
+
+		// check remote execution proof locally
+		let local_executor = test_client::NativeExecutor::new();
+		do_check_execution_proof(remote_block_storage_root, &local_executor, &RemoteCallRequest {
+			block: Default::default(),
+			method: "authorities".into(),
+			call_data: vec![],
+		}, remote_execution_proof).unwrap();
 	}
-
-	InMemoryStateBackend::default().update(changes)
 }

substrate/substrate/client/src/client.rs

@@ -232,12 +232,7 @@ impl<B, E, Block: BlockT> Client<B, E, Block> where
 	///
 	/// No changes are made.
 	pub fn execution_proof(&self, id: &BlockId<Block>, method: &str, call_data: &[u8]) -> error::Result<(Vec<u8>, Vec<Vec<u8>>)> {
-		use call_executor::state_to_execution_proof;
-
-		let result = self.executor.call(id, method, call_data);
-		let result = result?.return_data;
-		let proof = self.backend.state_at(*id).map(|state| state_to_execution_proof(&state))?;
-		Ok((result, proof))
+		self.state_at(id).and_then(|state| self.executor.prove_at_state(state, &mut Default::default(), method, call_data))
 	}
 
 	/// Set up the native execution environment to call into a native runtime code.

substrate/substrate/client/src/light.rs

@@ -19,7 +19,8 @@
 
 use std::sync::Arc;
 use futures::future::IntoFuture;
-use state_machine::CodeExecutor;
+use state_machine::{CodeExecutor, TryIntoTrieBackend as TryIntoStateTrieBackend,
+	TrieBackend as StateTrieBackend};
 use state_machine::backend::Backend as StateBackend;
 use runtime_primitives::generic::BlockId;
 use runtime_primitives::bft::Justification;
@@ -54,7 +55,7 @@ pub trait Fetcher<B: BlockT>: Send + Sync {
 /// Light client remote data checker.
 pub trait FetchChecker<B: BlockT>: Send + Sync {
 	/// Check remote method execution proof.
-	fn check_execution_proof(&self, request: &RemoteCallRequest<B::Hash>, remote_proof: (Vec<u8>, Vec<Vec<u8>>)) -> error::Result<CallResult>;
+	fn check_execution_proof(&self, request: &RemoteCallRequest<B::Hash>, remote_proof: Vec<Vec<u8>>) -> error::Result<CallResult>;
 }
 
 /// Light client backend.
@@ -202,12 +203,19 @@ impl<H: Clone> StateBackend for OnDemandState<H> {
 	}
 }
 
+impl<H> TryIntoStateTrieBackend for OnDemandState<H> {
+	fn try_into_trie_backend(self) -> Option<StateTrieBackend> {
+		None
+	}
+}
+
 impl<E, B> FetchChecker<B> for LightDataChecker<E, B>
 	where
 		E: CodeExecutor,
 		B: BlockT,
+		<<B as BlockT>::Header as HeaderT>::Hash: Into<[u8; 32]>, // TODO: remove when patricia_trie generic.
 {
-	fn check_execution_proof(&self, request: &RemoteCallRequest<B::Hash>, remote_proof: (Vec<u8>, Vec<Vec<u8>>)) -> error::Result<CallResult> {
+	fn check_execution_proof(&self, request: &RemoteCallRequest<B::Hash>, remote_proof: Vec<Vec<u8>>) -> error::Result<CallResult> {
 		check_execution_proof(&*self.backend, &self.executor, request, remote_proof)
 	}
 }

substrate/substrate/network/src/message.rs

@@ -162,8 +162,6 @@ pub enum Direction {
 pub struct RemoteCallResponse {
 	/// Id of a request this response was made for.
 	pub id: RequestId,
-	/// Method return value.
-	pub value: Vec<u8>,
 	/// Execution proof.
 	pub proof: Vec<Vec<u8>>,
 }

substrate/substrate/network/src/on_demand.rs

@@ -160,7 +160,7 @@ impl<B, E> OnDemandService for OnDemand<B, E> where
 	fn on_remote_response(&self, io: &mut SyncIo, peer: PeerId, response: message::RemoteCallResponse) {
 		let mut core = self.core.lock();
 		match core.remove(peer, response.id) {
-			Some(request) => match self.checker.check_execution_proof(&request.request, (response.value, response.proof)) {
+			Some(request) => match self.checker.check_execution_proof(&request.request, response.proof) {
 				Ok(response) => {
 					// we do not bother if receiver has been dropped already
 					let _ = request.sender.send(response);
@@ -312,10 +312,10 @@ mod tests {
 	}
 
 	impl FetchChecker<Block> for DummyFetchChecker {
-		fn check_execution_proof(&self, _request: &RemoteCallRequest<Hash>, remote_proof: (Vec<u8>, Vec<Vec<u8>>)) -> client::error::Result<client::CallResult> {
+		fn check_execution_proof(&self, _request: &RemoteCallRequest<Hash>, _remote_proof: Vec<Vec<u8>>) -> client::error::Result<client::CallResult> {
 			match self.ok {
 				true => Ok(client::CallResult {
-					return_data: remote_proof.0,
+					return_data: vec![42],
 					changes: Default::default(),
 				}),
 				false => Err(client::error::ErrorKind::Backend("Test error".into()).into()),
@@ -338,7 +338,6 @@ mod tests {
 	fn receive_response(on_demand: &OnDemand<Block, DummyExecutor>, network: &mut TestIo, peer: PeerId, id: message::RequestId) {
 		on_demand.on_remote_response(network, peer, message::RemoteCallResponse {
 			id: id,
-			value: vec![1],
 			proof: vec![vec![2]],
 		});
 	}
@@ -431,7 +430,7 @@ mod tests {
 		let response = on_demand.remote_call(RemoteCallRequest { block: Default::default(), method: "test".into(), call_data: vec![] });
 		let thread = ::std::thread::spawn(move || {
 			let result = response.wait().unwrap();
-			assert_eq!(result.return_data, vec![1]);
+			assert_eq!(result.return_data, vec![42]);
 		});
 
 		receive_response(&*on_demand, &mut network, 0, 0);

substrate/substrate/network/src/protocol.rs

@@ -507,17 +507,17 @@ impl<B: BlockT> Protocol<B> where
 
 	fn on_remote_call_request(&self, io: &mut SyncIo, peer_id: PeerId, request: message::RemoteCallRequest<B::Hash>) {
 		trace!(target: "sync", "Remote request {} from {} ({} at {})", request.id, peer_id, request.method, request.block);
-		let (value, proof) = match self.chain.execution_proof(&request.block, &request.method, &request.data) {
-			Ok((value, proof)) => (value, proof),
+		let proof = match self.chain.execution_proof(&request.block, &request.method, &request.data) {
+			Ok((_, proof)) => proof,
 			Err(error) => {
 				trace!(target: "sync", "Remote request {} from {} ({} at {}) failed with: {}",
 					request.id, peer_id, request.method, request.block, error);
-				(Default::default(), Default::default())
+				Default::default()
 			},
 		};
 
 		self.send_message(io, peer_id, GenericMessage::RemoteCallResponse(message::RemoteCallResponse {
-			id: request.id, value, proof,
+			id: request.id, proof,
 		}));
 	}
 

substrate/substrate/state-machine/Cargo.toml

@@ -5,7 +5,16 @@ authors = ["Parity Technologies <admin@parity.io>"]
 description = "Substrate State Machine"
 
 [dependencies]
-substrate-primitives = { path = "../primitives", version = "0.1.0" }
-triehash = "0.1.2"
 byteorder = "1.1"
+ethereum-types = "0.3"
 hex-literal = "0.1.0"
+log = "0.3"
+parking_lot = "0.4"
+triehash = "0.1"
+
+substrate-primitives = { path = "../primitives", version = "0.1.0" }
+
+hashdb = { git = "https://github.com/paritytech/parity.git" }
+kvdb = { git = "https://github.com/paritytech/parity.git" }
+memorydb = { git = "https://github.com/paritytech/parity.git" }
+patricia-trie = { git = "https://github.com/paritytech/parity.git" }

substrate/substrate/state-machine/src/backend.rs

@@ -19,12 +19,13 @@
 use std::{error, fmt};
 use std::collections::HashMap;
 use std::sync::Arc;
+use trie_backend::{TryIntoTrieBackend, TrieBackend};
 
 /// A state backend is used to read state data and can have changes committed
 /// to it.
 ///
-/// The clone operation should be cheap.
-pub trait Backend: Clone {
+/// The clone operation (if implemented) should be cheap.
+pub trait Backend: TryIntoTrieBackend {
 	/// An error type when fetching data is not possible.
 	type Error: super::Error;
 
@@ -124,3 +125,24 @@ impl Backend for InMemory {
 	}
 }
 
+impl TryIntoTrieBackend for InMemory {
+	fn try_into_trie_backend(self) -> Option<TrieBackend> {
+		use ethereum_types::H256 as TrieH256;
+		use memorydb::MemoryDB;
+		use patricia_trie::{TrieDBMut, TrieMut};
+
+		let mut root = TrieH256::default();
+		let mut mdb = MemoryDB::default();
+		{
+			let mut trie = TrieDBMut::new(&mut mdb, &mut root);
+			for (key, value) in self.inner.iter() {
+				if let Err(e) = trie.insert(&key, &value) {
+					warn!(target: "trie", "Failed to write to trie: {}", e);
+					return None;
+				}
+			}
+		}
+
+		Some(TrieBackend::with_memorydb(mdb, root))
+	}
+}

substrate/substrate/state-machine/src/lib.rs

@@ -20,8 +20,19 @@
 
 #[cfg_attr(test, macro_use)]
 extern crate hex_literal;
+
+#[macro_use]
+extern crate log;
+
+extern crate ethereum_types;
+extern crate kvdb;
+extern crate hashdb;
+extern crate memorydb;
 extern crate triehash;
+extern crate patricia_trie;
+
 extern crate byteorder;
+extern crate parking_lot;
 
 use std::collections::HashMap;
 use std::collections::hash_map::Drain;
@@ -30,10 +41,13 @@ use std::fmt;
 pub mod backend;
 mod ext;
 mod testing;
+mod proving_backend;
+mod trie_backend;
 
 pub use testing::TestExternalities;
 pub use ext::Ext;
 pub use backend::Backend;
+pub use trie_backend::{TryIntoTrieBackend, TrieBackend};
 
 /// The overlayed changes to state to be queried on top of the backend.
 ///
@@ -93,7 +107,11 @@ impl<E> Error for E where E: 'static + fmt::Debug + fmt::Display + Send {}
 #[derive(Debug, Eq, PartialEq)]
 pub enum ExecutionError {
 	/// The entry `:code` doesn't exist in storage so there's no way we can execute anything.
-	CodeEntryDoesNotExist
+	CodeEntryDoesNotExist,
+	/// Backend is incompatible with execution proof generation process.
+	UnableToGenerateProof,
+	/// Invalid execution proof.
+	InvalidProof,
 }
 
 impl fmt::Display for ExecutionError {
@@ -156,7 +174,6 @@ pub fn execute<B: backend::Backend, Exec: CodeExecutor>(
 	call_data: &[u8],
 ) -> Result<(Vec<u8>, B::Transaction), Box<Error>>
 {
-
 	let result = {
 		let mut externalities = ext::Ext::new(overlay, backend);
 		// make a copy.
@@ -184,12 +201,67 @@ pub fn execute<B: backend::Backend, Exec: CodeExecutor>(
 	}
 }
 
+/// Prove execution using the given state backend, overlayed changes, and call executor.
+/// Produces a state-backend-specific "transaction" which can be used to apply the changes
+/// to the backing store, such as the disk.
+/// Execution proof is the set of all 'touched' storage DBValues from the backend.
+///
+/// On an error, no prospective changes are written to the overlay.
+///
+/// Note: changes to code will be in place if this call is made again. For running partial
+/// blocks (e.g. a transaction at a time), ensure a different method is used.
+pub fn prove<B: TryIntoTrieBackend, Exec: CodeExecutor>(
+	backend: B,
+	overlay: &mut OverlayedChanges,
+	exec: &Exec,
+	method: &str,
+	call_data: &[u8],
+) -> Result<(Vec<u8>, Vec<Vec<u8>>, <TrieBackend as Backend>::Transaction), Box<Error>>
+{
+	let trie_backend = backend.try_into_trie_backend()
+		.ok_or_else(|| Box::new(ExecutionError::UnableToGenerateProof) as Box<Error>)?;
+	let proving_backend = proving_backend::ProvingBackend::new(trie_backend);
+	let (result, transaction) = execute(&proving_backend, overlay, exec, method, call_data)?;
+	let proof = proving_backend.extract_proof();
+	Ok((result, proof, transaction))
+}
+
+/// Check execution proof, generated by `prove` call.
+pub fn proof_check<Exec: CodeExecutor>(
+	root: [u8; 32],
+	proof: Vec<Vec<u8>>,
+	overlay: &mut OverlayedChanges,
+	exec: &Exec,
+	method: &str,
+	call_data: &[u8],
+) -> Result<(Vec<u8>, memorydb::MemoryDB), Box<Error>>
+{
+	let backend = proving_backend::create_proof_check_backend(root.into(), proof)?;
+	execute(&backend, overlay, exec, method, call_data)
+}
+
 #[cfg(test)]
 mod tests {
 	use super::*;
 	use super::backend::InMemory;
 	use super::ext::Ext;
 
+	struct DummyCodeExecutor;
+
+	impl CodeExecutor for DummyCodeExecutor {
+		type Error = u8;
+
+		fn call<E: Externalities>(
+			&self,
+			ext: &mut E,
+			_code: &[u8],
+			_method: &str,
+			_data: &[u8],
+		) -> Result<Vec<u8>, Self::Error> {
+			Ok(vec![ext.storage(b"value1").unwrap()[0] + ext.storage(b"value2").unwrap()[0]])
+		}
+	}
+
 	#[test]
 	fn overlayed_storage_works() {
 		let mut overlayed = OverlayedChanges::default();
@@ -248,4 +320,27 @@ mod tests {
 		const ROOT: [u8; 32] = hex!("8aad789dff2f538bca5d8ea56e8abe10f4c7ba3a5dea95fea4cd6e7c3a1168d3");
 		assert_eq!(ext.storage_root(), ROOT);
 	}
+
+	#[test]
+	fn execute_works() {
+		assert_eq!(execute(&trie_backend::tests::test_trie(),
+			&mut Default::default(), &DummyCodeExecutor, "test", &[]).unwrap().0, vec![66]);
+	}
+
+	#[test]
+	fn prove_and_proof_check_works() {
+		// fetch execution proof from 'remote' full node
+		let remote_backend = trie_backend::tests::test_trie();
+		let remote_root = remote_backend.storage_root(::std::iter::empty()).0;
+		let (remote_result, remote_proof, _) = prove(remote_backend,
+			&mut Default::default(), &DummyCodeExecutor, "test", &[]).unwrap();
+
+		// check proof locally
+		let (local_result, _) = proof_check(remote_root, remote_proof,
+			&mut Default::default(), &DummyCodeExecutor, "test", &[]).unwrap();
+
+		// check that both results are correct
+		assert_eq!(remote_result, vec![66]);
+		assert_eq!(remote_result, local_result);
+	}
 }

substrate/substrate/state-machine/src/proving_backend.rs

@@ -0,0 +1,165 @@
+// Copyright 2017 Parity Technologies (UK) Ltd.
+// This file is part of Substrate.
+
+// Substrate is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// Substrate is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with Substrate.  If not, see <http://www.gnu.org/licenses/>.
+
+//! Proving state machine backend.
+
+use std::cell::RefCell;
+use ethereum_types::H256 as TrieH256;
+use hashdb::HashDB;
+use memorydb::MemoryDB;
+use patricia_trie::{TrieDB, TrieError, Trie, Recorder};
+use trie_backend::{TrieBackend, Ephemeral};
+use {Error, ExecutionError, Backend, TryIntoTrieBackend};
+
+/// Patricia trie-based backend which also tracks all touched storage trie values.
+/// These can be sent to remote node and used as a proof of execution.
+pub struct ProvingBackend {
+	backend: TrieBackend,
+	proof_recorder: RefCell<Recorder>,
+}
+
+impl ProvingBackend {
+	/// Create new proving backend.
+	pub fn new(backend: TrieBackend) -> Self {
+		ProvingBackend {
+			backend,
+			proof_recorder: RefCell::new(Recorder::new()),
+		}
+	}
+
+	/// Consume the backend, extracting the gathered proof in lexicographical order
+	/// by value.
+	pub fn extract_proof(self) -> Vec<Vec<u8>> {
+		self.proof_recorder.into_inner().drain()
+			.into_iter()
+			.map(|n| n.data.to_vec())
+			.collect()
+	}
+}
+
+impl Backend for ProvingBackend {
+	type Error = String;
+	type Transaction = MemoryDB;
+
+	fn storage(&self, key: &[u8]) -> Result<Option<Vec<u8>>, Self::Error> {
+		let mut read_overlay = MemoryDB::default();
+		let eph = Ephemeral::new(
+			self.backend.backend_storage(),
+			&mut read_overlay,
+		);
+
+		let map_e = |e: Box<TrieError>| format!("Trie lookup error: {}", e);
+
+		let mut proof_recorder = self.proof_recorder.try_borrow_mut()
+			.expect("only fails when already borrowed; storage() is non-reentrant; qed");
+		TrieDB::new(&eph, &self.backend.root()).map_err(map_e)?
+			.get_with(key, &mut *proof_recorder).map(|x| x.map(|val| val.to_vec())).map_err(map_e)
+	}
+
+	fn pairs(&self) -> Vec<(Vec<u8>, Vec<u8>)> {
+		self.backend.pairs()
+	}
+
+	fn storage_root<I>(&self, delta: I) -> ([u8; 32], MemoryDB)
+		where I: IntoIterator<Item=(Vec<u8>, Option<Vec<u8>>)>
+	{
+		self.backend.storage_root(delta)
+	}
+}
+
+impl TryIntoTrieBackend for ProvingBackend {
+	fn try_into_trie_backend(self) -> Option<TrieBackend> {
+		None
+	}
+}
+
+/// Create proof check backend.
+pub fn create_proof_check_backend(root: TrieH256, proof: Vec<Vec<u8>>) -> Result<TrieBackend, Box<Error>> {
+	let mut db = MemoryDB::new();
+	for item in proof {
+		db.insert(&item);
+	}
+
+	if !db.contains(&root) {
+		return Err(Box::new(ExecutionError::InvalidProof) as Box<Error>);
+	}
+
+
+	Ok(TrieBackend::with_memorydb(db, root))
+}
+
+#[cfg(test)]
+mod tests {
+	use backend::{InMemory};
+	use trie_backend::tests::test_trie;
+	use super::*;
+
+	fn test_proving() -> ProvingBackend {
+		ProvingBackend::new(test_trie())
+	}
+
+	#[test]
+	fn proof_is_empty_until_value_is_read() {
+		assert!(test_proving().extract_proof().is_empty());
+	}
+
+	#[test]
+	fn proof_is_non_empty_after_value_is_read() {
+		let backend = test_proving();
+		assert_eq!(backend.storage(b"key").unwrap(), Some(b"value".to_vec()));
+		assert!(!backend.extract_proof().is_empty());
+	}
+
+	#[test]
+	fn proof_is_invalid_when_does_not_contains_root() {
+		assert!(create_proof_check_backend(1.into(), vec![]).is_err());
+	}
+
+	#[test]
+	fn passes_throgh_backend_calls() {
+		let trie_backend = test_trie();
+		let proving_backend = test_proving();
+		assert_eq!(trie_backend.storage(b"key").unwrap(), proving_backend.storage(b"key").unwrap());
+		assert_eq!(trie_backend.pairs(), proving_backend.pairs());
+		
+		let (trie_root, mut trie_mdb) = trie_backend.storage_root(::std::iter::empty());
+		let (proving_root, mut proving_mdb) = proving_backend.storage_root(::std::iter::empty());
+		assert_eq!(trie_root, proving_root);
+		assert_eq!(trie_mdb.drain(), proving_mdb.drain());
+	}
+
+	#[test]
+	fn proof_recorded_and_checked() {
+		let contents = (0..64).map(|i| (vec![i], Some(vec![i]))).collect::<Vec<_>>();
+		let in_memory = InMemory::default();
+		let in_memory = in_memory.update(contents);
+		let in_memory_root = in_memory.storage_root(::std::iter::empty()).0;
+		(0..64).for_each(|i| assert_eq!(in_memory.storage(&[i]).unwrap().unwrap(), vec![i]));
+
+		let trie = in_memory.try_into_trie_backend().unwrap();
+		let trie_root = trie.storage_root(::std::iter::empty()).0;
+		assert_eq!(in_memory_root, trie_root);
+		(0..64).for_each(|i| assert_eq!(trie.storage(&[i]).unwrap().unwrap(), vec![i]));
+
+		let proving = ProvingBackend::new(trie);
+		assert_eq!(proving.storage(&[42]).unwrap().unwrap(), vec![42]);
+
+		let proof = proving.extract_proof();
+
+		let proof_check = create_proof_check_backend(in_memory_root.into(), proof).unwrap();
+		assert_eq!(proof_check.storage(&[42]).unwrap().unwrap(), vec![42]);
+	}
+}

substrate/substrate/state-machine/src/trie_backend.rs

@@ -0,0 +1,290 @@
+// Copyright 2017 Parity Technologies (UK) Ltd.
+// This file is part of Substrate.
+
+// Substrate is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// Substrate is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with Substrate.  If not, see <http://www.gnu.org/licenses/>.
+
+//! Trie-based state machine backend.
+
+use std::collections::HashMap;
+use std::sync::Arc;
+use ethereum_types::H256 as TrieH256;
+use hashdb::{DBValue, HashDB};
+use kvdb::KeyValueDB;
+use memorydb::MemoryDB;
+use patricia_trie::{TrieDB, TrieDBMut, TrieError, Trie, TrieMut};
+use {Backend};
+
+/// Try convert into trie-based backend.
+pub trait TryIntoTrieBackend {
+	/// Try to convert self into trie backend.
+	fn try_into_trie_backend(self) -> Option<TrieBackend>;
+}
+
+/// Patricia trie-based backend. Transaction type is an overlay of changes to commit.
+#[derive(Clone)]
+pub struct TrieBackend {
+	storage: TrieBackendStorage,
+	root: TrieH256,
+}
+
+impl TrieBackend {
+	/// Create new trie-based backend.
+	pub fn with_kvdb(db: Arc<KeyValueDB>, storage_column: Option<u32>, root: TrieH256) -> Self {
+		TrieBackend {
+			storage: TrieBackendStorage::KeyValueDb(db, storage_column),
+			root,
+		}
+	}
+
+	/// Create new trie-based backend for genesis block.
+	pub fn with_kvdb_for_genesis(db: Arc<KeyValueDB>, storage_column: Option<u32>) -> Self {
+		let mut root = TrieH256::default();
+		let mut mdb = MemoryDB::default();
+		TrieDBMut::new(&mut mdb, &mut root);
+
+		Self::with_kvdb(db, storage_column, root)
+	}
+
+	/// Create new trie-based backend backed by MemoryDb storage.
+	pub fn with_memorydb(db: MemoryDB, root: TrieH256) -> Self {
+		// TODO: check that root is a part of db???
+		TrieBackend {
+			storage: TrieBackendStorage::MemoryDb(db),
+			root,
+		}
+	}
+
+	/// Get backend storage reference.
+	pub fn backend_storage(&self) -> &TrieBackendStorage {
+		&self.storage
+	}
+
+	/// Get trie root.
+	pub fn root(&self) -> &TrieH256 {
+		&self.root
+	}
+}
+
+impl Backend for TrieBackend {
+	type Error = String;
+	type Transaction = MemoryDB;
+
+	fn storage(&self, key: &[u8]) -> Result<Option<Vec<u8>>, Self::Error> {
+		let mut read_overlay = MemoryDB::default();
+		let eph = Ephemeral {
+			storage: &self.storage,
+			overlay: &mut read_overlay,
+		};
+
+		let map_e = |e: Box<TrieError>| format!("Trie lookup error: {}", e);
+
+		TrieDB::new(&eph, &self.root).map_err(map_e)?
+			.get(key).map(|x| x.map(|val| val.to_vec())).map_err(map_e)
+	}
+
+	fn pairs(&self) -> Vec<(Vec<u8>, Vec<u8>)> {
+		let mut read_overlay = MemoryDB::default();
+		let eph = Ephemeral {
+			storage: &self.storage,
+			overlay: &mut read_overlay,
+		};
+
+		let collect_all = || -> Result<_, Box<TrieError>> {
+			let trie = TrieDB::new(&eph, &self.root)?;
+			let mut v = Vec::new();
+			for x in trie.iter()? {
+				let (key, value) = x?;
+				v.push((key.to_vec(), value.to_vec()));
+			}
+
+			Ok(v)
+		};
+
+		match collect_all() {
+			Ok(v) => v,
+			Err(e) => {
+				debug!(target: "trie", "Error extracting trie values: {}", e);
+				Vec::new()
+			}
+		}
+	}
+
+	fn storage_root<I>(&self, delta: I) -> ([u8; 32], MemoryDB)
+		where I: IntoIterator<Item=(Vec<u8>, Option<Vec<u8>>)>
+	{
+		let mut write_overlay = MemoryDB::default();
+		let mut root = self.root;
+		{
+			let mut eph = Ephemeral {
+				storage: &self.storage,
+				overlay: &mut write_overlay,
+			};
+
+			let mut trie = TrieDBMut::from_existing(&mut eph, &mut root).expect("prior state root to exist"); // TODO: handle gracefully
+			for (key, change) in delta {
+				let result = match change {
+					Some(val) => trie.insert(&key, &val),
+					None => trie.remove(&key), // TODO: archive mode
+				};
+
+				if let Err(e) = result {
+					warn!(target: "trie", "Failed to write to trie: {}", e);
+				}
+			}
+		}
+
+		(root.0.into(), write_overlay)
+	}
+}
+
+impl TryIntoTrieBackend for TrieBackend {
+	fn try_into_trie_backend(self) -> Option<TrieBackend> {
+		Some(self)
+	}
+}
+
+pub struct Ephemeral<'a> {
+	storage: &'a TrieBackendStorage,
+	overlay: &'a mut MemoryDB,
+}
+
+impl<'a> Ephemeral<'a> {
+	pub fn new(storage: &'a TrieBackendStorage, overlay: &'a mut MemoryDB) -> Self {
+		Ephemeral {
+			storage,
+			overlay,
+		}
+	}
+}
+
+impl<'a> HashDB for Ephemeral<'a> {
+	fn keys(&self) -> HashMap<TrieH256, i32> {
+		self.overlay.keys() // TODO: iterate backing
+	}
+
+	fn get(&self, key: &TrieH256) -> Option<DBValue> {
+		match self.overlay.raw(key) {
+			Some((val, i)) => {
+				if i <= 0 {
+					None
+				} else {
+					Some(val)
+				}
+			}
+			None => match self.storage.get(&key.0[..]) {
+				Ok(x) => x,
+				Err(e) => {
+					warn!(target: "trie", "Failed to read from DB: {}", e);
+					None
+				},
+			},
+		}
+	}
+
+	fn contains(&self, key: &TrieH256) -> bool {
+		self.get(key).is_some()
+	}
+
+	fn insert(&mut self, value: &[u8]) -> TrieH256 {
+		self.overlay.insert(value)
+	}
+
+	fn emplace(&mut self, key: TrieH256, value: DBValue) {
+		self.overlay.emplace(key, value)
+	}
+
+	fn remove(&mut self, key: &TrieH256) {
+		self.overlay.remove(key)
+	}
+}
+
+#[derive(Clone)]
+pub enum TrieBackendStorage {
+	/// Key value db + storage column.
+	KeyValueDb(Arc<KeyValueDB>, Option<u32>),
+	/// Hash db.
+	MemoryDb(MemoryDB),
+}
+
+impl TrieBackendStorage {
+	pub fn get(&self, key: &[u8]) -> Result<Option<DBValue>, String> {
+		match *self {
+			TrieBackendStorage::KeyValueDb(ref db, storage_column) =>
+				db.get(storage_column, key)
+					.map_err(|e| format!("Trie lookup error: {}", e)),
+			TrieBackendStorage::MemoryDb(ref db) =>
+				Ok(db.get(&TrieH256::from_slice(key))),
+		}
+	}
+}
+
+#[cfg(test)]
+pub mod tests {
+	use super::*;
+
+	fn test_db() -> (MemoryDB, TrieH256) {
+		let mut root = TrieH256::default();
+		let mut mdb = MemoryDB::default();
+		{
+			let mut trie = TrieDBMut::new(&mut mdb, &mut root);
+			trie.insert(b"key", b"value").unwrap();
+			trie.insert(b"value1", &[42]).unwrap();
+			trie.insert(b"value2", &[24]).unwrap();
+			trie.insert(b":code", b"return 42").unwrap();
+		}
+		(mdb, root)
+	}
+
+	pub fn test_trie() -> TrieBackend {
+		let (mdb, root) = test_db();
+		TrieBackend::with_memorydb(mdb, root)
+	}
+
+	#[test]
+	fn read_from_storage_returns_some() {
+		assert_eq!(test_trie().storage(b"key").unwrap(), Some(b"value".to_vec()));
+	}
+
+	#[test]
+	fn read_from_storage_returns_none() {
+		assert_eq!(test_trie().storage(b"non-existing-key").unwrap(), None);
+	}
+
+	#[test]
+	fn pairs_are_not_empty_on_non_empty_storage() {
+		assert!(!test_trie().pairs().is_empty());
+	}
+
+	#[test]
+	fn pairs_are_empty_on_empty_storage() {
+		assert!(TrieBackend::with_memorydb(MemoryDB::new(), Default::default()).pairs().is_empty());
+	}
+
+	#[test]
+	fn storage_root_is_non_default() {
+		assert!(test_trie().storage_root(::std::iter::empty()).0 != [0; 32]);
+	}
+
+	#[test]
+	fn storage_root_transaction_is_empty() {
+		assert!(test_trie().storage_root(::std::iter::empty()).1.drain().is_empty());
+	}
+
+	#[test]
+	fn storage_root_transaction_is_non_empty() {
+		let (new_root, mut tx) = test_trie().storage_root(vec![(b"new-key".to_vec(), Some(b"new-value".to_vec()))]);
+		assert!(!tx.drain().is_empty());
+		assert!(new_root != test_trie().storage_root(::std::iter::empty()).0);
+	}
+}