Fetch changes trie roots + CHT-based proof for these roots (#896)

* build CHT for changes tries roots

* collect chnages tries roots proof in key_changes_proof

* flush check_changes_proof

* fixed compilation

* LightDataChecker now has a ref to the blockchain

* continue passing proofs

* new light db tests

* more CHT tests

* more tests for key changes proof when headers are missing

* lost files

substrate/core/state-machine/src/lib.rs

@@ -61,6 +61,7 @@ pub use changes_trie::{
 	key_changes, key_changes_proof, key_changes_proof_check,
 	prune as prune_changes_tries};
 pub use overlayed_changes::OverlayedChanges;
+pub use proving_backend::create_proof_check_backend_storage;
 pub use trie_backend_essence::Storage;
 pub use trie_backend::TrieBackend;
 
@@ -403,7 +404,7 @@ where
 {
 	let trie_backend = backend.try_into_trie_backend()
 		.ok_or_else(|| Box::new(ExecutionError::UnableToGenerateProof) as Box<Error>)?;
-	let proving_backend = proving_backend::ProvingBackend::new(trie_backend);
+	let proving_backend = proving_backend::ProvingBackend::new(&trie_backend);
 	let (result, _, _) = execute::<H, _, changes_trie::InMemoryStorage<H>, _>(
 		&proving_backend,
 		None,
@@ -444,11 +445,24 @@ pub fn prove_read<B, H>(
 where
 	B: Backend<H>,
 	H: Hasher,
-
 	H::Out: Ord + HeapSizeOf
 {
+
 	let trie_backend = backend.try_into_trie_backend()
 		.ok_or_else(|| Box::new(ExecutionError::UnableToGenerateProof) as Box<Error>)?;
+	prove_read_on_trie_backend(&trie_backend, key)
+}
+
+/// Generate storage read proof on pre-created trie backend.
+pub fn prove_read_on_trie_backend<S, H>(
+	trie_backend: &TrieBackend<S, H>,
+	key: &[u8]
+) -> Result<(Option<Vec<u8>>, Vec<Vec<u8>>), Box<Error>>
+where
+	S: trie_backend_essence::TrieBackendStorage<H>,
+	H: Hasher,
+	H::Out: Ord + HeapSizeOf
+{
 	let proving_backend = proving_backend::ProvingBackend::<_, H>::new(trie_backend);
 	let result = proving_backend.storage(key).map_err(|e| Box::new(e) as Box<Error>)?;
 	Ok((result, proving_backend.extract_proof()))
@@ -462,11 +476,22 @@ pub fn read_proof_check<H>(
 ) -> Result<Option<Vec<u8>>, Box<Error>>
 where
 	H: Hasher,
-
 	H::Out: Ord + HeapSizeOf
 {
-	let backend = proving_backend::create_proof_check_backend::<H>(root, proof)?;
-	backend.storage(key).map_err(|e| Box::new(e) as Box<Error>)
+	let proving_backend = proving_backend::create_proof_check_backend::<H>(root, proof)?;
+	read_proof_check_on_proving_backend(&proving_backend, key)
+}
+
+/// Check storage read proof on pre-created proving backend.
+pub fn read_proof_check_on_proving_backend<H>(
+	proving_backend: &TrieBackend<MemoryDB<H>, H>,
+	key: &[u8],
+) -> Result<Option<Vec<u8>>, Box<Error>>
+where
+	H: Hasher,
+	H::Out: Ord + HeapSizeOf
+{
+	proving_backend.storage(key).map_err(|e| Box::new(e) as Box<Error>)
 }
 
 /// Sets overlayed changes' changes trie configuration. Returns error if configuration

substrate/core/state-machine/src/proving_backend.rs

@@ -84,14 +84,14 @@ impl<'a, S, H> ProvingBackendEssence<'a, S, H>
 
 /// Patricia trie-based backend which also tracks all touched storage trie values.
 /// These can be sent to remote node and used as a proof of execution.
-pub struct ProvingBackend<S: TrieBackendStorage<H>, H: Hasher> {
-	backend: TrieBackend<S, H>,
+pub struct ProvingBackend<'a, S: 'a + TrieBackendStorage<H>, H: 'a + Hasher> {
+	backend: &'a TrieBackend<S, H>,
 	proof_recorder: RefCell<Recorder<H::Out>>,
 }
 
-impl<S: TrieBackendStorage<H>, H: Hasher> ProvingBackend<S, H> {
+impl<'a, S: 'a + TrieBackendStorage<H>, H: 'a + Hasher> ProvingBackend<'a, S, H> {
 	/// Create new proving backend.
-	pub fn new(backend: TrieBackend<S, H>) -> Self {
+	pub fn new(backend: &'a TrieBackend<S, H>) -> Self {
 		ProvingBackend {
 			backend,
 			proof_recorder: RefCell::new(Recorder::new()),
@@ -108,10 +108,10 @@ impl<S: TrieBackendStorage<H>, H: Hasher> ProvingBackend<S, H> {
 	}
 }
 
-impl<S, H> Backend<H> for ProvingBackend<S, H>
+impl<'a, S, H> Backend<H> for ProvingBackend<'a, S, H>
 	where
-		S: TrieBackendStorage<H>,
-		H: Hasher,
+		S: 'a + TrieBackendStorage<H>,
+		H: 'a + Hasher,
 		H::Out: Ord + HeapSizeOf,
 {
 	type Error = String;
@@ -174,10 +174,7 @@ where
 	H: Hasher,
 	H::Out: HeapSizeOf,
 {
-	let mut db = MemoryDB::default();	// TODO: use new for correctness
-	for item in proof {
-		db.insert(&item);
-	}
+	let db = create_proof_check_backend_storage(proof);
 
 	if !db.contains(&root) {
 		return Err(Box::new(ExecutionError::InvalidProof) as Box<Error>);
@@ -186,6 +183,21 @@ where
 	Ok(TrieBackend::new(db, root))
 }
 
+/// Create in-memory storage of proof check backend.
+pub fn create_proof_check_backend_storage<H>(
+	proof: Vec<Vec<u8>>
+) -> MemoryDB<H>
+where
+	H: Hasher,
+	H::Out: HeapSizeOf,
+{
+	let mut db = MemoryDB::default();	// TODO: use new for correctness
+	for item in proof {
+		db.insert(&item);
+	}
+	db
+}
+
 #[cfg(test)]
 mod tests {
 	use backend::{InMemory};
@@ -193,18 +205,20 @@ mod tests {
 	use super::*;
 	use primitives::{Blake2Hasher};
 
-	fn test_proving() -> ProvingBackend<MemoryDB<Blake2Hasher>, Blake2Hasher> {
-		ProvingBackend::new(test_trie())
+	fn test_proving<'a>(trie_backend: &'a TrieBackend<MemoryDB<Blake2Hasher>, Blake2Hasher>) -> ProvingBackend<'a, MemoryDB<Blake2Hasher>, Blake2Hasher> {
+		ProvingBackend::new(trie_backend)
 	}
 
 	#[test]
 	fn proof_is_empty_until_value_is_read() {
-		assert!(test_proving().extract_proof().is_empty());
+		let trie_backend = test_trie();
+		assert!(test_proving(&trie_backend).extract_proof().is_empty());
 	}
 
 	#[test]
 	fn proof_is_non_empty_after_value_is_read() {
-		let backend = test_proving();
+		let trie_backend = test_trie();
+		let backend = test_proving(&trie_backend);
 		assert_eq!(backend.storage(b"key").unwrap(), Some(b"value".to_vec()));
 		assert!(!backend.extract_proof().is_empty());
 	}
@@ -218,7 +232,7 @@ mod tests {
 	#[test]
 	fn passes_throgh_backend_calls() {
 		let trie_backend = test_trie();
-		let proving_backend = test_proving();
+		let proving_backend = test_proving(&trie_backend);
 		assert_eq!(trie_backend.storage(b"key").unwrap(), proving_backend.storage(b"key").unwrap());
 		assert_eq!(trie_backend.pairs(), proving_backend.pairs());
 
@@ -241,7 +255,7 @@ mod tests {
 		assert_eq!(in_memory_root, trie_root);
 		(0..64).for_each(|i| assert_eq!(trie.storage(&[i]).unwrap().unwrap(), vec![i]));
 
-		let proving = ProvingBackend::new(trie);
+		let proving = ProvingBackend::new(&trie);
 		assert_eq!(proving.storage(&[42]).unwrap().unwrap(), vec![42]);
 
 		let proof = proving.extract_proof();

substrate/core/state-machine/src/trie_backend.rs

@@ -49,6 +49,11 @@ impl<S: TrieBackendStorage<H>, H: Hasher> TrieBackend<S, H> where H::Out: HeapSi
 	pub fn root(&self) -> &H::Out {
 		self.essence.root()
 	}
+
+	/// Consumes self and returns underlying storage.
+	pub fn into_storage(self) -> S {
+		self.essence.into_storage()
+	}
 }
 
 impl super::Error for String {}

substrate/core/state-machine/src/trie_backend_essence.rs

@@ -56,6 +56,11 @@ impl<S: TrieBackendStorage<H>, H: Hasher> TrieBackendEssence<S, H> where H::Out:
 		&self.root
 	}
 
+	/// Consumes self and returns underlying storage.
+	pub fn into_storage(self) -> S {
+		self.storage
+	}
+
 	/// Get the value of storage at given key.
 	pub fn storage(&self, key: &[u8]) -> Result<Option<Vec<u8>>, String> {
 		let mut read_overlay = MemoryDB::default();