Light friendly storage tracking: changes trie + extending over ranges (#628)

* changes_trie

* changs_trie: continue

* changes_trie: adding tests

* fixed TODO

* removed obsolete ExtrinsicChanges

* encodable ChangesTrieConfiguration

* removed polkadot fle

* fixed grumbles

* ext_storage_changes_root returns u32

* moved changes trie root to digest

* removed commented code

* read storage values from native code

* fixed grumbles

* fixed grumbles

* missing comma

substrate/core/client/Cargo.toml

@@ -26,6 +26,7 @@ substrate-telemetry = { path = "../telemetry" }
 hashdb = "0.2.1"
 patricia-trie = "0.2.1"
 rlp = "0.2.4"
+memorydb = "0.2.1"
 
 [dev-dependencies]
 substrate-test-client = { path = "../test-client" }

substrate/core/client/db/src/lib.rs

@@ -57,7 +57,8 @@ use parking_lot::RwLock;
 use primitives::{H256, AuthorityId, Blake2Hasher, RlpCodec};
 use runtime_primitives::generic::BlockId;
 use runtime_primitives::bft::Justification;
-use runtime_primitives::traits::{Block as BlockT, Header as HeaderT, As, Hash, HashFor, NumberFor, Zero};
+use runtime_primitives::traits::{Block as BlockT, Header as HeaderT, As, Hash, HashFor,
+	NumberFor, Zero, Digest, DigestItem};
 use runtime_primitives::BuildStorage;
 use state_machine::backend::Backend as StateBackend;
 use executor::RuntimeInfo;
@@ -70,7 +71,7 @@ pub use state_db::PruningMode;
 const FINALIZATION_WINDOW: u64 = 32;
 
 /// DB-backed patricia trie state, transaction type is an overlay of changes to commit.
-pub type DbState = state_machine::TrieBackend<Blake2Hasher, RlpCodec>;
+pub type DbState = state_machine::TrieBackend<Arc<state_machine::Storage<Blake2Hasher>>, Blake2Hasher, RlpCodec>;
 
 /// Database settings.
 pub struct DatabaseSettings {
@@ -107,6 +108,7 @@ mod columns {
 	pub const HEADER: Option<u32> = Some(4);
 	pub const BODY: Option<u32> = Some(5);
 	pub const JUSTIFICATION: Option<u32> = Some(6);
+	pub const CHANGES_TRIE: Option<u32> = Some(7);
 }
 
 struct PendingBlock<Block: BlockT> {
@@ -230,6 +232,7 @@ impl<Block: BlockT> client::blockchain::Backend<Block> for BlockchainDb<Block> {
 pub struct BlockImportOperation<Block: BlockT, H: Hasher> {
 	old_state: DbState,
 	updates: MemoryDB<H>,
+	changes_trie_updates: MemoryDB<H>,
 	pending_block: Option<PendingBlock<Block>>,
 }
 
@@ -269,6 +272,11 @@ where Block: BlockT,
 		self.updates = update;
 		Ok(())
 	}
+
+	fn update_changes_trie(&mut self, update: MemoryDB<Blake2Hasher>) -> Result<(), client::error::Error> {
+		self.changes_trie_updates = update;
+		Ok(())
+	}
 }
 
 struct StorageDb<Block: BlockT> {
@@ -292,11 +300,55 @@ impl<Block: BlockT> state_db::HashDb for StorageDb<Block> {
 	}
 }
 
+struct DbGenesisStorage(pub H256);
+
+impl DbGenesisStorage {
+	pub fn new() -> Self {
+		let mut root = H256::default();
+		let mut mdb = MemoryDB::<Blake2Hasher>::new();
+		state_machine::TrieDBMut::<Blake2Hasher, RlpCodec>::new(&mut mdb, &mut root);
+		DbGenesisStorage(root)
+	}
+}
+
+impl state_machine::Storage<Blake2Hasher> for DbGenesisStorage {
+	fn get(&self, _key: &H256) -> Result<Option<DBValue>, String> {
+		Ok(None)
+	}
+}
+
+pub struct DbChangesTrieStorage<Block: BlockT> {
+	db: Arc<KeyValueDB>,
+	_phantom: ::std::marker::PhantomData<Block>,
+}
+
+impl<Block: BlockT> state_machine::ChangesTrieStorage<Blake2Hasher> for DbChangesTrieStorage<Block> {
+	fn root(&self, block: u64) -> Result<Option<H256>, String> {
+		Ok(read_db::<Block>(&*self.db, columns::BLOCK_INDEX, columns::HEADER, BlockId::Number(As::sa(block)))
+			.map_err(|err| format!("{}", err))
+			.and_then(|header| match header {
+				Some(header) => Block::Header::decode(&mut &header[..])
+					.ok_or_else(|| format!("Failed to parse header of block {}", block))
+					.map(Some),
+				None => Ok(None)
+			})?
+			.and_then(|header| header.digest().logs().iter()
+				.find(|log| log.as_changes_trie_root().is_some())
+				.and_then(DigestItem::as_changes_trie_root)
+				.map(|root| H256::from_slice(root.as_ref()))))
+	}
+
+	fn get(&self, key: &H256) -> Result<Option<DBValue>, String> {
+		self.db.get(columns::CHANGES_TRIE, &key[..])
+			.map_err(|err| format!("{}", err))
+	}
+}
 
 /// Disk backend. Keeps data in a key-value store. In archive mode, trie nodes are kept from all blocks.
 /// Otherwise, trie nodes are kept only from the most recent block.
 pub struct Backend<Block: BlockT> {
 	storage: Arc<StorageDb<Block>>,
+	tries_change_storage: DbChangesTrieStorage<Block>,
 	blockchain: BlockchainDb<Block>,
 	finalization_window: u64,
 }
@@ -323,12 +375,17 @@ impl<Block: BlockT> Backend<Block> {
 		let map_e = |e: state_db::Error<io::Error>| ::client::error::Error::from(format!("State database error: {:?}", e));
 		let state_db: StateDb<Block::Hash, H256> = StateDb::new(pruning, &StateMetaDb(&*db)).map_err(map_e)?;
 		let storage_db = StorageDb {
-			db,
+			db: db.clone(),
 			state_db,
 		};
+		let tries_change_storage = DbChangesTrieStorage {
+			db,
+			_phantom: Default::default(),
+		};
 
 		Ok(Backend {
 			storage: Arc::new(storage_db),
+			tries_change_storage: tries_change_storage,
 			blockchain,
 			finalization_window,
 		})
@@ -350,10 +407,17 @@ fn apply_state_commit(transaction: &mut DBTransaction, commit: state_db::CommitS
 	}
 }
 
+fn apply_changes_trie_commit(transaction: &mut DBTransaction, mut commit: MemoryDB<Blake2Hasher>) {
+	for (key, (val, _)) in commit.drain() {
+		transaction.put(columns::CHANGES_TRIE, &key[..], &val);
+	}
+}
+
 impl<Block> client::backend::Backend<Block, Blake2Hasher, RlpCodec> for Backend<Block> where Block: BlockT {
 	type BlockImportOperation = BlockImportOperation<Block, Blake2Hasher>;
 	type Blockchain = BlockchainDb<Block>;
 	type State = DbState;
+	type ChangesTrieStorage = DbChangesTrieStorage<Block>;
 
 	fn begin_operation(&self, block: BlockId<Block>) -> Result<Self::BlockImportOperation, client::error::Error> {
 		let state = self.state_at(block)?;
@@ -361,6 +425,7 @@ impl<Block> client::backend::Backend<Block, Blake2Hasher, RlpCodec> for Backend<
 			pending_block: None,
 			old_state: state,
 			updates: MemoryDB::default(),
+			changes_trie_updates: MemoryDB::default(),
 		})
 	}
 
@@ -393,6 +458,7 @@ impl<Block> client::backend::Backend<Block, Blake2Hasher, RlpCodec> for Backend<
 			let number_u64 = number.as_().into();
 			let commit = self.storage.state_db.insert_block(&hash, number_u64, &pending_block.header.parent_hash(), changeset);
 			apply_state_commit(&mut transaction, commit);
+			apply_changes_trie_commit(&mut transaction, operation.changes_trie_updates);
 
 			//finalize an older block
 			if number_u64 > self.finalization_window {
@@ -420,6 +486,10 @@ impl<Block> client::backend::Backend<Block, Blake2Hasher, RlpCodec> for Backend<
 		Ok(())
 	}
 
+	fn changes_trie_storage(&self) -> Option<&Self::ChangesTrieStorage> {
+		Some(&self.tries_change_storage)
+	}
+
 	fn revert(&self, n: NumberFor<Block>) -> Result<NumberFor<Block>, client::error::Error> {
 		use client::blockchain::HeaderBackend;
 		let mut best = self.blockchain.info()?.best_number;
@@ -459,15 +529,18 @@ impl<Block> client::backend::Backend<Block, Blake2Hasher, RlpCodec> for Backend<
 
 		// special case for genesis initialization
 		match block {
-			BlockId::Hash(h) if h == Default::default() =>
-				return Ok(DbState::with_storage_for_genesis(self.storage.clone())),
+			BlockId::Hash(h) if h == Default::default() => {
+				let genesis_storage = DbGenesisStorage::new();
+				let root = genesis_storage.0.clone();
+				return Ok(DbState::new(Arc::new(genesis_storage), root));
+			},
 			_ => {}
 		}
 
 		match self.blockchain.header(block) {
 			Ok(Some(ref hdr)) if !self.storage.state_db.is_pruned(hdr.number().as_()) => {
 				let root = H256::from_slice(hdr.state_root().as_ref());
-				Ok(DbState::with_storage(self.storage.clone(), root))
+				Ok(DbState::new(self.storage.clone(), root))
 			},
 			Err(e) => Err(e),
 			_ => Err(client::error::ErrorKind::UnknownBlock(format!("{:?}", block)).into()),
@@ -486,6 +559,7 @@ mod tests {
 	use client::backend::BlockImportOperation as Op;
 	use client::blockchain::HeaderBackend as BlockchainHeaderBackend;
 	use runtime_primitives::testing::{Header, Block as RawBlock};
+	use state_machine::{TrieMut, TrieDBMut, ChangesTrieStorage};
 
 	type Block = RawBlock<u64>;
 
@@ -711,4 +785,82 @@ mod tests {
 			assert!(backend.storage.db.get(::columns::STATE, &key.0[..]).unwrap().is_none());
 		}
 	}
+
+	#[test]
+	fn changes_trie_storage_works() {
+		let backend = Backend::<Block>::new_test(1000);
+
+		let prepare_changes = |changes: Vec<(Vec<u8>, Vec<u8>)>| {
+			let mut changes_root = H256::default();
+			let mut changes_trie_update = MemoryDB::<Blake2Hasher>::new();
+			{
+				let mut trie = TrieDBMut::<Blake2Hasher, RlpCodec>::new(
+					&mut changes_trie_update,
+					&mut changes_root
+				);
+				for (key, value) in changes {
+					trie.insert(&key, &value).unwrap();
+				}
+			}
+
+			(changes_root, changes_trie_update)
+		};
+
+		let insert_header = |number: u64, parent_hash: H256, changes: Vec<(Vec<u8>, Vec<u8>)>| {
+			use runtime_primitives::generic::DigestItem;
+			use runtime_primitives::testing::Digest;
+
+			let (changes_root, changes_trie_update) = prepare_changes(changes);
+			let digest = Digest {
+				logs: vec![
+					DigestItem::ChangesTrieRoot(changes_root),
+				],
+			};
+			let header = Header {
+				number,
+				parent_hash,
+				state_root: Default::default(),
+				digest,
+				extrinsics_root: Default::default(),
+			};
+			let header_hash = header.hash();
+
+			let block_id = if number == 0 {
+				BlockId::Hash(Default::default())
+			} else {
+				BlockId::Number(number - 1)
+			};
+			let mut op = backend.begin_operation(block_id).unwrap();
+			op.set_block_data(header, None, None, true).unwrap();
+			op.update_changes_trie(changes_trie_update).unwrap();
+			backend.commit_operation(op).unwrap();
+
+			header_hash
+		};
+
+		let check_changes = |backend: &Backend<Block>, block: u64, changes: Vec<(Vec<u8>, Vec<u8>)>| {
+			let (changes_root, mut changes_trie_update) = prepare_changes(changes);
+			assert_eq!(backend.tries_change_storage.root(block), Ok(Some(changes_root)));
+
+			for (key, (val, _)) in changes_trie_update.drain() {
+				assert_eq!(backend.changes_trie_storage().unwrap().get(&key), Ok(Some(val)));
+			}
+		};
+
+		let changes0 = vec![(b"key_at_0".to_vec(), b"val_at_0".to_vec())];
+		let changes1 = vec![
+			(b"key_at_1".to_vec(), b"val_at_1".to_vec()),
+			(b"another_key_at_1".to_vec(), b"another_val_at_1".to_vec()),
+		];
+		let changes2 = vec![(b"key_at_2".to_vec(), b"val_at_2".to_vec())];
+
+		let block0 = insert_header(0, Default::default(), changes0.clone());
+		let block1 = insert_header(1, block0, changes1.clone());
+		let _ = insert_header(2, block1, changes2.clone());
+
+		// check that the storage contains tries for all blocks
+		check_changes(&backend, 0, changes0);
+		check_changes(&backend, 1, changes1);
+		check_changes(&backend, 2, changes2);
+	}
 }

substrate/core/client/db/src/utils.rs

@@ -32,7 +32,7 @@ use DatabaseSettings;
 
 /// Number of columns in the db. Must be the same for both full && light dbs.
 /// Otherwise RocksDb will fail to open database && check its type.
-pub const NUM_COLUMNS: u32 = 7;
+pub const NUM_COLUMNS: u32 = 8;
 /// Meta column. The set of keys in the column is shared by full && light storages.
 pub const COLUMN_META: Option<u32> = Some(0);
 

substrate/core/client/src/backend.rs

@@ -22,8 +22,10 @@ use runtime_primitives::bft::Justification;
 use runtime_primitives::generic::BlockId;
 use runtime_primitives::traits::{Block as BlockT, NumberFor};
 use state_machine::backend::Backend as StateBackend;
+use state_machine::ChangesTrieStorage as StateChangesTrieStorage;
 use patricia_trie::NodeCodec;
 use hashdb::Hasher;
+use memorydb::MemoryDB;
 
 /// Block insertion operation. Keeps hold if the inserted block state and data.
 pub trait BlockImportOperation<Block, H, C>
@@ -53,6 +55,8 @@ where
 	fn update_storage(&mut self, update: <Self::State as StateBackend<H, C>>::Transaction) -> error::Result<()>;
 	/// Inject storage data into the database replacing any existing data.
 	fn reset_storage<I: Iterator<Item=(Vec<u8>, Vec<u8>)>>(&mut self, iter: I) -> error::Result<()>;
+	/// Inject changes trie data into the database.
+	fn update_changes_trie(&mut self, update: MemoryDB<H>) -> error::Result<()>;
 }
 
 /// Client backend. Manages the data layer.
@@ -75,6 +79,8 @@ where
 	type Blockchain: ::blockchain::Backend<Block>;
 	/// Associated state backend type.
 	type State: StateBackend<H, C>;
+	/// Changes trie storage.
+	type ChangesTrieStorage: StateChangesTrieStorage<H>;
 
 	/// Begin a new block insertion transaction with given parent block id.
 	/// When constructing the genesis, this is called with all-zero hash.
@@ -83,6 +89,8 @@ where
 	fn commit_operation(&self, transaction: Self::BlockImportOperation) -> error::Result<()>;
 	/// Returns reference to blockchain backend.
 	fn blockchain(&self) -> &Self::Blockchain;
+	/// Returns reference to changes trie storage.
+	fn changes_trie_storage(&self) -> Option<&Self::ChangesTrieStorage>;
 	/// Returns state backend with post-state of given block.
 	fn state_at(&self, block: BlockId<Block>) -> error::Result<Self::State>;
 	/// Attempts to revert the chain by `n` blocks. Returns the number of blocks that were

substrate/core/client/src/block_builder.rs

@@ -94,7 +94,7 @@ where
 	/// the error. Otherwise, it will return a mutable reference to self (in order to chain).
 	pub fn push(&mut self, xt: <Block as BlockT>::Extrinsic) -> error::Result<()> {
 		match self.executor.call_at_state(&self.state, &mut self.changes, "apply_extrinsic", &xt.encode(), native_when_possible()) {
-			Ok((result, _)) => {
+			Ok((result, _, _)) => {
 				match ApplyResult::decode(&mut result.as_slice()) {
 					Some(Ok(ApplyOutcome::Success)) | Some(Ok(ApplyOutcome::Fail)) => {
 						self.extrinsics.push(xt);
@@ -120,7 +120,7 @@ where
 
 	/// Consume the builder to return a valid `Block` containing all pushed extrinsics.
 	pub fn bake(mut self) -> error::Result<Block> {
-		let (output, _) = self.executor.call_at_state(
+		let (output, _, _) = self.executor.call_at_state(
 			&self.state,
 			&mut self.changes,
 			"finalise_block",

substrate/core/client/src/call_executor.rs

@@ -24,6 +24,7 @@ use executor::{RuntimeVersion, RuntimeInfo};
 use patricia_trie::NodeCodec;
 use hashdb::Hasher;
 use rlp::Encodable;
+use memorydb::MemoryDB;
 use codec::Decode;
 use primitives::{Blake2Hasher, RlpCodec};
 
@@ -76,7 +77,7 @@ where
 		method: &str,
 		call_data: &[u8],
 		manager: ExecutionManager<F>
-	) -> Result<(Vec<u8>, S::Transaction), error::Error>;
+	) -> Result<(Vec<u8>, S::Transaction, Option<MemoryDB<H>>), error::Error>;
 
 	/// Execute a call to a contract on top of given state, gathering execution proof.
 	///
@@ -129,7 +130,7 @@ where
 		call_data: &[u8],
 	) -> error::Result<CallResult> {
 		let mut changes = OverlayedChanges::default();
-		let (return_data, _) = self.call_at_state(
+		let (return_data, _, _) = self.call_at_state(
 			&self.backend.state_at(*id)?,
 			&mut changes,
 			method,
@@ -152,7 +153,8 @@ where
 			.and_then(|v| u64::decode(&mut &v[..]))
 			.unwrap_or(8) as usize;
 
-		self.executor.runtime_version(&mut Ext::new(&mut overlay, &state), heap_pages, &code)
+		let mut ext = Ext::new(&mut overlay, &state, self.backend.changes_trie_storage());
+		self.executor.runtime_version(&mut ext, heap_pages, &code)
 			.ok_or(error::ErrorKind::VersionInvalid.into())
 	}
 
@@ -165,9 +167,10 @@ where
 		method: &str,
 		call_data: &[u8],
 		manager: ExecutionManager<F>,
-	) -> error::Result<(Vec<u8>, S::Transaction)> {
+	) -> error::Result<(Vec<u8>, S::Transaction, Option<MemoryDB<Blake2Hasher>>)> {
 		state_machine::execute_using_consensus_failure_handler(
 			state,
+			self.backend.changes_trie_storage(),
 			changes,
 			&self.executor,
 			method,
@@ -189,7 +192,6 @@ where
 			method,
 			call_data,
 		)
-		.map(|(result, proof, _)| (result, proof))
 		.map_err(Into::into)
 	}
 

substrate/core/client/src/client.rs

@@ -24,7 +24,7 @@ use runtime_primitives::{bft::Justification, generic::{BlockId, SignedBlock, Blo
 use runtime_primitives::traits::{Block as BlockT, Header as HeaderT, Zero, One, As, NumberFor};
 use runtime_primitives::BuildStorage;
 use substrate_metadata::JsonMetadataDecodable;
-use primitives::{Blake2Hasher, RlpCodec};
+use primitives::{Blake2Hasher, RlpCodec, H256};
 use primitives::storage::{StorageKey, StorageData};
 use codec::{Encode, Decode};
 use state_machine::{
@@ -170,6 +170,7 @@ pub fn new_in_mem<E, Block, S>(
 		E: CodeExecutor<Blake2Hasher> + RuntimeInfo,
 		S: BuildStorage,
 		Block: BlockT,
+		H256: From<Block::Hash>,
 {
 	let backend = Arc::new(in_mem::Backend::new());
 	let executor = LocalCallExecutor::new(backend.clone(), executor);
@@ -359,7 +360,7 @@ impl<B, E, Block> Client<B, E, Block> where
 				&header.encode(),
 				execution_manager()
 			)?;
-			let (r, _) = args.using_encoded(|input|
+			let (r, _, _) = args.using_encoded(|input|
 				self.executor().call_at_state(
 				&state,
 				&mut overlay,
@@ -436,7 +437,7 @@ impl<B, E, Block> Client<B, E, Block> where
 		}
 
 		let mut transaction = self.backend.begin_operation(BlockId::Hash(parent_hash))?;
-		let (storage_update, storage_changes) = match transaction.state()? {
+		let (storage_update, changes_update, storage_changes) = match transaction.state()? {
 			Some(transaction_state) => {
 				let mut overlay = Default::default();
 				let mut r = self.executor.call_at_state(
@@ -462,11 +463,11 @@ impl<B, E, Block> Client<B, E, Block> where
 						}),
 					},
 				);
-				let (_, storage_update) = r?;
+				let (_, storage_update, changes_update) = r?;
 				overlay.commit_prospective();
-				(Some(storage_update), Some(overlay.into_committed()))
+				(Some(storage_update), Some(changes_update), Some(overlay.into_committed()))
 			},
-			None => (None, None)
+			None => (None, None, None)
 		};
 
 		let is_new_best = header.number() == &(self.backend.blockchain().info()?.best_number + One::one());
@@ -477,6 +478,9 @@ impl<B, E, Block> Client<B, E, Block> where
 		if let Some(storage_update) = storage_update {
 			transaction.update_storage(storage_update)?;
 		}
+		if let Some(Some(changes_update)) = changes_update {
+			transaction.update_changes_trie(changes_update)?;
+		}
 		self.backend.commit_operation(transaction)?;
 
 		if origin == BlockOrigin::NetworkBroadcast || origin == BlockOrigin::Own || origin == BlockOrigin::ConsensusBroadcast {

substrate/core/client/src/genesis.rs

@@ -45,7 +45,7 @@ mod tests {
 	use codec::{Encode, Decode, Joiner};
 	use keyring::Keyring;
 	use executor::NativeExecutionDispatch;
-	use state_machine::{execute, OverlayedChanges, ExecutionStrategy};
+	use state_machine::{execute, OverlayedChanges, ExecutionStrategy, InMemoryChangesTrieStorage};
 	use state_machine::backend::InMemory;
 	use test_client;
 	use test_client::runtime::genesismap::{GenesisConfig, additional_storage_with_genesis};
@@ -58,7 +58,13 @@ mod tests {
 		NativeExecutionDispatch::new()
 	}
 
-	fn construct_block(backend: &InMemory<Blake2Hasher, RlpCodec>, number: BlockNumber, parent_hash: Hash, state_root: Hash, txs: Vec<Transfer>) -> (Vec<u8>, Hash) {
+	fn construct_block(
+		backend: &InMemory<Blake2Hasher, RlpCodec>,
+		number: BlockNumber,
+		parent_hash: Hash,
+		state_root: Hash,
+		txs: Vec<Transfer>
+	) -> (Vec<u8>, Hash) {
 		use triehash::ordered_trie_root;
 
 		let transactions = txs.into_iter().map(|tx| {
@@ -83,6 +89,7 @@ mod tests {
 
 		execute(
 			backend,
+			Some(&InMemoryChangesTrieStorage::new()),
 			&mut overlay,
 			&executor(),
 			"initialise_block",
@@ -93,6 +100,7 @@ mod tests {
 		for tx in transactions.iter() {
 			execute(
 				backend,
+				Some(&InMemoryChangesTrieStorage::new()),
 				&mut overlay,
 				&executor(),
 				"apply_extrinsic",
@@ -101,8 +109,9 @@ mod tests {
 			).unwrap();
 		}
 
-		let (ret_data, _) = execute(
+		let (ret_data, _, _) = execute(
 			backend,
+			Some(&InMemoryChangesTrieStorage::new()),
 			&mut overlay,
 			&executor(),
 			"finalise_block",
@@ -145,6 +154,7 @@ mod tests {
 		let mut overlay = OverlayedChanges::default();
 		let _ = execute(
 			&backend,
+			Some(&InMemoryChangesTrieStorage::new()),
 			&mut overlay,
 			&executor(),
 			"execute_block",
@@ -168,6 +178,7 @@ mod tests {
 		let mut overlay = OverlayedChanges::default();
 		let _ = execute(
 			&backend,
+			Some(&InMemoryChangesTrieStorage::new()),
 			&mut overlay,
 			&executor(),
 			"execute_block",
@@ -192,6 +203,7 @@ mod tests {
 		let mut overlay = OverlayedChanges::default();
 		let _ = execute(
 			&backend,
+			Some(&InMemoryChangesTrieStorage::new()),
 			&mut overlay,
 			&Executor::new(),
 			"execute_block",

substrate/core/client/src/in_mem.rs

@@ -24,13 +24,16 @@ use backend;
 use light;
 use primitives::AuthorityId;
 use runtime_primitives::generic::BlockId;
-use runtime_primitives::traits::{Block as BlockT, Header as HeaderT, Zero, NumberFor, As};
+use runtime_primitives::traits::{Block as BlockT, Header as HeaderT, Zero,
+	NumberFor, As, Digest, DigestItem};
 use runtime_primitives::bft::Justification;
 use blockchain::{self, BlockStatus};
 use state_machine::backend::{Backend as StateBackend, InMemory};
+use state_machine::InMemoryChangesTrieStorage;
 use patricia_trie::NodeCodec;
 use hashdb::Hasher;
 use heapsize::HeapSizeOf;
+use memorydb::MemoryDB;
 
 struct PendingBlock<B: BlockT> {
 	block: StoredBlock<B>,
@@ -85,9 +88,9 @@ impl<B: BlockT> StoredBlock<B> {
 #[derive(Clone)]
 struct BlockchainStorage<Block: BlockT> {
 	blocks: HashMap<Block::Hash, StoredBlock<Block>>,
-	hashes: HashMap<<<Block as BlockT>::Header as HeaderT>::Number, Block::Hash>,
+	hashes: HashMap<NumberFor<Block>, Block::Hash>,
 	best_hash: Block::Hash,
-	best_number: <<Block as BlockT>::Header as HeaderT>::Number,
+	best_number: NumberFor<Block>,
 	genesis_hash: Block::Hash,
 	cht_roots: HashMap<NumberFor<Block>, Block::Hash>,
 }
@@ -275,6 +278,7 @@ pub struct BlockImportOperation<Block: BlockT, H: Hasher, C: NodeCodec<H>> {
 	pending_authorities: Option<Vec<AuthorityId>>,
 	old_state: InMemory<H, C>,
 	new_state: Option<InMemory<H, C>>,
+	changes_trie_update: Option<MemoryDB<H>>,
 }
 
 impl<Block, H, C> backend::BlockImportOperation<Block, H, C> for BlockImportOperation<Block, H, C>
@@ -314,6 +318,11 @@ where
 		Ok(())
 	}
 
+	fn update_changes_trie(&mut self, update: MemoryDB<H>) -> error::Result<()> {
+		self.changes_trie_update = Some(update);
+		Ok(())
+	}
+
 	fn reset_storage<I: Iterator<Item=(Vec<u8>, Vec<u8>)>>(&mut self, iter: I) -> error::Result<()> {
 		self.new_state = Some(InMemory::from(iter.collect::<HashMap<_, _>>()));
 		Ok(())
@@ -325,9 +334,11 @@ pub struct Backend<Block, H, C>
 where
 	Block: BlockT,
 	H: Hasher,
-	C: NodeCodec<H>
+	C: NodeCodec<H>,
+	H::Out: HeapSizeOf + From<Block::Hash>,
 {
 	states: RwLock<HashMap<Block::Hash, InMemory<H, C>>>,
+	changes_trie_storage: InMemoryChangesTrieStorage<H>,
 	blockchain: Blockchain<Block>,
 }
 
@@ -335,12 +346,14 @@ impl<Block, H, C> Backend<Block, H, C>
 where
 	Block: BlockT,
 	H: Hasher,
-	C: NodeCodec<H>
+	C: NodeCodec<H>,
+	H::Out: HeapSizeOf + From<Block::Hash>,
 {
 	/// Create a new instance of in-mem backend.
 	pub fn new() -> Backend<Block, H, C> {
 		Backend {
 			states: RwLock::new(HashMap::new()),
+			changes_trie_storage: InMemoryChangesTrieStorage::new(),
 			blockchain: Blockchain::new(),
 		}
 	}
@@ -350,12 +363,13 @@ impl<Block, H, C> backend::Backend<Block, H, C> for Backend<Block, H, C>
 where
 	Block: BlockT,
 	H: Hasher,
-	H::Out: HeapSizeOf,
+	H::Out: HeapSizeOf + From<Block::Hash>,
 	C: NodeCodec<H> + Send + Sync,
 {
 	type BlockImportOperation = BlockImportOperation<Block, H, C>;
 	type Blockchain = Blockchain<Block>;
 	type State = InMemory<H, C>;
+	type ChangesTrieStorage = InMemoryChangesTrieStorage<H>;
 
 	fn begin_operation(&self, block: BlockId<Block>) -> error::Result<Self::BlockImportOperation> {
 		let state = match block {
@@ -368,6 +382,7 @@ where
 			pending_authorities: None,
 			old_state: state,
 			new_state: None,
+			changes_trie_update: None,
 		})
 	}
 
@@ -375,10 +390,21 @@ where
 		if let Some(pending_block) = operation.pending_block {
 			let old_state = &operation.old_state;
 			let (header, body, justification) = pending_block.block.into_inner();
+
 			let hash = header.hash();
 			let parent_hash = *header.parent_hash();
 
 			self.states.write().insert(hash, operation.new_state.unwrap_or_else(|| old_state.clone()));
+			let changes_trie_root = header.digest().logs().iter()
+				.find(|log| log.as_changes_trie_root().is_some())
+				.and_then(DigestItem::as_changes_trie_root)
+				.cloned();
+			if let Some(changes_trie_root) = changes_trie_root {
+				if let Some(changes_trie_update) = operation.changes_trie_update {
+					let changes_trie_root: H::Out = changes_trie_root.into();
+					self.changes_trie_storage.insert(header.number().as_(), changes_trie_root, changes_trie_update);
+				}
+			}
 			self.blockchain.insert(hash, header, justification, body, pending_block.is_best);
 			// dumb implementation - store value for each block
 			if pending_block.is_best {
@@ -392,6 +418,10 @@ where
 		&self.blockchain
 	}
 
+	fn changes_trie_storage(&self) -> Option<&Self::ChangesTrieStorage> {
+		Some(&self.changes_trie_storage)
+	}
+
 	fn state_at(&self, block: BlockId<Block>) -> error::Result<Self::State> {
 		match self.blockchain.id(block).and_then(|id| self.states.read().get(&id).cloned()) {
 			Some(state) => Ok(state),
@@ -408,7 +438,7 @@ impl<Block, H, C> backend::LocalBackend<Block, H, C> for Backend<Block, H, C>
 where
 	Block: BlockT,
 	H: Hasher,
-	H::Out: HeapSizeOf,
+	H::Out: HeapSizeOf + From<Block::Hash>,
 	C: NodeCodec<H> + Send + Sync,
 {}
 

substrate/core/client/src/lib.rs

@@ -39,6 +39,7 @@ extern crate patricia_trie;
 extern crate hashdb;
 extern crate rlp;
 extern crate heapsize;
+extern crate memorydb;
 
 #[macro_use] extern crate error_chain;
 #[macro_use] extern crate log;

substrate/core/client/src/light/backend.rs

@@ -24,11 +24,7 @@ use parking_lot::RwLock;
 use primitives::AuthorityId;
 use runtime_primitives::{bft::Justification, generic::BlockId};
 use runtime_primitives::traits::{Block as BlockT, NumberFor};
-use state_machine::{
-	Backend as StateBackend,
-	TrieBackend as StateTrieBackend,
-	TryIntoTrieBackend as TryIntoStateTrieBackend
-};
+use state_machine::{Backend as StateBackend, InMemoryChangesTrieStorage, TrieBackend};
 
 use backend::{Backend as ClientBackend, BlockImportOperation, RemoteBackend};
 use blockchain::HeaderBackend as BlockchainHeaderBackend;
@@ -37,6 +33,8 @@ use light::blockchain::{Blockchain, Storage as BlockchainStorage};
 use light::fetcher::{Fetcher, RemoteReadRequest};
 use patricia_trie::NodeCodec;
 use hashdb::Hasher;
+use memorydb::MemoryDB;
+use heapsize::HeapSizeOf;
 
 /// Light client backend.
 pub struct Backend<S, F> {
@@ -77,10 +75,12 @@ impl<S, F, Block, H, C> ClientBackend<Block, H, C> for Backend<S, F> where
 	F: Fetcher<Block>,
 	H: Hasher,
 	C: NodeCodec<H>,
+	H::Out: HeapSizeOf,
 {
 	type BlockImportOperation = ImportOperation<Block, S, F>;
 	type Blockchain = Blockchain<S, F>;
 	type State = OnDemandState<Block, S, F>;
+	type ChangesTrieStorage = InMemoryChangesTrieStorage<H>;
 
 	fn begin_operation(&self, _block: BlockId<Block>) -> ClientResult<Self::BlockImportOperation> {
 		Ok(ImportOperation {
@@ -100,6 +100,10 @@ impl<S, F, Block, H, C> ClientBackend<Block, H, C> for Backend<S, F> where
 		&self.blockchain
 	}
 
+	fn changes_trie_storage(&self) -> Option<&Self::ChangesTrieStorage> {
+		None
+	}
+
 	fn state_at(&self, block: BlockId<Block>) -> ClientResult<Self::State> {
 		let block_hash = match block {
 			BlockId::Hash(h) => Some(h),
@@ -125,6 +129,7 @@ where
 	S: BlockchainStorage<Block>,
 	F: Fetcher<Block>,
 	H: Hasher,
+	H::Out: HeapSizeOf,
 	C: NodeCodec<H>,
 {}
 
@@ -164,6 +169,11 @@ where
 		Ok(())
 	}
 
+	fn update_changes_trie(&mut self, _update: MemoryDB<H>) -> ClientResult<()> {
+		// we're not storing anything locally => ignore changes
+		Ok(())
+	}
+
 	fn reset_storage<I: Iterator<Item=(Vec<u8>, Vec<u8>)>>(&mut self, _iter: I) -> ClientResult<()> {
 		// we're not storing anything locally => ignore changes
 		Ok(())
@@ -180,6 +190,7 @@ impl<Block, S, F, H, C> StateBackend<H, C> for OnDemandState<Block, S, F>
 {
 	type Error = ClientError;
 	type Transaction = ();
+	type TrieBackendStorage = MemoryDB<H>;
 
 	fn storage(&self, key: &[u8]) -> ClientResult<Option<Vec<u8>>> {
 		let mut header = self.cached_header.read().clone();
@@ -214,16 +225,8 @@ impl<Block, S, F, H, C> StateBackend<H, C> for OnDemandState<Block, S, F>
 		// whole state is not available on light node
 		Vec::new()
 	}
-}
 
-impl<Block, S, F, H, C> TryIntoStateTrieBackend<H, C> for OnDemandState<Block, S, F>
-where
-	Block: BlockT,
-	F: Fetcher<Block>,
-	H: Hasher,
-	C: NodeCodec<H>,
-{
-	fn try_into_trie_backend(self) -> Option<StateTrieBackend<H, C>> {
+	fn try_into_trie_backend(self) -> Option<TrieBackend<Self::TrieBackendStorage, H, C>> {
 		None
 	}
 }

substrate/core/client/src/light/call_executor.rs

@@ -17,6 +17,7 @@
 //! Light client call exector. Executes methods on remote full nodes, fetching
 //! execution proof and checking it locally.
 
+use std::marker::PhantomData;
 use std::sync::Arc;
 use futures::{IntoFuture, Future};
 
@@ -36,7 +37,7 @@ use light::fetcher::{Fetcher, RemoteCallRequest};
 use executor::RuntimeVersion;
 use codec::Decode;
 use heapsize::HeapSizeOf;
-use std::marker::PhantomData;
+use memorydb::MemoryDB;
 
 /// Call executor that executes methods on remote node, querying execution proof
 /// and checking proof by re-executing locally.
@@ -108,7 +109,7 @@ where
 		_method: &str,
 		_call_data: &[u8],
 		_m: ExecutionManager<FF>
-	) -> ClientResult<(Vec<u8>, S::Transaction)> {
+	) -> ClientResult<(Vec<u8>, S::Transaction, Option<MemoryDB<H>>)> {
 		Err(ClientErrorKind::NotAvailableOnLightClient.into())
 	}
 
@@ -143,7 +144,7 @@ pub fn check_execution_proof<Header, E, H, C>(
 	let local_state_root = request.header.state_root();
 
 	let mut changes = OverlayedChanges::default();
-	let (local_result, _) = execution_proof_check::<H, C, _>(
+	let local_result = execution_proof_check::<H, C, _>(
 		H256::from_slice(local_state_root.as_ref()).into(),
 		remote_proof,
 		&mut changes,