Light friendly storage tracking: changes trie + extending over ranges (#628)

* changes_trie

* changs_trie: continue

* changes_trie: adding tests

* fixed TODO

* removed obsolete ExtrinsicChanges

* encodable ChangesTrieConfiguration

* removed polkadot fle

* fixed grumbles

* ext_storage_changes_root returns u32

* moved changes trie root to digest

* removed commented code

* read storage values from native code

* fixed grumbles

* fixed grumbles

* missing comma

substrate/core/client/src/backend.rs

@@ -22,8 +22,10 @@ use runtime_primitives::bft::Justification;
 use runtime_primitives::generic::BlockId;
 use runtime_primitives::traits::{Block as BlockT, NumberFor};
 use state_machine::backend::Backend as StateBackend;
+use state_machine::ChangesTrieStorage as StateChangesTrieStorage;
 use patricia_trie::NodeCodec;
 use hashdb::Hasher;
+use memorydb::MemoryDB;
 
 /// Block insertion operation. Keeps hold if the inserted block state and data.
 pub trait BlockImportOperation<Block, H, C>
@@ -53,6 +55,8 @@ where
 	fn update_storage(&mut self, update: <Self::State as StateBackend<H, C>>::Transaction) -> error::Result<()>;
 	/// Inject storage data into the database replacing any existing data.
 	fn reset_storage<I: Iterator<Item=(Vec<u8>, Vec<u8>)>>(&mut self, iter: I) -> error::Result<()>;
+	/// Inject changes trie data into the database.
+	fn update_changes_trie(&mut self, update: MemoryDB<H>) -> error::Result<()>;
 }
 
 /// Client backend. Manages the data layer.
@@ -75,6 +79,8 @@ where
 	type Blockchain: ::blockchain::Backend<Block>;
 	/// Associated state backend type.
 	type State: StateBackend<H, C>;
+	/// Changes trie storage.
+	type ChangesTrieStorage: StateChangesTrieStorage<H>;
 
 	/// Begin a new block insertion transaction with given parent block id.
 	/// When constructing the genesis, this is called with all-zero hash.
@@ -83,6 +89,8 @@ where
 	fn commit_operation(&self, transaction: Self::BlockImportOperation) -> error::Result<()>;
 	/// Returns reference to blockchain backend.
 	fn blockchain(&self) -> &Self::Blockchain;
+	/// Returns reference to changes trie storage.
+	fn changes_trie_storage(&self) -> Option<&Self::ChangesTrieStorage>;
 	/// Returns state backend with post-state of given block.
 	fn state_at(&self, block: BlockId<Block>) -> error::Result<Self::State>;
 	/// Attempts to revert the chain by `n` blocks. Returns the number of blocks that were

substrate/core/client/src/block_builder.rs

@@ -94,7 +94,7 @@ where
 	/// the error. Otherwise, it will return a mutable reference to self (in order to chain).
 	pub fn push(&mut self, xt: <Block as BlockT>::Extrinsic) -> error::Result<()> {
 		match self.executor.call_at_state(&self.state, &mut self.changes, "apply_extrinsic", &xt.encode(), native_when_possible()) {
-			Ok((result, _)) => {
+			Ok((result, _, _)) => {
 				match ApplyResult::decode(&mut result.as_slice()) {
 					Some(Ok(ApplyOutcome::Success)) | Some(Ok(ApplyOutcome::Fail)) => {
 						self.extrinsics.push(xt);
@@ -120,7 +120,7 @@ where
 
 	/// Consume the builder to return a valid `Block` containing all pushed extrinsics.
 	pub fn bake(mut self) -> error::Result<Block> {
-		let (output, _) = self.executor.call_at_state(
+		let (output, _, _) = self.executor.call_at_state(
 			&self.state,
 			&mut self.changes,
 			"finalise_block",

substrate/core/client/src/call_executor.rs

@@ -24,6 +24,7 @@ use executor::{RuntimeVersion, RuntimeInfo};
 use patricia_trie::NodeCodec;
 use hashdb::Hasher;
 use rlp::Encodable;
+use memorydb::MemoryDB;
 use codec::Decode;
 use primitives::{Blake2Hasher, RlpCodec};
 
@@ -76,7 +77,7 @@ where
 		method: &str,
 		call_data: &[u8],
 		manager: ExecutionManager<F>
-	) -> Result<(Vec<u8>, S::Transaction), error::Error>;
+	) -> Result<(Vec<u8>, S::Transaction, Option<MemoryDB<H>>), error::Error>;
 
 	/// Execute a call to a contract on top of given state, gathering execution proof.
 	///
@@ -129,7 +130,7 @@ where
 		call_data: &[u8],
 	) -> error::Result<CallResult> {
 		let mut changes = OverlayedChanges::default();
-		let (return_data, _) = self.call_at_state(
+		let (return_data, _, _) = self.call_at_state(
 			&self.backend.state_at(*id)?,
 			&mut changes,
 			method,
@@ -152,7 +153,8 @@ where
 			.and_then(|v| u64::decode(&mut &v[..]))
 			.unwrap_or(8) as usize;
 
-		self.executor.runtime_version(&mut Ext::new(&mut overlay, &state), heap_pages, &code)
+		let mut ext = Ext::new(&mut overlay, &state, self.backend.changes_trie_storage());
+		self.executor.runtime_version(&mut ext, heap_pages, &code)
 			.ok_or(error::ErrorKind::VersionInvalid.into())
 	}
 
@@ -165,9 +167,10 @@ where
 		method: &str,
 		call_data: &[u8],
 		manager: ExecutionManager<F>,
-	) -> error::Result<(Vec<u8>, S::Transaction)> {
+	) -> error::Result<(Vec<u8>, S::Transaction, Option<MemoryDB<Blake2Hasher>>)> {
 		state_machine::execute_using_consensus_failure_handler(
 			state,
+			self.backend.changes_trie_storage(),
 			changes,
 			&self.executor,
 			method,
@@ -189,7 +192,6 @@ where
 			method,
 			call_data,
 		)
-		.map(|(result, proof, _)| (result, proof))
 		.map_err(Into::into)
 	}
 

substrate/core/client/src/client.rs

@@ -24,7 +24,7 @@ use runtime_primitives::{bft::Justification, generic::{BlockId, SignedBlock, Blo
 use runtime_primitives::traits::{Block as BlockT, Header as HeaderT, Zero, One, As, NumberFor};
 use runtime_primitives::BuildStorage;
 use substrate_metadata::JsonMetadataDecodable;
-use primitives::{Blake2Hasher, RlpCodec};
+use primitives::{Blake2Hasher, RlpCodec, H256};
 use primitives::storage::{StorageKey, StorageData};
 use codec::{Encode, Decode};
 use state_machine::{
@@ -170,6 +170,7 @@ pub fn new_in_mem<E, Block, S>(
 		E: CodeExecutor<Blake2Hasher> + RuntimeInfo,
 		S: BuildStorage,
 		Block: BlockT,
+		H256: From<Block::Hash>,
 {
 	let backend = Arc::new(in_mem::Backend::new());
 	let executor = LocalCallExecutor::new(backend.clone(), executor);
@@ -359,7 +360,7 @@ impl<B, E, Block> Client<B, E, Block> where
 				&header.encode(),
 				execution_manager()
 			)?;
-			let (r, _) = args.using_encoded(|input|
+			let (r, _, _) = args.using_encoded(|input|
 				self.executor().call_at_state(
 				&state,
 				&mut overlay,
@@ -436,7 +437,7 @@ impl<B, E, Block> Client<B, E, Block> where
 		}
 
 		let mut transaction = self.backend.begin_operation(BlockId::Hash(parent_hash))?;
-		let (storage_update, storage_changes) = match transaction.state()? {
+		let (storage_update, changes_update, storage_changes) = match transaction.state()? {
 			Some(transaction_state) => {
 				let mut overlay = Default::default();
 				let mut r = self.executor.call_at_state(
@@ -462,11 +463,11 @@ impl<B, E, Block> Client<B, E, Block> where
 						}),
 					},
 				);
-				let (_, storage_update) = r?;
+				let (_, storage_update, changes_update) = r?;
 				overlay.commit_prospective();
-				(Some(storage_update), Some(overlay.into_committed()))
+				(Some(storage_update), Some(changes_update), Some(overlay.into_committed()))
 			},
-			None => (None, None)
+			None => (None, None, None)
 		};
 
 		let is_new_best = header.number() == &(self.backend.blockchain().info()?.best_number + One::one());
@@ -477,6 +478,9 @@ impl<B, E, Block> Client<B, E, Block> where
 		if let Some(storage_update) = storage_update {
 			transaction.update_storage(storage_update)?;
 		}
+		if let Some(Some(changes_update)) = changes_update {
+			transaction.update_changes_trie(changes_update)?;
+		}
 		self.backend.commit_operation(transaction)?;
 
 		if origin == BlockOrigin::NetworkBroadcast || origin == BlockOrigin::Own || origin == BlockOrigin::ConsensusBroadcast {

substrate/core/client/src/genesis.rs

@@ -45,7 +45,7 @@ mod tests {
 	use codec::{Encode, Decode, Joiner};
 	use keyring::Keyring;
 	use executor::NativeExecutionDispatch;
-	use state_machine::{execute, OverlayedChanges, ExecutionStrategy};
+	use state_machine::{execute, OverlayedChanges, ExecutionStrategy, InMemoryChangesTrieStorage};
 	use state_machine::backend::InMemory;
 	use test_client;
 	use test_client::runtime::genesismap::{GenesisConfig, additional_storage_with_genesis};
@@ -58,7 +58,13 @@ mod tests {
 		NativeExecutionDispatch::new()
 	}
 
-	fn construct_block(backend: &InMemory<Blake2Hasher, RlpCodec>, number: BlockNumber, parent_hash: Hash, state_root: Hash, txs: Vec<Transfer>) -> (Vec<u8>, Hash) {
+	fn construct_block(
+		backend: &InMemory<Blake2Hasher, RlpCodec>,
+		number: BlockNumber,
+		parent_hash: Hash,
+		state_root: Hash,
+		txs: Vec<Transfer>
+	) -> (Vec<u8>, Hash) {
 		use triehash::ordered_trie_root;
 
 		let transactions = txs.into_iter().map(|tx| {
@@ -83,6 +89,7 @@ mod tests {
 
 		execute(
 			backend,
+			Some(&InMemoryChangesTrieStorage::new()),
 			&mut overlay,
 			&executor(),
 			"initialise_block",
@@ -93,6 +100,7 @@ mod tests {
 		for tx in transactions.iter() {
 			execute(
 				backend,
+				Some(&InMemoryChangesTrieStorage::new()),
 				&mut overlay,
 				&executor(),
 				"apply_extrinsic",
@@ -101,8 +109,9 @@ mod tests {
 			).unwrap();
 		}
 
-		let (ret_data, _) = execute(
+		let (ret_data, _, _) = execute(
 			backend,
+			Some(&InMemoryChangesTrieStorage::new()),
 			&mut overlay,
 			&executor(),
 			"finalise_block",
@@ -145,6 +154,7 @@ mod tests {
 		let mut overlay = OverlayedChanges::default();
 		let _ = execute(
 			&backend,
+			Some(&InMemoryChangesTrieStorage::new()),
 			&mut overlay,
 			&executor(),
 			"execute_block",
@@ -168,6 +178,7 @@ mod tests {
 		let mut overlay = OverlayedChanges::default();
 		let _ = execute(
 			&backend,
+			Some(&InMemoryChangesTrieStorage::new()),
 			&mut overlay,
 			&executor(),
 			"execute_block",
@@ -192,6 +203,7 @@ mod tests {
 		let mut overlay = OverlayedChanges::default();
 		let _ = execute(
 			&backend,
+			Some(&InMemoryChangesTrieStorage::new()),
 			&mut overlay,
 			&Executor::new(),
 			"execute_block",

substrate/core/client/src/in_mem.rs

@@ -24,13 +24,16 @@ use backend;
 use light;
 use primitives::AuthorityId;
 use runtime_primitives::generic::BlockId;
-use runtime_primitives::traits::{Block as BlockT, Header as HeaderT, Zero, NumberFor, As};
+use runtime_primitives::traits::{Block as BlockT, Header as HeaderT, Zero,
+	NumberFor, As, Digest, DigestItem};
 use runtime_primitives::bft::Justification;
 use blockchain::{self, BlockStatus};
 use state_machine::backend::{Backend as StateBackend, InMemory};
+use state_machine::InMemoryChangesTrieStorage;
 use patricia_trie::NodeCodec;
 use hashdb::Hasher;
 use heapsize::HeapSizeOf;
+use memorydb::MemoryDB;
 
 struct PendingBlock<B: BlockT> {
 	block: StoredBlock<B>,
@@ -85,9 +88,9 @@ impl<B: BlockT> StoredBlock<B> {
 #[derive(Clone)]
 struct BlockchainStorage<Block: BlockT> {
 	blocks: HashMap<Block::Hash, StoredBlock<Block>>,
-	hashes: HashMap<<<Block as BlockT>::Header as HeaderT>::Number, Block::Hash>,
+	hashes: HashMap<NumberFor<Block>, Block::Hash>,
 	best_hash: Block::Hash,
-	best_number: <<Block as BlockT>::Header as HeaderT>::Number,
+	best_number: NumberFor<Block>,
 	genesis_hash: Block::Hash,
 	cht_roots: HashMap<NumberFor<Block>, Block::Hash>,
 }
@@ -275,6 +278,7 @@ pub struct BlockImportOperation<Block: BlockT, H: Hasher, C: NodeCodec<H>> {
 	pending_authorities: Option<Vec<AuthorityId>>,
 	old_state: InMemory<H, C>,
 	new_state: Option<InMemory<H, C>>,
+	changes_trie_update: Option<MemoryDB<H>>,
 }
 
 impl<Block, H, C> backend::BlockImportOperation<Block, H, C> for BlockImportOperation<Block, H, C>
@@ -314,6 +318,11 @@ where
 		Ok(())
 	}
 
+	fn update_changes_trie(&mut self, update: MemoryDB<H>) -> error::Result<()> {
+		self.changes_trie_update = Some(update);
+		Ok(())
+	}
+
 	fn reset_storage<I: Iterator<Item=(Vec<u8>, Vec<u8>)>>(&mut self, iter: I) -> error::Result<()> {
 		self.new_state = Some(InMemory::from(iter.collect::<HashMap<_, _>>()));
 		Ok(())
@@ -325,9 +334,11 @@ pub struct Backend<Block, H, C>
 where
 	Block: BlockT,
 	H: Hasher,
-	C: NodeCodec<H>
+	C: NodeCodec<H>,
+	H::Out: HeapSizeOf + From<Block::Hash>,
 {
 	states: RwLock<HashMap<Block::Hash, InMemory<H, C>>>,
+	changes_trie_storage: InMemoryChangesTrieStorage<H>,
 	blockchain: Blockchain<Block>,
 }
 
@@ -335,12 +346,14 @@ impl<Block, H, C> Backend<Block, H, C>
 where
 	Block: BlockT,
 	H: Hasher,
-	C: NodeCodec<H>
+	C: NodeCodec<H>,
+	H::Out: HeapSizeOf + From<Block::Hash>,
 {
 	/// Create a new instance of in-mem backend.
 	pub fn new() -> Backend<Block, H, C> {
 		Backend {
 			states: RwLock::new(HashMap::new()),
+			changes_trie_storage: InMemoryChangesTrieStorage::new(),
 			blockchain: Blockchain::new(),
 		}
 	}
@@ -350,12 +363,13 @@ impl<Block, H, C> backend::Backend<Block, H, C> for Backend<Block, H, C>
 where
 	Block: BlockT,
 	H: Hasher,
-	H::Out: HeapSizeOf,
+	H::Out: HeapSizeOf + From<Block::Hash>,
 	C: NodeCodec<H> + Send + Sync,
 {
 	type BlockImportOperation = BlockImportOperation<Block, H, C>;
 	type Blockchain = Blockchain<Block>;
 	type State = InMemory<H, C>;
+	type ChangesTrieStorage = InMemoryChangesTrieStorage<H>;
 
 	fn begin_operation(&self, block: BlockId<Block>) -> error::Result<Self::BlockImportOperation> {
 		let state = match block {
@@ -368,6 +382,7 @@ where
 			pending_authorities: None,
 			old_state: state,
 			new_state: None,
+			changes_trie_update: None,
 		})
 	}
 
@@ -375,10 +390,21 @@ where
 		if let Some(pending_block) = operation.pending_block {
 			let old_state = &operation.old_state;
 			let (header, body, justification) = pending_block.block.into_inner();
+
 			let hash = header.hash();
 			let parent_hash = *header.parent_hash();
 
 			self.states.write().insert(hash, operation.new_state.unwrap_or_else(|| old_state.clone()));
+			let changes_trie_root = header.digest().logs().iter()
+				.find(|log| log.as_changes_trie_root().is_some())
+				.and_then(DigestItem::as_changes_trie_root)
+				.cloned();
+			if let Some(changes_trie_root) = changes_trie_root {
+				if let Some(changes_trie_update) = operation.changes_trie_update {
+					let changes_trie_root: H::Out = changes_trie_root.into();
+					self.changes_trie_storage.insert(header.number().as_(), changes_trie_root, changes_trie_update);
+				}
+			}
 			self.blockchain.insert(hash, header, justification, body, pending_block.is_best);
 			// dumb implementation - store value for each block
 			if pending_block.is_best {
@@ -392,6 +418,10 @@ where
 		&self.blockchain
 	}
 
+	fn changes_trie_storage(&self) -> Option<&Self::ChangesTrieStorage> {
+		Some(&self.changes_trie_storage)
+	}
+
 	fn state_at(&self, block: BlockId<Block>) -> error::Result<Self::State> {
 		match self.blockchain.id(block).and_then(|id| self.states.read().get(&id).cloned()) {
 			Some(state) => Ok(state),
@@ -408,7 +438,7 @@ impl<Block, H, C> backend::LocalBackend<Block, H, C> for Backend<Block, H, C>
 where
 	Block: BlockT,
 	H: Hasher,
-	H::Out: HeapSizeOf,
+	H::Out: HeapSizeOf + From<Block::Hash>,
 	C: NodeCodec<H> + Send + Sync,
 {}
 

substrate/core/client/src/lib.rs

@@ -39,6 +39,7 @@ extern crate patricia_trie;
 extern crate hashdb;
 extern crate rlp;
 extern crate heapsize;
+extern crate memorydb;
 
 #[macro_use] extern crate error_chain;
 #[macro_use] extern crate log;

substrate/core/client/src/light/backend.rs

@@ -24,11 +24,7 @@ use parking_lot::RwLock;
 use primitives::AuthorityId;
 use runtime_primitives::{bft::Justification, generic::BlockId};
 use runtime_primitives::traits::{Block as BlockT, NumberFor};
-use state_machine::{
-	Backend as StateBackend,
-	TrieBackend as StateTrieBackend,
-	TryIntoTrieBackend as TryIntoStateTrieBackend
-};
+use state_machine::{Backend as StateBackend, InMemoryChangesTrieStorage, TrieBackend};
 
 use backend::{Backend as ClientBackend, BlockImportOperation, RemoteBackend};
 use blockchain::HeaderBackend as BlockchainHeaderBackend;
@@ -37,6 +33,8 @@ use light::blockchain::{Blockchain, Storage as BlockchainStorage};
 use light::fetcher::{Fetcher, RemoteReadRequest};
 use patricia_trie::NodeCodec;
 use hashdb::Hasher;
+use memorydb::MemoryDB;
+use heapsize::HeapSizeOf;
 
 /// Light client backend.
 pub struct Backend<S, F> {
@@ -77,10 +75,12 @@ impl<S, F, Block, H, C> ClientBackend<Block, H, C> for Backend<S, F> where
 	F: Fetcher<Block>,
 	H: Hasher,
 	C: NodeCodec<H>,
+	H::Out: HeapSizeOf,
 {
 	type BlockImportOperation = ImportOperation<Block, S, F>;
 	type Blockchain = Blockchain<S, F>;
 	type State = OnDemandState<Block, S, F>;
+	type ChangesTrieStorage = InMemoryChangesTrieStorage<H>;
 
 	fn begin_operation(&self, _block: BlockId<Block>) -> ClientResult<Self::BlockImportOperation> {
 		Ok(ImportOperation {
@@ -100,6 +100,10 @@ impl<S, F, Block, H, C> ClientBackend<Block, H, C> for Backend<S, F> where
 		&self.blockchain
 	}
 
+	fn changes_trie_storage(&self) -> Option<&Self::ChangesTrieStorage> {
+		None
+	}
+
 	fn state_at(&self, block: BlockId<Block>) -> ClientResult<Self::State> {
 		let block_hash = match block {
 			BlockId::Hash(h) => Some(h),
@@ -125,6 +129,7 @@ where
 	S: BlockchainStorage<Block>,
 	F: Fetcher<Block>,
 	H: Hasher,
+	H::Out: HeapSizeOf,
 	C: NodeCodec<H>,
 {}
 
@@ -164,6 +169,11 @@ where
 		Ok(())
 	}
 
+	fn update_changes_trie(&mut self, _update: MemoryDB<H>) -> ClientResult<()> {
+		// we're not storing anything locally => ignore changes
+		Ok(())
+	}
+
 	fn reset_storage<I: Iterator<Item=(Vec<u8>, Vec<u8>)>>(&mut self, _iter: I) -> ClientResult<()> {
 		// we're not storing anything locally => ignore changes
 		Ok(())
@@ -180,6 +190,7 @@ impl<Block, S, F, H, C> StateBackend<H, C> for OnDemandState<Block, S, F>
 {
 	type Error = ClientError;
 	type Transaction = ();
+	type TrieBackendStorage = MemoryDB<H>;
 
 	fn storage(&self, key: &[u8]) -> ClientResult<Option<Vec<u8>>> {
 		let mut header = self.cached_header.read().clone();
@@ -214,16 +225,8 @@ impl<Block, S, F, H, C> StateBackend<H, C> for OnDemandState<Block, S, F>
 		// whole state is not available on light node
 		Vec::new()
 	}
-}
 
-impl<Block, S, F, H, C> TryIntoStateTrieBackend<H, C> for OnDemandState<Block, S, F>
-where
-	Block: BlockT,
-	F: Fetcher<Block>,
-	H: Hasher,
-	C: NodeCodec<H>,
-{
-	fn try_into_trie_backend(self) -> Option<StateTrieBackend<H, C>> {
+	fn try_into_trie_backend(self) -> Option<TrieBackend<Self::TrieBackendStorage, H, C>> {
 		None
 	}
 }

substrate/core/client/src/light/call_executor.rs

@@ -17,6 +17,7 @@
 //! Light client call exector. Executes methods on remote full nodes, fetching
 //! execution proof and checking it locally.
 
+use std::marker::PhantomData;
 use std::sync::Arc;
 use futures::{IntoFuture, Future};
 
@@ -36,7 +37,7 @@ use light::fetcher::{Fetcher, RemoteCallRequest};
 use executor::RuntimeVersion;
 use codec::Decode;
 use heapsize::HeapSizeOf;
-use std::marker::PhantomData;
+use memorydb::MemoryDB;
 
 /// Call executor that executes methods on remote node, querying execution proof
 /// and checking proof by re-executing locally.
@@ -108,7 +109,7 @@ where
 		_method: &str,
 		_call_data: &[u8],
 		_m: ExecutionManager<FF>
-	) -> ClientResult<(Vec<u8>, S::Transaction)> {
+	) -> ClientResult<(Vec<u8>, S::Transaction, Option<MemoryDB<H>>)> {
 		Err(ClientErrorKind::NotAvailableOnLightClient.into())
 	}
 
@@ -143,7 +144,7 @@ pub fn check_execution_proof<Header, E, H, C>(
 	let local_state_root = request.header.state_root();
 
 	let mut changes = OverlayedChanges::default();
-	let (local_result, _) = execution_proof_check::<H, C, _>(
+	let local_result = execution_proof_check::<H, C, _>(
 		H256::from_slice(local_state_root.as_ref()).into(),
 		remote_proof,
 		&mut changes,