Pin states in memory so that they are not pruned away while still referenced (#2761)

* State pinning in client

* Canonicalization queue

* Fixed prioritization queue

* possible fix of "hash mismatch"

* Check for pinned discarded states

* Release state before finalization

* Style

* Style

substrate/core/client/src/call_executor.rs

@@ -117,17 +117,17 @@ where
 	/// No changes are made.
 	fn prove_at_state<S: state_machine::Backend<H>>(
 		&self,
-		state: S,
+		mut state: S,
 		overlay: &mut OverlayedChanges,
 		method: &str,
 		call_data: &[u8]
 	) -> Result<(Vec<u8>, Vec<Vec<u8>>), error::Error> {
-		let trie_state = state.try_into_trie_backend()
+		let trie_state = state.as_trie_backend()
 			.ok_or_else(||
 				Box::new(state_machine::ExecutionError::UnableToGenerateProof)
 					as Box<state_machine::Error>
 			)?;
-		self.prove_at_trie_state(&trie_state, overlay, method, call_data)
+		self.prove_at_trie_state(trie_state, overlay, method, call_data)
 	}
 
 	/// Execute a call to a contract on top of given trie state, gathering execution proof.
@@ -239,18 +239,18 @@ where
 			_ => {},
 		}
 
-		let state = self.backend.state_at(*at)?;
+		let mut state = self.backend.state_at(*at)?;
 
 		match recorder {
 			Some(recorder) => {
-				let trie_state = state.try_into_trie_backend()
+				let trie_state = state.as_trie_backend()
 					.ok_or_else(||
 						Box::new(state_machine::ExecutionError::UnableToGenerateProof)
 							as Box<state_machine::Error>
 					)?;
 
 				let backend = state_machine::ProvingBackend::new_with_recorder(
-					&trie_state,
+					trie_state,
 					recorder.clone()
 				);
 

substrate/core/client/src/cht.rs

@@ -101,14 +101,14 @@ pub fn build_proof<Header, Hasher, BlocksI, HashesI>(
 		.into_iter()
 		.map(|(k, v)| (None, k, Some(v)))
 		.collect::<Vec<_>>();
-	let storage = InMemoryState::<Hasher>::default().update(transaction);
-	let trie_storage = storage.try_into_trie_backend()
-		.expect("InMemoryState::try_into_trie_backend always returns Some; qed");
+	let mut storage = InMemoryState::<Hasher>::default().update(transaction);
+	let trie_storage = storage.as_trie_backend()
+		.expect("InMemoryState::as_trie_backend always returns Some; qed");
 	let mut total_proof = HashSet::new();
 	for block in blocks.into_iter() {
 		debug_assert_eq!(block_to_cht_number(cht_size, block), Some(cht_num));
 
-		let (value, proof) = prove_read_on_trie_backend(&trie_storage, &encode_cht_key(block))?;
+		let (value, proof) = prove_read_on_trie_backend(trie_storage, &encode_cht_key(block))?;
 		assert!(value.is_some(), "we have just built trie that includes the value for block");
 		total_proof.extend(proof);
 	}

substrate/core/client/src/client.rs

@@ -1277,7 +1277,7 @@ impl<B, E, Block, RA> Client<B, E, Block, RA> where
 
 	/// Prepare in-memory header that is used in execution environment.
 	fn prepare_environment_block(&self, parent: &BlockId<Block>) -> error::Result<Block::Header> {
-		let parent_header = self.backend().blockchain().expect_header(*parent)?;
+		let parent_header = self.backend.blockchain().expect_header(*parent)?;
 		Ok(<<Block as BlockT>::Header as HeaderT>::new(
 			self.backend.blockchain().expect_block_number_from_id(parent)? + One::one(),
 			Default::default(),

substrate/core/client/src/light/backend.rs

@@ -38,13 +38,13 @@ use consensus::well_known_cache_keys;
 const IN_MEMORY_EXPECT_PROOF: &str = "InMemory state backend has Void error type and always suceeds; qed";
 
 /// Light client backend.
-pub struct Backend<S, F, H> {
+pub struct Backend<S, F, H: Hasher> {
 	blockchain: Arc<Blockchain<S, F>>,
 	genesis_state: RwLock<Option<InMemoryState<H>>>,
 }
 
 /// Light block (header and justification) import operation.
-pub struct ImportOperation<Block: BlockT, S, F, H> {
+pub struct ImportOperation<Block: BlockT, S, F, H: Hasher> {
 	header: Option<Block::Header>,
 	cache: HashMap<well_known_cache_keys::Id, Vec<u8>>,
 	leaf_state: NewBlockState,
@@ -64,14 +64,14 @@ pub struct OnDemandState<Block: BlockT, S, F> {
 }
 
 /// On-demand or in-memory genesis state.
-pub enum OnDemandOrGenesisState<Block: BlockT, S, F, H> {
+pub enum OnDemandOrGenesisState<Block: BlockT, S, F, H: Hasher> {
 	/// On-demand state - storage values are fetched from remote nodes.
 	OnDemand(OnDemandState<Block, S, F>),
 	/// Genesis state - storage values are stored in-memory.
 	Genesis(InMemoryState<H>),
 }
 
-impl<S, F, H> Backend<S, F, H> {
+impl<S, F, H: Hasher> Backend<S, F, H> {
 	/// Create new light backend.
 	pub fn new(blockchain: Arc<Blockchain<S, F>>) -> Self {
 		Self {
@@ -86,7 +86,7 @@ impl<S, F, H> Backend<S, F, H> {
 	}
 }
 
-impl<S: AuxStore, F, H> AuxStore for Backend<S, F, H> {
+impl<S: AuxStore, F, H: Hasher> AuxStore for Backend<S, F, H> {
 	fn insert_aux<
 		'a,
 		'b: 'a,
@@ -387,7 +387,7 @@ where
 		Vec::new()
 	}
 
-	fn try_into_trie_backend(self) -> Option<TrieBackend<Self::TrieBackendStorage, H>> {
+	fn as_trie_backend(&mut self) -> Option<&TrieBackend<Self::TrieBackendStorage, H>> {
 		None
 	}
 }
@@ -482,10 +482,10 @@ where
 		}
 	}
 
-	fn try_into_trie_backend(self) -> Option<TrieBackend<Self::TrieBackendStorage, H>> {
+	fn as_trie_backend(&mut self) -> Option<&TrieBackend<Self::TrieBackendStorage, H>> {
 		match self {
-			OnDemandOrGenesisState::OnDemand(state) => state.try_into_trie_backend(),
-			OnDemandOrGenesisState::Genesis(state) => state.try_into_trie_backend(),
+			OnDemandOrGenesisState::OnDemand(ref mut state) => state.as_trie_backend(),
+			OnDemandOrGenesisState::Genesis(ref mut state) => state.as_trie_backend(),
 		}
 	}
 }

substrate/core/client/src/light/call_executor.rs

@@ -388,7 +388,7 @@ impl<Block, B, Remote, Local> CallExecutor<Block, Blake2Hasher> for
 /// Method is executed using passed header as environment' current block.
 /// Proof includes both environment preparation proof and method execution proof.
 pub fn prove_execution<Block, S, E>(
-	state: S,
+	mut state: S,
 	header: Block::Header,
 	executor: &E,
 	method: &str,
@@ -399,13 +399,13 @@ pub fn prove_execution<Block, S, E>(
 		S: StateBackend<Blake2Hasher>,
 		E: CallExecutor<Block, Blake2Hasher>,
 {
-	let trie_state = state.try_into_trie_backend()
+	let trie_state = state.as_trie_backend()
 		.ok_or_else(|| Box::new(state_machine::ExecutionError::UnableToGenerateProof) as Box<state_machine::Error>)?;
 
 	// prepare execution environment + record preparation proof
 	let mut changes = Default::default();
 	let (_, init_proof) = executor.prove_at_trie_state(
-		&trie_state,
+		trie_state,
 		&mut changes,
 		"Core_initialize_block",
 		&header.encode(),