Trie simplification. (#2815)

* switch to simple codec, trie broken for now

* Actualy use trie_root_noext

* align some hash, failing test on EMCH comment

* Fix trie code over layout instead of hash, revert legacy code for legacy
mainnet ??

* stub behind LayOut

* fix no_std

* temp solution for legacy trie behind feature legacy-key in various crate

* use remote project

* rc client db need prefix

* update trie deps

* bum spec runtime version

* Removing legacy as default.

* Switch mode to non legacy.

* bump runtime version

* Remove legacy trie compatibility features.

* fix warning

* bump version

* change hash on new test.

* Move dependency (#11 trie PR) patched to a parity repo.
Bench reverted to correct hasher.
Some renaming and doc improvments.

* ChildBitmap renaming to BitMap.

* Renaming of LayOut to Layout.

* formatting.

* Removing abreviation such as _ix nb_ or bm.

* Update deps and apply renaming 'Buff' -> 'Buffer'.

* Align to latest trie crates naming changes.

* Update trie dependency.

* Update trie dependency.

* change block_import test hash

* update trie deps (trie use new scale codec but it does not seems to be
an issue).

* update to use latest trie version (no mgmt of multiple radix).

* tabify

* Restoring test to 10 000.

* Use published crate, trie bench is currently down until publishing
(require another pr to update version).

* Update trie-bench.

substrate/core/client/Cargo.toml

@@ -17,7 +17,7 @@ state-machine = { package = "substrate-state-machine", path = "../state-machine"
 keyring = { package = "substrate-keyring", path = "../keyring", optional = true }
 trie = { package = "substrate-trie", path = "../trie", optional = true }
 substrate-telemetry = { path = "../telemetry", optional = true }
-hash-db = { version = "0.14.0", default-features = false }
+hash-db = { version = "0.15.0", default-features = false }
 kvdb = { git = "https://github.com/paritytech/parity-common", optional = true, rev="b0317f649ab2c665b7987b8475878fc4d2e1f81d" }
 parity-codec = { version = "4.1.1", default-features = false, features = ["derive"] }
 primitives = { package = "substrate-primitives", path = "../primitives", default-features = false }

substrate/core/client/db/Cargo.toml

@@ -12,7 +12,7 @@ kvdb = { git = "https://github.com/paritytech/parity-common", rev="b0317f649ab2c
 kvdb-rocksdb = { git = "https://github.com/paritytech/parity-common", rev="b0317f649ab2c665b7987b8475878fc4d2e1f81d", optional = true }
 kvdb-memorydb = { git = "https://github.com/paritytech/parity-common", rev="b0317f649ab2c665b7987b8475878fc4d2e1f81d" }
 linked-hash-map = "0.5"
-hash-db = { version = "0.14.0" }
+hash-db = { version = "0.15.0" }
 primitives = { package = "substrate-primitives", path = "../../primitives" }
 sr-primitives = {  path = "../../sr-primitives" }
 client = { package = "substrate-client", path = "../../client" }

substrate/core/client/db/src/lib.rs

@@ -43,7 +43,7 @@ use client::blockchain::HeaderBackend;
 use client::ExecutionStrategies;
 use client::backend::{StorageCollection, ChildStorageCollection};
 use parity_codec::{Decode, Encode};
-use hash_db::Hasher;
+use hash_db::{Hasher, Prefix};
 use kvdb::{KeyValueDB, DBTransaction};
 use trie::{MemoryDB, PrefixedMemoryDB, prefixed_key};
 use parking_lot::{Mutex, RwLock};
@@ -507,7 +507,7 @@ struct StorageDb<Block: BlockT> {
 }
 
 impl<Block: BlockT> state_machine::Storage<Blake2Hasher> for StorageDb<Block> {
-	fn get(&self, key: &H256, prefix: &[u8]) -> Result<Option<DBValue>, String> {
+	fn get(&self, key: &H256, prefix: Prefix) -> Result<Option<DBValue>, String> {
 		let key = prefixed_key::<Blake2Hasher>(key, prefix);
 		self.state_db.get(&key, self).map(|r| r.map(|v| DBValue::from_slice(&v)))
 			.map_err(|e| format!("Database backend error: {:?}", e))
@@ -535,7 +535,7 @@ impl DbGenesisStorage {
 }
 
 impl state_machine::Storage<Blake2Hasher> for DbGenesisStorage {
-	fn get(&self, _key: &H256, _prefix: &[u8]) -> Result<Option<DBValue>, String> {
+	fn get(&self, _key: &H256, _prefix: Prefix) -> Result<Option<DBValue>, String> {
 		Ok(None)
 	}
 }
@@ -675,7 +675,7 @@ impl<Block> state_machine::ChangesTrieStorage<Blake2Hasher, NumberFor<Block>>
 where
 	Block: BlockT<Hash=H256>,
 {
-	fn get(&self, key: &H256, _prefix: &[u8]) -> Result<Option<DBValue>, String> {
+	fn get(&self, key: &H256, _prefix: Prefix) -> Result<Option<DBValue>, String> {
 		self.db.get(columns::CHANGES_TRIE, &key[..])
 			.map_err(|err| format!("{}", err))
 	}
@@ -1428,7 +1428,7 @@ where Block: BlockT<Hash=H256> {}
 
 #[cfg(test)]
 mod tests {
-	use hash_db::HashDB;
+	use hash_db::{HashDB, EMPTY_PREFIX};
 	use super::*;
 	use crate::columns;
 	use client::backend::Backend as BTrait;
@@ -1654,7 +1654,7 @@ mod tests {
 
 			op.reset_storage(storage.iter().cloned().collect(), Default::default()).unwrap();
 
-			key = op.db_updates.insert(&[], b"hello");
+			key = op.db_updates.insert(EMPTY_PREFIX, b"hello");
 			op.set_block_data(
 				header,
 				Some(vec![]),
@@ -1663,8 +1663,10 @@ mod tests {
 			).unwrap();
 
 			backend.commit_operation(op).unwrap();
-
-			assert_eq!(backend.storage.db.get(columns::STATE, key.as_bytes()).unwrap().unwrap(), &b"hello"[..]);
+			assert_eq!(backend.storage.db.get(
+				columns::STATE,
+				&trie::prefixed_key::<Blake2Hasher>(&key, EMPTY_PREFIX)
+			).unwrap().unwrap(), &b"hello"[..]);
 			hash
 		};
 
@@ -1688,8 +1690,8 @@ mod tests {
 			).0.into();
 			let hash = header.hash();
 
-			op.db_updates.insert(&[], b"hello");
-			op.db_updates.remove(&key, &[]);
+			op.db_updates.insert(EMPTY_PREFIX, b"hello");
+			op.db_updates.remove(&key, EMPTY_PREFIX);
 			op.set_block_data(
 				header,
 				Some(vec![]),
@@ -1698,8 +1700,10 @@ mod tests {
 			).unwrap();
 
 			backend.commit_operation(op).unwrap();
-
-			assert_eq!(backend.storage.db.get(columns::STATE, key.as_bytes()).unwrap().unwrap(), &b"hello"[..]);
+			assert_eq!(backend.storage.db.get(
+				columns::STATE,
+				&trie::prefixed_key::<Blake2Hasher>(&key, EMPTY_PREFIX)
+			).unwrap().unwrap(), &b"hello"[..]);
 			hash
 		};
 
@@ -1723,7 +1727,7 @@ mod tests {
 			).0.into();
 			let hash = header.hash();
 
-			op.db_updates.remove(&key, &[]);
+			op.db_updates.remove(&key, EMPTY_PREFIX);
 			op.set_block_data(
 				header,
 				Some(vec![]),
@@ -1733,7 +1737,11 @@ mod tests {
 
 			backend.commit_operation(op).unwrap();
 
-			assert!(backend.storage.db.get(columns::STATE, key.as_bytes()).unwrap().is_some());
+
+			assert!(backend.storage.db.get(
+				columns::STATE,
+				&trie::prefixed_key::<Blake2Hasher>(&key, EMPTY_PREFIX)
+			).unwrap().is_some());
 			hash
 		};
 
@@ -1764,14 +1772,19 @@ mod tests {
 			).unwrap();
 
 			backend.commit_operation(op).unwrap();
-
-			assert!(backend.storage.db.get(columns::STATE, key.as_bytes()).unwrap().is_none());
+			assert!(backend.storage.db.get(
+				columns::STATE,
+				&trie::prefixed_key::<Blake2Hasher>(&key, EMPTY_PREFIX)
+			).unwrap().is_none());
 		}
 
 		backend.finalize_block(BlockId::Number(1), None).unwrap();
 		backend.finalize_block(BlockId::Number(2), None).unwrap();
 		backend.finalize_block(BlockId::Number(3), None).unwrap();
-		assert!(backend.storage.db.get(columns::STATE, key.as_bytes()).unwrap().is_none());
+		assert!(backend.storage.db.get(
+			columns::STATE,
+			&trie::prefixed_key::<Blake2Hasher>(&key, EMPTY_PREFIX)
+		).unwrap().is_none());
 	}
 
 	#[test]
@@ -1789,7 +1802,7 @@ mod tests {
 			assert_eq!(backend.changes_tries_storage.root(&anchor, block), Ok(Some(changes_root)));
 
 			for (key, (val, _)) in changes_trie_update.drain() {
-				assert_eq!(backend.changes_trie_storage().unwrap().get(&key, &[]), Ok(Some(val)));
+				assert_eq!(backend.changes_trie_storage().unwrap().get(&key, EMPTY_PREFIX), Ok(Some(val)));
 			}
 		};
 
@@ -1915,23 +1928,23 @@ mod tests {
 		let mut tx = DBTransaction::new();
 		backend.changes_tries_storage.prune(&config, &mut tx, Default::default(), 12);
 		backend.storage.db.write(tx).unwrap();
-		assert!(backend.changes_tries_storage.get(&root1, &[]).unwrap().is_none());
-		assert!(backend.changes_tries_storage.get(&root2, &[]).unwrap().is_none());
-		assert!(backend.changes_tries_storage.get(&root3, &[]).unwrap().is_none());
-		assert!(backend.changes_tries_storage.get(&root4, &[]).unwrap().is_none());
-		assert!(backend.changes_tries_storage.get(&root5, &[]).unwrap().is_some());
-		assert!(backend.changes_tries_storage.get(&root6, &[]).unwrap().is_some());
-		assert!(backend.changes_tries_storage.get(&root7, &[]).unwrap().is_some());
-		assert!(backend.changes_tries_storage.get(&root8, &[]).unwrap().is_some());
+		assert!(backend.changes_tries_storage.get(&root1, EMPTY_PREFIX).unwrap().is_none());
+		assert!(backend.changes_tries_storage.get(&root2, EMPTY_PREFIX).unwrap().is_none());
+		assert!(backend.changes_tries_storage.get(&root3, EMPTY_PREFIX).unwrap().is_none());
+		assert!(backend.changes_tries_storage.get(&root4, EMPTY_PREFIX).unwrap().is_none());
+		assert!(backend.changes_tries_storage.get(&root5, EMPTY_PREFIX).unwrap().is_some());
+		assert!(backend.changes_tries_storage.get(&root6, EMPTY_PREFIX).unwrap().is_some());
+		assert!(backend.changes_tries_storage.get(&root7, EMPTY_PREFIX).unwrap().is_some());
+		assert!(backend.changes_tries_storage.get(&root8, EMPTY_PREFIX).unwrap().is_some());
 
 		// now simulate finalization of block#16, causing prune of tries at #5..#8
 		let mut tx = DBTransaction::new();
 		backend.changes_tries_storage.prune(&config, &mut tx, Default::default(), 16);
 		backend.storage.db.write(tx).unwrap();
-		assert!(backend.changes_tries_storage.get(&root5, &[]).unwrap().is_none());
-		assert!(backend.changes_tries_storage.get(&root6, &[]).unwrap().is_none());
-		assert!(backend.changes_tries_storage.get(&root7, &[]).unwrap().is_none());
-		assert!(backend.changes_tries_storage.get(&root8, &[]).unwrap().is_none());
+		assert!(backend.changes_tries_storage.get(&root5, EMPTY_PREFIX).unwrap().is_none());
+		assert!(backend.changes_tries_storage.get(&root6, EMPTY_PREFIX).unwrap().is_none());
+		assert!(backend.changes_tries_storage.get(&root7, EMPTY_PREFIX).unwrap().is_none());
+		assert!(backend.changes_tries_storage.get(&root8, EMPTY_PREFIX).unwrap().is_none());
 
 		// now "change" pruning mode to archive && simulate finalization of block#20
 		// => no changes tries are pruned, because we never prune in archive mode
@@ -1939,10 +1952,10 @@ mod tests {
 		let mut tx = DBTransaction::new();
 		backend.changes_tries_storage.prune(&config, &mut tx, Default::default(), 20);
 		backend.storage.db.write(tx).unwrap();
-		assert!(backend.changes_tries_storage.get(&root9, &[]).unwrap().is_some());
-		assert!(backend.changes_tries_storage.get(&root10, &[]).unwrap().is_some());
-		assert!(backend.changes_tries_storage.get(&root11, &[]).unwrap().is_some());
-		assert!(backend.changes_tries_storage.get(&root12, &[]).unwrap().is_some());
+		assert!(backend.changes_tries_storage.get(&root9, EMPTY_PREFIX).unwrap().is_some());
+		assert!(backend.changes_tries_storage.get(&root10, EMPTY_PREFIX).unwrap().is_some());
+		assert!(backend.changes_tries_storage.get(&root11, EMPTY_PREFIX).unwrap().is_some());
+		assert!(backend.changes_tries_storage.get(&root12, EMPTY_PREFIX).unwrap().is_some());
 	}
 
 	#[test]
@@ -1981,15 +1994,15 @@ mod tests {
 		let mut tx = DBTransaction::new();
 		backend.changes_tries_storage.prune(&config, &mut tx, block5, 5);
 		backend.storage.db.write(tx).unwrap();
-		assert!(backend.changes_tries_storage.get(&root1, &[]).unwrap().is_none());
-		assert!(backend.changes_tries_storage.get(&root2, &[]).unwrap().is_some());
+		assert!(backend.changes_tries_storage.get(&root1, EMPTY_PREFIX).unwrap().is_none());
+		assert!(backend.changes_tries_storage.get(&root2, EMPTY_PREFIX).unwrap().is_some());
 
 		// now simulate finalization of block#6, causing prune of tries at #2
 		let mut tx = DBTransaction::new();
 		backend.changes_tries_storage.prune(&config, &mut tx, block6, 6);
 		backend.storage.db.write(tx).unwrap();
-		assert!(backend.changes_tries_storage.get(&root2, &[]).unwrap().is_none());
-		assert!(backend.changes_tries_storage.get(&root3, &[]).unwrap().is_some());
+		assert!(backend.changes_tries_storage.get(&root2, EMPTY_PREFIX).unwrap().is_none());
+		assert!(backend.changes_tries_storage.get(&root3, EMPTY_PREFIX).unwrap().is_some());
 	}
 
 	#[test]

substrate/core/client/src/cht.rs

@@ -78,7 +78,8 @@ pub fn compute_root<Header, Hasher, I>(
 		Hasher::Out: Ord,
 		I: IntoIterator<Item=ClientResult<Option<Header::Hash>>>,
 {
-	Ok(trie::trie_root::<Hasher, _, _, _>(
+	use trie::TrieConfiguration;
+	Ok(trie::trie_types::Layout::<Hasher>::trie_root(
 		build_pairs::<Header, I>(cht_size, cht_num, hashes)?
 	))
 }

substrate/core/client/src/client.rs

@@ -57,7 +57,7 @@ use state_machine::{
 	ChangesTrieRootsStorage, ChangesTrieStorage,
 	key_changes, key_changes_proof, OverlayedChanges, NeverOffchainExt,
 };
-use hash_db::Hasher;
+use hash_db::{Hasher, Prefix};
 
 use crate::backend::{
 	self, BlockImportOperation, PrunableStateChangesTrieStorage,
@@ -614,7 +614,7 @@ impl<B, E, Block, RA> Client<B, E, Block, RA> where
 		}
 
 		impl<'a, Block: BlockT> ChangesTrieStorage<Blake2Hasher, NumberFor<Block>> for AccessedRootsRecorder<'a, Block> {
-			fn get(&self, key: &H256, prefix: &[u8]) -> Result<Option<DBValue>, String> {
+			fn get(&self, key: &H256, prefix: Prefix) -> Result<Option<DBValue>, String> {
 				self.storage.get(key, prefix)
 			}
 		}

substrate/core/client/src/genesis.rs

@@ -71,11 +71,12 @@ mod tests {
 		state_root: Hash,
 		txs: Vec<Transfer>
 	) -> (Vec<u8>, Hash) {
-		use trie::ordered_trie_root;
+		use trie::{TrieConfiguration, trie_types::Layout};
 
 		let transactions = txs.into_iter().map(|tx| tx.into_signed_tx()).collect::<Vec<_>>();
 
-		let extrinsics_root = ordered_trie_root::<Blake2Hasher, _, _>(transactions.iter().map(Encode::encode)).into();
+		let iter = transactions.iter().map(Encode::encode);
+		let extrinsics_root = Layout::<Blake2Hasher>::ordered_trie_root(iter).into();
 
 		let mut header = Header {
 			parent_hash,

substrate/core/client/src/in_mem.rs

@@ -25,7 +25,7 @@ use sr_primitives::traits::{Block as BlockT, Header as HeaderT, Zero, NumberFor}
 use sr_primitives::{Justification, StorageOverlay, ChildrenStorageOverlay};
 use state_machine::backend::{Backend as StateBackend, InMemory};
 use state_machine::{self, InMemoryChangesTrieStorage, ChangesTrieAnchorBlockId};
-use hash_db::Hasher;
+use hash_db::{Hasher, Prefix};
 use trie::MemoryDB;
 use consensus::well_known_cache_keys::Id as CacheKeyId;
 
@@ -755,8 +755,8 @@ impl<Block, H> state_machine::ChangesTrieStorage<H, NumberFor<Block>> for Change
 		Block: BlockT,
 		H: Hasher,
 {
-	fn get(&self, _key: &H::Out, _prefix: &[u8]) -> Result<Option<state_machine::DBValue>, String> {
-		Err("Dummy implementation".into())
+	fn get(&self, key: &H::Out, prefix: Prefix) -> Result<Option<state_machine::DBValue>, String> {
+		self.0.get(key, prefix)
 	}
 }
 

substrate/core/client/src/light/fetcher.rs

@@ -21,7 +21,7 @@ use std::collections::BTreeMap;
 use std::marker::PhantomData;
 use std::future::Future;
 
-use hash_db::{HashDB, Hasher};
+use hash_db::{HashDB, Hasher, EMPTY_PREFIX};
 use parity_codec::{Decode, Encode};
 use primitives::{ChangesTrieConfiguration, convert_hash};
 use sr_primitives::traits::{
@@ -333,7 +333,7 @@ impl<E, H, B: BlockT, S: BlockchainStorage<B>, F> LightDataChecker<E, H, B, S, F
 				// we share the storage for multiple checks, do it here
 				let mut cht_root = H::Out::default();
 				cht_root.as_mut().copy_from_slice(local_cht_root.as_ref());
-				if !storage.contains(&cht_root, &[]) {
+				if !storage.contains(&cht_root, EMPTY_PREFIX) {
 					return Err(ClientError::InvalidCHTProof.into());
 				}
 
@@ -610,8 +610,9 @@ pub mod tests {
 	}
 
 	fn header_with_computed_extrinsics_root(extrinsics: Vec<Extrinsic>) -> Header {
-		let extrinsics_root =
-			trie::ordered_trie_root::<Blake2Hasher, _, _>(extrinsics.iter().map(Encode::encode));
+		use trie::{TrieConfiguration, trie_types::Layout};
+		let iter = extrinsics.iter().map(Encode::encode);
+		let extrinsics_root = Layout::<Blake2Hasher>::ordered_trie_root(iter);
 
 		// only care about `extrinsics_root`
 		Header::new(0, extrinsics_root, H256::zero(), H256::zero(), Default::default())