no_std support for substrate trie (#2146)

* no_std trie compile in test_runtime (require to set nightly feature due
to the way hashbrown currently works).

* No nightly with hashmap_core.

* using crate elastic-array

* switch to publish trie crates

* fix default array decl

* bump impl_version for ci

* set all semver when possible wasm, and remove redundant code.

* Actually test use_trie function

* impl version +1

* Bump impl version

substrate/core/client/Cargo.toml

@@ -18,7 +18,7 @@ state-machine = { package = "substrate-state-machine", path = "../state-machine"
 keyring = { package = "substrate-keyring", path = "../keyring", optional = true }
 trie = { package = "substrate-trie", path = "../trie", optional = true }
 substrate-telemetry = { path = "../telemetry", optional = true }
-hash-db = { version = "0.12", optional = true }
+hash-db = { version = "0.12.2", default-features = false }
 kvdb = { git = "https://github.com/paritytech/parity-common", optional = true, rev="b0317f649ab2c665b7987b8475878fc4d2e1f81d" }
 parity-codec = { version = "3.3", default-features = false, features = ["derive"] }
 primitives = { package = "substrate-primitives", path = "../primitives", default-features = false }
@@ -35,10 +35,14 @@ kvdb-memorydb = { git = "https://github.com/paritytech/parity-common", rev="b031
 [features]
 default = ["std"]
 std = [
+	"rstd/std",
 	"parity-codec/std",
-	"consensus",
 	"primitives/std",
 	"inherents/std",
+	"runtime-primitives/std",
+	"runtime-version/std",
+	"hash-db/std",
+	"consensus",
 	"parking_lot",
 	"error-chain",
 	"fnv",
@@ -47,13 +51,9 @@ std = [
 	"futures",
 	"heapsize",
 	"executor",
-	"runtime-primitives/std",
-	"runtime-version/std",
-	"rstd/std",
 	"state-machine",
 	"keyring",
 	"trie",
 	"substrate-telemetry",
-	"hash-db",
 	"kvdb"
 ]

substrate/core/sr-api-macros/tests/runtime_calls.rs

@@ -106,3 +106,11 @@ fn calling_with_native_else_wasm_and_fail_on_native_should_work() {
 	let block_id = BlockId::Number(client.info().unwrap().chain.best_number);
 	assert_eq!(runtime_api.fail_on_native(&block_id).unwrap(), 1);
 }
+
+#[test]
+fn use_trie_function() {
+	let client = test_client::new_with_execution_strategy(ExecutionStrategy::AlwaysWasm);
+	let runtime_api = client.runtime_api();
+	let block_id = BlockId::Number(client.info().unwrap().chain.best_number);
+	assert_eq!(runtime_api.use_trie(&block_id).unwrap(), 2);
+}

substrate/core/sr-io/Cargo.toml

@@ -25,11 +25,10 @@ std = [
 	"primitives/std",
 	"parity-codec/std",
 	"rstd/std",
-
 	"hash-db/std",
+	"trie",
 	"environmental",
 	"substrate-state-machine",
-	"trie",
 	"libsecp256k1",
 	"tiny-keccak"
 ]

substrate/core/sr-std/with_std.rs

@@ -30,6 +30,7 @@ pub use std::ptr;
 pub use std::rc;
 pub use std::slice;
 pub use std::vec;
+pub use std::default;
 pub use std::result;
 
 pub mod collections {

substrate/core/sr-std/without_std.rs

@@ -61,6 +61,7 @@ pub use core::num;
 pub use core::ops;
 pub use core::ptr;
 pub use core::slice;
+pub use core::default;
 pub use core::result;
 // We are trying to avoid certain things here, such as `core::string`
 // (if you need `String` you most probably doing something wrong, since

substrate/core/test-runtime/Cargo.toml

@@ -20,6 +20,9 @@ runtime_io = { package = "sr-io", path = "../sr-io", default-features = false }
 runtime_primitives = { package = "sr-primitives", path = "../sr-primitives", default-features = false }
 runtime_version = { package = "sr-version", path = "../sr-version", default-features = false }
 runtime_support = { package = "srml-support", path = "../../srml/support", default-features = false }
+substrate-trie = { path = "../trie", default-features = false }
+trie-db = { version = "0.12", default-features = false }
+memory-db = { version = "0.12", default-features = false }
 offchain-primitives = { package = "substrate-offchain-primitives", path = "../offchain/primitives", default-features = false}
 executive = { package = "srml-executive", path = "../../srml/executive", default-features = false }
 cfg-if = "0.1.6"
@@ -47,6 +50,10 @@ std = [
 	"runtime_primitives/std",
 	"runtime_version/std",
 	"consensus_aura/std",
+	"primitives/std",
+	"substrate-trie/std",
+	"trie-db/std",
+	"memory-db/std",
 	"offchain-primitives/std",
 	"executive/std",
 ]

substrate/core/test-runtime/src/lib.rs

@@ -25,6 +25,10 @@ pub mod system;
 use rstd::{prelude::*, marker::PhantomData};
 use parity_codec::{Encode, Decode, Input};
 
+use primitives::Blake2Hasher;
+use trie_db::{TrieMut, Trie};
+use substrate_trie::{TrieDB, TrieDBMut, PrefixedMemoryDB};
+
 use substrate_client::{
 	runtime_api as client_api, block_builder::api as block_builder_api, decl_runtime_apis,
 	impl_runtime_apis,
@@ -232,6 +236,8 @@ cfg_if! {
 				fn function_signature_changed() -> u64;
 				fn fail_on_native() -> u64;
 				fn fail_on_wasm() -> u64;
+				/// trie no_std testing
+				fn use_trie() -> u64;
 				fn benchmark_indirect_call() -> u64;
 				fn benchmark_direct_call() -> u64;
 			}
@@ -254,6 +260,8 @@ cfg_if! {
 				fn function_signature_changed() -> Vec<u64>;
 				fn fail_on_native() -> u64;
 				fn fail_on_wasm() -> u64;
+				/// trie no_std testing
+				fn use_trie() -> u64;
 				fn benchmark_indirect_call() -> u64;
 				fn benchmark_direct_call() -> u64;
 			}
@@ -281,6 +289,37 @@ fn benchmark_add_one(i: u64) -> u64 {
 #[cfg(not(feature = "std"))]
 static BENCHMARK_ADD_ONE: runtime_io::ExchangeableFunction<fn(u64) -> u64> = runtime_io::ExchangeableFunction::new(benchmark_add_one);
 
+fn code_using_trie() -> u64 {
+	let pairs = [
+		(b"0103000000000000000464".to_vec(), b"0400000000".to_vec()),
+		(b"0103000000000000000469".to_vec(), b"0401000000".to_vec()),
+	].to_vec();
+
+	let mut mdb = PrefixedMemoryDB::default();
+	let mut root = rstd::default::Default::default();
+	let _ = {
+		let v = &pairs;
+		let mut t = TrieDBMut::<Blake2Hasher>::new(&mut mdb, &mut root);
+		for i in 0..v.len() {
+			let key: &[u8]= &v[i].0;
+			let val: &[u8] = &v[i].1;
+			t.insert(key, val).expect("static input");
+		}
+		t
+	};
+
+	let trie = TrieDB::<Blake2Hasher>::new(&mdb, &root).expect("on memory with static content");
+
+	let iter = trie.iter().expect("static input");
+	let mut iter_pairs = Vec::new();
+	for pair in iter {
+		let (key, value) = pair.expect("on memory with static content");
+		iter_pairs.push((key, value.to_vec()));
+	}
+	iter_pairs.len() as u64
+}
+
+
 cfg_if! {
 	if #[cfg(feature = "std")] {
 		impl_runtime_apis! {
@@ -367,6 +406,11 @@ cfg_if! {
 				fn fail_on_wasm() -> u64 {
 					1
 				}
+
+				fn use_trie() -> u64 {
+					code_using_trie()
+				}
+
 				fn benchmark_indirect_call() -> u64 {
 					let function = benchmark_add_one;
 					(0..1000).fold(0, |p, i| p + function(i))
@@ -483,6 +527,10 @@ cfg_if! {
 					panic!("Failing because we are on wasm")
 				}
 
+				fn use_trie() -> u64 {
+					code_using_trie()
+				}
+
 				fn benchmark_indirect_call() -> u64 {
 					(0..10000).fold(0, |p, i| p + BENCHMARK_ADD_ONE.get()(i))
 				}
@@ -492,6 +540,8 @@ cfg_if! {
 				}
 			}
 
+
+
 			impl consensus_aura::AuraApi<Block> for Runtime {
 				fn slot_duration() -> u64 { 1 }
 			}
@@ -510,4 +560,4 @@ cfg_if! {
 			}
 		}
 	}
-}
+}

substrate/core/trie/Cargo.toml

@@ -12,11 +12,12 @@ name = "bench"
 harness = false
 
 [dependencies]
-codec = { package = "parity-codec", version = "3.2" }
-hash-db = { version = "0.12", default-features = false }
-trie-db = { version = "0.12", optional = true }
-trie-root = { version = "0.12", default-features = false }
-memory-db = { version = "0.12", optional = true }
+codec = { package = "parity-codec", version = "3.2", default-features = false }
+rstd = { package = "sr-std", path = "../sr-std", default-features = false }
+hash-db = { version = "0.12.2", default-features = false }
+trie-db = { version = "0.12.2", default-features = false }
+trie-root = { version = "0.12.2", default-features = false }
+memory-db = { version = "0.12.2", default-features = false }
 
 [dev-dependencies]
 substrate-primitives = { path = "../primitives" }
@@ -29,8 +30,10 @@ hex-literal = "0.1.0"
 [features]
 default = ["std"]
 std = [
+	"rstd/std",
+  "codec/std",
 	"hash-db/std",
-	"memory-db",
-	"trie-db",
+	"memory-db/std",
+	"trie-db/std",
 	"trie-root/std"
 ]

substrate/core/trie/src/error.rs

@@ -6,7 +6,9 @@
 // option. This file may not be copied, modified, or distributed
 // except according to those terms.
 
+#[cfg(feature="std")]
 use std::fmt;
+#[cfg(feature="std")]
 use std::error::Error as StdError;
 
 #[derive(Debug, PartialEq, Eq, Clone)]
@@ -16,12 +18,14 @@ pub enum Error {
 	BadFormat,
 }
 
+#[cfg(feature="std")]
 impl StdError for Error {
 	fn description(&self) -> &str {
 		"codec error"
 	}
 }
 
+#[cfg(feature="std")]
 impl fmt::Display for Error {
 	fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
 		fmt::Debug::fmt(&self, f)

substrate/core/trie/src/lib.rs

@@ -16,13 +16,16 @@
 
 //! Utility functions to interact with Substrate's Base-16 Modified Merkle Patricia tree ("trie").
 
-// FIXME: no_std - https://github.com/paritytech/substrate/issues/1574
+#![cfg_attr(not(feature = "std"), no_std)]
+#![cfg_attr(not(feature = "std"), feature(alloc))]
 
 mod error;
 mod node_header;
 mod node_codec;
 mod trie_stream;
 
+use rstd::boxed::Box;
+use rstd::vec::Vec;
 use hash_db::Hasher;
 /// Our `NodeCodec`-specific error.
 pub use error::Error;
@@ -290,7 +293,7 @@ fn take<'a>(input: &mut &'a[u8], count: usize) -> Option<&'a[u8]> {
 fn partial_to_key(partial: &[u8], offset: u8, big: u8) -> Vec<u8> {
 	let nibble_count = (partial.len() - 1) * 2 + if partial[0] & 16 == 16 { 1 } else { 0 };
 	let (first_byte_small, big_threshold) = (offset, (big - offset) as usize);
-	let mut output = vec![first_byte_small + nibble_count.min(big_threshold) as u8];
+	let mut output = [first_byte_small + nibble_count.min(big_threshold) as u8].to_vec();
 	if nibble_count >= big_threshold { output.push((nibble_count - big_threshold) as u8) }
 	if nibble_count % 2 == 1 {
 		output.push(partial[0] & 0x0f);

substrate/core/trie/src/node_codec.rs

@@ -16,7 +16,8 @@
 
 //! `NodeCodec` implementation for Substrate's trie format.
 
-use std::marker::PhantomData;
+use rstd::marker::PhantomData;
+use rstd::vec::Vec;
 use codec::{Encode, Decode, Compact};
 use hash_db::Hasher;
 use trie_db::{self, DBValue, NibbleSlice, node::Node, ChildReference};
@@ -28,10 +29,6 @@ use super::{EMPTY_TRIE, LEAF_NODE_OFFSET, LEAF_NODE_BIG, EXTENSION_NODE_OFFSET,
 #[derive(Default, Clone)]
 pub struct NodeCodec<H: Hasher>(PhantomData<H>);
 
-// NOTE: what we'd really like here is:
-// `impl<H: Hasher> NodeCodec<H> for RlpNodeCodec<H> where H::Out: Decodable`
-// but due to the current limitations of Rust const evaluation we can't
-// do `const HASHED_NULL_NODE: H::Out = H::Out( … … )`. Perhaps one day soon?
 impl<H: Hasher> trie_db::NodeCodec<H> for NodeCodec<H> {
 	type Error = Error;
 
@@ -39,7 +36,7 @@ impl<H: Hasher> trie_db::NodeCodec<H> for NodeCodec<H> {
 		H::hash(&[0u8][..])
 	}
 
-	fn decode(data: &[u8]) -> ::std::result::Result<Node, Self::Error> {
+	fn decode(data: &[u8]) -> ::rstd::result::Result<Node, Self::Error> {
 		use Error::BadFormat;
 		let input = &mut &*data;
 		match NodeHeader::decode(input).ok_or(BadFormat)? {
@@ -92,7 +89,7 @@ impl<H: Hasher> trie_db::NodeCodec<H> for NodeCodec<H> {
 		data == &[EMPTY_TRIE][..]
 	}
 	fn empty_node() -> Vec<u8> {
-		vec![EMPTY_TRIE]
+		[EMPTY_TRIE].to_vec()
 	}
 
 	// FIXME: refactor this so that `partial` isn't already encoded with HPE. Should just be an `impl Iterator<Item=u8>`.
@@ -117,7 +114,7 @@ impl<H: Hasher> trie_db::NodeCodec<H> for NodeCodec<H> {
 	fn branch_node<I>(children: I, maybe_value: Option<DBValue>) -> Vec<u8>
 		where I: IntoIterator<Item=Option<ChildReference<H::Out>>> + Iterator<Item=Option<ChildReference<H::Out>>>
 	{
-		let mut output = vec![0, 0, 0];
+		let mut output = [0, 0, 0].to_vec();
 		let have_value = if let Some(value) = maybe_value {
 			(&*value).encode_to(&mut output);
 			true

substrate/core/trie/src/trie_stream.rs

@@ -16,10 +16,11 @@
 
 //! `TrieStream` implementation for Substrate's trie format. 
 
-use std::iter::once;
+use rstd::iter::once;
 use hash_db::Hasher;
 use trie_root;
 use codec::Encode;
+use rstd::vec::Vec;
 
 use super::{EMPTY_TRIE, LEAF_NODE_OFFSET, LEAF_NODE_BIG, EXTENSION_NODE_OFFSET,
 	EXTENSION_NODE_BIG, branch_node};