Start using trie db

cumulus/runtime/Cargo.toml

@@ -8,8 +8,14 @@ edition = "2018"
 codec = { package = "parity-codec", version = "3.1", default-features = false, features = [ "derive" ] }
 rstd = { package = "sr-std", git = "https://github.com/paritytech/substrate", default-features = false, branch = "polkadot-master" }
 runtime-primitives = { package = "sr-primitives", git = "https://github.com/paritytech/substrate", default-features = false, branch = "polkadot-master" }
+primitives = { package = "substrate-primitives", git = "https://github.com/paritytech/substrate", default-features = false, branch = "polkadot-master" }
 rio = { package = "sr-io", git = "https://github.com/paritytech/substrate", default-features = false, branch = "polkadot-master" }
 executive = { package = "srml-executive", git = "https://github.com/paritytech/substrate", default-features = false, branch = "polkadot-master" }
+substrate-trie = { git = "https://github.com/paritytech/substrate", default-features = false, branch = "polkadot-master" }
+memory-db = { version = "0.12.2", default-features = false }
+hash-db = { version = "0.12.2", default-features = false }
+trie-db = { version = "0.12.2", default-features = false }
+hashbrown = "0.1"
 
 [dev-dependencies]
 keyring = { package = "substrate-keyring", git = "https://github.com/paritytech/substrate", branch = "polkadot-master" }
@@ -24,5 +30,10 @@ std = [
 	"rstd/std",
 	"rio/std",
 	"runtime-primitives/std",
+	"primitives/std",
 	"executive/std",
+	"memory-db/std",
+	"hash-db/std",
+	"trie-db/std",
 ]
+wasm = ["hashbrown/nightly"]

cumulus/runtime/src/lib.rs

@@ -16,7 +16,7 @@
 
 #![cfg_attr(not(feature = "std"), no_std)]
 
-use rstd::{vec::Vec, collections::btree_map::BTreeMap};
+use rstd::vec::Vec;
 use codec::{Encode, Decode};
 use runtime_primitives::traits::Block as BlockT;
 
@@ -27,19 +27,23 @@ pub use rstd::slice;
 #[macro_use]
 pub mod validate_block;
 
-type WitnessData = BTreeMap<Vec<u8>, Vec<u8>>;
+/// The type of the witness data.
+type WitnessData = Vec<Vec<u8>>;
 
 /// The parachain block that is created on a collator and validated by a validator.
 #[derive(Encode, Decode)]
-struct ParachainBlock<B: BlockT> {
+struct ParachainBlockData<B: BlockT> {
 	extrinsics: Vec<<B as BlockT>::Extrinsic>,
 	/// The data that is required to emulate the storage accesses executed by all extrinsics.
 	witness_data: WitnessData,
 }
 
-impl<B: BlockT> ParachainBlock<B> {
+impl<B: BlockT> ParachainBlockData<B> {
 	#[cfg(test)]
-	fn new(extrinsics: Vec<<B as BlockT>::Extrinsic>, witness_data: WitnessData) -> Self {
+	fn new(
+		extrinsics: Vec<<B as BlockT>::Extrinsic>,
+		witness_data: WitnessData
+	) -> Self {
 		Self {
 			extrinsics,
 			witness_data,
@@ -47,11 +51,11 @@ impl<B: BlockT> ParachainBlock<B> {
 	}
 }
 
-impl<B: BlockT> Default for ParachainBlock<B> {
+impl<B: BlockT> Default for ParachainBlockData<B> {
 	fn default() -> Self {
 		Self {
 			extrinsics: Vec::default(),
-			witness_data: BTreeMap::default(),
+			witness_data: WitnessData::default(),
 		}
 	}
 }

cumulus/runtime/src/validate_block/implementation.rs

@@ -0,0 +1,205 @@
+// Copyright 2019 Parity Technologies (UK) Ltd.
+// This file is part of Substrate.
+
+// Substrate is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// Substrate is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with Cumulus. If not, see <http://www.gnu.org/licenses/>.
+
+//! The actual implementation of the validate block functionality.
+
+use crate::WitnessData;
+use runtime_primitives::traits::{
+	Block as BlockT, Header as HeaderT, One, Hash as HashT
+};
+use executive::ExecuteBlock;
+
+use substrate_trie::{MemoryDB, read_trie_value};
+
+use rstd::{slice, ptr, cmp, vec::Vec, boxed::Box, mem};
+
+use hash_db::HashDB;
+
+/// Extract the hashing algorithm type from the given block type.
+type HashingOf<B> = <<B as BlockT>::Header as HeaderT>::Hashing;
+/// Extract the hash type from the given block type.
+type HashOf<B> = <<B as BlockT>::Header as HeaderT>::Hash;
+
+/// Abstract the storage into a trait without `Block` generic.
+trait StorageT {
+	/// Retrieve the value for the given key.
+	fn get(&self, key: &[u8]) -> Option<Vec<u8>>;
+
+	/// Insert the given key and value.
+	fn insert(&mut self, key: &[u8], value: &[u8]);
+
+	/// Remove key and value.
+	fn remove(&mut self, key: &[u8]);
+}
+
+static mut STORAGE: Option<Box<StorageT>> = None;
+/// The message to use as expect message while accessing the `STORAGE`.
+const STORAGE_SET_EXPECT: &str =
+	"`STORAGE` needs to be set before calling this function.";
+
+/// Validate a given parachain block on a validator.
+#[cfg(not(feature = "std"))]
+#[doc(hidden)]
+pub fn validate_block<B: BlockT, E: ExecuteBlock<B>>(
+	mut block: &[u8],
+	mut prev_head: &[u8]
+) {
+	use codec::Decode;
+
+	let block = crate::ParachainBlockData::<B>::decode(&mut block)
+		.expect("Could not decode parachain block.");
+	let parent_header = <<B as BlockT>::Header as Decode>::decode(&mut prev_head)
+		.expect("Could not decode parent header.");
+	let storage = Storage::<B>::new(
+		block.witness_data,
+		parent_header.state_root().clone()
+	).expect("Create storage out of witness data.");
+
+	let _guard = unsafe {
+		STORAGE = Some(Box::new(storage));
+		(
+			// Replace storage calls with our own implementations
+			rio::ext_get_allocated_storage.replace_implementation(ext_get_allocated_storage),
+			rio::ext_get_storage_into.replace_implementation(ext_get_storage_into),
+			rio::ext_set_storage.replace_implementation(ext_set_storage),
+			rio::ext_exists_storage.replace_implementation(ext_exists_storage),
+			rio::ext_clear_storage.replace_implementation(ext_clear_storage),
+		)
+	};
+
+	let block_number = *parent_header.number() + One::one();
+	//E::execute_extrinsics_without_checks(block_number, block.extrinsics);
+}
+
+/// The storage implementation used when validating a block.
+struct Storage<B: BlockT> {
+	witness_data: MemoryDB<<HashingOf<B> as HashT>::Hasher>,
+	overlay: hashbrown::HashMap<Vec<u8>, Option<Vec<u8>>>,
+	storage_root: HashOf<B>,
+}
+
+impl<B: BlockT> Storage<B> {
+	/// Initialize from the given witness data and storage root.
+	///
+	/// Returns an error if given storage root was not found in the witness data.
+	fn new(
+		data: WitnessData,
+		storage_root: HashOf<B>
+	) -> Result<Self, &'static str> {
+		let mut db = MemoryDB::default();
+		data.into_iter().for_each(|i| { db.insert(&[], &i); });
+
+		if !db.contains(&storage_root, &[]) {
+			return Err("Witness data does not contain given storage root.")
+		}
+
+		Ok(Self {
+			witness_data: db,
+			overlay: Default::default(),
+			storage_root,
+		})
+	}
+}
+
+impl<B: BlockT> StorageT for Storage<B> {
+	fn get(&self, key: &[u8]) -> Option<Vec<u8>> {
+		self.overlay.get(key).cloned().or_else(|| {
+			read_trie_value(
+				&self.witness_data,
+				&self.storage_root, key
+			).ok()
+		}).unwrap_or(None)
+	}
+
+	fn insert(&mut self, key: &[u8], value: &[u8]) {
+		self.overlay.insert(key.to_vec(), Some(value.to_vec()));
+	}
+
+	fn remove(&mut self, key: &[u8]) {
+		self.overlay.insert(key.to_vec(), None);
+	}
+}
+
+pub unsafe fn ext_get_allocated_storage(
+	key_data: *const u8,
+	key_len: u32,
+	written_out: *mut u32,
+) -> *mut u8 {
+	let key = slice::from_raw_parts(key_data, key_len as usize);
+	match STORAGE.as_mut().expect(STORAGE_SET_EXPECT).get(key) {
+		Some(mut value) => {
+			*written_out = value.len() as u32;
+			let ptr = value.as_mut_ptr();
+			mem::forget(ptr);
+			ptr
+		},
+		None => {
+			*written_out = u32::max_value();
+			ptr::null_mut()
+		}
+	}
+}
+
+pub unsafe fn ext_set_storage(
+	key_data: *const u8,
+	key_len: u32,
+	value_data: *const u8,
+	value_len: u32,
+) {
+	let key = slice::from_raw_parts(key_data, key_len as usize);
+	let value = slice::from_raw_parts(value_data, value_len as usize);
+
+	STORAGE.as_mut().expect(STORAGE_SET_EXPECT).insert(key, value);
+}
+
+pub unsafe fn ext_get_storage_into(
+	key_data: *const u8,
+	key_len: u32,
+	value_data: *mut u8,
+	value_len: u32,
+	value_offset: u32,
+) -> u32 {
+	let key = slice::from_raw_parts(key_data, key_len as usize);
+	let out_value = slice::from_raw_parts_mut(value_data, value_len as usize);
+
+	match STORAGE.as_mut().expect(STORAGE_SET_EXPECT).get(key) {
+		Some(value) => {
+			let value = &value[value_offset as usize..];
+			let len = cmp::min(value_len as usize, value.len());
+			out_value[..len].copy_from_slice(&value[..len]);
+			len as u32
+		},
+		None => {
+			u32::max_value()
+		}
+	}
+}
+
+pub unsafe fn ext_exists_storage(key_data: *const u8, key_len: u32) -> u32 {
+	let key = slice::from_raw_parts(key_data, key_len as usize);
+
+	if STORAGE.as_mut().expect(STORAGE_SET_EXPECT).get(key).is_some() {
+		1
+	} else {
+		0
+	}
+}
+
+pub unsafe fn ext_clear_storage(key_data: *const u8, key_len: u32) {
+	let key = slice::from_raw_parts(key_data, key_len as usize);
+
+	STORAGE.as_mut().expect(STORAGE_SET_EXPECT).remove(key);
+}

cumulus/runtime/src/validate_block/mod.rs

@@ -16,16 +16,11 @@
 
 //! A module that enables a runtime to work as parachain.
 
-#[cfg(not(feature = "std"))]
-use runtime_primitives::traits::{Block as BlockT, Header as HeaderT, One};
-#[cfg(not(feature = "std"))]
-use executive::ExecuteBlock;
-
-#[cfg(not(feature = "std"))]
-#[doc(hidden)]
-pub mod storage_functions;
 #[cfg(test)]
 mod tests;
+#[cfg(not(feature = "std"))]
+#[doc(hidden)]
+pub mod implementation;
 
 /// Register the `validate_block` function that is used by parachains to validate blocks on a validator.
 ///
@@ -65,7 +60,7 @@ macro_rules! register_validate_block_impl {
 				let block = $crate::slice::from_raw_parts(block, block_len as usize);
 				let prev_head = $crate::slice::from_raw_parts(prev_head, prev_head_len as usize);
 
-				$crate::validate_block::validate_block::<$block, $block_executor>(block, prev_head);
+				$crate::validate_block::implementation::validate_block::<$block, $block_executor>(block, prev_head);
 			}
 		}
 	};
@@ -77,30 +72,4 @@ macro_rules! register_validate_block_impl {
 #[macro_export]
 macro_rules! register_validate_block_impl {
 	($block:ty, $block_executor:ty) => {};
-}
-
-/// Validate a given parachain block on a validator.
-#[cfg(not(feature = "std"))]
-#[doc(hidden)]
-pub fn validate_block<Block: BlockT, E: ExecuteBlock<Block>>(mut block: &[u8], mut prev_head: &[u8]) {
-	use codec::Decode;
-
-	let block = crate::ParachainBlock::<Block>::decode(&mut block).expect("Could not decode parachain block.");
-	let parent_header = <<Block as BlockT>::Header as Decode>::decode(&mut prev_head).expect("Could not decode parent header.");
-
-	let _guard = unsafe {
-		use storage_functions as storage;
-		storage::STORAGE = Some(block.witness_data);
-		(
-			// Replace storage calls with our own implementations
-			rio::ext_get_allocated_storage.replace_implementation(storage::ext_get_allocated_storage),
-			rio::ext_get_storage_into.replace_implementation(storage::ext_get_storage_into),
-			rio::ext_set_storage.replace_implementation(storage::ext_set_storage),
-			rio::ext_exists_storage.replace_implementation(storage::ext_exists_storage),
-			rio::ext_clear_storage.replace_implementation(storage::ext_clear_storage),
-		)
-	};
-
-	let block_number = *parent_header.number() + One::one();
-	E::execute_extrinsics_without_checks(block_number, block.extrinsics);
 }

cumulus/runtime/src/validate_block/storage_functions.rs

@@ -1,79 +0,0 @@
-// Copyright 2019 Parity Technologies (UK) Ltd.
-// This file is part of Substrate.
-
-// Substrate is free software: you can redistribute it and/or modify
-// it under the terms of the GNU General Public License as published by
-// the Free Software Foundation, either version 3 of the License, or
-// (at your option) any later version.
-
-// Substrate is distributed in the hope that it will be useful,
-// but WITHOUT ANY WARRANTY; without even the implied warranty of
-// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-// GNU General Public License for more details.
-
-// You should have received a copy of the GNU General Public License
-// along with Cumulus.  If not, see <http://www.gnu.org/licenses/>.
-
-//! All storage functions that are replaced by `validate_block` in the Substrate runtime.
-
-use crate::WitnessData;
-use rstd::{slice, ptr, cmp};
-
-pub static mut STORAGE: Option<WitnessData> = None;
-const STORAGE_SET_EXPECT: &str = "`STORAGE` needs to be set before calling this function.";
-
-pub unsafe fn ext_get_allocated_storage(key_data: *const u8, key_len: u32, written_out: *mut u32) -> *mut u8 {
-	let key = slice::from_raw_parts(key_data, key_len as usize);
-	match STORAGE.as_mut().expect(STORAGE_SET_EXPECT).get_mut(key) {
-		Some(value) => {
-			*written_out = value.len() as u32;
-			value.as_mut_ptr()
-		},
-		None => {
-			*written_out = u32::max_value();
-			ptr::null_mut()
-		}
-	}
-}
-
-pub unsafe fn ext_set_storage(key_data: *const u8, key_len: u32, value_data: *const u8, value_len: u32) {
-	let key = slice::from_raw_parts(key_data, key_len as usize);
-	let value = slice::from_raw_parts(value_data, value_len as usize);
-
-	STORAGE.as_mut().map(|s| {
-		s.insert(key.to_vec(), value.to_vec());
-	});
-}
-
-pub unsafe fn ext_get_storage_into(key_data: *const u8, key_len: u32, value_data: *mut u8, value_len: u32, value_offset: u32) -> u32 {
-	let key = slice::from_raw_parts(key_data, key_len as usize);
-	let out_value = slice::from_raw_parts_mut(value_data, value_len as usize);
-
-	match STORAGE.as_mut().expect(STORAGE_SET_EXPECT).get_mut(key) {
-		Some(value) => {
-			let value = &value[value_offset as usize..];
-			let len = cmp::min(value_len as usize, value.len());
-			out_value[..len].copy_from_slice(&value[..len]);
-			len as u32
-		},
-		None => {
-			u32::max_value()
-		}
-	}
-}
-
-pub unsafe fn ext_exists_storage(key_data: *const u8, key_len: u32) -> u32 {
-	let key = slice::from_raw_parts(key_data, key_len as usize);
-
-	if STORAGE.as_mut().expect(STORAGE_SET_EXPECT).contains_key(key) {
-		1
-	} else {
-		0
-	}
-}
-
-pub unsafe fn ext_clear_storage(key_data: *const u8, key_len: u32) {
-	let key = slice::from_raw_parts(key_data, key_len as usize);
-
-	STORAGE.as_mut().expect(STORAGE_SET_EXPECT).remove(key);
-}

cumulus/runtime/src/validate_block/tests.rs

@@ -27,7 +27,7 @@ use std::collections::BTreeMap;
 
 use codec::{KeyedVec, Encode};
 
-const WASM_CODE: &'static [u8] =
+const WASM_CODE: &[u8] =
 	include_bytes!("../../test-runtime/wasm/target/wasm32-unknown-unknown/release/cumulus_test_runtime.compact.wasm");
 
 fn create_witness_data() -> BTreeMap<Vec<u8>, Vec<u8>> {

cumulus/runtime/test-runtime/Cargo.toml

@@ -14,3 +14,4 @@ std = [
 	"runtime/std",
 	"substrate-test-runtime/std",
 ]
+wasm = ["runtime/wasm"]

cumulus/runtime/test-runtime/wasm/Cargo.toml

@@ -12,10 +12,8 @@ crate-type = ["cdylib"]
 cumulus-test-runtime = { path = "..", default-features = false }
 
 [features]
-default = []
+default = ["wasm"]
-std = [
+wasm = ["cumulus-test-runtime/wasm"]
-	"cumulus-test-runtime/std",
-]
 
 [profile.release]
 panic = "abort"