Introduce first groundwork for Wasm executor (#27)

* Introduce first groundwork for Wasm executor.

* Remove old Rust-runtime code.

* Avoid commiting compled files.

* Add runtime precompile.

* Rename so module makes more sense.

* Further renaming.

* Ensure tests work.

* Allow bringing in of externalities.

- Add util functions/macros.
- Add uncompacted runtime.
- Add some external crates from pwasm-std for managing allocs/memory
stuff.

* Nice macros for imports.

* Allow passing in of data through allocators.

Make memcpy and malloc work.
Basic allocator.

* Can now pass in bytes to WasmExecutor.

* Additional cleanup.

* Switch usages of `OutData` to `u64`

No need to be able to return bytes anymore.

* convert to safe but extremely verbose type conversion.

@rphmeier any more concise way of doing this?

* Remove StaticExternalities distinction.

* Remove another unused use.

* Refactor wasm utils out

* Remove extraneous copies that weren't really testing anything.

* Try to use wasm 0.15

* Make it work!

* Call-time externalities working.

* Add basic externalities.

* Fix grumbles and note unwraps to be sorted.

* Test storage externality.

Unforunately had to change signatures of externalities to avoid
immutable function returning a reference. Not sure what to do about
this...

* Fix nits.

* Compile collation logic.

* Move back to refs. Yey.

* Remove "object" id for storage access.

* Fix test.

* Fix up rest of tests.

* remove unwrap.

* Expose set/get code in externalities

Also improve tests and add nice wrappers in rust-wasm.

* Add validator set.

* Introduce validator set into externalities and test.

* Add another external function.

* Remove code and validators; use storage for everything.

* Introduce validators function.

* Tests (and a fix) for the validators getter.

* Allow calls into runtime to return data.

* Remove unneeded trace.

* Make runtime printing a bit nicer.

* Create separate runtimes for testing and polkadot.

* Remove commented code.

* Use new path.

* Refactor into shared support module.

* Fix warning.

* Remove unwraps.

* Make macro a little less unhygenic.

* Add wasm files.

substrate/state_machine/src/backend.rs

@@ -17,8 +17,6 @@
 //! State machine backends. These manage the code and storage of contracts.
 
 use std::{error, fmt};
-
-use primitives::Address;
 use primitives::hash::H256;
 use triehash::sec_trie_root;
 
@@ -26,8 +24,6 @@ use super::{Update, MemoryState};
 
 /// Output of a commit.
 pub struct Committed {
-	/// Root of the code tree after changes committed.
-	pub code_tree_root: H256,
 	/// Root of the storage tree after changes committed.
 	pub storage_tree_root: H256,
 }
@@ -38,11 +34,8 @@ pub trait Backend {
 	/// An error type when fetching data is not possible.
 	type Error: super::Error;
 
-	/// Get code associated with specific address.
-	fn code(&self, address: &Address) -> Result<&[u8], Self::Error>;
-
 	/// Get keyed storage associated with specific address.
-	fn storage(&self, address: &Address, key: &H256) -> Result<&[u8], Self::Error>;
+	fn storage(&self, key: &[u8]) -> Result<&[u8], Self::Error>;
 
 	/// Commit updates to the backend and get new state.
 	fn commit<I>(&mut self, changes: I) -> Committed
@@ -74,12 +67,8 @@ pub struct InMemory {
 impl Backend for InMemory {
 	type Error = Void;
 
-	fn code(&self, address: &Address) -> Result<&[u8], Void> {
-		Ok(self.inner.code(address).unwrap_or(&[]))
-	}
-
-	fn storage(&self, address: &Address, key: &H256) -> Result<&[u8], Void> {
-		Ok(self.inner.storage(address, key).unwrap_or(&[]))
+	fn storage(&self, key: &[u8]) -> Result<&[u8], Void> {
+		Ok(self.inner.storage(key).unwrap_or(&[]))
 	}
 
 	fn commit<I>(&mut self, changes: I) -> Committed
@@ -88,22 +77,13 @@ impl Backend for InMemory {
 		self.inner.update(changes);
 
 		// fully recalculate trie roots.
-
-		let storage_roots = self.inner.storage.iter().map(|(addr, storage)| {
-			let flat_trie = storage.iter().map(|(k, v)| (k.to_vec(), v.clone())).collect();
-			(addr.to_vec(), sec_trie_root(flat_trie).to_vec())
-		}).collect();
-
-		let storage_tree_root = H256(sec_trie_root(storage_roots).0);
-
-		let code_tree_root = sec_trie_root(
-			self.inner.code.iter().map(|(k, v)| (k.to_vec(), v.clone())).collect()
-		);
-
-		let code_tree_root = H256(code_tree_root.0);
+		let storage_tree_root = H256(sec_trie_root(
+			self.inner.storage.iter()
+				.map(|(k, v)| (k.to_vec(), v.clone()))
+				.collect()
+			).0);
 
 		Committed {
-			code_tree_root,
 			storage_tree_root,
 		}
 	}

substrate/state_machine/src/ext.rs

@@ -19,17 +19,16 @@
 use std::{error, fmt};
 
 use backend::Backend;
-use primitives::Address;
-use primitives::contract::{CallData, OutData};
-use primitives::hash::H256;
-use {Externalities, CodeExecutor, StaticExternalities, OverlayedChanges};
+use {Externalities, OverlayedChanges};
 
 /// Errors that can occur when interacting with the externalities.
 #[derive(Debug, Copy, Clone)]
 pub enum Error<B, E> {
 	/// Failure to load state data from the backend.
+	#[allow(unused)]
 	Backend(B),
 	/// Failure to execute a function.
+	#[allow(unused)]
 	Executor(E),
 }
 
@@ -52,122 +51,26 @@ impl<B: error::Error, E: error::Error> error::Error for Error<B, E> {
 }
 
 /// Wraps a read-only backend, call executor, and current overlayed changes.
-pub struct Ext<'a, B: 'a, E: 'a> {
+pub struct Ext<'a, B: 'a> {
 	/// The overlayed changes to write to.
 	pub overlay: &'a mut OverlayedChanges,
 	/// The storage backend to read from.
 	pub backend: &'a B,
-	/// Contract code executor.
-	pub exec: &'a E,
-	/// Contract address.
-	pub local: Address,
 }
 
-impl<'a, B: 'a, E: 'a> StaticExternalities<E> for Ext<'a, B, E>
-	where B: Backend, E: CodeExecutor
+impl<'a, B: 'a> Externalities for Ext<'a, B>
+	where B: Backend
 {
-	type Error = Error<B::Error, E::Error>;
+	type Error = B::Error;
 
-	fn storage(&self, key: &H256) -> Result<&[u8], Self::Error> {
-		match self.overlay.storage(&self.local, key) {
+	fn storage(&self, key: &[u8]) -> Result<&[u8], Self::Error> {
+		match self.overlay.storage(key) {
 			Some(x) => Ok(x),
-			None => self.backend.storage(&self.local, key).map_err(Error::Backend)
+			None => self.backend.storage(key)
 		}
 	}
 
-	fn call_static(&self, address: &Address, method: &str, data: &CallData) -> Result<OutData, Self::Error> {
-		let inner_ext = StaticExt {
-			backend: self.backend,
-			exec: self.exec,
-			local: address.clone(),
-			overlay: self.overlay,
-		};
-
-		let code = match self.overlay.code(address) {
-			Some(x) => x,
-			None => self.backend.code(address).map_err(Error::Backend)?,
-		};
-
-		self.exec.call_static(
-			&inner_ext,
-			code,
-			method,
-			data,
-		).map_err(Error::Executor)
-	}
-}
-
-impl<'a, B: 'a, E: 'a> Externalities<E> for Ext<'a, B, E>
-	where B: Backend, E: CodeExecutor
-{
-	fn set_storage(&mut self, key: H256, value: Vec<u8>) {
-		self.overlay.set_storage(self.local, key, value);
-	}
-
-	fn call(&mut self, address: &Address, method: &str, data: &CallData) -> Result<OutData, Self::Error> {
-		let code = {
-			let code = match self.overlay.code(address) {
-				Some(x) => x,
-				None => self.backend.code(address).map_err(Error::Backend)?,
-			};
-
-			code.to_owned()
-		};
-
-		let mut inner_ext = Ext {
-			backend: self.backend,
-			exec: self.exec,
-			local: address.clone(),
-			overlay: &mut *self.overlay,
-		};
-
-		self.exec.call(
-			&mut inner_ext,
-			&code[..],
-			method,
-			data,
-		).map_err(Error::Executor)
-	}
-}
-
-// Static externalities
-struct StaticExt<'a, B: 'a, E: 'a> {
-	overlay: &'a OverlayedChanges,
-	backend: &'a B,
-	exec: &'a E,
-	local: Address,
-}
-
-impl<'a, B: 'a, E: 'a> StaticExternalities<E> for StaticExt<'a, B, E>
-	where B: Backend, E: CodeExecutor
-{
-	type Error = Error<B::Error, E::Error>;
-
-	fn storage(&self, key: &H256) -> Result<&[u8], Self::Error> {
-		match self.overlay.storage(&self.local, key) {
-			Some(x) => Ok(x),
-			None => self.backend.storage(&self.local, key).map_err(Error::Backend)
-		}
-	}
-
-	fn call_static(&self, address: &Address, method: &str, data: &CallData) -> Result<OutData, Self::Error> {
-		let inner_ext = StaticExt {
-			backend: self.backend,
-			exec: self.exec,
-			local: address.clone(),
-			overlay: self.overlay,
-		};
-
-		let code = match self.overlay.code(address) {
-			Some(x) => x,
-			None => self.backend.code(address).map_err(Error::Backend)?,
-		};
-
-		self.exec.call_static(
-			&inner_ext,
-			code,
-			method,
-			data,
-		).map_err(Error::Executor)
+	fn set_storage(&mut self, key: Vec<u8>, value: Vec<u8>) {
+		self.overlay.set_storage(key, value);
 	}
 }

substrate/state_machine/src/lib.rs

@@ -27,76 +27,45 @@ extern crate keccak_hash;
 extern crate patricia_trie;
 extern crate triehash;
 
+extern crate byteorder;
+
 use std::collections::HashMap;
 use std::fmt;
 
-use primitives::Address;
-use primitives::contract::{CallData, OutData};
-use primitives::hash::H256;
+use primitives::contract::{CallData};
 
 pub mod backend;
 mod ext;
 
 /// Updates to be committed to the state.
 pub enum Update {
-	/// Set storage of address at given key -- empty is deletion.
-	Storage(Address, H256, Vec<u8>),
-	/// Set code of address -- empty is deletion.
-	Code(Address, Vec<u8>),
+	/// Set storage of object at given key -- empty is deletion.
+	Storage(Vec<u8>, Vec<u8>),
 }
 
 // in-memory section of the state.
 #[derive(Default)]
 struct MemoryState {
-	code: HashMap<Address, Vec<u8>>,
-	storage: HashMap<Address, HashMap<H256, Vec<u8>>>,
+	storage: HashMap<Vec<u8>, Vec<u8>>,
 }
 
 impl MemoryState {
-	fn code(&self, address: &Address) -> Option<&[u8]> {
-		self.code.get(address).map(|v| &v[..])
+	fn storage(&self, key: &[u8]) -> Option<&[u8]> {
+		self.storage.get(key).map(|v| &v[..])
 	}
 
-	fn storage(&self, address: &Address, key: &H256) -> Option<&[u8]> {
-		self.storage.get(address)
-			.and_then(|m| m.get(key))
-			.map(|v| &v[..])
-	}
-
-	#[allow(unused)]
-	fn set_code(&mut self, address: Address, code: Vec<u8>) {
-		self.code.insert(address, code);
-	}
-
-	fn set_storage(&mut self, address: Address, key: H256, val: Vec<u8>) {
-		self.storage.entry(address)
-			.or_insert_with(HashMap::new)
-			.insert(key, val);
+	fn set_storage(&mut self, key: Vec<u8>, val: Vec<u8>) {
+		self.storage.insert(key, val);
 	}
 
 	fn update<I>(&mut self, changes: I) where I: IntoIterator<Item=Update> {
 		for update in changes {
 			match update {
-				Update::Storage(addr, key, val) => {
+				Update::Storage(key, val) => {
 					if val.is_empty() {
-						let mut empty = false;
-						if let Some(s) = self.storage.get_mut(&addr) {
-							s.remove(&key);
-							empty = s.is_empty();
-						};
-
-						if empty { self.storage.remove(&addr); }
+						self.storage.remove(&key);
 					} else {
-						self.storage.entry(addr)
-							.or_insert_with(HashMap::new)
-							.insert(key, val);
-					}
-				}
-				Update::Code(addr, code) => {
-					if code.is_empty() {
-						self.code.remove(&addr);
-					} else {
-						self.code.insert(addr, code);
+						self.storage.insert(key, val);
 					}
 				}
 			}
@@ -115,43 +84,27 @@ pub struct OverlayedChanges {
 }
 
 impl OverlayedChanges {
-	fn code(&self, address: &Address) -> Option<&[u8]> {
-		self.prospective.code(address)
-			.or_else(|| self.committed.code(address))
+	fn storage(&self, key: &[u8]) -> Option<&[u8]> {
+		self.prospective.storage(key)
+			.or_else(|| self.committed.storage(key))
 			.and_then(|v| if v.is_empty() { None } else { Some(v) })
 	}
 
-	fn storage(&self, address: &Address, key: &H256) -> Option<&[u8]> {
-		self.prospective.storage(address, key)
-			.or_else(|| self.committed.storage(address, key))
-			.and_then(|v| if v.is_empty() { None } else { Some(v) })
-	}
-
-	#[allow(unused)]
-	fn set_code(&mut self, address: Address, code: Vec<u8>) {
-		self.prospective.set_code(address, code);
-	}
-
-	fn set_storage(&mut self, address: Address, key: H256, val: Vec<u8>) {
-		self.prospective.set_storage(address, key, val);
+	fn set_storage(&mut self, key: Vec<u8>, val: Vec<u8>) {
+		self.prospective.set_storage(key, val);
 	}
 
 	/// Discard prospective changes to state.
 	pub fn discard_prospective(&mut self) {
-		self.prospective.code.clear();
 		self.prospective.storage.clear();
 	}
 
 	/// Commit prospective changes to state.
 	pub fn commit_prospective(&mut self) {
-		let code_updates = self.prospective.code.drain()
-			.map(|(addr, code)| Update::Code(addr, code));
-
 		let storage_updates = self.prospective.storage.drain()
-			.flat_map(|(addr, storages)| storages.into_iter().map(move |(k, v)| (addr, k, v)))
-			.map(|(addr, key, value)| Update::Storage(addr, key, value));
+			.map(|(key, value)| Update::Storage(key, value));
 
-		self.committed.update(code_updates.chain(storage_updates));
+		self.committed.update(storage_updates);
 	}
 }
 
@@ -161,89 +114,76 @@ impl OverlayedChanges {
 pub trait Error: 'static + fmt::Debug + fmt::Display + Send {}
 impl<E> Error for E where E: 'static + fmt::Debug + fmt::Display + Send {}
 
-/// Externalities: pinned to specific active address.
-pub trait Externalities<CodeExecutor>: StaticExternalities<CodeExecutor> {
-	/// Read storage of current contract being called.
-	fn storage(&self, key: &H256) -> Result<&[u8], Self::Error> {
-		StaticExternalities::storage(self, key)
-	}
-
-	/// Set storage of current contract being called.
-	fn set_storage(&mut self, key: H256, value: Vec<u8>);
-
-	/// Make a sub-call to another contract.
-	fn call(&mut self, address: &Address, method: &str, data: &CallData) -> Result<OutData, Self::Error>;
-
-	/// Make a static (read-only) call to another contract.
-	fn call_static(&self, address: &Address, method: &str, data: &CallData) -> Result<OutData, Self::Error> {
-		StaticExternalities::call_static(self, address, method, data)
+fn value_vec(mut value: usize, initial: Vec<u8>) -> Vec<u8> {
+	let mut acc = initial;
+	while value > 0 {
+		acc.push(value as u8);
+		value /= 256;
 	}
+	acc
 }
 
-/// Static externalities: used only for read-only requests.
-pub trait StaticExternalities<CodeExecutor> {
+/// Externalities: pinned to specific active address.
+pub trait Externalities {
 	/// Externalities error type.
 	type Error: Error;
 
 	/// Read storage of current contract being called.
-	fn storage(&self, key: &H256) -> Result<&[u8], Self::Error>;
+	fn storage(&self, key: &[u8]) -> Result<&[u8], Self::Error>;
 
-	/// Make a static (read-only) call to another contract.
-	fn call_static(&self, address: &Address, method: &str, data: &CallData) -> Result<OutData, Self::Error>;
+	/// Set storage of current contract being called (effective immediately).
+	fn set_storage(&mut self, key: Vec<u8>, value: Vec<u8>);
+
+	/// Get the current set of validators.
+	fn validators(&self) -> Result<Vec<&[u8]>, Self::Error> {
+		(0..self.storage(b"\0validator_count")?.into_iter()
+				.rev()
+				.fold(0, |acc, &i| (acc << 8) + (i as usize)))
+			.map(|i| self.storage(&value_vec(i, b"\0validator".to_vec())))
+			.collect()
+	}
 }
 
-/// Contract code executor.
+/// Code execution engine.
 pub trait CodeExecutor: Sized {
-	/// Error type for contract execution.
+	/// Externalities error type.
 	type Error: Error;
 
-	/// Execute a contract in read-only mode.
-	/// The execution is not allowed to modify the state.
-	fn call_static<E: StaticExternalities<Self>>(
-		&self,
-		ext: &E,
-		code: &[u8],
-		method: &str,
-		data: &CallData,
-	) -> Result<OutData, Self::Error>;
-
-	/// Execute a contract.
-	fn call<E: Externalities<Self>>(
+	/// Call a given method in the runtime.
+	fn call<E: Externalities>(
 		&self,
 		ext: &mut E,
 		code: &[u8],
 		method: &str,
 		data: &CallData,
-	) -> Result<OutData, Self::Error>;
+	) -> Result<Vec<u8>, Self::Error>;
 }
 
 /// Execute a call using the given state backend, overlayed changes, and call executor.
 ///
 /// On an error, no prospective changes are written to the overlay.
+///
+/// Note: changes to code will be in place if this call is made again. For running partial
+/// blocks (e.g. a transaction at a time), ensure a differrent method is used.
 pub fn execute<B: backend::Backend, Exec: CodeExecutor>(
 	backend: &B,
 	overlay: &mut OverlayedChanges,
 	exec: &Exec,
-	address: &Address,
 	method: &str,
 	call_data: &CallData,
-) -> Result<OutData, Box<Error>> {
-	let code = match overlay.code(address) {
-		Some(x) => x.to_owned(),
-		None => backend.code(address).map_err(|e| Box::new(e) as _)?.to_owned(),
-	};
+) -> Result<Vec<u8>, Box<Error>> {
 
 	let result = {
 		let mut externalities = ext::Ext {
 			backend,
-			exec,
-			overlay: &mut *overlay,
-			local: *address,
+			overlay: &mut *overlay
 		};
+		// make a copy.
+		let code = externalities.storage(b"\0code").unwrap_or(&[]).to_vec();
 
 		exec.call(
 			&mut externalities,
-			&code[..],
+			&code,
 			method,
 			call_data,
 		)
@@ -263,59 +203,67 @@ pub fn execute<B: backend::Backend, Exec: CodeExecutor>(
 
 #[cfg(test)]
 mod tests {
-	use super::OverlayedChanges;
-
-	use primitives::hash::H256;
-	use primitives::Address;
+	use std::collections::HashMap;
+	use super::{OverlayedChanges, Externalities};
 
 	#[test]
 	fn overlayed_storage_works() {
 		let mut overlayed = OverlayedChanges::default();
 
-		let key = H256::random();
-		let addr = Address::random();
+		let key = vec![42, 69, 169, 142];
 
-		assert!(overlayed.storage(&addr, &key).is_none());
+		assert!(overlayed.storage(&key).is_none());
 
-		overlayed.set_storage(addr, key, vec![1, 2, 3]);
-		assert_eq!(overlayed.storage(&addr, &key).unwrap(), &[1, 2, 3]);
+		overlayed.set_storage(key.clone(), vec![1, 2, 3]);
+		assert_eq!(overlayed.storage(&key).unwrap(), &[1, 2, 3]);
 
 		overlayed.commit_prospective();
-		assert_eq!(overlayed.storage(&addr, &key).unwrap(), &[1, 2, 3]);
+		assert_eq!(overlayed.storage(&key).unwrap(), &[1, 2, 3]);
 
-		overlayed.set_storage(addr, key, vec![]);
-		assert!(overlayed.storage(&addr, &key).is_none());
+		overlayed.set_storage(key.clone(), vec![]);
+		assert!(overlayed.storage(&key).is_none());
 
 		overlayed.discard_prospective();
-		assert_eq!(overlayed.storage(&addr, &key).unwrap(), &[1, 2, 3]);
+		assert_eq!(overlayed.storage(&key).unwrap(), &[1, 2, 3]);
 
-		overlayed.set_storage(addr, key, vec![]);
+		overlayed.set_storage(key.clone(), vec![]);
 		overlayed.commit_prospective();
-		assert!(overlayed.storage(&addr, &key).is_none());
+		assert!(overlayed.storage(&key).is_none());
+	}
+
+	#[derive(Debug, Default)]
+	struct TestExternalities {
+		storage: HashMap<Vec<u8>, Vec<u8>>,
+	}
+	impl Externalities for TestExternalities {
+		type Error = u8;
+
+		fn storage(&self, key: &[u8]) -> Result<&[u8], Self::Error> {
+			Ok(self.storage.get(&key.to_vec()).map_or(&[] as &[u8], Vec::as_slice))
+		}
+
+		fn set_storage(&mut self, key: Vec<u8>, value: Vec<u8>) {
+			self.storage.insert(key, value);
+		}
 	}
 
 	#[test]
-	fn overlayed_code_works() {
-		let mut overlayed = OverlayedChanges::default();
+	fn validators_call_works() {
+		let mut ext = TestExternalities::default();
 
-		let addr = Address::random();
+		assert_eq!(ext.validators(), Ok(vec![]));
 
-		assert!(overlayed.code(&addr).is_none());
+		ext.set_storage(b"\0validator_count".to_vec(), vec![]);
+		assert_eq!(ext.validators(), Ok(vec![]));
 
-		overlayed.set_code(addr, vec![1, 2, 3]);
-		assert_eq!(overlayed.code(&addr).unwrap(), &[1, 2, 3]);
+		ext.set_storage(b"\0validator_count".to_vec(), vec![1]);
+		assert_eq!(ext.validators(), Ok(vec![&[][..]]));
 
-		overlayed.commit_prospective();
-		assert_eq!(overlayed.code(&addr).unwrap(), &[1, 2, 3]);
+		ext.set_storage(b"\0validator".to_vec(), b"first".to_vec());
+		assert_eq!(ext.validators(), Ok(vec![&b"first"[..]]));
 
-		overlayed.set_code(addr, vec![]);
-		assert!(overlayed.code(&addr).is_none());
-
-		overlayed.discard_prospective();
-		assert_eq!(overlayed.code(&addr).unwrap(), &[1, 2, 3]);
-
-		overlayed.set_code(addr, vec![]);
-		overlayed.commit_prospective();
-		assert!(overlayed.code(&addr).is_none());
+		ext.set_storage(b"\0validator_count".to_vec(), vec![2]);
+		ext.set_storage(b"\0validator\x01".to_vec(), b"second".to_vec());
+		assert_eq!(ext.validators(), Ok(vec![&b"first"[..], &b"second"[..]]));
 	}
 }