Block builder (substrate) (#73)

* Block builder (substrate)

* Fix wasm build

* Bulid on any block

* Test for block builder.

* Block import tests for client.

* Tidy ups

* clean up block builder instantiation

* Propert block generation for tests

* Fixed rpc tests

substrate/Cargo.lock

@@ -1423,8 +1423,8 @@ dependencies = [
  "substrate-executor 0.1.0",
  "substrate-keyring 0.1.0",
  "substrate-primitives 0.1.0",
+ "substrate-runtime-io 0.1.0",
  "substrate-runtime-support 0.1.0",
- "substrate-serializer 0.1.0",
  "substrate-state-machine 0.1.0",
  "substrate-test-runtime 0.1.0",
  "triehash 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)",
@@ -1483,10 +1483,14 @@ dependencies = [
  "serde_derive 1.0.27 (registry+https://github.com/rust-lang/crates.io-index)",
  "serde_json 1.0.9 (registry+https://github.com/rust-lang/crates.io-index)",
  "substrate-client 0.1.0",
+ "substrate-codec 0.1.0",
  "substrate-executor 0.1.0",
+ "substrate-keyring 0.1.0",
  "substrate-primitives 0.1.0",
+ "substrate-runtime-support 0.1.0",
  "substrate-serializer 0.1.0",
  "substrate-state-machine 0.1.0",
+ "substrate-test-runtime 0.1.0",
 ]
 
 [[package]]
@@ -1519,6 +1523,7 @@ dependencies = [
  "substrate-client 0.1.0",
  "substrate-executor 0.1.0",
  "substrate-primitives 0.1.0",
+ "substrate-runtime-support 0.1.0",
  "substrate-state-machine 0.1.0",
 ]
 

substrate/substrate/client/Cargo.toml

@@ -13,8 +13,10 @@ ed25519 = { path = "../ed25519" }
 substrate-codec = { path = "../codec" }
 substrate-executor = { path = "../executor" }
 substrate-primitives = { path = "../primitives" }
+substrate-runtime-io = { path = "../runtime-io" }
 substrate-runtime-support = { path = "../runtime-support" }
-substrate-serializer = { path = "../serializer" }
 substrate-state-machine = { path = "../state-machine" }
-substrate-test-runtime = { path = "../test-runtime" }
 substrate-keyring = { path = "../../substrate/keyring" }
+
+[dev-dependencies]
+substrate-test-runtime = { path = "../test-runtime" }

substrate/substrate/client/src/backend.rs

@@ -21,22 +21,24 @@ use error;
 use primitives::block;
 use blockchain::{self, BlockId};
 
-/// Block insertion transction. Keeps hold if the inserted block state and data.
+/// Block insertion operation. Keeps hold if the inserted block state and data.
 pub trait BlockImportOperation {
 	/// Associated state backend type.
 	type State: state_machine::backend::Backend;
 
 	/// Returns pending state.
-	fn state(&self) -> error::Result<Self::State>;
+	fn state(&self) -> error::Result<&Self::State>;
 	/// Append block data to the transaction.
-	fn import_block(&mut self, header: block::Header, body: Option<block::Body>, is_new_best: bool) -> error::Result<()>;
+	fn set_block_data(&mut self, header: block::Header, body: Option<block::Body>, is_new_best: bool) -> error::Result<()>;
+	/// Inject storage data into the database.
+	fn set_storage<I: Iterator<Item=(Vec<u8>, Vec<u8>)>>(&mut self, iter: I) -> error::Result<()>;
 	/// Inject storage data into the database.
 	fn reset_storage<I: Iterator<Item=(Vec<u8>, Vec<u8>)>>(&mut self, iter: I) -> error::Result<()>;
 }
 
 /// Client backend. Manages the data layer.
 pub trait Backend {
-	/// Associated block insertion transaction type.
+	/// Associated block insertion operation type.
 	type BlockImportOperation: BlockImportOperation;
 	/// Associated blockchain backend type.
 	type Blockchain: blockchain::Backend;
@@ -44,9 +46,9 @@ pub trait Backend {
 	type State: state_machine::backend::Backend;
 
 	/// Begin a new block insertion transaction with given parent block id.
-	fn begin_transaction(&self, block: BlockId) -> error::Result<Self::BlockImportOperation>;
+	fn begin_operation(&self, block: BlockId) -> error::Result<Self::BlockImportOperation>;
 	/// Commit block insertion.
-	fn commit_transaction(&self, transaction: Self::BlockImportOperation) -> error::Result<()>;
+	fn commit_operation(&self, transaction: Self::BlockImportOperation) -> error::Result<()>;
 	/// Returns reference to blockchain backend.
 	fn blockchain(&self) -> &Self::Blockchain;
 	/// Returns state backend for specified block.

substrate/substrate/client/src/block_builder.rs

@@ -0,0 +1,90 @@
+// Copyright 2017 Parity Technologies (UK) Ltd.
+// This file is part of Substrate.
+
+// Substrate is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// Substrate is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with Substrate.  If not, see <http://www.gnu.org/licenses/>.
+
+//! Utility struct to build a block.
+
+use std::vec::Vec;
+use codec::{Joiner, Slicable};
+use state_machine::{self, CodeExecutor};
+use primitives::{Header, Block};
+use primitives::block::Transaction;
+use {backend, error, BlockId, Client};
+use triehash::ordered_trie_root;
+
+/// Utility for building new (valid) blocks from a stream of transactions.
+pub struct BlockBuilder<B, E> where
+	B: backend::Backend,
+	E: CodeExecutor + Clone,
+	error::Error: From<<<B as backend::Backend>::State as state_machine::backend::Backend>::Error>,
+{
+	header: Header,
+	transactions: Vec<Transaction>,
+	executor: E,
+	state: B::State,
+	changes: state_machine::OverlayedChanges,
+}
+
+impl<B, E> BlockBuilder<B, E> where
+	B: backend::Backend,
+	E: CodeExecutor + Clone,
+	error::Error: From<<<B as backend::Backend>::State as state_machine::backend::Backend>::Error>,
+{
+	/// Create a new instance of builder from the given client, building on the latest block.
+	pub fn new(client: &Client<B, E>) -> error::Result<Self> {
+		client.info().and_then(|i| Self::at_block(&BlockId::Hash(i.chain.best_hash), client))
+	}
+
+	/// Create a new instance of builder from the given client using a particular block's ID to
+	/// build upon.
+	pub fn at_block(block_id: &BlockId, client: &Client<B, E>) -> error::Result<Self> {
+		Ok(BlockBuilder {
+			header: Header {
+				number: client.block_number_from_id(block_id)?.ok_or(error::ErrorKind::UnknownBlock(*block_id))? + 1,
+				parent_hash: client.block_hash_from_id(block_id)?.ok_or(error::ErrorKind::UnknownBlock(*block_id))?,
+				state_root: Default::default(),
+				transaction_root: Default::default(),
+				digest: Default::default(),
+			},
+			transactions: Default::default(),
+			executor: client.clone_executor(),
+			state: client.state_at(block_id)?,
+			changes: Default::default(),
+		})
+	}
+
+	/// Push a transaction onto the block's list of transactions. This will ensure the transaction
+	/// can be validly executed (by executing it); if it is invalid, it'll be returned along with
+	/// the error. Otherwise, it will return a mutable reference to self (in order to chain).
+	pub fn push(&mut self, tx: Transaction) -> error::Result<()> {
+		let output = state_machine::execute(&self.state, &mut self.changes, &self.executor, "execute_transaction",
+			&vec![].and(&self.header).and(&tx))?;
+		self.header = Header::decode(&mut &output[..]).expect("Header came straight out of runtime so must be valid");
+		self.transactions.push(tx);
+		Ok(())
+	}
+
+	/// Consume the builder to return a valid `Block` containing all pushed transactions.
+	pub fn bake(mut self) -> error::Result<Block> {
+		self.header.transaction_root = ordered_trie_root(self.transactions.iter().map(Slicable::encode)).0.into();
+		let output = state_machine::execute(&self.state, &mut self.changes, &self.executor, "finalise_block",
+			&self.header.encode())?;
+		self.header = Header::decode(&mut &output[..]).expect("Header came straight out of runtime so must be valid");
+		Ok(Block {
+			header: self.header,
+			transactions: self.transactions,
+		})
+	}
+}

substrate/substrate/client/src/in_mem.rs

@@ -21,13 +21,13 @@ use parking_lot::RwLock;
 use state_machine;
 use error;
 use backend;
-use primitives;
-use ser;
+use runtime_support::Hashable;
 use primitives::block::{self, HeaderHash};
 use blockchain::{self, BlockId, BlockStatus};
+use state_machine::backend::Backend as StateBackend;
 
 fn header_hash(header: &block::Header) -> block::HeaderHash {
-	primitives::hashing::blake2_256(&ser::encode(header)).into()
+	header.blake2_256().into()
 }
 
 struct PendingBlock {
@@ -41,7 +41,7 @@ struct Block {
 	body: Option<block::Body>,
 }
 
-/// In-memory transaction.
+/// In-memory operation.
 pub struct BlockImportOperation {
 	pending_block: Option<PendingBlock>,
 	pending_state: state_machine::backend::InMemory,
@@ -156,12 +156,12 @@ impl blockchain::Backend for Blockchain {
 impl backend::BlockImportOperation for BlockImportOperation {
 	type State = state_machine::backend::InMemory;
 
-	fn state(&self) -> error::Result<Self::State> {
-		Ok(self.pending_state.clone())
+	fn state(&self) -> error::Result<&Self::State> {
+		Ok(&self.pending_state)
 	}
 
-	fn import_block(&mut self, header: block::Header, body: Option<block::Body>, is_new_best: bool) -> error::Result<()> {
-		assert!(self.pending_block.is_none(), "Only one block per transaction is allowed");
+	fn set_block_data(&mut self, header: block::Header, body: Option<block::Body>, is_new_best: bool) -> error::Result<()> {
+		assert!(self.pending_block.is_none(), "Only one block per operation is allowed");
 		self.pending_block = Some(PendingBlock {
 			block: Block {
 				header: header,
@@ -172,6 +172,11 @@ impl backend::BlockImportOperation for BlockImportOperation {
 		Ok(())
 	}
 
+	fn set_storage<I: Iterator<Item=(Vec<u8>, Vec<u8>)>>(&mut self, changes: I) -> error::Result<()> {
+		self.pending_state.commit(changes);
+		Ok(())
+	}
+
 	fn reset_storage<I: Iterator<Item=(Vec<u8>, Vec<u8>)>>(&mut self, iter: I) -> error::Result<()> {
 		self.pending_state = state_machine::backend::InMemory::from(iter.collect());
 		Ok(())
@@ -192,39 +197,6 @@ impl Backend {
 			blockchain: Blockchain::new(),
 		}
 	}
-
-	/// Generate and import a sequence of blocks. A user supplied function is allowed to modify each block header. Useful for testing.
-	pub fn generate_blocks<F>(&self, count: usize, edit_header: F) where F: Fn(&mut block::Header) {
-		use backend::{Backend, BlockImportOperation};
-		let info = blockchain::Backend::info(&self.blockchain).expect("In-memory backend never fails");
-		let mut best_num = info.best_number;
-		let mut best_hash = info.best_hash;
-		let state_root = blockchain::Backend::header(&self.blockchain, BlockId::Hash(best_hash))
-			.expect("In-memory backend never fails")
-			.expect("Best header always exists in the blockchain")
-			.state_root;
-		for _ in 0 .. count {
-			best_num = best_num + 1;
-			let mut header = block::Header {
-				parent_hash: best_hash,
-				number: best_num,
-				state_root: state_root,
-				transaction_root: Default::default(),
-				digest: Default::default(),
-			};
-			edit_header(&mut header);
-
-			let mut tx = self.begin_transaction(BlockId::Hash(best_hash)).expect("In-memory backend does not fail");
-			best_hash = header_hash(&header);
-			tx.import_block(header, None, true).expect("In-memory backend does not fail");
-			self.commit_transaction(tx).expect("In-memory backend does not fail");
-		}
-	}
-
-	/// Generate and import a sequence of blocks. Useful for testing.
-	pub fn push_blocks(&self, count: usize) {
-		self.generate_blocks(count, |_| {})
-	}
 }
 
 impl backend::Backend for Backend {
@@ -232,7 +204,7 @@ impl backend::Backend for Backend {
 	type Blockchain = Blockchain;
 	type State = state_machine::backend::InMemory;
 
-	fn begin_transaction(&self, block: BlockId) -> error::Result<Self::BlockImportOperation> {
+	fn begin_operation(&self, block: BlockId) -> error::Result<Self::BlockImportOperation> {
 		let state = match block {
 			BlockId::Hash(h) if h.is_zero() => Self::State::default(),
 			_ => self.state_at(block)?,
@@ -244,10 +216,10 @@ impl backend::Backend for Backend {
 		})
 	}
 
-	fn commit_transaction(&self, transaction: Self::BlockImportOperation) -> error::Result<()> {
-		if let Some(pending_block) = transaction.pending_block {
+	fn commit_operation(&self, operation: Self::BlockImportOperation) -> error::Result<()> {
+		if let Some(pending_block) = operation.pending_block {
 			let hash = header_hash(&pending_block.block.header);
-			self.states.write().insert(hash, transaction.pending_state);
+			self.states.write().insert(hash, operation.pending_state);
 			self.blockchain.insert(hash, pending_block.block.header, pending_block.block.body, pending_block.is_best);
 		}
 		Ok(())

substrate/substrate/client/src/lib.rs

@@ -18,13 +18,13 @@
 
 #![warn(missing_docs)]
 
+extern crate substrate_runtime_support as runtime_support;
+extern crate substrate_runtime_io as runtime_io;
 extern crate substrate_primitives as primitives;
 extern crate substrate_state_machine as state_machine;
-extern crate substrate_serializer as ser;
 extern crate substrate_codec as codec;
 #[cfg(test)] #[macro_use] extern crate substrate_executor as executor;
 extern crate ed25519;
-#[cfg(test)] extern crate substrate_runtime_support as runtime_support;
 #[cfg(test)] extern crate substrate_test_runtime as test_runtime;
 #[cfg(test)] extern crate substrate_keyring as keyring;
 
@@ -39,6 +39,7 @@ pub mod blockchain;
 pub mod backend;
 pub mod in_mem;
 pub mod genesis;
+pub mod block_builder;
 
 pub use blockchain::Info as ChainInfo;
 pub use blockchain::BlockId;
@@ -50,6 +51,8 @@ use codec::{KeyedVec, Slicable};
 use blockchain::Backend as BlockchainBackend;
 use backend::BlockImportOperation;
 use state_machine::backend::Backend as StateBackend;
+use state_machine::{Ext, OverlayedChanges};
+use runtime_support::Hashable;
 
 /// Polkadot Client
 #[derive(Debug)]
@@ -59,6 +62,7 @@ pub struct Client<B, E> where B: backend::Backend {
 }
 
 /// Client info
+// TODO: split queue info from chain info and amalgamate into single struct.
 #[derive(Debug)]
 pub struct ClientInfo {
 	/// Best block hash.
@@ -134,10 +138,10 @@ impl<B, E> Client<B, E> where
 		if backend.blockchain().header(BlockId::Number(0))?.is_none() {
 			trace!("Empty database, writing genesis block");
 			let (genesis_header, genesis_store) = build_genesis();
-			let mut tx = backend.begin_transaction(BlockId::Hash(block::HeaderHash::default()))?;
-			tx.reset_storage(genesis_store.into_iter())?;
-			tx.import_block(genesis_header, None, true)?;
-			backend.commit_transaction(tx)?;
+			let mut op = backend.begin_operation(BlockId::Hash(block::HeaderHash::default()))?;
+			op.reset_storage(genesis_store.into_iter())?;
+			op.set_block_data(genesis_header, Some(vec![]), true)?;
+			backend.commit_operation(op)?;
 		}
 		Ok(Client {
 			backend,
@@ -167,6 +171,11 @@ impl<B, E> Client<B, E> where
 		self.storage(id, &StorageKey(b":code".to_vec())).map(|data| data.0)
 	}
 
+	/// Clone a new instance of Executor.
+	pub fn clone_executor(&self) -> E where E: Clone {
+		self.executor.clone()
+	}
+
 	/// Get the current set of authorities from storage.
 	pub fn authorities_at(&self, id: &BlockId) -> error::Result<Vec<AuthorityId>> {
 		let state = self.state_at(id)?;
@@ -183,10 +192,8 @@ impl<B, E> Client<B, E> where
 	/// No changes are made.
 	pub fn call(&self, id: &BlockId, method: &str, call_data: &[u8]) -> error::Result<CallResult> {
 		let mut changes = state_machine::OverlayedChanges::default();
-		let state = self.state_at(id)?;
-
 		let return_data = state_machine::execute(
-			&state,
+			&self.state_at(id)?,
 			&mut changes,
 			&self.executor,
 			method,
@@ -195,6 +202,33 @@ impl<B, E> Client<B, E> where
 		Ok(CallResult { return_data, changes })
 	}
 
+	/// Set up the native execution environment to call into a native runtime code.
+	pub fn using_environment<F: FnOnce() -> T, T>(
+		&self, f: F
+	) -> error::Result<T> {
+		self.using_environment_at(&BlockId::Number(self.info()?.chain.best_number), &mut Default::default(), f)
+	}
+
+	/// Set up the native execution environment to call into a native runtime code.
+	pub fn using_environment_at<F: FnOnce() -> T, T>(
+		&self,
+		id: &BlockId,
+		overlay: &mut OverlayedChanges,
+		f: F
+	) -> error::Result<T> {
+		Ok(runtime_io::with_externalities(&mut Ext { backend: &self.state_at(id)?, overlay }, f))
+	}
+
+	/// Create a new block, built on the head of the chain.
+	pub fn new_block(&self) -> error::Result<block_builder::BlockBuilder<B, E>> where E: Clone {
+		block_builder::BlockBuilder::new(self)
+	}
+
+	/// Create a new block, built on top of `parent`.
+	pub fn new_block_at(&self, parent: &BlockId) -> error::Result<block_builder::BlockBuilder<B, E>> where E: Clone {
+		block_builder::BlockBuilder::at_block(parent, &self)
+	}
+
 	/// Queue a block for import.
 	pub fn import_block(&self, header: block::Header, body: Option<block::Body>) -> error::Result<ImportResult> {
 		// TODO: import lock
@@ -204,13 +238,22 @@ impl<B, E> Client<B, E> where
 			blockchain::BlockStatus::Unknown => return Ok(ImportResult::UnknownParent),
 		}
 
-		let mut transaction = self.backend.begin_transaction(BlockId::Hash(header.parent_hash))?;
-		let mut _state = transaction.state()?;
-		// TODO: execute block on _state
+		let mut transaction = self.backend.begin_operation(BlockId::Hash(header.parent_hash))?;
+		let mut overlay = OverlayedChanges::default();
+
+		state_machine::execute(
+			transaction.state()?,
+			&mut overlay,
+			&self.executor,
+			"execute_block",
+			&block::Block { header: header.clone(), transactions: body.clone().unwrap_or_default().clone() }.encode()
+		)?;
 
 		let is_new_best = header.number == self.backend.blockchain().info()?.best_number + 1;
-		transaction.import_block(header, body, is_new_best)?;
-		self.backend.commit_transaction(transaction)?;
+		trace!("Imported {}, (#{}), best={}", block::HeaderHash::from(header.blake2_256()), header.number, is_new_best);
+		transaction.set_block_data(header, body, is_new_best)?;
+		transaction.set_storage(overlay.drain())?;
+		self.backend.commit_operation(transaction)?;
 		Ok(ImportResult::Queued)
 	}
 
@@ -238,6 +281,22 @@ impl<B, E> Client<B, E> where
 		self.backend.blockchain().hash(block_number)
 	}
 
+	/// Convert an arbitrary block ID into a block hash.
+	pub fn block_hash_from_id(&self, id: &BlockId) -> error::Result<Option<block::HeaderHash>> {
+		match *id {
+			BlockId::Hash(h) => Ok(Some(h)),
+			BlockId::Number(n) => self.block_hash(n),
+		}
+	}
+
+	/// Convert an arbitrary block ID into a block hash.
+	pub fn block_number_from_id(&self, id: &BlockId) -> error::Result<Option<block::Number>> {
+		match *id {
+			BlockId::Hash(_) => Ok(self.header(id)?.map(|h| h.number)),
+			BlockId::Number(n) => Ok(Some(n)),
+		}
+	}
+
 	/// Get block header by id.
 	pub fn header(&self, id: &BlockId) -> error::Result<Option<block::Header>> {
 		self.backend.blockchain().header(*id)
@@ -254,11 +313,38 @@ mod tests {
 	use super::*;
 	use codec::Slicable;
 	use keyring::Keyring;
+	use primitives::block::Transaction as PrimitiveTransaction;
 	use test_runtime::genesismap::{GenesisConfig, additional_storage_with_genesis};
+	use test_runtime::{UncheckedTransaction, Transaction};
 	use test_runtime;
 
 	native_executor_instance!(Executor, test_runtime::api::dispatch, include_bytes!("../../test-runtime/wasm/target/wasm32-unknown-unknown/release/substrate_test_runtime.compact.wasm"));
 
+	fn genesis_config() -> GenesisConfig {
+		GenesisConfig::new_simple(vec![
+			Keyring::Alice.to_raw_public(),
+			Keyring::Bob.to_raw_public(),
+			Keyring::Charlie.to_raw_public()
+		], 1000)
+	}
+
+	fn prepare_genesis() -> (primitives::block::Header, Vec<(Vec<u8>, Vec<u8>)>) {
+		let mut storage = genesis_config().genesis_map();
+		let block = genesis::construct_genesis_block(&storage);
+		storage.extend(additional_storage_with_genesis(&block));
+		(primitives::block::Header::decode(&mut block.header.encode().as_ref()).expect("to_vec() always gives a valid serialisation; qed"), storage.into_iter().collect())
+	}
+
+	#[test]
+	fn client_initialises_from_genesis_ok() {
+		let client = new_in_mem(Executor::new(), prepare_genesis).unwrap();
+		let genesis_hash = client.block_hash(0).unwrap().unwrap();
+
+		assert_eq!(client.using_environment(|| test_runtime::system::latest_block_hash()).unwrap(), genesis_hash);
+		assert_eq!(client.using_environment(|| test_runtime::system::balance_of(Keyring::Alice.to_raw_public())).unwrap(), 1000);
+		assert_eq!(client.using_environment(|| test_runtime::system::balance_of(Keyring::Ferdie.to_raw_public())).unwrap(), 0);
+	}
+
 	#[test]
 	fn authorities_call_works() {
 		let genesis_config = GenesisConfig::new_simple(vec![
@@ -275,10 +361,79 @@ mod tests {
 		};
 		let client = new_in_mem(Executor::new(), prepare_genesis).unwrap();
 
+		assert_eq!(client.info().unwrap().chain.best_number, 0);
 		assert_eq!(client.authorities_at(&BlockId::Number(0)).unwrap(), vec![
 			Keyring::Alice.to_raw_public(),
 			Keyring::Bob.to_raw_public(),
 			Keyring::Charlie.to_raw_public()
 		]);
 	}
+
+	#[test]
+	fn block_builder_works_with_no_transactions() {
+		let genesis_config = GenesisConfig::new_simple(vec![
+			Keyring::Alice.to_raw_public(),
+			Keyring::Bob.to_raw_public(),
+			Keyring::Charlie.to_raw_public()
+		], 1000);
+
+		let prepare_genesis = || {
+			let mut storage = genesis_config.genesis_map();
+			let block = genesis::construct_genesis_block(&storage);
+			storage.extend(additional_storage_with_genesis(&block));
+			(primitives::block::Header::decode(&mut block.header.encode().as_ref()).expect("to_vec() always gives a valid serialisation; qed"), storage.into_iter().collect())
+		};
+		let client = new_in_mem(Executor::new(), prepare_genesis).unwrap();
+
+		let builder = client.new_block().unwrap();
+		let block = builder.bake().unwrap();
+
+		client.import_block(block.header, Some(block.transactions)).unwrap();
+
+		assert_eq!(client.info().unwrap().chain.best_number, 1);
+		assert_eq!(client.using_environment(|| test_runtime::system::latest_block_hash()).unwrap(), client.block_hash(1).unwrap().unwrap());
+	}
+
+	trait Signable {
+		fn signed(self) -> PrimitiveTransaction;
+	}
+	impl Signable for Transaction {
+		fn signed(self) -> PrimitiveTransaction {
+			let signature = Keyring::from_raw_public(self.from.clone()).unwrap().sign(&self.encode());
+			PrimitiveTransaction::decode(&mut UncheckedTransaction { signature, tx: self }.encode().as_ref()).unwrap()
+		}
+	}
+
+	#[test]
+	fn block_builder_works_with_transactions() {
+		let genesis_config = GenesisConfig::new_simple(vec![
+			Keyring::Alice.to_raw_public(),
+			Keyring::Bob.to_raw_public(),
+			Keyring::Charlie.to_raw_public()
+		], 1000);
+
+		let prepare_genesis = || {
+			let mut storage = genesis_config.genesis_map();
+			let block = genesis::construct_genesis_block(&storage);
+			storage.extend(additional_storage_with_genesis(&block));
+			(primitives::block::Header::decode(&mut block.header.encode().as_ref()).expect("to_vec() always gives a valid serialisation; qed"), storage.into_iter().collect())
+		};
+		let client = new_in_mem(Executor::new(), prepare_genesis).unwrap();
+
+		let mut builder = client.new_block().unwrap();
+
+		builder.push(Transaction {
+			from: Keyring::Alice.to_raw_public(),
+			to: Keyring::Ferdie.to_raw_public(),
+			amount: 42,
+			nonce: 0
+		}.signed()).unwrap();
+		let block = builder.bake().unwrap();
+		client.import_block(block.header, Some(block.transactions)).unwrap();
+
+		assert_eq!(client.info().unwrap().chain.best_number, 1);
+		assert!(client.state_at(&BlockId::Number(1)).unwrap() != client.state_at(&BlockId::Number(0)).unwrap());
+		assert_eq!(client.using_environment(|| test_runtime::system::balance_of(Keyring::Alice.to_raw_public())).unwrap(), 958);
+		assert_eq!(client.using_environment(|| test_runtime::system::balance_of(Keyring::Ferdie.to_raw_public())).unwrap(), 42);
+	}
 }

substrate/substrate/executor/src/native_executor.rs

@@ -45,12 +45,18 @@ pub trait NativeExecutionDispatch {
 
 /// A generic `CodeExecutor` implementation that uses a delegate to determine wasm code equivalence
 /// and dispatch to native code when possible, falling back on `WasmExecutor` when not.
-#[derive(Default)]
+#[derive(Debug, Default)]
 pub struct NativeExecutor<D: NativeExecutionDispatch + Sync + Send> {
 	/// Dummy field to avoid the compiler complaining about us not using `D`.
 	pub _dummy: ::std::marker::PhantomData<D>,
 }
 
+impl<D: NativeExecutionDispatch + Sync + Send> Clone for NativeExecutor<D> {
+	fn clone(&self) -> Self {
+		NativeExecutor { _dummy: Default::default() }
+	}
+}
+
 impl<D: NativeExecutionDispatch + Sync + Send> CodeExecutor for NativeExecutor<D> {
 	type Error = Error;
 

substrate/substrate/executor/src/wasm_executor.rs

@@ -276,7 +276,7 @@ impl_function_executor!(this: FunctionExecutor<'e, E>,
 /// Wasm rust executor for contracts.
 ///
 /// Executes the provided code in a sandboxed wasm runtime.
-#[derive(Debug, Default)]
+#[derive(Debug, Default, Clone)]
 pub struct WasmExecutor;
 
 impl CodeExecutor for WasmExecutor {

substrate/substrate/network/Cargo.toml

@@ -22,7 +22,11 @@ substrate-primitives = { path = "../../substrate/primitives" }
 substrate-client = { path = "../../substrate/client" }
 substrate-state-machine = { path = "../../substrate/state-machine" }
 substrate-serializer = { path = "../../substrate/serializer" }
+substrate-runtime-support = { path = "../../substrate/runtime-support" }
 
 [dev-dependencies]
+substrate-test-runtime = { path = "../test-runtime" }
 substrate-executor = { path = "../../substrate/executor" }
+substrate-keyring = { path = "../../substrate/keyring" }
+substrate-codec = { path = "../../substrate/codec" }
 env_logger = "0.4"

substrate/substrate/network/src/lib.rs

@@ -27,6 +27,7 @@ extern crate substrate_primitives as primitives;
 extern crate substrate_state_machine as state_machine;
 extern crate substrate_serializer as ser;
 extern crate substrate_client as client;
+extern crate substrate_runtime_support as runtime_support;
 extern crate serde;
 extern crate serde_json;
 #[macro_use] extern crate serde_derive;
@@ -34,6 +35,12 @@ extern crate serde_json;
 #[macro_use] extern crate bitflags;
 #[macro_use] extern crate error_chain;
 
+#[cfg(test)] extern crate env_logger;
+#[cfg(test)] extern crate substrate_test_runtime as test_runtime;
+#[cfg(test)] extern crate substrate_keyring as keyring;
+#[cfg(test)] #[macro_use] extern crate substrate_executor as executor;
+#[cfg(test)] extern crate substrate_codec as codec;
+
 mod service;
 mod sync;
 mod protocol;
@@ -44,13 +51,7 @@ mod config;
 mod chain;
 mod blocks;
 
-#[cfg(test)]
-mod test;
-
-#[cfg(test)]
-extern crate substrate_executor;
-#[cfg(test)]
-extern crate env_logger;
+#[cfg(test)] mod test;
 
 pub use service::Service;
 pub use protocol::{ProtocolStatus};
@@ -59,5 +60,6 @@ pub use network::{NonReservedPeerMode, ConnectionFilter, ConnectionDirection, Ne
 
 // TODO: move it elsewhere
 fn header_hash(header: &primitives::Header) -> primitives::block::HeaderHash {
-	primitives::hashing::blake2_256(&ser::encode(header)).into()
+	use runtime_support::Hashable;
+	header.blake2_256().into()
 }

substrate/substrate/network/src/test/mod.rs

@@ -19,13 +19,21 @@ mod sync;
 use std::collections::{VecDeque, HashSet, HashMap};
 use std::sync::Arc;
 use parking_lot::RwLock;
-use client::{self, BlockId};
-use primitives::block;
-use substrate_executor as executor;
+use client::{self, BlockId, genesis};
+use client::block_builder::BlockBuilder;
+use primitives;
+use executor;
 use io::SyncIo;
 use protocol::Protocol;
 use config::ProtocolConfig;
 use network::{PeerId, SessionInfo, Error as NetworkError};
+use test_runtime::genesismap::{GenesisConfig, additional_storage_with_genesis};
+use runtime_support::Hashable;
+use test_runtime;
+use keyring::Keyring;
+use codec::Slicable;
+
+native_executor_instance!(Executor, test_runtime::api::dispatch, include_bytes!("../../../test-runtime/wasm/target/wasm32-unknown-unknown/release/substrate_test_runtime.compact.wasm"));
 
 pub struct TestIo<'p> {
 	pub queue: &'p RwLock<VecDeque<TestPacket>>,
@@ -92,7 +100,7 @@ pub struct TestPacket {
 }
 
 pub struct Peer {
-	pub chain: Arc<client::Client<client::in_mem::Backend, executor::WasmExecutor>>,
+	chain: Arc<client::Client<client::in_mem::Backend, executor::NativeExecutor<Executor>>>,
 	pub sync: Protocol,
 	pub queue: RwLock<VecDeque<TestPacket>>,
 }
@@ -149,6 +157,36 @@ impl Peer {
 
 	fn flush(&self) {
 	}
+
+	fn generate_blocks<F>(&self, count: usize, mut edit_block: F) where F: FnMut(&mut BlockBuilder<client::in_mem::Backend, executor::NativeExecutor<Executor>>) {
+		for _ in 0 .. count {
+			let mut builder = self.chain.new_block().unwrap();
+			edit_block(&mut builder);
+			let block = builder.bake().unwrap();
+			trace!("Generating {}, (#{})", primitives::block::HeaderHash::from(block.header.blake2_256()), block.header.number);
+			self.chain.import_block(block.header, Some(block.transactions)).unwrap();
+		}
+	}
+
+	fn push_blocks(&self, count: usize, with_tx: bool) {
+		let mut nonce = 0;
+		if with_tx {
+			self.generate_blocks(count, |builder| {
+				let tx = test_runtime::Transaction {
+					from: Keyring::Alice.to_raw_public(),
+					to: Keyring::Alice.to_raw_public(),
+					amount: 1,
+					nonce: nonce,
+				};
+				let signature = Keyring::from_raw_public(tx.from.clone()).unwrap().sign(&tx.encode());
+				let tx = primitives::block::Transaction::decode(&mut test_runtime::UncheckedTransaction { signature, tx: tx }.encode().as_ref()).unwrap();
+				builder.push(tx).unwrap();
+				nonce = nonce + 1;
+			});
+		} else {
+			self.generate_blocks(count, |_| ());
+		}
+	}
 }
 
 pub struct TestNet {
@@ -158,6 +196,19 @@ pub struct TestNet {
 }
 
 impl TestNet {
+	fn genesis_config() -> GenesisConfig {
+		GenesisConfig::new_simple(vec![
+			Keyring::Alice.to_raw_public(),
+		], 1000)
+	}
+
+	fn prepare_genesis() -> (primitives::block::Header, Vec<(Vec<u8>, Vec<u8>)>) {
+		let mut storage = Self::genesis_config().genesis_map();
+		let block = genesis::construct_genesis_block(&storage);
+		storage.extend(additional_storage_with_genesis(&block));
+		(primitives::block::Header::decode(&mut block.header.encode().as_ref()).expect("to_vec() always gives a valid serialisation; qed"), storage.into_iter().collect())
+	}
+
 	pub fn new(n: usize) -> Self {
 		Self::new_with_config(n, ProtocolConfig::default())
 	}
@@ -168,17 +219,9 @@ impl TestNet {
 			started: false,
 			disconnect_events: Vec::new(),
 		};
-		let test_genesis_block = block::Header {
-			parent_hash: 0.into(),
-			number: 0,
-			state_root: 0.into(),
-			transaction_root: Default::default(),
-			digest: Default::default(),
-		};
 
 		for _ in 0..n {
-			let chain = Arc::new(client::new_in_mem(executor::WasmExecutor,
-				|| (test_genesis_block.clone(), vec![])).unwrap());
+			let chain = Arc::new(client::new_in_mem(Executor::new(), Self::prepare_genesis).unwrap());
 			let sync = Protocol::new(config.clone(), chain.clone()).unwrap();
 			net.peers.push(Arc::new(Peer {
 				sync: sync,

substrate/substrate/network/src/test/sync.rs

@@ -22,8 +22,8 @@ use super::*;
 fn sync_from_two_peers_works() {
 	::env_logger::init().ok();
 	let mut net = TestNet::new(3);
-	net.peer(1).chain.backend().push_blocks(100);
-	net.peer(2).chain.backend().push_blocks(100);
+	net.peer(1).push_blocks(100, false);
+	net.peer(2).push_blocks(100, false);
 	net.sync();
 	assert!(net.peer(0).chain.backend().blockchain().equals_to(net.peer(1).chain.backend().blockchain()));
 	let status = net.peer(0).sync.status();
@@ -34,9 +34,9 @@ fn sync_from_two_peers_works() {
 fn sync_from_two_peers_with_ancestry_search_works() {
 	::env_logger::init().ok();
 	let mut net = TestNet::new(3);
-	net.peer(0).chain.backend().generate_blocks(10, |header| header.state_root = 42.into());
-	net.peer(1).chain.backend().push_blocks(100);
-	net.peer(2).chain.backend().push_blocks(100);
+	net.peer(0).push_blocks(10, true);
+	net.peer(1).push_blocks(100, false);
+	net.peer(2).push_blocks(100, false);
 	net.restart_peer(0);
 	net.sync();
 	assert!(net.peer(0).chain.backend().blockchain().canon_equals_to(net.peer(1).chain.backend().blockchain()));
@@ -45,7 +45,7 @@ fn sync_from_two_peers_with_ancestry_search_works() {
 #[test]
 fn sync_long_chain_works() {
 	let mut net = TestNet::new(2);
-	net.peer(1).chain.backend().push_blocks(5000);
+	net.peer(1).push_blocks(500, false);
 	net.sync_steps(3);
 	assert_eq!(net.peer(0).sync.status().sync.state, SyncState::Downloading);
 	net.sync();
@@ -56,8 +56,8 @@ fn sync_long_chain_works() {
 fn sync_no_common_longer_chain_fails() {
 	::env_logger::init().ok();
 	let mut net = TestNet::new(3);
-	net.peer(0).chain.backend().generate_blocks(200, |header| header.state_root = 42.into());
-	net.peer(1).chain.backend().push_blocks(200);
+	net.peer(0).push_blocks(20, true);
+	net.peer(1).push_blocks(20, false);
 	net.sync();
 	assert!(!net.peer(0).chain.backend().blockchain().canon_equals_to(net.peer(1).chain.backend().blockchain()));
 }
@@ -66,16 +66,16 @@ fn sync_no_common_longer_chain_fails() {
 fn sync_after_fork_works() {
 	::env_logger::init().ok();
 	let mut net = TestNet::new(3);
-	net.peer(0).chain.backend().push_blocks(30);
-	net.peer(1).chain.backend().push_blocks(30);
-	net.peer(2).chain.backend().push_blocks(30);
+	net.peer(0).push_blocks(30, false);
+	net.peer(1).push_blocks(30, false);
+	net.peer(2).push_blocks(30, false);
 
-	net.peer(0).chain.backend().generate_blocks(10, |header| header.state_root = 42.into()); // fork
-	net.peer(1).chain.backend().push_blocks(20);
-	net.peer(2).chain.backend().push_blocks(20);
+	net.peer(0).push_blocks(10, true);
+	net.peer(1).push_blocks(20, false);
+	net.peer(2).push_blocks(20, false);
 
-	net.peer(1).chain.backend().generate_blocks(10, |header| header.state_root = 42.into()); // second fork between 1 and 2
-	net.peer(2).chain.backend().push_blocks(1);
+	net.peer(1).push_blocks(10, true);
+	net.peer(2).push_blocks(1, false);
 
 	// peer 1 has the best chain
 	let peer1_chain = net.peer(1).chain.backend().blockchain().clone();

substrate/substrate/primitives/src/block.rs

@@ -16,20 +16,20 @@
 
 //! Block and header type definitions.
 
+use rstd::vec::Vec;
 #[cfg(feature = "std")]
 use bytes;
-use rstd::vec::Vec;
+use Hash;
 use codec::{Input, Slicable};
-use hash::H256;
 
 /// Used to refer to a block number.
 pub type Number = u64;
 
 /// Hash used to refer to a block hash.
-pub type HeaderHash = H256;
+pub type HeaderHash = Hash;
 
 /// Hash used to refer to a transaction hash.
-pub type TransactionHash = H256;
+pub type TransactionHash = Hash;
 
 /// Simple generic transaction type.
 #[derive(PartialEq, Eq, Clone)]
@@ -127,9 +127,9 @@ pub struct Header {
 	/// Block number.
 	pub number: Number,
 	/// State root after this transition.
-	pub state_root: H256,
+	pub state_root: Hash,
 	/// The root of the trie that represents this block's transactions, indexed by a 32-byte integer.
-	pub transaction_root: H256,
+	pub transaction_root: Hash,
 	/// The digest of activity on the block.
 	pub digest: Digest,
 }

substrate/substrate/primitives/src/lib.rs

@@ -91,8 +91,10 @@ mod tests;
 
 pub use self::hash::{H160, H256};
 pub use self::uint::{U256, U512};
-
 pub use block::{Block, Header};
 
+/// General hash type.
+pub type Hash = H256;
+
 /// An identifier for an authority in the consensus algorithm. The same as ed25519::Public.
 pub type AuthorityId = [u8; 32];

substrate/substrate/rpc/Cargo.toml

@@ -15,3 +15,4 @@ substrate-executor = { path = "../executor" }
 [dev-dependencies]
 assert_matches = "1.1"
 substrate-executor = { path = "../executor" }
+substrate-runtime-support = { path = "../runtime-support" }

substrate/substrate/rpc/src/chain/tests.rs

@@ -16,6 +16,7 @@
 
 use substrate_executor as executor;
 use client;
+use runtime_support::Hashable;
 use super::*;
 
 #[test]
@@ -31,7 +32,7 @@ fn should_return_header() {
 	let client = client::new_in_mem(executor::WasmExecutor, || (test_genesis_block.clone(), vec![])).unwrap();
 
 	assert_matches!(
-		ChainApi::header(&client, "af65e54217fb213853703d57b80fc5b2bb834bf923046294d7a49bff62f0a8b2".into()),
+		ChainApi::header(&client, test_genesis_block.blake2_256().into()),
 		Ok(Some(ref x)) if x == &block::Header {
 			parent_hash: 0.into(),
 			number: 0,

substrate/substrate/rpc/src/lib.rs

@@ -33,6 +33,8 @@ extern crate substrate_executor;
 #[cfg(test)]
 #[macro_use]
 extern crate assert_matches;
+#[cfg(test)]
+extern crate substrate_runtime_support as runtime_support;
 
 pub mod chain;
 pub mod state;

substrate/substrate/rpc/src/state/tests.rs

@@ -17,6 +17,7 @@
 use super::*;
 use substrate_executor as executor;
 use self::error::{Error, ErrorKind};
+use runtime_support::Hashable;
 use client;
 
 #[test]
@@ -30,7 +31,7 @@ fn should_return_storage() {
 	};
 
 	let client = client::new_in_mem(executor::WasmExecutor, || (test_genesis_block.clone(), vec![])).unwrap();
-	let genesis_hash = "af65e54217fb213853703d57b80fc5b2bb834bf923046294d7a49bff62f0a8b2".into();
+	let genesis_hash = test_genesis_block.blake2_256().into();
 
 	assert_matches!(
 		StateApi::storage(&client, StorageKey(vec![10]), genesis_hash),
@@ -51,7 +52,7 @@ fn should_call_contract() {
 	};
 
 	let client = client::new_in_mem(executor::WasmExecutor, || (test_genesis_block.clone(), vec![])).unwrap();
-	let genesis_hash = "af65e54217fb213853703d57b80fc5b2bb834bf923046294d7a49bff62f0a8b2".into();
+	let genesis_hash = test_genesis_block.blake2_256().into();
 
 	assert_matches!(
 		StateApi::call(&client, "balanceOf".into(), vec![1,2,3], genesis_hash),

substrate/substrate/state-machine/src/backend.rs

@@ -57,7 +57,7 @@ impl error::Error for Void {
 
 /// In-memory backend. Fully recomputes tries on each commit but useful for
 /// tests.
-#[derive(Default, Clone)]
+#[derive(Debug, PartialEq, Default, Clone)]
 pub struct InMemory {
 	inner: MemoryState, // keeps all the state in memory.
 }

substrate/substrate/state-machine/src/lib.rs

@@ -42,13 +42,10 @@ pub use testing::TestExternalities;
 pub use ext::Ext;
 
 /// Updates to be committed to the state.
-pub enum Update {
-	/// Set storage of object at given key -- empty is deletion.
-	Storage(Vec<u8>, Vec<u8>),
-}
+pub type Update = (Vec<u8>, Vec<u8>);
 
 // in-memory section of the state.
-#[derive(Default, Clone)]
+#[derive(Debug, PartialEq, Default, Clone)]
 struct MemoryState {
 	storage: HashMap<Vec<u8>, Vec<u8>>,
 }
@@ -63,9 +60,7 @@ impl MemoryState {
 	}
 
 	fn update<I>(&mut self, changes: I) where I: IntoIterator<Item=Update> {
-		for update in changes {
-			match update {
-				Update::Storage(key, val) => {
+		for (key, val) in changes {
 			if val.is_empty() {
 				self.storage.remove(&key);
 			} else {
@@ -74,8 +69,6 @@ impl MemoryState {
 		}
 	}
 }
-	}
-}
 
 /// The overlayed changes to state to be queried on top of the backend.
 ///
@@ -105,10 +98,12 @@ impl OverlayedChanges {
 
 	/// Commit prospective changes to state.
 	pub fn commit_prospective(&mut self) {
-		let storage_updates = self.prospective.storage.drain()
-			.map(|(key, value)| Update::Storage(key, value));
+		self.committed.update(self.prospective.storage.drain());
+	}
 
-		self.committed.update(storage_updates);
+	/// Drain prospective changes to an iterator.
+	pub fn drain(&mut self) -> ::std::collections::hash_map::Drain<std::vec::Vec<u8>, std::vec::Vec<u8>> {
+		self.committed.storage.drain()
 	}
 }
 

substrate/substrate/test-runtime/src/genesismap.rs

@@ -62,7 +62,10 @@ macro_rules! map {
 
 pub fn additional_storage_with_genesis(genesis_block: &Block) -> HashMap<Vec<u8>, Vec<u8>> {
 	use codec::Slicable;
+	use primitives::hexdisplay::HexDisplay;
+	println!("genesis hash {}", HexDisplay::from(&genesis_block.header.blake2_256()));
+	println!("genesis {}", HexDisplay::from(&genesis_block.header.encode()));
 	map![
-		twox_128(&b"latest"[..]).encode() => genesis_block.header.blake2_256().encode()
+		twox_128(&b"latest"[..]).to_vec() => genesis_block.header.blake2_256().to_vec()
 	]
 }