Merge branch 'master' into author-relay-block

substrate/Cargo.toml

@@ -6,6 +6,7 @@ authors = ["Parity Technologies <admin@parity.io>"]
 [dependencies]
 error-chain = "0.11"
 polkadot-cli = { path = "cli", version = "0.1" }
+polkadot-network = { path = "network" }
 
 [workspace]
 members = [
@@ -15,6 +16,7 @@ members = [
 	"environmental",
 	"executor",
 	"native-runtime",
+	"network",
 	"primitives",
 	"rpc-servers",
 	"rpc",

substrate/cli/src/error.rs

@@ -16,9 +16,14 @@
 
 //! Initialization errors.
 
+use client;
+
 error_chain! {
 	foreign_links {
 		Io(::std::io::Error) #[doc="IO error"];
 		Cli(::clap::Error) #[doc="CLI error"];
 	}
+	links {
+		Client(client::error::Error, client::error::ErrorKind) #[doc="Client error"];
+    }
 }

substrate/cli/src/lib.rs

@@ -48,14 +48,13 @@ pub fn run<I, T>(args: I) -> error::Result<()> where
 	let yaml = load_yaml!("./cli.yml");
 	let matches = clap::App::from_yaml(yaml).version(crate_version!()).get_matches_from_safe(args)?;
 
-	// TODO [ToDr] Split paremeters parsing from actual execution.
+	// TODO [ToDr] Split parameters parsing from actual execution.
 	let log_pattern = matches.value_of("log").unwrap_or("");
 	init_logger(log_pattern);
 
 	// Create client
-	let blockchain = DummyBlockchain;
 	let executor = executor::executor();
-	let client = client::Client::new(blockchain, executor);
+	let client = client::new_in_mem(executor)?;
 
 	let address = "127.0.0.1:9933".parse().unwrap();
 	let handler = rpc::rpc_handler(client);
@@ -96,17 +95,3 @@ fn init_logger(pattern: &str) {
 	builder.init().expect("Logger initialized only once.");
 }
 
-#[derive(Debug, Default)]
-struct DummyBlockchain;
-
-impl client::Blockchain for DummyBlockchain {
-	type Error = ();
-
-	fn latest_hash(&self) -> Result<primitives::block::HeaderHash, Self::Error> {
-		Ok(0.into())
-	}
-
-	fn header(&self, _hash: &primitives::block::HeaderHash) -> Result<Option<primitives::block::Header>, Self::Error> {
-		Ok(None)
-	}
-}

substrate/client/Cargo.toml

@@ -5,5 +5,8 @@ authors = ["Parity Technologies <admin@parity.io>"]
 
 [dependencies]
 error-chain = "0.11"
+log = "0.3"
+parking_lot = "0.4"
 polkadot-primitives = { path = "../primitives", version = "0.1" }
 polkadot-state-machine = { path = "../state-machine", version = "0.1" }
+polkadot-serializer = { path = "../serializer", version = "0.1" }

substrate/client/src/backend.rs

@@ -0,0 +1,53 @@
+// Copyright 2017 Parity Technologies (UK) Ltd.
+// This file is part of Polkadot.
+
+// Polkadot is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// Polkadot is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with Polkadot.  If not, see <http://www.gnu.org/licenses/>.
+
+//! Polkadot Client data backend
+
+use state_machine;
+use error;
+use primitives::block;
+use blockchain::{self, BlockId};
+
+/// Block insertion transction. Keeps hold if the inseted block state and data.
+pub trait BlockImportOperation {
+	/// Associated state backend type.
+	type State: state_machine::backend::Backend;
+
+	/// Returns pending state.
+	fn state(&self) -> error::Result<Self::State>;
+	/// Append block data to the transaction.
+	fn import_block(&mut self, header: block::Header, body: Option<block::Body>, is_new_best: bool) -> error::Result<()>;
+}
+
+/// Client backend. Manages the data layer.
+pub trait Backend {
+	/// Associated block insertion transaction type.
+	type BlockImportOperation: BlockImportOperation;
+	/// Associated blockchain backend type.
+	type Blockchain: blockchain::Backend;
+	/// Associated state backend type.
+	type State: state_machine::backend::Backend;
+
+	/// Begin a new block insertion transaction with given parent block id.
+	fn begin_transaction(&self, block: BlockId) -> error::Result<Self::BlockImportOperation>;
+	/// Commit block insertion.
+	fn commit_transaction(&self, transaction: Self::BlockImportOperation) -> error::Result<()>;
+	/// Returns reference to blockchain backend.
+	fn blockchain(&self) -> &Self::Blockchain;
+	/// Returns state backend for specified block.
+	fn state_at(&self, block: BlockId) -> error::Result<Self::State>;
+}
+

substrate/client/src/blockchain.rs

@@ -0,0 +1,86 @@
+// Copyright 2017 Parity Technologies (UK) Ltd.
+// This file is part of Polkadot.
+
+// Polkadot is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// Polkadot is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with Polkadot.  If not, see <http://www.gnu.org/licenses/>.
+
+//! Polkadot blockchain trait
+
+use std::fmt::{Display, Formatter, Error as FmtError};
+use primitives::block;
+use error::Result;
+
+/// Block indentification.
+#[derive(Debug, Clone, Copy)]
+pub enum BlockId {
+	/// Identify by block header hash.
+	Hash(block::HeaderHash),
+	/// Identify by block number.
+	Number(block::Number),
+}
+
+impl Display for BlockId {
+	fn fmt(&self, f: &mut Formatter) -> ::std::result::Result<(), FmtError> {
+		match *self {
+			BlockId::Hash(h) => h.fmt(f),
+			BlockId::Number(n) => n.fmt(f),
+		}
+	}
+}
+
+/// Blockchain database backend. Does not perform any validation.
+pub trait Backend: Send + Sync {
+	/// Get block header. Returns `None` if block is not found.
+	fn header(&self, id: BlockId) -> Result<Option<block::Header>>;
+	/// Get block body. Returns `None` if block is not found.
+	fn body(&self, id: BlockId) -> Result<Option<block::Body>>;
+	/// Get blockchain info.
+	fn info(&self) -> Result<Info>;
+	/// Get block status.
+	fn status(&self, id: BlockId) -> Result<BlockStatus>;
+	/// Get block hash by number. Returns `None` if the header is not in the chain.
+	fn hash(&self, number: block::Number) -> Result<Option<block::HeaderHash>>;
+}
+
+/// Block import outcome
+pub enum ImportResult<E> {
+	/// Imported successfully.
+	Imported,
+	/// Block already exists, skippped.
+	AlreadyInChain,
+	/// Unknown parent.
+	UnknownParent,
+	/// Other errror.
+	Err(E),
+}
+
+/// Blockchain info
+#[derive(Debug)]
+pub struct Info {
+	/// Best block hash.
+	pub best_hash: block::HeaderHash,
+	/// Best block number.
+	pub best_number: block::Number,
+	/// Genesis block hash.
+	pub genesis_hash: block::HeaderHash,
+}
+
+/// Block status.
+#[derive(Debug, PartialEq, Eq)]
+pub enum BlockStatus {
+	/// Already in the blockchain.
+	InChain,
+	/// Not in the queue or the blockchain.
+	Unknown,
+}
+

substrate/client/src/error.rs

@@ -16,8 +16,9 @@
 
 //! Polkadot client possible errors.
 
-use primitives::block;
+use std;
 use state_machine;
+use blockchain;
 
 error_chain! {
 	errors {
@@ -28,7 +29,7 @@ error_chain! {
 		}
 
 		/// Unknown block.
-		UnknownBlock(h: block::HeaderHash) {
+		UnknownBlock(h: blockchain::BlockId) {
 			description("unknown block"),
 			display("UnknownBlock: {}", h),
 		}
@@ -38,6 +39,12 @@ error_chain! {
 			description("execution error"),
 			display("Execution: {}", e),
 		}
+
+		/// Blockchain error.
+		Blockchain(e: Box<std::error::Error + Send>) {
+			description("Blockchain error"),
+			display("Blockchain: {}", e),
+		}
 	}
 }
 
@@ -47,3 +54,16 @@ impl From<Box<state_machine::Error>> for Error {
 		ErrorKind::Execution(e).into()
 	}
 }
+
+impl From<state_machine::backend::Void> for Error {
+	fn from(_e: state_machine::backend::Void) -> Self {
+		unreachable!()
+	}
+}
+
+impl Error {
+	/// Chain a blockchain error.
+	pub fn from_blockchain(e: Box<std::error::Error + Send>) -> Self {
+		ErrorKind::Blockchain(e).into()
+	}
+}

substrate/client/src/in_mem.rs

@@ -0,0 +1,204 @@
+// Copyright 2017 Parity Technologies (UK) Ltd.
+// This file is part of Polkadot.
+
+// Polkadot is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// Polkadot is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with Polkadot.  If not, see <http://www.gnu.org/licenses/>.
+
+//! In memory client backend
+
+use std::collections::HashMap;
+use parking_lot::RwLock;
+use state_machine;
+use error;
+use backend;
+use primitives;
+use ser;
+use primitives::block::{self, HeaderHash};
+use blockchain::{self, BlockId, BlockStatus};
+
+fn header_hash(header: &primitives::block::Header) -> primitives::block::HeaderHash {
+	primitives::hash(&ser::to_vec(header))
+}
+
+struct PendingBlock {
+	block: Block,
+	is_best: bool,
+}
+
+struct Block {
+	header: block::Header,
+	body: Option<block::Body>,
+}
+
+/// In-memory transaction.
+pub struct BlockImportOperation {
+	pending_block: Option<PendingBlock>,
+	pending_state: state_machine::backend::InMemory,
+}
+
+struct BlockchainStorage {
+	blocks: HashMap<HeaderHash, Block>,
+	hashes: HashMap<block::Number, HeaderHash>,
+	best_hash: HeaderHash,
+	best_number: block::Number,
+	genesis_hash: HeaderHash,
+}
+
+/// In-memory blockchain. Supports concurrent reads.
+pub struct Blockchain {
+	storage: RwLock<BlockchainStorage>,
+}
+
+impl Blockchain {
+	fn id(&self, id: BlockId) -> Option<HeaderHash> {
+		match id {
+			BlockId::Hash(h) => Some(h),
+			BlockId::Number(n) => self.storage.read().hashes.get(&n).cloned(),
+		}
+	}
+
+	fn new() -> Blockchain {
+		Blockchain {
+			storage: RwLock::new(
+				BlockchainStorage {
+					blocks: HashMap::new(),
+					hashes: HashMap::new(),
+					best_hash: HeaderHash::default(),
+					best_number: 0,
+					genesis_hash: HeaderHash::default(),
+				})
+		}
+	}
+
+	fn insert(&self, hash: HeaderHash, header: block::Header, body: Option<block::Body>, is_new_best: bool) {
+		let number = header.number;
+		let mut storage = self.storage.write();
+		storage.blocks.insert(hash, Block {
+			header: header,
+			body: body,
+		});
+		storage.hashes.insert(number, hash);
+		if is_new_best {
+			storage.best_hash = hash;
+			storage.best_number = number;
+		}
+		if number == 0 {
+			storage.genesis_hash = hash;
+		}
+	}
+}
+
+impl blockchain::Backend for Blockchain {
+	fn header(&self, id: BlockId) -> error::Result<Option<block::Header>> {
+		Ok(self.id(id).and_then(|hash| self.storage.read().blocks.get(&hash).map(|b| b.header.clone())))
+	}
+
+	fn body(&self, id: BlockId) -> error::Result<Option<block::Body>> {
+		Ok(self.id(id).and_then(|hash| self.storage.read().blocks.get(&hash).and_then(|b| b.body.clone())))
+	}
+
+	fn info(&self) -> error::Result<blockchain::Info> {
+		let storage = self.storage.read();
+		Ok(blockchain::Info {
+			best_hash: storage.best_hash,
+			best_number: storage.best_number,
+			genesis_hash: storage.genesis_hash,
+		})
+	}
+
+	fn status(&self, id: BlockId) -> error::Result<BlockStatus> {
+		match self.id(id).map_or(false, |hash| self.storage.read().blocks.contains_key(&hash)) {
+			true => Ok(BlockStatus::InChain),
+			false => Ok(BlockStatus::Unknown),
+		}
+	}
+
+	fn hash(&self, number: block::Number) -> error::Result<Option<block::HeaderHash>> {
+		Ok(self.id(BlockId::Number(number)))
+	}
+}
+
+impl backend::BlockImportOperation for BlockImportOperation {
+	type State = state_machine::backend::InMemory;
+
+	fn state(&self) -> error::Result<Self::State> {
+		Ok(self.pending_state.clone())
+	}
+
+	fn import_block(&mut self, header: block::Header, body: Option<block::Body>, is_new_best: bool) -> error::Result<()> {
+		assert!(self.pending_block.is_none(), "Only one block per transaction is allowed");
+		self.pending_block = Some(PendingBlock {
+			block: Block {
+				header: header,
+				body: body,
+			},
+			is_best: is_new_best,
+		});
+		Ok(())
+	}
+}
+
+/// In-memory backend. Keeps all states and blocks in memory. Useful for testing.
+pub struct Backend {
+	states: RwLock<HashMap<block::HeaderHash, state_machine::backend::InMemory>>,
+	blockchain: Blockchain,
+}
+
+impl Backend {
+	/// Create a new instance of in-mem backend.
+	pub fn new() -> Backend {
+		Backend {
+			states: RwLock::new(HashMap::new()),
+			blockchain: Blockchain::new(),
+		}
+	}
+}
+
+impl backend::Backend for Backend {
+	type BlockImportOperation = BlockImportOperation;
+	type Blockchain = Blockchain;
+	type State = state_machine::backend::InMemory;
+
+	fn begin_transaction(&self, block: BlockId) -> error::Result<Self::BlockImportOperation> {
+		let state = match block {
+			BlockId::Hash(h) if h.is_zero() => Self::State::default(),
+			_ => self.state_at(block)?,
+		};
+
+		Ok(BlockImportOperation {
+			pending_block: None,
+			pending_state: state,
+		})
+	}
+
+	fn commit_transaction(&self, transaction: Self::BlockImportOperation) -> error::Result<()> {
+		if let Some(pending_block) = transaction.pending_block {
+			let hash = header_hash(&pending_block.block.header);
+			self.states.write().insert(hash, transaction.pending_state);
+			self.blockchain.insert(hash, pending_block.block.header, pending_block.block.body, pending_block.is_best);
+		}
+		Ok(())
+	}
+
+	fn blockchain(&self) -> &Blockchain {
+		&self.blockchain
+	}
+
+	fn state_at(&self, block: BlockId) -> error::Result<Self::State> {
+		match self.blockchain.id(block).and_then(|id| self.states.read().get(&id).cloned()) {
+			Some(state) => Ok(state),
+			None => Err(error::ErrorKind::UnknownBlock(block).into()),
+		}
+	}
+}
+

substrate/client/src/lib.rs

@@ -20,28 +20,43 @@
 
 extern crate polkadot_primitives as primitives;
 extern crate polkadot_state_machine as state_machine;
+extern crate polkadot_serializer as ser;
 
-#[macro_use]
-extern crate error_chain;
+extern crate parking_lot;
+#[macro_use] extern crate error_chain;
+#[macro_use] extern crate log;
 
 pub mod error;
+pub mod blockchain;
+pub mod backend;
+pub mod in_mem;
+
+pub use blockchain::Info as ChainInfo;
+pub use blockchain::BlockId;
 
 use primitives::{block};
 use primitives::contract::{CallData, StorageKey, StorageData};
-use state_machine::backend::Backend;
 
-use self::error::ResultExt;
+use blockchain::Backend as BlockchainBackend;
+use backend::BlockImportOperation;
+use state_machine::backend::Backend as StateBackend;
 
-/// Blockchain access
-pub trait Blockchain {
-	/// Error Type
-	type Error;
+/// Polkadot Client
+#[derive(Debug)]
+pub struct Client<B, E> where B: backend::Backend {
+	backend: B,
+	executor: E,
+}
 
-	/// Returns the hash of latest block.
-	fn latest_hash(&self) -> Result<block::HeaderHash, Self::Error>;
-
-	/// Given a hash return a header
-	fn header(&self, hash: &block::HeaderHash) -> Result<Option<block::Header>, Self::Error>;
+/// Client info
+#[derive(Debug)]
+pub struct ClientInfo {
+	/// Best block hash.
+	pub chain: ChainInfo,
+	/// Best block number in the queue.
+	pub best_queued_number: Option<block::Number>,
+	/// Best queued block hash.
+	pub best_queued_hash: Option<block::HeaderHash>,
 }
 
 /// Information regarding the result of a call.
@@ -52,39 +67,77 @@ pub struct CallResult {
 	pub changes: state_machine::OverlayedChanges,
 }
 
-/// Polkadot Client
+/// Block import result.
 #[derive(Debug)]
-pub struct Client<B,  E> {
-	blockchain: B,
-	executor: E,
+pub enum ImportResult {
+	/// Added to the import queue.
+	Queued,
+	/// Already in the import queue.
+	AlreadyQueued,
+	/// Already in the blockchain.
+	AlreadyInChain,
+	/// Block or parent is known to be bad.
+	KnownBad,
+	/// Block parent is not in the chain.
+	UnknownParent,
 }
 
-impl<B, E> Client<B, E> {
-	/// Creates new Polkadot Client with given blockchain and code executor.
-	pub fn new(blockchain: B, executor: E) -> Self {
-		Client {
-			blockchain,
-			executor,
-		}
-	}
+/// Block status.
+#[derive(Debug, PartialEq, Eq)]
+pub enum BlockStatus {
+	/// Added to the import queue.
+	Queued,
+	/// Already in the blockchain.
+	InChain,
+	/// Block or parent is known to be bad.
+	KnownBad,
+	/// Not in the queue or the blockchain.
+	Unknown,
+}
+
+
+/// Create an instance of in-memory client.
+pub fn new_in_mem<E>(executor: E) -> error::Result<Client<in_mem::Backend, E>> where E: state_machine::CodeExecutor {
+	Client::new(in_mem::Backend::new(), executor)
 }
 
 impl<B, E> Client<B, E> where
-	B: Blockchain,
+	B: backend::Backend,
 	E: state_machine::CodeExecutor,
+	error::Error: From<<<B as backend::Backend>::State as state_machine::backend::Backend>::Error>,
 {
+	/// Creates new Polkadot Client with given blockchain and code executor.
+	pub fn new(backend: B, executor: E) -> error::Result<Self> {
+		if backend.blockchain().header(BlockId::Number(0))?.is_none() {
+			trace!("Empty database, writing genesis block");
+			// TODO: spec
+			let genesis_header = block::Header {
+				parent_hash: Default::default(),
+				number: 0,
+				state_root: Default::default(),
+				parachain_activity: Default::default(),
+				logs: Default::default(),
+			};
 
-	fn state_at(&self, _hash: &block::HeaderHash) -> error::Result<state_machine::backend::InMemory> {
-		// TODO [ToDr] Actually retrieve the state.
-		Ok(state_machine::backend::InMemory::default())
+			let mut tx = backend.begin_transaction(BlockId::Hash(block::HeaderHash::default()))?;
+			tx.import_block(genesis_header, None, true)?;
+			backend.commit_transaction(tx)?;
+		}
+		Ok(Client {
+			backend,
+			executor,
+		})
+	}
+
+	fn state_at(&self, hash: &block::HeaderHash) -> error::Result<B::State> {
+		self.backend.state_at(BlockId::Hash(*hash))
 	}
 
 	/// Return single storage entry of contract under given address in state in a block of given hash.
 	pub fn storage(&self, hash: &block::HeaderHash, key: &StorageKey) -> error::Result<StorageData> {
-		self.state_at(hash)?
+		Ok(self.state_at(hash)?
 			.storage(&key.0)
-			.map(|x| StorageData(x.to_vec()))
-			.chain_err(|| error::ErrorKind::Backend)
+			.map(|x| StorageData(x.to_vec()))?)
 	}
 
 	/// Execute a call to a contract on top of state in a block of given hash.
@@ -103,4 +156,52 @@ impl<B, E> Client<B, E> where
 		)?;
 		Ok(CallResult { return_data: vec![], changes })
 	}
+
+	/// Queue a block for import.
+	pub fn import_block(&self, header: block::Header, body: Option<block::Body>) -> error::Result<ImportResult> {
+		// TODO: import lock
+		// TODO: validate block
+		match self.backend.blockchain().status(BlockId::Hash(header.parent_hash))? {
+			blockchain::BlockStatus::InChain => (),
+			blockchain::BlockStatus::Unknown => return Ok(ImportResult::UnknownParent),
+		}
+
+		let mut transaction = self.backend.begin_transaction(BlockId::Number(header.number))?;
+		let mut _state = transaction.state()?;
+		// TODO: execute block on _state
+
+		let is_new_best = header.number == self.backend.blockchain().info()?.best_number + 1;
+		transaction.import_block(header, body, is_new_best)?;
+		self.backend.commit_transaction(transaction)?;
+		Ok(ImportResult::Queued)
+	}
+
+	/// Get blockchain info.
+	pub fn info(&self) -> error::Result<ClientInfo> {
+		let info = self.backend.blockchain().info().map_err(|e| error::Error::from_blockchain(Box::new(e)))?;
+		Ok(ClientInfo {
+			chain: info,
+			best_queued_hash: None,
+			best_queued_number: None,
+		})
+	}
+
+	/// Get block status.
+	pub fn block_status(&self, hash: &block::HeaderHash) -> error::Result<BlockStatus> {
+		// TODO: more efficient implementation
+		match self.backend.blockchain().header(BlockId::Hash(*hash)).map_err(|e| error::Error::from_blockchain(Box::new(e)))?.is_some() {
+			true => Ok(BlockStatus::InChain),
+			false => Ok(BlockStatus::Unknown),
+		}
+	}
+
+	/// Get block hash by number.
+	pub fn block_hash(&self, block_number: block::Number) -> error::Result<Option<block::HeaderHash>> {
+		self.backend.blockchain().hash(block_number)
+	}
+
+	/// Get block header by hash.
+	pub fn header(&self, hash: &block::HeaderHash) -> error::Result<Option<block::Header>> {
+		self.backend.blockchain().header(BlockId::Hash(*hash))
+	}
 }

substrate/executor/Cargo.toml

@@ -16,8 +16,7 @@ parity-wasm = "0.15.0"
 byteorder = "1.1"
 rustc-hex = "1.0.0"
 native-runtime = { path = "../native-runtime", version = "0.1" }
-
-libc = { version = "0.2.33" }
+triehash = "0.1.0"
 
 [dev-dependencies]
 assert_matches = "1.1"

substrate/executor/src/lib.rs

@@ -38,7 +38,7 @@ extern crate parity_wasm;
 extern crate byteorder;
 extern crate rustc_hex;
 extern crate native_runtime;
-extern crate libc;
+extern crate triehash;
 
 #[macro_use]
 extern crate error_chain;

substrate/executor/src/native_executor.rs

@@ -42,8 +42,9 @@ impl CodeExecutor for NativeExecutor {
 mod tests {
 	use super::*;
 	use codec::KeyedVec;
-	use native_runtime::support::{one, two, TestExternalities, StaticHexInto};
+	use native_runtime::support::{one, two, StaticHexInto};
 	use native_runtime::runtime::staking::balance;
+	use state_machine::TestExternalities;
 	use primitives::twox_128;
 
 	const BLOATY_CODE: &[u8] = include_bytes!("../../wasm-runtime/target/wasm32-unknown-unknown/release/runtime_polkadot.wasm");

substrate/executor/src/wasm_executor.rs

@@ -29,6 +29,7 @@ use wasm_utils::{MemoryInstance, UserDefinedElements,
 	AddModuleWithoutFullDependentInstance};
 use primitives::{ed25519, blake2_256, twox_128, twox_256};
 use primitives::hexdisplay::HexDisplay;
+use triehash::ordered_trie_root;
 
 struct Heap {
 	end: u32,
@@ -66,15 +67,26 @@ impl<'e, E: Externalities> FunctionExecutor<'e, E> {
 }
 
 trait WritePrimitive<T: Sized> {
-	fn write_primitive(&self, offset: u32, t: T);
+	fn write_primitive(&self, offset: u32, t: T) -> ::std::result::Result<(), DummyUserError>;
 }
 
 impl WritePrimitive<u32> for MemoryInstance {
-	fn write_primitive(&self, offset: u32, t: u32) {
+	fn write_primitive(&self, offset: u32, t: u32) -> ::std::result::Result<(), DummyUserError> {
 		use byteorder::{LittleEndian, ByteOrder};
 		let mut r = [0u8; 4];
 		LittleEndian::write_u32(&mut r, t);
-		let _ = self.set(offset, &r);
+		self.set(offset, &r).map_err(|_| DummyUserError)
+	}
+}
+
+trait ReadPrimitive<T: Sized> {
+	fn read_primitive(&self, offset: u32) -> ::std::result::Result<T, DummyUserError>;
+}
+
+impl ReadPrimitive<u32> for MemoryInstance {
+	fn read_primitive(&self, offset: u32) -> ::std::result::Result<u32, DummyUserError> {
+		use byteorder::{LittleEndian, ByteOrder};
+		Ok(LittleEndian::read_u32(&self.get(offset, 4).map_err(|_| DummyUserError)?))
 	}
 }
 
@@ -95,29 +107,29 @@ impl_function_executor!(this: FunctionExecutor<'e, E>,
 		println!("Runtime: {}", number);
 	},
 	ext_memcmp(s1: *const u8, s2: *const u8, n: usize) -> i32 => {
-		if let (Ok(sl1), Ok(sl2))
-			= (this.memory.get(s1, n as usize), this.memory.get(s2, n as usize)) {
-			match sl1.cmp(&sl2) {
-				Ordering::Greater => 1,
-				Ordering::Less => -1,
-				Ordering::Equal => 0,
-			}
-		} else {
-			return Err(DummyUserError.into());
+		let sl1 = this.memory.get(s1, n as usize).map_err(|_| DummyUserError)?;
+		let sl2 = this.memory.get(s2, n as usize).map_err(|_| DummyUserError)?;
+		match sl1.cmp(&sl2) {
+			Ordering::Greater => 1,
+			Ordering::Less => -1,
+			Ordering::Equal => 0,
 		}
 	},
 	ext_memcpy(dest: *mut u8, src: *const u8, count: usize) -> *mut u8 => {
-		let _ = this.memory.copy_nonoverlapping(src as usize, dest as usize, count as usize);
+		this.memory.copy_nonoverlapping(src as usize, dest as usize, count as usize)
+			.map_err(|_| DummyUserError)?;
 		println!("memcpy {} from {}, {} bytes", dest, src, count);
 		dest
 	},
 	ext_memmove(dest: *mut u8, src: *const u8, count: usize) -> *mut u8 => {
-		let _ = this.memory.copy(src as usize, dest as usize, count as usize);
+		this.memory.copy(src as usize, dest as usize, count as usize)
+			.map_err(|_| DummyUserError)?;
 		println!("memmove {} from {}, {} bytes", dest, src, count);
 		dest
 	},
 	ext_memset(dest: *mut u8, val: u32, count: usize) -> *mut u8 => {
-		let _ = this.memory.clear(dest as usize, val as u8, count as usize);
+		this.memory.clear(dest as usize, val as u8, count as usize)
+			.map_err(|_| DummyUserError)?;
 		println!("memset {} with {}, {} bytes", dest, val, count);
 		dest
 	},
@@ -136,90 +148,80 @@ impl_function_executor!(this: FunctionExecutor<'e, E>,
 		}
 	},
 	ext_get_allocated_storage(key_data: *const u8, key_len: u32, written_out: *mut u32) -> *mut u8 => {
-		let (offset, written) = if let Ok(key) = this.memory.get(key_data, key_len as usize) {
-			if let Ok(value) = this.ext.storage(&key) {
-				let offset = this.heap.allocate(value.len() as u32) as u32;
-				let _ = this.memory.set(offset, &value);
-				(offset, value.len() as u32)
-			} else { (0, 0) }
-		} else { (0, 0) };
+		let key = this.memory.get(key_data, key_len as usize).map_err(|_| DummyUserError)?;
+		let value = this.ext.storage(&key).map_err(|_| DummyUserError)?;
 
-		this.memory.write_primitive(written_out, written);
-		offset as u32
+		let offset = this.heap.allocate(value.len() as u32) as u32;
+		this.memory.set(offset, &value).map_err(|_| DummyUserError)?;
+
+		this.memory.write_primitive(written_out, value.len() as u32)?;
+		offset
 	},
 	ext_get_storage_into(key_data: *const u8, key_len: u32, value_data: *mut u8, value_len: u32, value_offset: u32) -> u32 => {
-		if let Ok(key) = this.memory.get(key_data, key_len as usize) {
-			if let Ok(value) = this.ext.storage(&key) {
-				let value = &value[value_offset as usize..];
-				let written = ::std::cmp::min(value_len as usize, value.len());
-				let _ = this.memory.set(value_data, &value[..written]);
-				written as u32
-			} else { 0 }
-		} else { 0 }
+		let key = this.memory.get(key_data, key_len as usize).map_err(|_| DummyUserError)?;
+		let value = this.ext.storage(&key).map_err(|_| DummyUserError)?;
+		let value = &value[value_offset as usize..];
+		let written = ::std::cmp::min(value_len as usize, value.len());
+		this.memory.set(value_data, &value[..written]).map_err(|_| DummyUserError)?;
+		written as u32
+	},
+	ext_storage_root(result: *mut u8) => {
+		let r = this.ext.storage_root();
+		this.memory.set(result, &r[..]).map_err(|_| DummyUserError)?;
+	},
+	ext_enumerated_trie_root(values_data: *const u8, _values_len: u32, lens_data: *const u32, lens_len: u32, result: *mut u8) => {
+		let values = (0..lens_len)
+			.map(|i| this.memory.read_primitive(lens_data + i * 4))
+			.collect::<::std::result::Result<Vec<u32>, DummyUserError>>()?
+			.into_iter()
+			.scan(0u32, |acc, v| { let o = *acc; *acc += v; Some((o, v)) })
+			.map(|(offset, len)|
+				this.memory.get(values_data + offset, len as usize)
+					.map_err(|_| DummyUserError)
+			)
+			.collect::<::std::result::Result<Vec<_>, DummyUserError>>()?;
+		let r = ordered_trie_root(values.into_iter());
+		this.memory.set(result, &r[..]).map_err(|_| DummyUserError)?;
 	},
 	ext_chain_id() -> u64 => {
 		this.ext.chain_id()
 	},
 	ext_twox_128(data: *const u8, len: u32, out: *mut u8) => {
-		let maybe_value = if len == 0 {
-			Ok(vec![])
+		let result = if len == 0 {
+			twox_128(&[0u8; 0])
 		} else {
-			this.memory.get(data, len as usize)
+			twox_128(&this.memory.get(data, len as usize).map_err(|_| DummyUserError)?)
 		};
-		let result = if let Ok(value) = maybe_value {
-			twox_128(&value)
-		} else {
-			[0; 16]
-		};
-		let _ = this.memory.set(out, &result);
+		this.memory.set(out, &result).map_err(|_| DummyUserError)?;
 	},
 	ext_twox_256(data: *const u8, len: u32, out: *mut u8) => {
-		let maybe_value = if len == 0 {
-			Ok(vec![])
+		let result = if len == 0 {
+			twox_256(&[0u8; 0])
 		} else {
-			this.memory.get(data, len as usize)
+			twox_256(&this.memory.get(data, len as usize).map_err(|_| DummyUserError)?)
 		};
-		let result = if let Ok(value) = maybe_value {
-			twox_256(&value)
-		} else {
-			[0; 32]
-		};
-		let _ = this.memory.set(out, &result);
+		this.memory.set(out, &result).map_err(|_| DummyUserError)?;
 	},
 	ext_blake2_256(data: *const u8, len: u32, out: *mut u8) => {
-		let maybe_value = if len == 0 {
-			Ok(vec![])
+		let result = if len == 0 {
+			blake2_256(&[0u8; 0])
 		} else {
-			this.memory.get(data, len as usize)
+			blake2_256(&this.memory.get(data, len as usize).map_err(|_| DummyUserError)?)
 		};
-		let result = if let Ok(value) = maybe_value {
-			blake2_256(&value)
-		} else {
-			[0; 32]
-		};
-		let _ = this.memory.set(out, &result);
+		this.memory.set(out, &result).map_err(|_| DummyUserError)?;
 	},
 	ext_ed25519_verify(msg_data: *const u8, msg_len: u32, sig_data: *const u8, pubkey_data: *const u8) -> u32 => {
-		(||{
-			let mut sig = [0u8; 64];
-			if let Err(_) = this.memory.get_into(sig_data, &mut sig[..]) {
-				return 2;
-			};
-			let mut pubkey = [0u8; 32];
-			if let Err(_) = this.memory.get_into(pubkey_data, &mut pubkey[..]) {
-				return 3;
-			};
+		let mut sig = [0u8; 64];
+		this.memory.get_into(sig_data, &mut sig[..]).map_err(|_| DummyUserError)?;
+		let mut pubkey = [0u8; 32];
+		this.memory.get_into(pubkey_data, &mut pubkey[..]).map_err(|_| DummyUserError)?;
+		let msg = this.memory.get(msg_data, msg_len as usize).map_err(|_| DummyUserError)?;
 
-			if let Ok(msg) = this.memory.get(msg_data, msg_len as usize) {
-				if ed25519::Signature::from(sig).verify(&msg, &ed25519::Public::from(pubkey)) {
-					0
-				} else {
-					5
-				}
-			} else {
-				4
-			}
-		})()
+		if ed25519::Signature::from(sig).verify(&msg, &ed25519::Public::from(pubkey)) {
+			0
+		} else {
+			5
+		}
 	}
 	=> <'e, E: Externalities + 'e>
 );
@@ -281,7 +283,8 @@ mod tests {
 	use primitives::{blake2_256, twox_128};
 	use runtime_std;
 	use codec::KeyedVec;
-	use native_runtime::support::{one, two, StaticHexInto, TestExternalities};
+	use state_machine::TestExternalities;
+	use native_runtime::support::{one, two, StaticHexInto};
 	use native_runtime::runtime::staking::balance;
 
 	#[test]
@@ -380,6 +383,16 @@ mod tests {
 		);
 	}
 
+	#[test]
+	fn enumerated_trie_root_should_work() {
+		let mut ext = TestExternalities::default();
+		let test_code = include_bytes!("../../wasm-runtime/target/wasm32-unknown-unknown/release/runtime_test.compact.wasm");
+		assert_eq!(
+			WasmExecutor.call(&mut ext, &test_code[..], "test_enumerated_trie_root", &CallData(vec![])).unwrap(),
+			ordered_trie_root(vec![b"zero".to_vec(), b"one".to_vec(), b"two".to_vec()]).0.to_vec()
+		);
+	}
+
 	fn tx() -> Vec<u8> { "679fcf0a846b4224c84ecad7d91a26241c46d00cb53d6480a363274e8965ee34b0b80b4b2e3836d3d8f8f12c0c1aef7350af587d9aee3883561d11726068ac0a2f8c6129d816cf51c374bc7f08c3e63ed156cf78aefb4a6550d97b87997977ee00000000000000000228000000d75a980182b10ab7d54bfed3c964073a0ee172f3daa62325af021a68f707511a4500000000000000".convert() }
 
 	#[test]

substrate/native-runtime/Cargo.toml

@@ -7,6 +7,7 @@ authors = ["Parity Technologies <admin@parity.io>"]
 polkadot-runtime-codec = { path = "../runtime-codec", version = "0.1" }
 polkadot-runtime-std = { path = "../runtime-std", version = "0.1" }
 rustc-hex = "1.0"
+hex-literal = "0.1.0"
 
 [features]
 default = ["std"]

substrate/network/Cargo.toml

@@ -0,0 +1,29 @@
+[package]
+description = "Polkadot network protocol"
+name = "polkadot-network"
+version = "0.1.0"
+license = "GPL-3.0"
+authors = ["Parity Technologies <admin@parity.io>"]
+
+[lib]
+
+[dependencies]
+ethcore-network = { git = "https://github.com/paritytech/parity.git" }
+ethcore-io = { git = "https://github.com/paritytech/parity.git" }
+polkadot-primitives = { path = "../primitives" }
+polkadot-client = { path = "../client" }
+polkadot-state-machine = { path = "../state-machine" }
+polkadot-serializer = { path = "../serializer" }
+log = "0.3"
+env_logger = "0.4"
+rand = "0.3"
+heapsize = "0.4"
+semver = "0.6"
+smallvec = { version = "0.4", features = ["heapsizeof"] }
+parking_lot = "0.4"
+ipnetwork = "0.12"
+error-chain = "0.11"
+bitflags = "1.0"
+serde = "1.0"
+serde_derive = "1.0"
+serde_json = "1.0"

substrate/network/src/blocks.rs

@@ -0,0 +1,263 @@
+// Copyright 2017 Parity Technologies (UK) Ltd.
+// This file is part of Polkadot.
+
+// Polkadot is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// Polkadot is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with Polkadot.  If not, see <http://www.gnu.org/licenses/>.?
+
+use std::mem;
+use std::cmp;
+use std::ops::Range;
+use std::collections::{HashMap, BTreeMap};
+use std::collections::hash_map::Entry;
+use network::PeerId;
+use primitives::block::{Number as BlockNumber};
+use message;
+
+const MAX_PARALLEL_DOWNLOADS: u32 = 1;
+
+/// Block data with origin.
+#[derive(Debug, Clone, PartialEq, Eq)]
+pub struct BlockData {
+	pub block: message::BlockData,
+	pub origin: PeerId,
+}
+
+#[derive(Debug)]
+enum BlockRangeState {
+	Downloading {
+		len: BlockNumber,
+		downloading: u32,
+	},
+	Complete(Vec<BlockData>),
+}
+
+impl BlockRangeState {
+	pub fn len(&self) -> BlockNumber {
+		match *self {
+			BlockRangeState::Downloading { len, .. } => len,
+			BlockRangeState::Complete(ref blocks) => blocks.len() as BlockNumber,
+		}
+	}
+}
+
+/// A collection of blocks being downloaded.
+#[derive(Default)]
+pub struct BlockCollection {
+	/// Downloaded blocks.
+	blocks: BTreeMap<BlockNumber, BlockRangeState>,
+	peer_requests: HashMap<PeerId, BlockNumber>,
+}
+
+impl BlockCollection {
+	/// Create a new instance.
+	pub fn new() -> BlockCollection {
+		BlockCollection {
+			blocks: BTreeMap::new(),
+			peer_requests: HashMap::new(),
+		}
+	}
+
+	/// Clear everything.
+	pub fn clear(&mut self) {
+		self.blocks.clear();
+		self.peer_requests.clear();
+	}
+
+	/// Insert a set of blocks into collection.
+	pub fn insert(&mut self, start: BlockNumber, blocks: Vec<message::BlockData>, peer_id: PeerId) {
+		if blocks.is_empty() {
+			return;
+		}
+
+		match self.blocks.get(&start) {
+			Some(&BlockRangeState::Downloading { .. }) => {
+				trace!(target: "sync", "Ignored block data still marked as being downloaded: {}", start);
+				debug_assert!(false);
+				return;
+			},
+			Some(&BlockRangeState::Complete(ref existing)) if existing.len() >= blocks.len() => {
+				trace!(target: "sync", "Ignored block data already downloaded: {}", start);
+				return;
+			},
+			_ => (),
+		}
+
+		self.blocks.insert(start, BlockRangeState::Complete(blocks.into_iter().map(|b| BlockData { origin: peer_id, block: b }).collect()));
+	}
+
+	/// Returns a set of block hashes that require a header download. The returned set is marked as being downloaded.
+	pub fn needed_blocks(&mut self, peer_id: PeerId, count: usize, peer_best: BlockNumber, common: BlockNumber) -> Option<Range<BlockNumber>> {
+		// First block number that we need to download
+		let first_different = common + 1;
+		let count = count as BlockNumber;
+		let (mut range, downloading) = {
+			let mut downloading_iter = self.blocks.iter().peekable();
+			let mut prev: Option<(&BlockNumber, &BlockRangeState)> = None;
+			loop {
+				let next = downloading_iter.next();
+				break match &(prev, next) {
+					&(Some((start, &BlockRangeState::Downloading { ref len, downloading })), _) if downloading < MAX_PARALLEL_DOWNLOADS  =>
+						(*start .. *start + *len, downloading),
+					&(Some((start, r)), Some((next_start, _))) if start + r.len() < *next_start =>
+						(*start + r.len() .. cmp::min(*next_start, *start + count), 0), // gap
+					&(Some((start, r)), None) =>
+						(start + r.len() .. start + r.len() + count, 0), // last range
+					&(None, None) =>
+						(first_different .. first_different + count, 0), // empty
+					&(None, Some((start, _))) if *start > first_different =>
+						(first_different .. cmp::min(first_different + count, *start), 0), // gap at the start
+					_ => {
+						prev = next;
+						continue
+					},
+				}
+			}
+		};
+
+		// crop to peers best
+		if range.start >= peer_best {
+			return None;
+		}
+		range.end = cmp::min(peer_best, range.end);
+
+		self.peer_requests.insert(peer_id, range.start);
+		self.blocks.insert(range.start, BlockRangeState::Downloading{ len: range.end - range.start, downloading: downloading + 1 });
+		Some(range)
+	}
+
+	/// Get a valid chain of blocks ordered in descending order and ready for importing into blockchain.
+	pub fn drain(&mut self, from: BlockNumber) -> Vec<BlockData> {
+		let mut drained = Vec::new();
+		let mut ranges = Vec::new();
+		{
+			let mut prev = from;
+			for (start, range_data) in &mut self.blocks {
+				match range_data {
+					&mut BlockRangeState::Complete(ref mut blocks) if *start <= prev => {
+							prev = *start + blocks.len() as BlockNumber;
+							let mut blocks = mem::replace(blocks, Vec::new());
+							drained.append(&mut blocks);
+							ranges.push(*start);
+					},
+					_ => break,
+				}
+			}
+		}
+		for r in ranges {
+			self.blocks.remove(&r);
+		}
+		trace!(target: "sync", "Drained {} blocks", drained.len());
+		drained
+	}
+
+	pub fn clear_peer_download(&mut self, peer_id: PeerId) {
+		match self.peer_requests.entry(peer_id) {
+			Entry::Occupied(entry) => {
+				let start = entry.remove();
+				let remove = match self.blocks.get_mut(&start) {
+					Some(&mut BlockRangeState::Downloading { ref mut downloading, .. }) if *downloading > 1 => {
+						*downloading = *downloading - 1;
+						false
+					},
+					Some(&mut BlockRangeState::Downloading { .. })  => {
+						true
+					},
+					_ => {
+						debug_assert!(false);
+						false
+					}
+				};
+				if remove {
+					self.blocks.remove(&start);
+				}
+			},
+			_ => (),
+		}
+	}
+}
+
+#[cfg(test)]
+mod test {
+	use super::{BlockCollection, BlockData};
+	use message;
+	use primitives::block::{HeaderHash};
+
+	fn is_empty(bc: &BlockCollection) -> bool {
+		bc.blocks.is_empty() &&
+		bc.peer_requests.is_empty()
+	}
+
+	fn generate_blocks(n: usize) -> Vec<message::BlockData> {
+		(0 .. n).map(|_| message::BlockData {
+			hash: HeaderHash::random(),
+			header: None,
+			body: None,
+			message_queue: None,
+			receipt: None,
+		}).collect()
+	}
+
+	#[test]
+	fn create_clear() {
+		let mut bc = BlockCollection::new();
+		assert!(is_empty(&bc));
+		bc.insert(1, generate_blocks(100),  0);
+		assert!(!is_empty(&bc));
+		bc.clear();
+		assert!(is_empty(&bc));
+	}
+
+	#[test]
+	fn insert_blocks() {
+		let mut bc = BlockCollection::new();
+		assert!(is_empty(&bc));
+		let peer0 = 0;
+		let peer1 = 1;
+		let peer2 = 2;
+
+		let blocks = generate_blocks(150);
+		assert_eq!(bc.needed_blocks(peer0, 40, 150, 0), Some(1 .. 41));
+		assert_eq!(bc.needed_blocks(peer1, 40, 150, 0), Some(41 .. 81));
+		assert_eq!(bc.needed_blocks(peer2, 40, 150, 0), Some(81 .. 121));
+
+		bc.clear_peer_download(peer1);
+		bc.insert(41, blocks[41..81].to_vec(), peer1);
+		assert_eq!(bc.drain(1), vec![]);
+		assert_eq!(bc.needed_blocks(peer1, 40, 150, 0), Some(121 .. 150));
+		bc.clear_peer_download(peer0);
+		bc.insert(1, blocks[1..11].to_vec(), peer0);
+
+		assert_eq!(bc.needed_blocks(peer0, 40, 150, 0), Some(11 .. 41));
+		assert_eq!(bc.drain(1), blocks[1..11].iter().map(|b| BlockData { block: b.clone(), origin: 0 }).collect::<Vec<_>>());
+
+		bc.clear_peer_download(peer0);
+		bc.insert(11, blocks[11..41].to_vec(), peer0);
+
+		let drained = bc.drain(12);
+		assert_eq!(drained[..30], blocks[11..41].iter().map(|b| BlockData { block: b.clone(), origin: 0 }).collect::<Vec<_>>()[..]);
+		assert_eq!(drained[30..], blocks[41..81].iter().map(|b| BlockData { block: b.clone(), origin: 1 }).collect::<Vec<_>>()[..]);
+
+		bc.clear_peer_download(peer2);
+		assert_eq!(bc.needed_blocks(peer2, 40, 150, 80), Some(81 .. 121));
+		bc.clear_peer_download(peer2);
+		bc.insert(81, blocks[81..121].to_vec(), peer2);
+		bc.clear_peer_download(peer1);
+		bc.insert(121, blocks[121..150].to_vec(), peer1);
+
+		assert_eq!(bc.drain(80), vec![]);
+		let drained = bc.drain(81);
+		assert_eq!(drained[..40], blocks[81..121].iter().map(|b| BlockData { block: b.clone(), origin: 2 }).collect::<Vec<_>>()[..]);
+		assert_eq!(drained[40..], blocks[121..150].iter().map(|b| BlockData { block: b.clone(), origin: 1 }).collect::<Vec<_>>()[..]);
+	}
+}
+

substrate/network/src/chain.rs

@@ -0,0 +1,59 @@
+// Copyright 2017 Parity Technologies (UK) Ltd.
+// This file is part of Polkadot.
+
+// Polkadot is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// Polkadot is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with Polkadot.  If not, see <http://www.gnu.org/licenses/>.
+
+//! Blockchain access trait
+
+use client::{self, Client as PolkadotClient, ImportResult, ClientInfo, BlockStatus};
+use client::error::Error;
+use state_machine;
+use primitives::block;
+
+pub trait Client : Send + Sync {
+	/// Given a hash return a header
+	fn import(&self, header: block::Header, body: Option<block::Body>) -> Result<ImportResult, Error>;
+
+	/// Get blockchain info.
+	fn info(&self) -> Result<ClientInfo, Error>;
+
+	/// Get block status.
+	fn block_status(&self, hash: &block::HeaderHash) -> Result<BlockStatus, Error>;
+
+	/// Get block hash by number.
+	fn block_hash(&self, block_number: block::Number) -> Result<Option<block::HeaderHash>, Error>;
+}
+
+impl<B, E> Client for PolkadotClient<B, E> where
+	B: client::backend::Backend + Send + Sync + 'static,
+	E: state_machine::CodeExecutor + Send + Sync + 'static,
+{
+	fn import(&self, header: block::Header, body: Option<block::Body>) -> Result<ImportResult, Error> {
+		(self as &Client).import(header, body)
+	}
+
+	fn info(&self) -> Result<ClientInfo, Error> {
+		(self as &Client).info()
+	}
+
+	fn block_status(&self, hash: &block::HeaderHash) -> Result<BlockStatus, Error> {
+		(self as &Client).block_status(hash)
+	}
+
+	fn block_hash(&self, block_number: block::Number) -> Result<Option<block::HeaderHash>, Error> {
+		(self as &Client).block_hash(block_number)
+	}
+}
+
+

substrate/network/src/config.rs

@@ -0,0 +1,30 @@
+// Copyright 2017 Parity Technologies (UK) Ltd.
+// This file is part of Polkadot.
+
+// Polkadot is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// Polkadot is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with Polkadot.  If not, see <http://www.gnu.org/licenses/>.?
+
+use service::Role;
+
+/// Protocol configuration
+pub struct ProtocolConfig {
+	pub roles: Role,
+}
+
+impl Default for ProtocolConfig {
+	fn default() -> ProtocolConfig {
+		ProtocolConfig {
+			roles: Role::FULL,
+		}
+	}
+}

substrate/network/src/error.rs

@@ -12,34 +12,20 @@
 // GNU General Public License for more details.
 
 // You should have received a copy of the GNU General Public License
-// along with Polkadot.  If not, see <http://www.gnu.org/licenses/>.
+// along with Polkadot.  If not, see <http://www.gnu.org/licenses/>.?
 
+use network::Error as NetworkError;
 use client;
-use primitives::{block, parachain};
 
-/// Temporary dummy blockchain implementation for tests.
-#[derive(Debug, Default)]
-pub struct Blockchain;
-
-impl client::Blockchain for Blockchain {
-	type Error = ::std::io::Error;
-
-	fn latest_hash(&self) -> Result<block::HeaderHash, Self::Error> {
-		Ok(0.into())
+error_chain! {
+	foreign_links {
+		Network(NetworkError) #[doc = "Devp2p error."];
 	}
 
-	fn header(&self, hash: &block::HeaderHash) -> Result<Option<block::Header>, Self::Error> {
-		Ok(if hash != &0.into() {
-			None
-		} else {
-			Some(block::Header {
-				parent_hash: 0.into(),
-				number: 0,
-				state_root: 0.into(),
-				parachain_activity: parachain::Activity(vec![0]),
-				logs: vec![],
-			})
-		})
+	links {
+		Client(client::error::Error, client::error::ErrorKind);
+	}
+
+	errors {
 	}
 }
-

substrate/network/src/io.rs

@@ -0,0 +1,78 @@
+// Copyright 2017 Parity Technologies (UK) Ltd.
+// This file is part of Polkadot.
+
+// Polkadot is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// Polkadot is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with Polkadot.  If not, see <http://www.gnu.org/licenses/>.?
+
+use network::{NetworkContext, PeerId, Error as NetworkError, SessionInfo};
+
+/// IO interface for the syncing handler.
+/// Provides peer connection management and an interface to the blockchain client.
+pub trait SyncIo {
+	/// Disable a peer
+	fn disable_peer(&mut self, peer_id: PeerId);
+	/// Disconnect peer
+	fn disconnect_peer(&mut self, peer_id: PeerId);
+	/// Send a packet to a peer.
+	fn send(&mut self, peer_id: PeerId, data: Vec<u8>) -> Result<(), NetworkError>;
+	/// Returns peer identifier string
+	fn peer_info(&self, peer_id: PeerId) -> String {
+		peer_id.to_string()
+	}
+	/// Returns information on p2p session
+	fn peer_session_info(&self, peer_id: PeerId) -> Option<SessionInfo>;
+	/// Check if the session is expired
+	fn is_expired(&self) -> bool;
+}
+
+/// Wraps `NetworkContext` and the blockchain client
+pub struct NetSyncIo<'s, 'h> where 'h: 's {
+	network: &'s NetworkContext<'h>,
+}
+
+impl<'s, 'h> NetSyncIo<'s, 'h> {
+	/// Creates a new instance from the `NetworkContext` and the blockchain client reference.
+	pub fn new(network: &'s NetworkContext<'h>) -> NetSyncIo<'s, 'h> {
+		NetSyncIo {
+			network: network,
+		}
+	}
+}
+
+impl<'s, 'h> SyncIo for NetSyncIo<'s, 'h> {
+	fn disable_peer(&mut self, peer_id: PeerId) {
+		self.network.disable_peer(peer_id);
+	}
+
+	fn disconnect_peer(&mut self, peer_id: PeerId) {
+		self.network.disconnect_peer(peer_id);
+	}
+
+	fn send(&mut self, peer_id: PeerId, data: Vec<u8>) -> Result<(), NetworkError>{
+		self.network.send(peer_id, 0, data)
+	}
+
+	fn peer_session_info(&self, peer_id: PeerId) -> Option<SessionInfo> {
+		self.network.session_info(peer_id)
+	}
+
+	fn is_expired(&self) -> bool {
+		self.network.is_expired()
+	}
+
+	fn peer_info(&self, peer_id: PeerId) -> String {
+		self.network.peer_client_version(peer_id)
+	}
+}
+
+

substrate/network/src/lib.rs

@@ -0,0 +1,62 @@
+// Copyright 2017 Parity Technologies (UK) Ltd.
+// This file is part of Polkadot.
+
+// Polkadot is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// Polkadot is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with Polkadot.  If not, see <http://www.gnu.org/licenses/>.?
+
+#![warn(missing_docs)]
+
+//! Implements polkadot protocol version as specified here:
+//! https://github.com/paritytech/polkadot/wiki/Network-protocol
+
+extern crate ethcore_network as network;
+extern crate ethcore_io as core_io;
+extern crate env_logger;
+extern crate rand;
+extern crate semver;
+extern crate parking_lot;
+extern crate smallvec;
+extern crate ipnetwork;
+extern crate polkadot_primitives as primitives;
+extern crate polkadot_client as client;
+extern crate polkadot_state_machine as state_machine;
+extern crate polkadot_serializer as ser;
+extern crate serde;
+extern crate serde_json;
+#[macro_use] extern crate serde_derive;
+#[macro_use] extern crate log;
+#[macro_use] extern crate bitflags;
+#[macro_use] extern crate error_chain;
+
+mod service;
+mod sync;
+mod protocol;
+mod io;
+mod message;
+mod error;
+mod config;
+mod chain;
+mod blocks;
+
+#[cfg(test)]
+mod test;
+
+pub use service::Service;
+pub use protocol::{ProtocolStatus};
+pub use network::{NonReservedPeerMode, ConnectionFilter, ConnectionDirection, NetworkConfiguration};
+
+// TODO: move it elsewhere
+fn header_hash(header: &primitives::block::Header) -> primitives::block::HeaderHash {
+	primitives::hash(&ser::to_vec(header))
+}
+

substrate/network/src/message.rs

@@ -0,0 +1,189 @@
+// Copyright 2017 Parity Technologies (UK) Ltd.
+// This file is part of Polkadot.
+
+// Polkadot is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// Polkadot is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with Polkadot.  If not, see <http://www.gnu.org/licenses/>.?
+
+//! Network packet message types. These get serialized and put into the lower level protocol payload.
+
+use std::borrow::Borrow;
+use primitives::parachain::Id as ParachainId;
+use primitives::Address;
+use primitives::block::{Number as BlockNumber, HeaderHash, Header, Body};
+use service::Role as RoleFlags;
+
+pub type RequestId = u64;
+type Bytes = Vec<u8>;
+
+type Signature = ::primitives::hash::H256; //TODO:
+
+/// Configured node role.
+#[derive(Debug, PartialEq, Eq, Serialize, Deserialize)]
+pub enum Role {
+	/// Full relay chain client with no additional responsibilities.
+	Full,
+	/// Relay chain light client.
+	Light,
+	/// Parachain validator.
+	Validator,
+	/// Parachain collator.
+	Collator,
+}
+
+impl<T> From<T> for RoleFlags where T: Borrow<[Role]> {
+	fn from(roles: T) -> RoleFlags {
+		let mut flags = RoleFlags::NONE;
+		let roles: &[Role] = roles.borrow();
+		for r in roles {
+			match *r {
+				Role::Full => flags = flags | RoleFlags::FULL,
+				Role::Light => flags = flags | RoleFlags::LIGHT,
+				Role::Validator => flags = flags | RoleFlags::VALIDATOR,
+				Role::Collator => flags = flags | RoleFlags::COLLATOR,
+			}
+		}
+		flags
+	}
+}
+
+impl From<RoleFlags> for Vec<Role> where {
+	fn from(flags: RoleFlags) -> Vec<Role> {
+		let mut roles = Vec::new();
+		if !(flags & RoleFlags::FULL).is_empty() {
+			roles.push(Role::Full);
+		}
+		if !(flags & RoleFlags::LIGHT).is_empty() {
+			roles.push(Role::Light);
+		}
+		if !(flags & RoleFlags::VALIDATOR).is_empty() {
+			roles.push(Role::Validator);
+		}
+		if !(flags & RoleFlags::COLLATOR).is_empty() {
+			roles.push(Role::Collator);
+		}
+		roles
+	}
+}
+
+#[derive(Debug, PartialEq, Eq, Serialize, Deserialize, Copy, Clone)]
+/// Bits of block data and associated artefacts to request.
+pub enum BlockAttribute {
+	/// Include block header.
+	Header,
+	/// Include block body.
+	Body,
+	/// Include block receipt.
+	Receipt,
+	/// Include block message queue.
+	MessageQueue,
+}
+
+#[derive(Debug, PartialEq, Eq, Clone, Serialize, Deserialize)]
+/// Block data sent in the response.
+pub struct BlockData {
+	/// Block header hash.
+	pub hash: HeaderHash,
+	/// Block header if requested.
+	pub header: Option<Header>,
+	/// Block body if requested.
+	pub body: Option<Body>,
+	/// Block receipt if requested.
+	pub receipt: Option<Bytes>,
+	/// Block message queue if requested.
+	pub message_queue: Option<Bytes>,
+}
+
+#[serde(untagged)]
+#[derive(Debug, PartialEq, Eq, Clone, Serialize, Deserialize)]
+/// Identifies starting point of a block sequence.
+pub enum FromBlock {
+	/// Start with given hash.
+	Hash(HeaderHash),
+	/// Start with given block number.
+	Number(BlockNumber),
+}
+
+#[derive(Debug, PartialEq, Eq, Clone, Serialize, Deserialize)]
+/// Block enumeration direction.
+pub enum Direction {
+	/// Enumerate in ascending order (from child to parent).
+	Ascending,
+	/// Enumerate in descendfing order (from parent to canonical child).
+	Descending,
+}
+
+#[derive(Debug, PartialEq, Eq, Serialize, Deserialize)]
+/// A network message.
+pub enum Message {
+	/// Status packet.
+	Status(Status),
+	/// Block request.
+	BlockRequest(BlockRequest),
+	/// Block response.
+	BlockResponse(BlockResponse),
+	/// Block announce.
+	BlockAnnounce(BlockAnnounce),
+}
+
+#[derive(Debug, PartialEq, Eq, Serialize, Deserialize)]
+pub struct Status {
+	/// Protocol version.
+	pub version: u32,
+	/// Supported roles.
+	pub roles: Vec<Role>,
+	/// Best block number.
+	pub best_number: BlockNumber,
+	/// Best block hash.
+	pub best_hash: HeaderHash,
+	/// Genesis block hash.
+	pub genesis_hash: HeaderHash,
+	/// Signatue of `best_hash` made with validator address. Required for the validator role.
+	pub validator_signature: Option<Signature>,
+	/// Validator address. Required for the validator role.
+	pub validator_id: Option<Address>,
+	/// Parachain id. Required for the collator role.
+	pub parachain_id: Option<ParachainId>,
+}
+
+#[derive(Debug, PartialEq, Eq, Clone, Serialize, Deserialize)]
+/// Request block data from a peer.
+pub struct BlockRequest {
+	/// Unique request id.
+	pub id: RequestId,
+	/// Bits of block data to request.
+	pub fields: Vec<BlockAttribute>,
+	/// Start from this block.
+	pub from: FromBlock,
+	/// End at this block. An implementation defined maximum is used when unspecified.
+	pub to: Option<HeaderHash>,
+	/// Sequence direction.
+	pub direction: Direction,
+	/// Maximum number of block to return. An implementation defined maximum is used when unspecified.
+	pub max: Option<u32>,
+}
+
+#[derive(Debug, PartialEq, Eq, Serialize, Deserialize)]
+/// Response to `BlockRequest`
+pub struct BlockResponse {
+	/// Id of a request this response was made for.
+	pub id: RequestId,
+	/// Block data for the requested sequence.
+	pub blocks: Vec<BlockData>,
+}
+
+#[derive(Debug, PartialEq, Eq, Serialize, Deserialize)]
+/// Announce a new complete relay chain block on the network.
+pub struct BlockAnnounce {
+	/// New block header.
+	pub header: Header,
+}

substrate/network/src/protocol.rs

@@ -0,0 +1,347 @@
+// Copyright 2017 Parity Technologies (UK) Ltd.
+// This file is part of Polkadot.
+
+// Polkadot is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// Polkadot is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with Polkadot.  If not, see <http://www.gnu.org/licenses/>.?
+
+use std::collections::{HashMap, HashSet, BTreeMap};
+use std::mem;
+use std::sync::Arc;
+use parking_lot::RwLock;
+use serde_json;
+use std::time;
+use primitives::block::{HeaderHash, TransactionHash, Number as BlockNumber, Header};
+use network::{PeerId, NodeId};
+
+use message::{self, Message};
+use sync::{ChainSync, Status as SyncStatus};
+use service::Role;
+use config::ProtocolConfig;
+use chain::Client;
+use io::SyncIo;
+use error;
+
+const REQUEST_TIMEOUT_SEC: u64 = 15;
+const PROTOCOL_VERSION: u32 = 0;
+
+// Lock must always be taken in order declared here.
+pub struct Protocol {
+	config: ProtocolConfig,
+	chain: Arc<Client>,
+	genesis_hash: HeaderHash,
+	sync: RwLock<ChainSync>,
+	/// All connected peers
+	peers: RwLock<HashMap<PeerId, Peer>>,
+	/// Connected peers pending Status message.
+	handshaking_peers: RwLock<HashMap<PeerId, time::Instant>>,
+}
+
+/// Syncing status and statistics
+#[derive(Clone, Copy)]
+pub struct ProtocolStatus {
+	/// Sync status.
+	pub sync: SyncStatus,
+	/// Total number of connected peers
+	pub num_peers: usize,
+	/// Total number of active peers.
+	pub num_active_peers: usize,
+}
+
+/// Peer information
+struct Peer {
+	/// Protocol version
+	protocol_version: u32,
+	/// Roles
+	roles: Role,
+	/// Peer best block hash
+	best_hash: HeaderHash,
+	/// Peer best block number
+	best_number: BlockNumber,
+	/// Pending block request if any
+	block_request: Option<message::BlockRequest>,
+	/// Request timestamp
+	request_timestamp: Option<time::Instant>,
+	/// Holds a set of transactions recently sent to this peer to avoid spamming.
+	_last_sent_transactions: HashSet<TransactionHash>,
+	/// Request counter,
+	request_id: message::RequestId,
+}
+
+#[derive(Debug)]
+pub struct PeerInfo {
+	/// Roles
+	pub roles: Role,
+	/// Protocol version
+	pub protocol_version: u32,
+	/// Peer best block hash
+	pub best_hash: HeaderHash,
+	/// Peer best block number
+	pub best_number: BlockNumber,
+}
+
+/// Transaction stats
+#[derive(Debug)]
+pub struct TransactionStats {
+	/// Block number where this TX was first seen.
+	pub first_seen: u64,
+	/// Peers it was propagated to.
+	pub propagated_to: BTreeMap<NodeId, usize>,
+}
+
+impl Protocol {
+	/// Create a new instance.
+	pub fn new(config: ProtocolConfig, chain: Arc<Client>) -> error::Result<Protocol>  {
+		let info = chain.info()?;
+		let protocol = Protocol {
+			config: config,
+			chain: chain,
+			genesis_hash: info.chain.genesis_hash,
+			sync: RwLock::new(ChainSync::new(&info)),
+			peers: RwLock::new(HashMap::new()),
+			handshaking_peers: RwLock::new(HashMap::new()),
+		};
+		Ok(protocol)
+	}
+
+	/// Returns protocol status
+	pub fn status(&self) -> ProtocolStatus {
+		let sync = self.sync.read();
+		let peers = self.peers.read();
+		ProtocolStatus {
+			sync: sync.status(),
+			num_peers: peers.values().count(),
+			num_active_peers: peers.values().filter(|p| p.block_request.is_some()).count(),
+		}
+	}
+
+	pub fn handle_packet(&self, io: &mut SyncIo, peer_id: PeerId, data: &[u8]) {
+		let message: Message = match serde_json::from_slice(data) {
+			Ok(m) => m,
+			Err(e) => {
+				debug!("Invalid packet from {}: {}", peer_id, e);
+				io.disable_peer(peer_id);
+				return;
+			}
+		};
+
+		match message {
+			Message::Status(s) => self.on_status_message(io, peer_id, s),
+			Message::BlockRequest(r) => self.on_block_request(io, peer_id, r),
+			Message::BlockResponse(r) => {
+				let request = {
+					let mut peers = self.peers.write();
+					if let Some(ref mut peer) = peers.get_mut(&peer_id) {
+						peer.request_timestamp = None;
+						match mem::replace(&mut peer.block_request, None) {
+							Some(r) => r,
+							None => {
+								debug!("Unexpected response packet from {}", peer_id);
+								io.disable_peer(peer_id);
+								return;
+							}
+						}
+					} else {
+						debug!("Unexpected packet from {}", peer_id);
+						io.disable_peer(peer_id);
+						return;
+					}
+				};
+				if request.id != r.id {
+					trace!("Ignoring mismatched response packet from {}", peer_id);
+					return;
+				}
+				self.on_block_response(io, peer_id, request, r);
+			},
+			Message::BlockAnnounce(announce) => {
+				self.on_block_announce(io, peer_id, announce);
+			}
+		}
+	}
+
+	pub fn send_message(&self, io: &mut SyncIo, peer_id: PeerId, mut message: Message) {
+		let mut peers = self.peers.write();
+		if let Some(ref mut peer) = peers.get_mut(&peer_id) {
+			match &mut message {
+				&mut Message::BlockRequest(ref mut r) => {
+					peer.block_request = Some(r.clone());
+					peer.request_timestamp = Some(time::Instant::now());
+					r.id = peer.request_id;
+					peer.request_id = peer.request_id + 1;
+				},
+				_ => (),
+			}
+		}
+		let data = serde_json::to_vec(&message).expect("Serializer is infallible; qed");
+		if let Err(e) = io.send(peer_id, data) {
+			debug!(target:"sync", "Error sending message: {:?}", e);
+			io.disconnect_peer(peer_id);
+		}
+	}
+
+	/// Called when a new peer is connected
+	pub fn on_peer_connected(&self, io: &mut SyncIo, peer_id: PeerId) {
+		trace!(target: "sync", "Connected {}: {}", peer_id, io.peer_info(peer_id));
+		self.handshaking_peers.write().insert(peer_id, time::Instant::now());
+		self.send_status(io, peer_id);
+	}
+
+	/// Called by peer when it is disconnecting
+	pub fn on_peer_disconnected(&self, io: &mut SyncIo, peer: PeerId) {
+		trace!(target: "sync", "Disconnecting {}: {}", peer, io.peer_info(peer));
+		let removed = {
+			let mut peers = self.peers.write();
+			let mut handshaking_peers = self.handshaking_peers.write();
+			handshaking_peers.remove(&peer);
+			peers.remove(&peer).is_some()
+		};
+		if removed {
+			self.sync.write().peer_disconnected(io, self, peer);
+		}
+	}
+
+	pub fn on_block_request(&self, _io: &mut SyncIo, _peer_id: PeerId, _request: message::BlockRequest) {
+	}
+
+	pub fn on_block_response(&self, io: &mut SyncIo, peer_id: PeerId, request: message::BlockRequest, response: message::BlockResponse) {
+		// TODO: validate response
+		self.sync.write().on_block_data(io, self, peer_id, request, response);
+	}
+
+	pub fn tick(&self, io: &mut SyncIo) {
+		self.maintain_peers(io);
+	}
+
+	fn maintain_peers(&self, io: &mut SyncIo) {
+		let tick = time::Instant::now();
+		let mut aborting = Vec::new();
+		{
+			let peers = self.peers.read();
+			let handshaking_peers = self.handshaking_peers.read();
+			for (peer_id, timestamp) in peers.iter()
+				.filter_map(|(id, peer)| peer.request_timestamp.as_ref().map(|r| (id, r)))
+				.chain(handshaking_peers.iter()) {
+				if (tick - *timestamp).as_secs() > REQUEST_TIMEOUT_SEC {
+					trace!(target:"sync", "Timeout {}", peer_id);
+					io.disconnect_peer(*peer_id);
+					aborting.push(*peer_id);
+				}
+			}
+		}
+		for p in aborting {
+			self.on_peer_disconnected(io, p);
+		}
+	}
+
+	pub fn peer_info(&self, peer: PeerId) -> Option<PeerInfo> {
+		self.peers.read().get(&peer).map(|p| {
+			PeerInfo {
+				roles: p.roles,
+				protocol_version: p.protocol_version,
+				best_hash: p.best_hash,
+				best_number: p.best_number,
+			}
+		})
+	}
+
+	/// Called by peer to report status
+	fn on_status_message(&self, io: &mut SyncIo, peer_id: PeerId, status: message::Status) {
+		trace!(target: "sync", "New peer {} {:?}", peer_id, status);
+		if io.is_expired() {
+			trace!(target: "sync", "Status packet from expired session {}:{}", peer_id, io.peer_info(peer_id));
+			return;
+		}
+
+		{
+			let mut peers = self.peers.write();
+			let mut handshaking_peers = self.handshaking_peers.write();
+			if peers.contains_key(&peer_id) {
+				debug!(target: "sync", "Unexpected status packet from {}:{}", peer_id, io.peer_info(peer_id));
+				return;
+			}
+			if status.genesis_hash != self.genesis_hash {
+				io.disable_peer(peer_id);
+				trace!(target: "sync", "Peer {} genesis hash mismatch (ours: {}, theirs: {})", peer_id, self.genesis_hash, status.genesis_hash);
+				return;
+			}
+			if status.version != PROTOCOL_VERSION {
+				io.disable_peer(peer_id);
+				trace!(target: "sync", "Peer {} unsupported eth protocol ({})", peer_id, status.version);
+				return;
+			}
+
+			let peer = Peer {
+				protocol_version: status.version,
+				roles: status.roles.into(),
+				best_hash: status.best_hash,
+				best_number: status.best_number,
+				block_request: None,
+				request_timestamp: None,
+				_last_sent_transactions: HashSet::new(),
+				request_id: 0,
+			};
+			peers.insert(peer_id.clone(), peer);
+			handshaking_peers.remove(&peer_id);
+			debug!(target: "sync", "Connected {} {}", peer_id, io.peer_info(peer_id));
+		}
+		self.sync.write().new_peer(io, self, peer_id);
+	}
+
+	/// Send Status message
+	fn send_status(&self, io: &mut SyncIo, peer_id: PeerId) {
+		if let Ok(info) = self.chain.info() {
+			let status = message::Status {
+				version: PROTOCOL_VERSION,
+				genesis_hash: info.chain.genesis_hash,
+				roles: self.config.roles.into(),
+				best_number: info.chain.best_number,
+				best_hash: info.chain.best_hash,
+				validator_signature: None,
+				validator_id: None,
+				parachain_id: None,
+			};
+			self.send_message(io, peer_id, Message::Status(status))
+		}
+	}
+
+	pub fn abort(&self) {
+		let mut sync = self.sync.write();
+		let mut peers = self.peers.write();
+		let mut handshaking_peers = self.handshaking_peers.write();
+		sync.clear();
+		peers.clear();
+		handshaking_peers.clear();
+	}
+
+	pub fn on_block_announce(&self, io: &mut SyncIo, peer_id: PeerId, announce: message::BlockAnnounce) {
+		let header = announce.header;
+		self.sync.write().on_block_announce(io, self, peer_id, &header);
+	}
+
+	pub fn on_block_imported(&self, header: &Header) {
+		self.sync.write().update_chain_info(&header);
+	}
+
+	pub fn on_new_transactions(&self) {
+	}
+
+	pub fn transactions_stats(&self) -> BTreeMap<TransactionHash, TransactionStats> {
+		BTreeMap::new()
+	}
+
+	pub fn chain(&self) -> &Client {
+		&*self.chain
+	}
+}
+
+
+

substrate/network/src/service.rs

@@ -0,0 +1,239 @@
+// Copyright 2017 Parity Technologies (UK) Ltd.
+// This file is part of Polkadot.
+
+// Polkadot is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// Polkadot is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with Polkadot.  If not, see <http://www.gnu.org/licenses/>.?
+
+use std::sync::Arc;
+use std::collections::{BTreeMap};
+use std::io;
+use network::{NetworkProtocolHandler, NetworkService, NetworkContext, HostInfo, PeerId, ProtocolId,
+NetworkConfiguration , NonReservedPeerMode, ErrorKind};
+use primitives::block::{TransactionHash, Header};
+use core_io::{TimerToken};
+use io::NetSyncIo;
+use protocol::{Protocol, ProtocolStatus, PeerInfo as ProtocolPeerInfo, TransactionStats};
+use config::{ProtocolConfig};
+use error::Error;
+use chain::Client;
+
+/// Polkadot devp2p protocol id
+pub const DOT_PROTOCOL_ID: ProtocolId = *b"dot";
+
+bitflags! {
+	pub struct Role: u32 {
+		const NONE = 0b00000000;
+		const FULL = 0b00000001;
+		const LIGHT = 0b00000010;
+		const VALIDATOR = 0b00000100;
+		const COLLATOR = 0b00001000;
+	}
+}
+
+/// Sync status
+pub trait SyncProvider: Send + Sync {
+	/// Get sync status
+	fn status(&self) -> ProtocolStatus;
+	/// Get peers information
+	fn peers(&self) -> Vec<PeerInfo>;
+	/// Get this node id if available.
+	fn node_id(&self) -> Option<String>;
+	/// Returns propagation count for pending transactions.
+	fn transactions_stats(&self) -> BTreeMap<TransactionHash, TransactionStats>;
+}
+
+/// Peer connection information
+#[derive(Debug)]
+pub struct PeerInfo {
+	/// Public node id
+	pub id: Option<String>,
+	/// Node client ID
+	pub client_version: String,
+	/// Capabilities
+	pub capabilities: Vec<String>,
+	/// Remote endpoint address
+	pub remote_address: String,
+	/// Local endpoint address
+	pub local_address: String,
+	/// Dot protocol info.
+	pub dot_info: Option<ProtocolPeerInfo>,
+}
+
+/// Service initialization parameters.
+pub struct Params {
+	/// Configuration.
+	pub config: ProtocolConfig,
+	/// Network layer configuration.
+	pub network_config: NetworkConfiguration,
+	/// Polkadot relay chain access point.
+	pub chain: Arc<Client>,
+}
+
+/// Polkadot network service. Handles network IO and manages connectivity.
+pub struct Service {
+	/// Network service
+	network: NetworkService,
+	/// Devp2p protocol handler
+	handler: Arc<ProtocolHandler>,
+}
+
+impl Service {
+	/// Creates and register protocol with the network service
+	pub fn new(params: Params) -> Result<Arc<Service>, Error> {
+
+		let service = NetworkService::new(params.network_config.clone(), None)?;
+
+		let sync = Arc::new(Service {
+			network: service,
+			handler: Arc::new(ProtocolHandler {
+				protocol: Protocol::new(params.config, params.chain.clone())?,
+			}),
+		});
+
+		Ok(sync)
+	}
+
+	/// Called when a new block is imported by the client.
+	pub fn on_block_imported(&self, header: &Header) {
+		self.handler.protocol.on_block_imported(header)
+	}
+
+	/// Called when new transactons are imported by the client.
+	pub fn on_new_transactions(&self) {
+		self.handler.protocol.on_new_transactions()
+	}
+
+	fn start(&self) {
+		match self.network.start().map_err(Into::into) {
+			Err(ErrorKind::Io(ref e)) if  e.kind() == io::ErrorKind::AddrInUse =>
+				warn!("Network port {:?} is already in use, make sure that another instance of Polkadot client is not running or change the port using the --port option.", self.network.config().listen_address.expect("Listen address is not set.")),
+			Err(err) => warn!("Error starting network: {}", err),
+			_ => {},
+		};
+		self.network.register_protocol(self.handler.clone(), DOT_PROTOCOL_ID, 1, &[0u8])
+			.unwrap_or_else(|e| warn!("Error registering polkadot protocol: {:?}", e));
+	}
+
+	fn stop(&self) {
+		self.handler.protocol.abort();
+		self.network.stop().unwrap_or_else(|e| warn!("Error stopping network: {:?}", e));
+	}
+}
+
+impl SyncProvider for Service {
+	/// Get sync status
+	fn status(&self) -> ProtocolStatus {
+		self.handler.protocol.status()
+	}
+
+	/// Get sync peers
+	fn peers(&self) -> Vec<PeerInfo> {
+		self.network.with_context_eval(DOT_PROTOCOL_ID, |ctx| {
+			let peer_ids = self.network.connected_peers();
+
+			peer_ids.into_iter().filter_map(|peer_id| {
+				let session_info = match ctx.session_info(peer_id) {
+					None => return None,
+					Some(info) => info,
+				};
+
+				Some(PeerInfo {
+					id: session_info.id.map(|id| id.hex()),
+					client_version: session_info.client_version,
+					capabilities: session_info.peer_capabilities.into_iter().map(|c| c.to_string()).collect(),
+					remote_address: session_info.remote_address,
+					local_address: session_info.local_address,
+					dot_info: self.handler.protocol.peer_info(peer_id),
+				})
+			}).collect()
+		}).unwrap_or_else(Vec::new)
+	}
+
+	fn node_id(&self) -> Option<String> {
+		self.network.external_url()
+	}
+
+	fn transactions_stats(&self) -> BTreeMap<TransactionHash, TransactionStats> {
+		self.handler.protocol.transactions_stats()
+	}
+}
+
+struct ProtocolHandler {
+	/// Protocol handler
+	protocol: Protocol,
+}
+
+impl NetworkProtocolHandler for ProtocolHandler {
+	fn initialize(&self, io: &NetworkContext, _host_info: &HostInfo) {
+		io.register_timer(0, 1000).expect("Error registering sync timer");
+	}
+
+	fn read(&self, io: &NetworkContext, peer: &PeerId, _packet_id: u8, data: &[u8]) {
+		self.protocol.handle_packet(&mut NetSyncIo::new(io), *peer, data);
+	}
+
+	fn connected(&self, io: &NetworkContext, peer: &PeerId) {
+		self.protocol.on_peer_connected(&mut NetSyncIo::new(io), *peer);
+	}
+
+	fn disconnected(&self, io: &NetworkContext, peer: &PeerId) {
+		self.protocol.on_peer_disconnected(&mut NetSyncIo::new(io), *peer);
+	}
+
+	fn timeout(&self, io: &NetworkContext, _timer: TimerToken) {
+		self.protocol.tick(&mut NetSyncIo::new(io));
+	}
+}
+
+/// Trait for managing network
+pub trait ManageNetwork : Send + Sync {
+	/// Set to allow unreserved peers to connect
+	fn accept_unreserved_peers(&self);
+	/// Set to deny unreserved peers to connect
+	fn deny_unreserved_peers(&self);
+	/// Remove reservation for the peer
+	fn remove_reserved_peer(&self, peer: String) -> Result<(), String>;
+	/// Add reserved peer
+	fn add_reserved_peer(&self, peer: String) -> Result<(), String>;
+	/// Start network
+	fn start_network(&self);
+	/// Stop network
+	fn stop_network(&self);
+}
+
+
+impl ManageNetwork for Service {
+	fn accept_unreserved_peers(&self) {
+		self.network.set_non_reserved_mode(NonReservedPeerMode::Accept);
+	}
+
+	fn deny_unreserved_peers(&self) {
+		self.network.set_non_reserved_mode(NonReservedPeerMode::Deny);
+	}
+
+	fn remove_reserved_peer(&self, peer: String) -> Result<(), String> {
+		self.network.remove_reserved_peer(&peer).map_err(|e| format!("{:?}", e))
+	}
+
+	fn add_reserved_peer(&self, peer: String) -> Result<(), String> {
+		self.network.add_reserved_peer(&peer).map_err(|e| format!("{:?}", e))
+	}
+
+	fn start_network(&self) {
+		self.start();
+	}
+
+	fn stop_network(&self) {
+		self.stop();
+	}
+}

substrate/network/src/sync.rs

@@ -0,0 +1,372 @@
+// Copyright 2017 Parity Technologies (UK) Ltd.
+// This file is part of Polkadot.
+
+// Polkadot is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// Polkadot is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with Polkadot.  If not, see <http://www.gnu.org/licenses/>.?
+
+use std::collections::HashMap;
+use io::SyncIo;
+use protocol::Protocol;
+use network::PeerId;
+use client::{ImportResult, BlockStatus, ClientInfo};
+use primitives::block::{HeaderHash, Number as BlockNumber, Header};
+use blocks::{self, BlockCollection};
+use message::{self, Message};
+use super::header_hash;
+
+// Maximum parallel requests for a block.
+const MAX_BLOCK_DOWNLOAD: usize = 1;
+
+struct PeerSync {
+	pub common_hash: HeaderHash,
+	pub common_number: BlockNumber,
+	pub best_hash: HeaderHash,
+	pub best_number: BlockNumber,
+	pub state: PeerSyncState,
+}
+
+#[derive(Copy, Clone, Eq, PartialEq, Debug)]
+pub enum PeerSyncState {
+    AncestorSearch(BlockNumber),
+    Available,
+    DownloadingNew(BlockNumber),
+    DownloadingStale(HeaderHash),
+}
+
+/// Relay chain sync strategy.
+pub struct ChainSync {
+	genesis_hash: HeaderHash,
+	peers: HashMap<PeerId, PeerSync>,
+	blocks: BlockCollection,
+	best_queued_number: BlockNumber,
+	best_queued_hash: HeaderHash,
+	required_block_attributes: Vec<message::BlockAttribute>,
+}
+
+/// Syncing status and statistics
+#[derive(Clone, Copy)]
+pub struct Status;
+
+impl ChainSync {
+	pub fn new(info: &ClientInfo) -> ChainSync {
+		ChainSync {
+			genesis_hash: info.chain.genesis_hash,
+			peers: HashMap::new(),
+			blocks: BlockCollection::new(),
+			best_queued_hash: info.best_queued_hash.unwrap_or(info.chain.best_hash),
+			best_queued_number: info.best_queued_number.unwrap_or(info.chain.best_number),
+			required_block_attributes: vec![message::BlockAttribute::Header, message::BlockAttribute::Body],
+		}
+	}
+
+	/// Returns sync status
+	pub fn status(&self) -> Status {
+		Status
+	}
+
+	pub fn new_peer(&mut self, io: &mut SyncIo, protocol: &Protocol, peer_id: PeerId) {
+		if let Some(info) = protocol.peer_info(peer_id) {
+			match (protocol.chain().block_status(&info.best_hash), info.best_number) {
+				(Err(e), _) => {
+					debug!(target:"sync", "Error reading blockchain: {:?}", e);
+					io.disconnect_peer(peer_id);
+				},
+				(Ok(BlockStatus::KnownBad), _) => {
+					debug!(target:"sync", "New peer with known bad best block {} ({}).", info.best_hash, info.best_number);
+					io.disable_peer(peer_id);
+				},
+				(Ok(BlockStatus::Unknown), 0) => {
+					debug!(target:"sync", "New peer with unkown genesis hash {} ({}).", info.best_hash, info.best_number);
+					io.disable_peer(peer_id);
+				},
+				(Ok(BlockStatus::Unknown), _) => {
+					debug!(target:"sync", "New peer with unkown best hash {} ({}), searching for common ancestor.", info.best_hash, info.best_number);
+					self.peers.insert(peer_id, PeerSync {
+						common_hash: self.genesis_hash,
+						common_number: 0,
+						best_hash: info.best_hash,
+						best_number: info.best_number,
+						state: PeerSyncState::AncestorSearch(info.best_number - 1),
+					});
+					Self::request_ancestry(io, protocol, peer_id, info.best_number - 1)
+				},
+				(Ok(BlockStatus::Queued), _) | (Ok(BlockStatus::InChain), _) => {
+					debug!(target:"sync", "New peer with known best hash {} ({}).", info.best_hash, info.best_number);
+					self.peers.insert(peer_id, PeerSync {
+						common_hash: info.best_hash,
+						common_number: info.best_number,
+						best_hash: info.best_hash,
+						best_number: info.best_number,
+						state: PeerSyncState::Available,
+					});
+				}
+			}
+		}
+	}
+
+	pub fn on_block_data(&mut self, io: &mut SyncIo, protocol: &Protocol, peer_id: PeerId, _request: message::BlockRequest, response: message::BlockResponse) {
+		let count = response.blocks.len();
+		let mut imported: usize = 0;
+		let new_blocks = if let Some(ref mut peer) = self.peers.get_mut(&peer_id) {
+			match peer.state {
+				PeerSyncState::DownloadingNew(start_block) => {
+					self.blocks.clear_peer_download(peer_id);
+					peer.state = PeerSyncState::Available;
+
+					self.blocks.insert(start_block, response.blocks, peer_id);
+					self.blocks.drain(self.best_queued_number + 1)
+				},
+				PeerSyncState::DownloadingStale(_) => {
+					peer.state = PeerSyncState::Available;
+					response.blocks.into_iter().map(|b| blocks::BlockData {
+						origin: peer_id,
+						block: b
+					}).collect()
+				},
+				PeerSyncState::AncestorSearch(n) => {
+					match response.blocks.get(0) {
+						Some(ref block) => {
+							match protocol.chain().block_hash(n) {
+								Ok(Some(block_hash)) if block_hash == block.hash => {
+									peer.common_hash = block.hash;
+									peer.common_number = n;
+									peer.state = PeerSyncState::Available;
+									trace!(target:"sync", "Found common ancestor for peer {}: {} ({})", peer_id, block.hash, n);
+									vec![]
+								},
+								Ok(_) if n > 0 => {
+									let n = n - 1;
+									peer.state = PeerSyncState::AncestorSearch(n);
+									Self::request_ancestry(io, protocol, peer_id, n);
+									return;
+								},
+								Ok(_) => { // genesis mismatch
+									io.disable_peer(peer_id);
+									return;
+								},
+								Err(e) => {
+									debug!(target:"sync", "Error reading blockchain: {:?}", e);
+									io.disconnect_peer(peer_id);
+									return;
+								}
+							}
+						},
+						None => {
+							trace!(target:"sync", "Invalid response when searching for ancestor from {}", peer_id);
+							io.disconnect_peer(peer_id);
+							return;
+						}
+					}
+				},
+				PeerSyncState::Available => Vec::new(),
+			}
+		} else {
+			vec![]
+		};
+
+		// Blocks in the response/drain should be in ascending order.
+		for block in new_blocks {
+			let origin = block.origin;
+			let block = block.block;
+			if let Some(header) = block.header {
+				let number = header.number;
+				let hash = header_hash(&header);
+				let parent = header.parent_hash;
+				let result = protocol.chain().import(header, block.body);
+				match result {
+					Ok(ImportResult::AlreadyInChain) => {
+						trace!(target: "sync", "Block already in chain {}: {:?}", number, hash);
+						self.block_imported(&hash, number);
+					},
+					Ok(ImportResult::AlreadyQueued) => {
+						trace!(target: "sync", "Block already queued {}: {:?}", number, hash);
+						self.block_imported(&hash, number);
+					},
+					Ok(ImportResult::Queued) => {
+						trace!(target: "sync", "Block queued {}: {:?}", number, hash);
+						self.block_imported(&hash, number);
+						imported = imported + 1;
+					},
+					Ok(ImportResult::UnknownParent) => {
+						debug!(target: "sync", "Block with unknown parent {}: {:?}, parent: {:?}", number, hash, parent);
+						self.restart(io, protocol);
+						return;
+					},
+					Ok(ImportResult::KnownBad) => {
+						debug!(target: "sync", "Bad block {}: {:?}", number, hash);
+						io.disable_peer(origin); //TODO: use persistent ID
+						self.restart(io, protocol);
+						return;
+					}
+					Err(e) => {
+						debug!(target: "sync", "Error importing block {}: {:?}: {:?}", number, hash, e);
+						self.restart(io, protocol);
+						return;
+					}
+				}
+			}
+		}
+		trace!(target: "sync", "Imported {} of {}", imported, count);
+		self.maintain_sync(io, protocol);
+	}
+
+	fn maintain_sync(&mut self, io: &mut SyncIo, protocol: &Protocol) {
+		let peers: Vec<PeerId> = self.peers.keys().map(|p| *p).collect();
+		for peer in peers {
+			self.download_new(io, protocol, peer);
+		}
+	}
+
+	fn block_imported(&mut self, hash: &HeaderHash, number: BlockNumber) {
+		if number > self.best_queued_number {
+			self.best_queued_number = number;
+			self.best_queued_hash = *hash;
+		}
+		// Update common blocks
+		for (_, peer) in self.peers.iter_mut() {
+			if peer.best_number >= number {
+				peer.common_number = number;
+				peer.common_hash = *hash;
+			}
+		}
+	}
+
+	pub fn update_chain_info(&mut self, best_header: &Header ) {
+		let hash = header_hash(&best_header);
+		self.block_imported(&hash, best_header.number)
+	}
+
+	pub fn on_block_announce(&mut self, io: &mut SyncIo, protocol: &Protocol, peer_id: PeerId, header: &Header) {
+		let hash = header_hash(&header);
+		if let Some(ref mut peer) = self.peers.get_mut(&peer_id) {
+			if header.number > peer.best_number {
+				peer.best_number = header.number;
+				peer.best_hash = hash;
+			}
+		} else {
+			return;
+		}
+
+		if !self.is_known_or_already_downloading(protocol, &hash) {
+			let stale = header.number <= self.best_queued_number;
+			if stale {
+				if !self.is_known_or_already_downloading(protocol, &header.parent_hash) {
+					trace!(target: "sync", "Ignoring unkown stale block announce from {}: {} {:?}", peer_id, hash, header);
+				} else {
+					trace!(target: "sync", "Downloading new stale block announced from {}: {} {:?}", peer_id, hash, header);
+					self.download_stale(io, protocol, peer_id, &hash);
+				}
+			} else {
+				trace!(target: "sync", "Downloading new block announced from {}: {} {:?}", peer_id, hash, header);
+				self.download_new(io, protocol, peer_id);
+			}
+		} else {
+			trace!(target: "sync", "Known block announce from {}: {}", peer_id, hash);
+		}
+	}
+
+	fn is_known_or_already_downloading(&self, protocol: &Protocol, hash: &HeaderHash) -> bool {
+		self.peers.iter().any(|(_, p)| p.state == PeerSyncState::DownloadingStale(*hash))
+			|| protocol.chain().block_status(hash).ok().map_or(false, |s| s != BlockStatus::Unknown)
+	}
+
+	pub fn peer_disconnected(&mut self, io: &mut SyncIo, protocol: &Protocol, peer_id: PeerId) {
+		self.blocks.clear_peer_download(peer_id);
+		self.peers.remove(&peer_id);
+		self.maintain_sync(io, protocol);
+	}
+
+	pub fn restart(&mut self, io: &mut SyncIo, protocol: &Protocol) {
+		self.blocks.clear();
+		let ids: Vec<PeerId> = self.peers.keys().map(|p| *p).collect();
+		for id in ids {
+			self.new_peer(io, protocol, id);
+		}
+		match protocol.chain().info() {
+			Ok(info) => {
+				self.best_queued_hash = info.best_queued_hash.unwrap_or(info.chain.best_hash);
+				self.best_queued_number = info.best_queued_number.unwrap_or(info.chain.best_number);
+			},
+			Err(e) => {
+				debug!(target:"sync", "Error reading blockchain: {:?}", e);
+				self.best_queued_hash = self.genesis_hash;
+				self.best_queued_number = 0;
+			}
+		}
+	}
+
+	pub fn clear(&mut self) {
+		self.blocks.clear();
+		self.peers.clear();
+	}
+
+	// Download old block.
+	fn download_stale(&mut self, io: &mut SyncIo, protocol: &Protocol, peer_id: PeerId, hash: &HeaderHash) {
+		if let Some(ref mut peer) = self.peers.get_mut(&peer_id) {
+			match peer.state {
+				PeerSyncState::Available => {
+					let request = message::BlockRequest {
+						id: 0,
+						fields: self.required_block_attributes.clone(),
+						from: message::FromBlock::Hash(*hash),
+						to: None,
+						direction: message::Direction::Ascending,
+						max: Some(1),
+					};
+					peer.state = PeerSyncState::DownloadingStale(*hash);
+					protocol.send_message(io, peer_id, Message::BlockRequest(request));
+				},
+				_ => (),
+			}
+		}
+	}
+
+	// Issue a request for a peer to download new blocks, if any are available
+	fn download_new(&mut self, io: &mut SyncIo, protocol: &Protocol, peer_id: PeerId) {
+		if let Some(ref mut peer) = self.peers.get_mut(&peer_id) {
+			match peer.state {
+				PeerSyncState::Available => {
+					if let Some(range) = self.blocks.needed_blocks(peer_id, MAX_BLOCK_DOWNLOAD, peer.common_number, peer.best_number) {
+						let request = message::BlockRequest {
+							id: 0,
+							fields: self.required_block_attributes.clone(),
+							from: message::FromBlock::Number(range.start),
+							to: None,
+							direction: message::Direction::Ascending,
+							max: Some((range.end - range.start) as u32),
+						};
+						peer.state = PeerSyncState::DownloadingNew(range.start);
+						protocol.send_message(io, peer_id, Message::BlockRequest(request));
+					}
+				},
+				_ => (),
+			}
+		}
+	}
+
+	fn request_ancestry(io: &mut SyncIo, protocol: &Protocol, peer_id: PeerId, block: BlockNumber) {
+		let request = message::BlockRequest {
+			id: 0,
+			fields: vec![message::BlockAttribute::Header],
+			from: message::FromBlock::Number(block),
+			to: None,
+			direction: message::Direction::Ascending,
+			max: Some(1),
+		};
+		protocol.send_message(io, peer_id, Message::BlockRequest(request));
+	}
+}
+
+
+
+

substrate/primitives/src/block.rs

@@ -20,24 +20,30 @@ use bytes;
 use hash::H256;
 use parachain;
 
+/// Used to refer to a block number.
+pub type Number = u64;
+
 /// Hash used to refer to a block hash.
 pub type HeaderHash = H256;
 
+/// Hash used to refer to a transaction hash.
+pub type TransactionHash = H256;
+
 /// Execution log (event)
-#[derive(Debug, PartialEq, Eq, Serialize, Deserialize)]
+#[derive(Debug, PartialEq, Eq, Clone, Serialize, Deserialize)]
 pub struct Log(#[serde(with="bytes")] pub Vec<u8>);
 
 /// A relay chain block header.
 ///
 /// https://github.com/w3f/polkadot-spec/blob/master/spec.md#header
-#[derive(Debug, PartialEq, Eq, Serialize, Deserialize)]
+#[derive(Debug, PartialEq, Eq, Clone, Serialize, Deserialize)]
 #[serde(rename_all = "camelCase")]
 #[serde(deny_unknown_fields)]
 pub struct Header {
 	/// Block parent's hash.
 	pub parent_hash: HeaderHash,
 	/// Block number.
-	pub number: u64,
+	pub number: Number,
 	/// State root after this transition.
 	pub state_root: H256,
 	/// Parachain activity bitfield
@@ -50,7 +56,7 @@ pub struct Header {
 ///
 /// Included candidates should be sorted by parachain ID, and without duplicate
 /// IDs.
-#[derive(Debug, PartialEq, Eq, Serialize, Deserialize)]
+#[derive(Debug, PartialEq, Eq, Clone, Serialize, Deserialize)]
 #[serde(rename_all = "camelCase")]
 #[serde(deny_unknown_fields)]
 pub struct Body {

substrate/primitives/src/hash.rs

@@ -37,11 +37,11 @@ macro_rules! impl_serde {
 	}
 }
 
-impl_hash!(H160, 20);
+construct_hash!(H160, 20);
 impl_serde!(H160, 20);
-impl_hash!(H256, 32);
+construct_hash!(H256, 32);
 impl_serde!(H256, 32);
-impl_hash!(H512, 64);
+construct_hash!(H512, 64);
 impl_serde!(H512, 64);
 
 #[cfg(test)]

substrate/primitives/src/parachain.rs

@@ -98,7 +98,7 @@ pub struct HeadData(#[serde(with="bytes")] pub Vec<u8>);
 pub struct ValidationCode(#[serde(with="bytes")] pub Vec<u8>);
 
 /// Activitiy bit field
-#[derive(Debug, PartialEq, Eq, Serialize, Deserialize)]
+#[derive(Debug, PartialEq, Eq, Clone, Default, Serialize, Deserialize)]
 pub struct Activity(#[serde(with="bytes")] pub Vec<u8>);
 
 #[cfg(test)]

substrate/rpc/src/chain/mod.rs

@@ -18,6 +18,7 @@
 
 use primitives::block;
 use client;
+use state_machine;
 
 mod error;
 
@@ -35,11 +36,12 @@ build_rpc_trait! {
 	}
 }
 
-impl<B> ChainApi for B where
-	B: client::Blockchain + Send + Sync + 'static,
-	B::Error: ::std::error::Error + Send,
+impl<B, E> ChainApi for client::Client<B, E> where
+	B: client::backend::Backend + Send + Sync + 'static,
+	E: state_machine::CodeExecutor + Send + Sync + 'static,
+	client::error::Error: From<<<B as client::backend::Backend>::State as state_machine::backend::Backend>::Error>,
 {
 	fn header(&self, hash: block::HeaderHash) -> Result<Option<block::Header>> {
-		self.header(&hash).chain_err(|| "Blockchain error")
+		client::Client::header(self, &hash).chain_err(|| "Blockchain error")
 	}
 }

substrate/rpc/src/chain/tests.rs

@@ -14,28 +14,27 @@
 // You should have received a copy of the GNU General Public License
 // along with Polkadot.  If not, see <http://www.gnu.org/licenses/>.
 
-use primitives::parachain;
+use polkadot_executor as executor;
+use client;
 use super::*;
 
-use test_helpers::Blockchain;
-
 #[test]
 fn should_return_header() {
-	let state = Blockchain::default();
+	let client = client::new_in_mem(executor::executor()).unwrap();
 
 	assert_matches!(
-		ChainApi::header(&state, 0.into()),
+		ChainApi::header(&client, "11265ce45dd2baaaf071f6df8c5a44f0ed1d85a50e71451ff2d4345e57d12e3a".into()),
 		Ok(Some(ref x)) if x == &block::Header {
 			parent_hash: 0.into(),
 			number: 0,
 			state_root: 0.into(),
-			parachain_activity: parachain::Activity(vec![0]),
+			parachain_activity: Default::default(),
 			logs: vec![],
 		}
 	);
 
 	assert_matches!(
-		ChainApi::header(&state, 5.into()),
+		ChainApi::header(&client, 5.into()),
 		Ok(None)
 	);
 }

substrate/rpc/src/lib.rs

@@ -36,6 +36,3 @@ extern crate assert_matches;
 
 pub mod chain;
 pub mod state;
-
-#[cfg(test)]
-mod test_helpers;

substrate/rpc/src/state/mod.rs

@@ -42,8 +42,9 @@ build_rpc_trait! {
 }
 
 impl<B, E> StateApi for Client<B, E> where
-	B: client::Blockchain + Send + Sync + 'static,
+	B: client::backend::Backend + Send + Sync + 'static,
 	E: state_machine::CodeExecutor + Send + Sync + 'static,
+	client::error::Error: From<<<B as client::backend::Backend>::State as state_machine::backend::Backend>::Error>,
 {
 	fn storage(&self, key: StorageKey, block: block::HeaderHash) -> Result<StorageData> {
 		Ok(self.storage(&block, &key)?)

substrate/rpc/src/state/tests.rs

@@ -18,14 +18,15 @@ use super::*;
 use polkadot_executor as executor;
 
 use self::error::{Error, ErrorKind};
-use test_helpers::Blockchain;
+use client;
 
 #[test]
 fn should_return_storage() {
-	let client = Client::new(Blockchain::default(), executor::executor());
+	let client = client::new_in_mem(executor::executor()).unwrap();
+	let genesis_hash = "11265ce45dd2baaaf071f6df8c5a44f0ed1d85a50e71451ff2d4345e57d12e3a".into();
 
 	assert_matches!(
-		StateApi::storage(&client, StorageKey(vec![10]), 0.into()),
+		StateApi::storage(&client, StorageKey(vec![10]), genesis_hash),
 		Ok(ref x) if x.0.is_empty()
 	)
 }
@@ -34,10 +35,11 @@ fn should_return_storage() {
 #[ignore]	// TODO: [ToDr] reenable once we can properly mock the wasm executor env
 fn should_call_contract() {
 	// TODO [ToDr] Fix test after we are able to mock state.
-	let client = Client::new(Blockchain::default(), executor::executor());
+	let client = client::new_in_mem(executor::executor()).unwrap();
+	let genesis_hash = "11265ce45dd2baaaf071f6df8c5a44f0ed1d85a50e71451ff2d4345e57d12e3a".into();
 
 	assert_matches!(
-		StateApi::call(&client, "balanceOf".into(), CallData(vec![1,2,3]), 0.into()),
+		StateApi::call(&client, "balanceOf".into(), CallData(vec![1,2,3]), genesis_hash),
 		Err(Error(ErrorKind::Client(client::error::ErrorKind::Execution(_)), _))
 	)
 }

substrate/runtime-std/Cargo.toml

@@ -13,9 +13,10 @@ pwasm-libc = { path = "../wasm-runtime/pwasm-libc", version = "0.1" }
 environmental = { path = "../environmental", version = "0.1", optional = true }
 polkadot-state-machine = { path = "../state-machine", version = "0.1", optional = true }
 polkadot-primitives = { path = "../primitives", version = "0.1", optional = true }
+triehash = { version = "0.1", optional = true }
 
 [features]
 default = ["std"]
-std = ["environmental", "polkadot-state-machine", "polkadot-primitives"]
+std = ["environmental", "polkadot-state-machine", "polkadot-primitives", "triehash"]
 nightly = []
 strict = []

substrate/runtime-std/with_std.rs

@@ -20,6 +20,7 @@ extern crate environmental;
 
 extern crate polkadot_state_machine;
 extern crate polkadot_primitives as primitives;
+extern crate triehash;
 
 use std::fmt;
 use primitives::ed25519;
@@ -31,7 +32,7 @@ pub use std::boxed;
 pub use std::slice;
 pub use std::mem;
 
-pub use polkadot_state_machine::{Externalities, ExternalitiesError};
+pub use polkadot_state_machine::{Externalities, ExternalitiesError, TestExternalities};
 use primitives::hexdisplay::HexDisplay;
 
 // TODO: use the real error, not NoError.
@@ -84,6 +85,18 @@ pub fn chain_id() -> u64 {
 	).unwrap_or(0)
 }
 
+/// "Commit" all existing operations and get the resultant storage root.
+pub fn storage_root() -> [u8; 32] {
+	ext::with(|ext|
+		ext.storage_root()
+	).unwrap_or([0u8; 32])
+}
+
+/// "Commit" all existing operations and get the resultant storage root.
+pub fn enumerated_trie_root(serialised_values: &[&[u8]]) -> [u8; 32] {
+	triehash::ordered_trie_root(serialised_values.iter().map(|s| s.to_vec())).0
+}
+
 /// Conduct a Keccak-256 hash of the given data.
 pub use primitives::{blake2_256, twox_128, twox_256};
 
@@ -134,23 +147,6 @@ macro_rules! impl_stubs {
 #[cfg(test)]
 mod tests {
 	use super::*;
-	use std::collections::HashMap;
-
-	#[derive(Debug, Default)]
-	struct TestExternalities {
-		storage: HashMap<Vec<u8>, Vec<u8>>,
-	}
-	impl Externalities for TestExternalities {
-		fn storage(&self, key: &[u8]) -> Result<&[u8], ExternalitiesError> {
-			Ok(self.storage.get(&key.to_vec()).map_or(&[] as &[u8], Vec::as_slice))
-		}
-
-		fn set_storage(&mut self, key: Vec<u8>, value: Vec<u8>) {
-			self.storage.insert(key, value);
-		}
-
-		fn chain_id(&self) -> u64 { 42 }
-	}
 
 	macro_rules! map {
 		($( $name:expr => $value:expr ),*) => (

substrate/runtime-std/without_std.rs

@@ -33,6 +33,8 @@ extern "C" {
 	fn ext_set_storage(key_data: *const u8, key_len: u32, value_data: *const u8, value_len: u32);
 	fn ext_get_allocated_storage(key_data: *const u8, key_len: u32, written_out: *mut u32) -> *mut u8;
 	fn ext_get_storage_into(key_data: *const u8, key_len: u32, value_data: *mut u8, value_len: u32, value_offset: u32) -> u32;
+	fn ext_storage_root(result: *mut u8);
+	fn ext_enumerated_trie_root(values_data: *const u8, values_len: u32, lens_data: *const u32, lens_len: u32, result: *mut u8);
 	fn ext_chain_id() -> u64;
 	fn ext_blake2_256(data: *const u8, len: u32, out: *mut u8);
 	fn ext_twox_128(data: *const u8, len: u32, out: *mut u8);
@@ -67,6 +69,30 @@ pub fn read_storage(key: &[u8], value_out: &mut [u8], value_offset: usize) -> us
 	}
 }
 
+/// The current storage's root.
+pub fn storage_root() -> [u8; 32] {
+	let mut result: [u8; 32] = Default::default();
+	unsafe {
+		ext_storage_root(result.as_mut_ptr());
+	}
+	result
+}
+
+/// A trie root calculated from enumerated values.
+pub fn enumerated_trie_root(values: &[&[u8]]) -> [u8; 32] {
+	let lens = values.iter().map(|v| (v.len() as u32).to_le()).collect::<Vec<_>>();
+	let values = values.iter().fold(Vec::new(), |mut acc, sl| { acc.extend_from_slice(sl); acc });
+	let mut result: [u8; 32] = Default::default();
+	unsafe {
+		ext_enumerated_trie_root(
+			values.as_ptr(), values.len() as u32,
+			lens.as_ptr(), lens.len() as u32,
+			result.as_mut_ptr()
+		);
+	}
+	result
+}
+
 /// The current relay chain identifier.
 pub fn chain_id() -> u64 {
 	unsafe {
@@ -77,7 +103,6 @@ pub fn chain_id() -> u64 {
 /// Conduct a 256-bit Blake2 hash.
 pub fn blake2_256(data: &[u8]) -> [u8; 32] {
 	let mut result: [u8; 32] = Default::default();
-	// guaranteed to write into result.
 	unsafe {
 		ext_blake2_256(data.as_ptr(), data.len() as u32, result.as_mut_ptr());
 	}
@@ -87,7 +112,6 @@ pub fn blake2_256(data: &[u8]) -> [u8; 32] {
 /// Conduct four XX hashes to give a 256-bit result.
 pub fn twox_256(data: &[u8]) -> [u8; 32] {
 	let mut result: [u8; 32] = Default::default();
-	// guaranteed to write into result.
 	unsafe {
 		ext_twox_256(data.as_ptr(), data.len() as u32, result.as_mut_ptr());
 	}
@@ -97,7 +121,6 @@ pub fn twox_256(data: &[u8]) -> [u8; 32] {
 /// Conduct two XX hashes to give a 128-bit result.
 pub fn twox_128(data: &[u8]) -> [u8; 16] {
 	let mut result: [u8; 16] = Default::default();
-	// guaranteed to write into result.
 	unsafe {
 		ext_twox_128(data.as_ptr(), data.len() as u32, result.as_mut_ptr());
 	}

substrate/state-machine/Cargo.toml

@@ -11,3 +11,4 @@ patricia-trie = "0.1.0"
 memorydb = "0.1.1"
 triehash = "0.1"
 byteorder = "1.1"
+hex-literal = "0.1.0"

substrate/state-machine/src/backend.rs

@@ -40,6 +40,9 @@ pub trait Backend {
 	/// Commit updates to the backend and get new state.
 	fn commit<I>(&mut self, changes: I) -> Committed
 		where I: IntoIterator<Item=Update>;
+
+	/// Get all key/value pairs into a Vec.
+	fn pairs(&self) -> Vec<(&[u8], &[u8])>;
 }
 
 /// Error impossible.
@@ -59,15 +62,27 @@ impl error::Error for Void {
 
 /// In-memory backend. Fully recomputes tries on each commit but useful for
 /// tests.
-#[derive(Default)]
+#[derive(Default, Clone)]
 pub struct InMemory {
 	inner: MemoryState, // keeps all the state in memory.
 }
 
+#[cfg(test)]
+impl InMemory {
+	/// Create a new instance from a given storage map.
+	pub fn from(storage: ::std::collections::HashMap<Vec<u8>, Vec<u8>>) -> Self {
+		InMemory {
+			inner: MemoryState {
+				storage
+			}
+		}
+	}
+}
+
 impl Backend for InMemory {
 	type Error = Void;
 
-	fn storage(&self, key: &[u8]) -> Result<&[u8], Void> {
+	fn storage(&self, key: &[u8]) -> Result<&[u8], Self::Error> {
 		Ok(self.inner.storage(key).unwrap_or(&[]))
 	}
 
@@ -87,6 +102,10 @@ impl Backend for InMemory {
 			storage_tree_root,
 		}
 	}
+
+	fn pairs(&self) -> Vec<(&[u8], &[u8])> {
+		self.inner.storage.iter().map(|(k, v)| (&k[..], &v[..])).collect()
+	}
 }
 
 // TODO: DB-based backend

substrate/state-machine/src/ext.rs

@@ -17,7 +17,7 @@
 //! Conrete externalities implementation.
 
 use std::{error, fmt};
-
+use triehash::trie_root;
 use backend::Backend;
 use {Externalities, ExternalitiesError, OverlayedChanges};
 
@@ -75,4 +75,15 @@ impl<'a, B: 'a> Externalities for Ext<'a, B>
 	fn chain_id(&self) -> u64 {
 		42
 	}
+
+	fn storage_root(&self) -> [u8; 32] {
+		let mut all_pairs = self.backend.pairs();
+		all_pairs.extend(
+			self.overlay.committed.storage.iter()
+				.chain(self.overlay.prospective.storage.iter())
+				.map(|(k, v)| (&k[..], &v[..]))
+		);
+
+		trie_root(all_pairs.into_iter().map(|(k, v)| (k.to_vec(), v.to_vec()))).0
+	}
 }

substrate/state-machine/src/lib.rs

@@ -19,6 +19,8 @@
 #![warn(missing_docs)]
 
 extern crate polkadot_primitives as primitives;
+#[macro_use]
+extern crate hex_literal;
 
 extern crate hashdb;
 extern crate memorydb;
@@ -35,6 +37,9 @@ use primitives::contract::{CallData};
 
 pub mod backend;
 mod ext;
+mod testing;
+
+pub use testing::TestExternalities;
 
 /// Updates to be committed to the state.
 pub enum Update {
@@ -43,7 +48,7 @@ pub enum Update {
 }
 
 // in-memory section of the state.
-#[derive(Default)]
+#[derive(Default, Clone)]
 struct MemoryState {
 	storage: HashMap<Vec<u8>, Vec<u8>>,
 }
@@ -141,6 +146,9 @@ pub trait Externalities {
 	/// Get the identity of the chain.
 	fn chain_id(&self) -> u64;
 
+	/// Get the trie root of the current storage map.
+	fn storage_root(&self) -> [u8; 32];
+
 	/// Get the current set of authorities from storage.
 	fn authorities(&self) -> Result<Vec<&[u8]>, ExternalitiesError> {
 		(0..self.storage(b"con:aut:len")?.into_iter()
@@ -152,7 +160,7 @@ pub trait Externalities {
 }
 
 /// Code execution engine.
-pub trait CodeExecutor: Sized {
+pub trait CodeExecutor: Sized + Send + Sync {
 	/// Externalities error type.
 	type Error: Error;
 
@@ -210,8 +218,9 @@ pub fn execute<B: backend::Backend, Exec: CodeExecutor>(
 
 #[cfg(test)]
 mod tests {
-	use std::collections::HashMap;
-	use super::{OverlayedChanges, Externalities, ExternalitiesError};
+	use super::*;
+	use super::backend::InMemory;
+	use super::ext::Ext;
 
 	#[test]
 	fn overlayed_storage_works() {
@@ -238,22 +247,6 @@ mod tests {
 		assert!(overlayed.storage(&key).is_none());
 	}
 
-	#[derive(Debug, Default)]
-	struct TestExternalities {
-		storage: HashMap<Vec<u8>, Vec<u8>>,
-	}
-	impl Externalities for TestExternalities {
-		fn storage(&self, key: &[u8]) -> Result<&[u8], ExternalitiesError> {
-			Ok(self.storage.get(&key.to_vec()).map_or(&[] as &[u8], Vec::as_slice))
-		}
-
-		fn set_storage(&mut self, key: Vec<u8>, value: Vec<u8>) {
-			self.storage.insert(key, value);
-		}
-
-		fn chain_id(&self) -> u64 { 42 }
-	}
-
 	#[test]
 	fn authorities_call_works() {
 		let mut ext = TestExternalities::default();
@@ -273,4 +266,34 @@ mod tests {
 		ext.set_storage(b"con:aut:\x01\0\0\0".to_vec(), b"second".to_vec());
 		assert_eq!(ext.authorities(), Ok(vec![&b"first"[..], &b"second"[..]]));
 	}
+
+	macro_rules! map {
+		($( $name:expr => $value:expr ),*) => (
+			vec![ $( ( $name, $value ) ),* ].into_iter().collect()
+		)
+	}
+
+	#[test]
+	fn overlayed_storage_root_works() {
+		let mut backend = InMemory::from(map![
+			b"doe".to_vec() => b"reindeer".to_vec(),
+			b"dog".to_vec() => b"puppyXXX".to_vec(),
+			b"dogglesworth".to_vec() => b"catXXX".to_vec()
+		]);
+		let mut overlay = OverlayedChanges {
+			committed: MemoryState { storage: map![
+				b"dog".to_vec() => b"puppy".to_vec(),
+				b"dogglesworth".to_vec() => b"catYYY".to_vec()
+			], },
+			prospective: MemoryState { storage: map![
+				b"dogglesworth".to_vec() => b"cat".to_vec()
+			], },
+		};
+		let ext = Ext {
+			backend: &mut backend,
+			overlay: &mut overlay,
+		};
+		const ROOT: [u8; 32] = hex!("8aad789dff2f538bca5d8ea56e8abe10f4c7ba3a5dea95fea4cd6e7c3a1168d3");
+		assert_eq!(ext.storage_root(), ROOT);
+	}
 }

substrate/state-machine/src/testing.rs

@@ -0,0 +1,68 @@
+// Copyright 2017 Parity Technologies (UK) Ltd.
+// This file is part of Polkadot.
+
+// Polkadot is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// Polkadot is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with Polkadot.  If not, see <http://www.gnu.org/licenses/>.
+
+//! Test implementation for Externalities.
+
+use std::collections::HashMap;
+use super::{Externalities, ExternalitiesError};
+use triehash::trie_root;
+
+/// Simple HashMap based Externalities impl.
+#[derive(Debug, Default)]
+pub struct TestExternalities {
+	/// The storage.
+	pub storage: HashMap<Vec<u8>, Vec<u8>>,
+}
+
+impl TestExternalities {
+	/// Create a new instance with empty storage.
+	pub fn new() -> Self {
+		TestExternalities {
+			storage: HashMap::new(),
+		}
+	}
+}
+
+impl Externalities for TestExternalities {
+	fn storage(&self, key: &[u8]) -> Result<&[u8], ExternalitiesError> {
+		Ok(self.storage.get(&key.to_vec()).map_or(&[] as &[u8], Vec::as_slice))
+	}
+
+	fn set_storage(&mut self, key: Vec<u8>, value: Vec<u8>) {
+		self.storage.insert(key, value);
+	}
+
+	fn chain_id(&self) -> u64 { 42 }
+
+	fn storage_root(&self) -> [u8; 32] {
+		trie_root(self.storage.clone()).0
+	}
+}
+
+#[cfg(test)]
+mod tests {
+	use super::*;
+
+	#[test]
+	fn commit_should_work() {
+		let mut ext = TestExternalities::new();
+		ext.set_storage(b"doe".to_vec(), b"reindeer".to_vec());
+		ext.set_storage(b"dog".to_vec(), b"puppy".to_vec());
+		ext.set_storage(b"dogglesworth".to_vec(), b"cat".to_vec());
+		const ROOT: [u8; 32] = hex!("8aad789dff2f538bca5d8ea56e8abe10f4c7ba3a5dea95fea4cd6e7c3a1168d3");
+		assert_eq!(ext.storage_root(), ROOT);
+	}
+}

substrate/wasm-runtime/Cargo.lock

@@ -194,6 +194,23 @@ dependencies = [
  "winapi 0.3.4 (registry+https://github.com/rust-lang/crates.io-index)",
 ]
 
+[[package]]
+name = "hex-literal"
+version = "0.1.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+dependencies = [
+ "hex-literal-impl 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)",
+ "proc-macro-hack 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)",
+]
+
+[[package]]
+name = "hex-literal-impl"
+version = "0.1.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+dependencies = [
+ "proc-macro-hack 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)",
+]
+
 [[package]]
 name = "isatty"
 version = "0.1.6"
@@ -381,6 +398,7 @@ dependencies = [
  "pwasm-alloc 0.1.0",
  "pwasm-libc 0.1.0",
  "rustc_version 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)",
+ "triehash 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)",
 ]
 
 [[package]]
@@ -389,12 +407,26 @@ version = "0.1.0"
 dependencies = [
  "byteorder 1.2.1 (registry+https://github.com/rust-lang/crates.io-index)",
  "hashdb 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)",
+ "hex-literal 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)",
  "memorydb 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)",
  "patricia-trie 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)",
  "polkadot-primitives 0.1.0",
  "triehash 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)",
 ]
 
+[[package]]
+name = "proc-macro-hack"
+version = "0.4.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+dependencies = [
+ "proc-macro-hack-impl 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)",
+]
+
+[[package]]
+name = "proc-macro-hack-impl"
+version = "0.4.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+
 [[package]]
 name = "pwasm-alloc"
 version = "0.1.0"
@@ -748,6 +780,8 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 "checksum gcc 0.3.54 (registry+https://github.com/rust-lang/crates.io-index)" = "5e33ec290da0d127825013597dbdfc28bee4964690c7ce1166cbc2a7bd08b1bb"
 "checksum hashdb 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)" = "d97be07c358c5b461268b4ce60304024c5fa5acfd4bd8cd743639f0252003cf5"
 "checksum heapsize 0.4.2 (registry+https://github.com/rust-lang/crates.io-index)" = "1679e6ea370dee694f91f1dc469bf94cf8f52051d147aec3e1f9497c6fc22461"
+"checksum hex-literal 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)" = "bd546ef520ab3745f1aae5f2cdc6de9e6498e94d1ab138b9eb3ddfbf335847fb"
+"checksum hex-literal-impl 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)" = "2ea76da4c7f1a54d01d54985566d3fdd960b2bbd7b970da024821c883c2d9631"
 "checksum isatty 0.1.6 (registry+https://github.com/rust-lang/crates.io-index)" = "8f2a233726c7bb76995cec749d59582e5664823b7245d4970354408f1d79a7a2"
 "checksum keccak-hash 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)" = "1f300c1f149cd9ca5214eed24f6e713a597517420fb8b15499824aa916259ec1"
 "checksum kernel32-sys 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)" = "7507624b29483431c0ba2d82aece8ca6cdba9382bff4ddd0f7490560c056098d"
@@ -766,6 +800,8 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 "checksum parking_lot_core 0.2.10 (registry+https://github.com/rust-lang/crates.io-index)" = "9f35048d735bb93dd115a0030498785971aab3234d311fbe273d020084d26bd8"
 "checksum patricia-trie 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)" = "f1e2f638d79aba5c4a71a4f373df6e3cd702250a53b7f0ed4da1e2a7be9737ae"
 "checksum plain_hasher 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)" = "83ae80873992f511142c07d0ec6c44de5636628fdb7e204abd655932ea79d995"
+"checksum proc-macro-hack 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)" = "3ba8d4f9257b85eb6cdf13f055cea3190520aab1409ca2ab43493ea4820c25f0"
+"checksum proc-macro-hack-impl 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)" = "d5cb6f960ad471404618e9817c0e5d10b1ae74cfdf01fab89ea0641fe7fb2892"
 "checksum quote 0.3.15 (registry+https://github.com/rust-lang/crates.io-index)" = "7a6e920b65c65f10b2ae65c831a81a073a89edd28c7cce89475bff467ab4167a"
 "checksum rand 0.3.20 (registry+https://github.com/rust-lang/crates.io-index)" = "512870020642bb8c221bf68baa1b2573da814f6ccfe5c9699b1c303047abe9b1"
 "checksum rand 0.4.2 (registry+https://github.com/rust-lang/crates.io-index)" = "eba5f8cb59cc50ed56be8880a5c7b496bfd9bd26394e176bc67884094145c2c5"

substrate/wasm-runtime/polkadot/src/lib.rs

@@ -26,6 +26,10 @@ extern crate rustc_hex;
 
 extern crate polkadot_runtime_codec as codec;
 
+#[cfg(test)]
+#[macro_use]
+extern crate hex_literal;
+
 #[macro_use]
 pub mod support;
 pub mod primitives;

substrate/wasm-runtime/polkadot/src/runtime/governance.rs

@@ -114,9 +114,9 @@ pub mod internal {
 #[cfg(test)]
 mod tests {
 	use super::*;
-	use runtime_std::{with_externalities, twox_128};
+	use runtime_std::{with_externalities, twox_128, TestExternalities};
 	use codec::{KeyedVec, Joiner};
-	use support::{one, two, TestExternalities, with_env};
+	use support::{one, two, with_env};
 	use primitives::{AccountID, InternalFunction};
 	use runtime::{staking, session};
 

substrate/wasm-runtime/polkadot/src/runtime/session.rs

@@ -132,9 +132,9 @@ mod tests {
 	use super::public::*;
 	use super::privileged::*;
 	use super::internal::*;
-	use runtime_std::{with_externalities, twox_128};
+	use runtime_std::{with_externalities, twox_128, TestExternalities};
 	use codec::{KeyedVec, Joiner};
-	use support::{one, two, TestExternalities, with_env};
+	use support::{one, two, with_env};
 	use primitives::AccountID;
 	use runtime::{consensus, session};
 

substrate/wasm-runtime/polkadot/src/runtime/staking.rs

@@ -206,9 +206,9 @@ mod tests {
 	use super::public::*;
 	use super::privileged::*;
 
-	use runtime_std::{with_externalities, twox_128};
+	use runtime_std::{with_externalities, twox_128, TestExternalities};
 	use codec::{KeyedVec, Joiner};
-	use support::{one, two, TestExternalities, with_env};
+	use support::{one, two, with_env};
 	use primitives::AccountID;
 	use runtime::{staking, session};
 

substrate/wasm-runtime/polkadot/src/runtime/system.rs

@@ -18,8 +18,8 @@
 //! and depositing logs.
 
 use runtime_std::prelude::*;
-use runtime_std::{mem, print};
-use codec::KeyedVec;
+use runtime_std::{mem, print, storage_root, enumerated_trie_root};
+use codec::{KeyedVec, Slicable};
 use support::{Hashable, storage, with_env};
 use primitives::{Block, BlockNumber, Hash, UncheckedTransaction, TxOrder};
 use runtime::{staking, session};
@@ -74,13 +74,11 @@ pub mod internal {
 			"Parent hash should be valid."
 		);
 
-		// TODO: check transaction trie root represents the transactions.
-		// this requires non-trivial changes to the externals API or compiling trie rooting into wasm
-		// so will wait until a little later.
-
-		// store the header hash in storage.
-		let header_hash_key = header.number.to_keyed_vec(BLOCK_HASH_AT);
-		storage::put(&header_hash_key, &header.blake2_256());
+		// check transaction trie root represents the transactions.
+		let txs = block.transactions.iter().map(Slicable::to_vec).collect::<Vec<_>>();
+		let txs_root = enumerated_trie_root(&txs.iter().map(Vec::as_slice).collect::<Vec<_>>());
+//		println!("TR: {}", ::support::HexDisplay::from(&txs_root));
+		assert!(header.transaction_root == txs_root, "Transaction trie root must be valid.");
 
 		// execute transactions
 		block.transactions.iter().for_each(execute_transaction);
@@ -91,9 +89,14 @@ pub mod internal {
 		// any final checks
 		final_checks(&block);
 
-		// TODO: check storage root.
-		// this requires non-trivial changes to the externals API or compiling trie rooting into wasm
-		// so will wait until a little later.
+
+		// check storage root.
+		assert!(header.state_root == storage_root(), "Storage root must match that calculated.");
+
+		// store the header hash in storage; we can't do it before otherwise there would be a
+		// cyclic dependency.
+		let header_hash_key = header.number.to_keyed_vec(BLOCK_HASH_AT);
+		storage::put(&header_hash_key, &header.blake2_256());
 	}
 
 	/// Execute a given transaction.
@@ -127,10 +130,10 @@ mod tests {
 	use super::*;
 	use super::internal::*;
 
-	use runtime_std::{with_externalities, twox_128};
+	use runtime_std::{with_externalities, twox_128, TestExternalities};
 	use codec::{Joiner, KeyedVec, Slicable};
-	use support::{StaticHexInto, TestExternalities, HexDisplay, one, two};
-	use primitives::{UncheckedTransaction, Transaction, Function};
+	use support::{StaticHexInto, HexDisplay, one, two};
+	use primitives::{UncheckedTransaction, Transaction, Function, Header, Digest};
 	use runtime::staking;
 
 	#[test]
@@ -162,4 +165,105 @@ mod tests {
 			assert_eq!(staking::balance(&two), 69);
 		});
 	}
+
+	fn new_test_ext() -> TestExternalities {
+		let one = one();
+		let two = two();
+		let three = [3u8; 32];
+
+		TestExternalities { storage: map![
+			twox_128(&0u64.to_keyed_vec(b"sys:old:")).to_vec() => [69u8; 32].to_vec(),
+			twox_128(b"gov:apr").to_vec() => vec![].join(&667u32),
+			twox_128(b"ses:len").to_vec() => vec![].join(&2u64),
+			twox_128(b"ses:val:len").to_vec() => vec![].join(&3u32),
+			twox_128(&0u32.to_keyed_vec(b"ses:val:")).to_vec() => one.to_vec(),
+			twox_128(&1u32.to_keyed_vec(b"ses:val:")).to_vec() => two.to_vec(),
+			twox_128(&2u32.to_keyed_vec(b"ses:val:")).to_vec() => three.to_vec(),
+			twox_128(b"sta:wil:len").to_vec() => vec![].join(&3u32),
+			twox_128(&0u32.to_keyed_vec(b"sta:wil:")).to_vec() => one.to_vec(),
+			twox_128(&1u32.to_keyed_vec(b"sta:wil:")).to_vec() => two.to_vec(),
+			twox_128(&2u32.to_keyed_vec(b"sta:wil:")).to_vec() => three.to_vec(),
+			twox_128(b"sta:spe").to_vec() => vec![].join(&2u64),
+			twox_128(b"sta:vac").to_vec() => vec![].join(&3u64),
+			twox_128(b"sta:era").to_vec() => vec![].join(&0u64),
+			twox_128(&one.to_keyed_vec(b"sta:bal:")).to_vec() => vec![111u8, 0, 0, 0, 0, 0, 0, 0]
+		], }
+	}
+
+	#[test]
+	fn block_import_works() {
+		let one = one();
+		let two = two();
+
+		let mut t = new_test_ext();
+
+		let tx = UncheckedTransaction {
+			transaction: Transaction {
+				signed: one.clone(),
+				nonce: 0,
+				function: Function::StakingTransfer,
+				input_data: vec![].join(&two).join(&69u64),
+			},
+			signature: "679fcf0a846b4224c84ecad7d91a26241c46d00cb53d6480a363274e8965ee34b0b80b4b2e3836d3d8f8f12c0c1aef7350af587d9aee3883561d11726068ac0a".convert(),
+		};
+
+		let h = Header {
+			parent_hash: [69u8; 32],
+			number: 1,
+			state_root: hex!("2481853da20b9f4322f34650fea5f240dcbfb266d02db94bfa0153c31f4a29db"),
+			transaction_root: hex!("91fab88ad8c30a6d05ad8e0cf9ab139bf1b8cdddc69abd51cdfa6d2699038af1"),
+			digest: Digest { logs: vec![], },
+		};
+
+		let b = Block {
+			header: h,
+			transactions: vec![tx],
+		};
+
+		with_externalities(&mut t, || {
+			execute_block(b);
+			assert_eq!(staking::balance(&one), 42);
+			assert_eq!(staking::balance(&two), 69);
+		});
+	}
+
+	#[test]
+	#[should_panic]
+	fn block_import_of_bad_state_root_fails() {
+		let one = one();
+		let two = two();
+
+		let mut t = new_test_ext();
+
+		let tx = UncheckedTransaction {
+			transaction: Transaction {
+				signed: one.clone(),
+				nonce: 0,
+				function: Function::StakingTransfer,
+				input_data: vec![].join(&two).join(&69u64),
+			},
+			signature: "679fcf0a846b4224c84ecad7d91a26241c46d00cb53d6480a363274e8965ee34b0b80b4b2e3836d3d8f8f12c0c1aef7350af587d9aee3883561d11726068ac0a".convert(),
+		};
+		// tx: 2f8c6129d816cf51c374bc7f08c3e63ed156cf78aefb4a6550d97b87997977ee00000000000000000228000000d75a980182b10ab7d54bfed3c964073a0ee172f3daa62325af021a68f707511a4500000000000000
+		// sig: 679fcf0a846b4224c84ecad7d91a26241c46d00cb53d6480a363274e8965ee34b0b80b4b2e3836d3d8f8f12c0c1aef7350af587d9aee3883561d11726068ac0a
+
+		let h = Header {
+			parent_hash: [69u8; 32],
+			number: 1,
+			state_root: [0u8; 32],
+			transaction_root: [0u8; 32],		// Unchecked currently.
+			digest: Digest { logs: vec![], },
+		};
+
+		let b = Block {
+			header: h,
+			transactions: vec![tx],
+		};
+
+		with_externalities(&mut t, || {
+			execute_block(b);
+			assert_eq!(staking::balance(&one), 42);
+			assert_eq!(staking::balance(&two), 69);
+		});
+	}
 }

substrate/wasm-runtime/polkadot/src/runtime/timestamp.rs

@@ -42,10 +42,9 @@ mod tests {
 	use super::*;
 	use super::public::*;
 
-	use runtime_std::{with_externalities, twox_128};
+	use runtime_std::{with_externalities, twox_128, TestExternalities};
 	use runtime::timestamp;
 	use codec::{Joiner, KeyedVec};
-	use support::TestExternalities;
 
 	#[test]
 	fn timestamp_works() {

substrate/wasm-runtime/polkadot/src/support/mod.rs

@@ -28,7 +28,9 @@ mod testing;
 pub use self::environment::with_env;
 pub use self::storage::StorageVec;
 pub use self::hashable::Hashable;
+
 #[cfg(feature = "std")]
 pub use self::statichex::{StaticHexConversion, StaticHexInto};
+
 #[cfg(feature = "std")]
-pub use self::testing::{AsBytesRef, HexDisplay, TestExternalities, one, two};
+pub use self::testing::{AsBytesRef, HexDisplay, one, two};

substrate/wasm-runtime/polkadot/src/support/storage.rs

@@ -146,9 +146,8 @@ pub trait StorageVec {
 mod tests {
 	use super::*;
 	use std::collections::HashMap;
-	use runtime_std::with_externalities;
-	use support::{TestExternalities, HexDisplay};
-	use runtime_std::{storage, twox_128};
+	use support::HexDisplay;
+	use runtime_std::{storage, twox_128, TestExternalities, with_externalities};
 
 	#[test]
 	fn integers_can_be_stored() {

substrate/wasm-runtime/polkadot/src/support/testing.rs

@@ -16,30 +16,9 @@
 
 //! Testing helpers.
 
-use std::collections::HashMap;
-use runtime_std::{Externalities, ExternalitiesError};
 use primitives::AccountID;
 use super::statichex::StaticHexInto;
 
-#[derive(Debug, Default)]
-/// Simple externaties implementation.
-pub struct TestExternalities {
-	/// The storage map.
-	pub storage: HashMap<Vec<u8>, Vec<u8>>,
-}
-
-impl Externalities for TestExternalities {
-	fn storage(&self, key: &[u8]) -> Result<&[u8], ExternalitiesError> {
-		Ok(self.storage.get(&key.to_vec()).map_or(&[] as &[u8], Vec::as_slice))
-	}
-
-	fn set_storage(&mut self, key: Vec<u8>, value: Vec<u8>) {
-		self.storage.insert(key, value);
-	}
-
-	fn chain_id(&self) -> u64 { 42 }
-}
-
 #[macro_export]
 macro_rules! map {
 	($( $name:expr => $value:expr ),*) => (

substrate/wasm-runtime/test/src/lib.rs

@@ -8,7 +8,10 @@ use alloc::vec::Vec;
 
 #[macro_use]
 extern crate polkadot_runtime_std as runtime_std;
-use runtime_std::{set_storage, storage, print, blake2_256, twox_128, twox_256, ed25519_verify};
+use runtime_std::{
+	set_storage, storage, print, blake2_256,
+	twox_128, twox_256, ed25519_verify, enumerated_trie_root
+};
 
 fn test_blake2_256(input: &[u8]) -> Vec<u8> {
 	blake2_256(&input).to_vec()
@@ -29,6 +32,10 @@ fn test_ed25519_verify(input: &[u8]) -> Vec<u8> {
 	[ed25519_verify(sig, &msg[..], pubkey) as u8].to_vec()
 }
 
+fn test_enumerated_trie_root(_input: &[u8]) -> Vec<u8> {
+	enumerated_trie_root(&[&b"zero"[..], &b"one"[..], &b"two"[..]]).to_vec()
+}
+
 fn test_data_in(input: &[u8]) -> Vec<u8> {
 	print("set_storage");
 	set_storage(b"input", &input);
@@ -59,4 +66,4 @@ fn test_conditional_panic(input: &[u8]) -> Vec<u8> {
 }
 
 impl_stubs!(test_data_in, test_empty_return, test_panic, test_conditional_panic,
-	test_blake2_256, test_twox_256, test_twox_128, test_ed25519_verify);
+	test_blake2_256, test_twox_256, test_twox_128, test_ed25519_verify, test_enumerated_trie_root);