Consensus Engines Implementation: Aura (#911)

* Generalize BlockImport

 - move ImportBlock, BlockOrigin, ImportResult into shared sr-primitives
 - let Consensus provide  and  traits again
 - update consensus traits to latest development
 - implement traits on client::Client, test_client::TestClient
 - update RHD to use the new import_block API

* Move ImportBlock into consensus-common
* Send import notification in aura tests
* Integrating aura into service
* Make Signatures more generic
* Aura Block Production with the given key
* run aura on the thread pool
* start at exact step start in aura
* Add needed wasm blob, in leiu of better solutions.
* Make API ids consistent with traits and bring upstream for sharing.
* Add decrease_free_balance to Balances module
* Encode `Metadata` once instead of two times
* Bitops include xor
* Upgrade key module.
* Default pages to somewhat bigger.
* Introduce upgrade key into node
* Add `Created` event

substrate/core/client/src/call_executor.rs

@@ -150,7 +150,7 @@ where
 		let heap_pages = state.storage(well_known_keys::HEAP_PAGES)
 			.map_err(|e| error::ErrorKind::Execution(Box::new(e)))?
 			.and_then(|v| u64::decode(&mut &v[..]))
-			.unwrap_or(8) as usize;
+			.unwrap_or(1024) as usize;
 
 		let mut ext = Ext::new(&mut overlay, &state, self.backend.changes_trie_storage());
 		self.executor.runtime_version(&mut ext, heap_pages, &code)

substrate/core/client/src/client.rs

@@ -26,6 +26,7 @@ use runtime_primitives::{
 	generic::{BlockId, SignedBlock, Block as RuntimeBlock},
 	transaction_validity::{TransactionValidity, TransactionTag},
 };
+use consensus::{ImportBlock, ImportResult, BlockOrigin};
 use runtime_primitives::traits::{Block as BlockT, Header as HeaderT, Zero, As, NumberFor, CurrentHeight, BlockNumberToHash};
 use runtime_primitives::{ApplyResult, BuildStorage};
 use runtime_api as api;
@@ -44,7 +45,7 @@ use blockchain::{self, Info as ChainInfo, Backend as ChainBackend, HeaderBackend
 use call_executor::{CallExecutor, LocalCallExecutor};
 use executor::{RuntimeVersion, RuntimeInfo};
 use notifications::{StorageNotifications, StorageEventStream};
-use {cht, error, in_mem, block_builder, genesis};
+use {cht, error, in_mem, block_builder, genesis, consensus};
 
 /// Type that implements `futures::Stream` of block import events.
 pub type ImportNotifications<Block> = mpsc::UnboundedReceiver<BlockImportNotification<Block>>;
@@ -106,21 +107,6 @@ pub struct ClientInfo<Block: BlockT> {
 	pub best_queued_hash: Option<Block::Hash>,
 }
 
-/// Block import result.
-#[derive(Debug)]
-pub enum ImportResult {
-	/// Added to the import queue.
-	Queued,
-	/// Already in the import queue.
-	AlreadyQueued,
-	/// Already in the blockchain.
-	AlreadyInChain,
-	/// Block or parent is known to be bad.
-	KnownBad,
-	/// Block parent is not in the chain.
-	UnknownParent,
-}
-
 /// Block status.
 #[derive(Debug, PartialEq, Eq)]
 pub enum BlockStatus {
@@ -134,70 +120,6 @@ pub enum BlockStatus {
 	Unknown,
 }
 
-/// Block data origin.
-#[derive(Debug, PartialEq, Eq, Clone, Copy)]
-pub enum BlockOrigin {
-	/// Genesis block built into the client.
-	Genesis,
-	/// Block is part of the initial sync with the network.
-	NetworkInitialSync,
-	/// Block was broadcasted on the network.
-	NetworkBroadcast,
-	/// Block that was received from the network and validated in the consensus process.
-	ConsensusBroadcast,
-	/// Block that was collated by this node.
-	Own,
-	/// Block was imported from a file.
-	File,
-}
-
-/// Data required to import a Block
-pub struct ImportBlock<Block: BlockT> {
-	/// Origin of the Block
-	pub origin: BlockOrigin,
-	/// Header
-	pub header: Block::Header,
-	/// Justification provided for this block from the outside
-	pub external_justification: Justification,
-	/// Internal Justification for the block
-	pub internal_justification: Vec<u8>, // Block::Digest::DigestItem?
-	/// Block's body
-	pub body: Option<Vec<Block::Extrinsic>>,
-	/// Is this block finalized already?
-	/// `true` implies instant finality.
-	pub finalized: bool,
-	/// Auxiliary consensus data produced by the block.
-	/// Contains a list of key-value pairs. If values are `None`, the keys
-	/// will be deleted.
-	pub auxiliary: Vec<(Vec<u8>, Option<Vec<u8>>)>,
-}
-
-impl<Block: BlockT> ImportBlock<Block> {
-	/// Deconstruct the justified header into parts.
-	pub fn into_inner(self)
-		-> (
-			BlockOrigin,
-			<Block as BlockT>::Header,
-			Justification,
-			Justification,
-			Option<Vec<<Block as BlockT>::Extrinsic>>,
-			bool,
-			Vec<(Vec<u8>, Option<Vec<u8>>)>,
-		) {
-		(
-			self.origin,
-			self.header,
-			self.external_justification,
-			self.internal_justification,
-			self.body,
-			self.finalized,
-			self.auxiliary,
-		)
-	}
-}
-
-
-
 /// Summary of an imported block
 #[derive(Clone, Debug)]
 pub struct BlockImportNotification<Block: BlockT> {
@@ -222,6 +144,41 @@ pub struct FinalityNotification<Block: BlockT> {
 	pub header: Block::Header,
 }
 
+// used in importing a block, where additional changes are made after the runtime
+// executed.
+enum PrePostHeader<H> {
+	// they are the same: no post-runtime digest items.
+	Same(H),
+	// different headers (pre, post).
+	Different(H, H),
+}
+
+impl<H> PrePostHeader<H> {
+	// get a reference to the "pre-header" -- the header as it should be just after the runtime.
+	fn pre(&self) -> &H {
+		match *self {
+			PrePostHeader::Same(ref h) => h,
+			PrePostHeader::Different(ref h, _) => h,
+		}
+	}
+
+	// get a reference to the "post-header" -- the header as it should be after all changes are applied.
+	fn post(&self) -> &H {
+		match *self {
+			PrePostHeader::Same(ref h) => h,
+			PrePostHeader::Different(_, ref h) => h,
+		}
+	}
+
+	// convert to the "post-header" -- the header as it should be after all changes are applied.
+	fn into_post(self) -> H {
+		match self {
+			PrePostHeader::Same(h) => h,
+			PrePostHeader::Different(_, h) => h,
+		}
+	}
+}
+
 /// Create an instance of in-memory client.
 pub fn new_in_mem<E, Block, S>(
 	executor: E,
@@ -513,52 +470,6 @@ impl<B, E, Block> Client<B, E, Block> where
 		)
 	}
 
-	/// Import a checked and validated block
-	pub fn import_block(
-		&self,
-		import_block: ImportBlock<Block>,
-		new_authorities: Option<Vec<AuthorityId>>,
-	) -> error::Result<ImportResult> {
-
-		let (
-			origin,
-			header,
-			_,
-			justification,
-			body,
-			finalized,
-			_aux, // TODO: write this to DB also
-		) = import_block.into_inner();
-		let parent_hash = header.parent_hash().clone();
-
-		match self.backend.blockchain().status(BlockId::Hash(parent_hash))? {
-			blockchain::BlockStatus::InChain => {},
-			blockchain::BlockStatus::Unknown => return Ok(ImportResult::UnknownParent),
-		}
-		let hash = header.hash();
-		let _import_lock = self.import_lock.lock();
-		let height: u64 = header.number().as_();
-		*self.importing_block.write() = Some(hash);
-
-		let result = self.execute_and_import_block(
-			origin,
-			hash,
-			header,
-			justification,
-			body,
-			new_authorities,
-			finalized,
-		);
-
-		*self.importing_block.write() = None;
-		telemetry!("block.import";
-			"height" => height,
-			"best" => ?hash,
-			"origin" => ?origin
-		);
-		result
-	}
-
 	// TODO [ToDr] Optimize and re-use tags from the pool.
 	fn transaction_tags(&self, at: Block::Hash, body: &Option<Vec<Block::Extrinsic>>) -> error::Result<Vec<TransactionTag>> {
 		let id = BlockId::Hash(at);
@@ -587,13 +498,13 @@ impl<B, E, Block> Client<B, E, Block> where
 		&self,
 		origin: BlockOrigin,
 		hash: Block::Hash,
-		header: Block::Header,
+		import_headers: PrePostHeader<Block::Header>,
 		justification: Justification,
 		body: Option<Vec<Block::Extrinsic>>,
 		authorities: Option<Vec<AuthorityId>>,
 		finalized: bool,
 	) -> error::Result<ImportResult> {
-		let parent_hash = header.parent_hash().clone();
+		let parent_hash = import_headers.post().parent_hash().clone();
 		match self.backend.blockchain().status(BlockId::Hash(hash))? {
 			blockchain::BlockStatus::InChain => return Ok(ImportResult::AlreadyInChain),
 			blockchain::BlockStatus::Unknown => {},
@@ -627,12 +538,13 @@ impl<B, E, Block> Client<B, E, Block> where
 					transaction_state,
 					&mut overlay,
 					"execute_block",
-					&<Block as BlockT>::new(header.clone(), body.clone().unwrap_or_default()).encode(),
+					&<Block as BlockT>::new(import_headers.pre().clone(), body.clone().unwrap_or_default()).encode(),
 					match (origin, self.block_execution_strategy) {
 						(BlockOrigin::NetworkInitialSync, _) | (_, ExecutionStrategy::NativeWhenPossible) =>
 							ExecutionManager::NativeWhenPossible,
 						(_, ExecutionStrategy::AlwaysWasm) => ExecutionManager::AlwaysWasm,
 						_ => ExecutionManager::Both(|wasm_result, native_result| {
+							let header = import_headers.post();
 							warn!("Consensus error between wasm and native block execution at block {}", hash);
 							warn!("   Header {:?}", header);
 							warn!("   Native result {:?}", native_result);
@@ -654,7 +566,7 @@ impl<B, E, Block> Client<B, E, Block> where
 		};
 
 		// TODO: non longest-chain rule.
-		let is_new_best = finalized || header.number() > &last_best_number;
+		let is_new_best = finalized || import_headers.post().number() > &last_best_number;
 		let leaf_state = if finalized {
 			::backend::NewBlockState::Final
 		} else if is_new_best {
@@ -663,10 +575,10 @@ impl<B, E, Block> Client<B, E, Block> where
 			::backend::NewBlockState::Normal
 		};
 
-		trace!("Imported {}, (#{}), best={}, origin={:?}", hash, header.number(), is_new_best, origin);
+		trace!("Imported {}, (#{}), best={}, origin={:?}", hash, import_headers.post().number(), is_new_best, origin);
 
 		transaction.set_block_data(
-			header.clone(),
+			import_headers.post().clone(),
 			body,
 			Some(justification),
 			leaf_state,
@@ -693,7 +605,7 @@ impl<B, E, Block> Client<B, E, Block> where
 			if finalized {
 				let notification = FinalityNotification::<Block> {
 					hash,
-					header: header.clone(),
+					header: import_headers.post().clone(),
 				};
 
 				self.finality_notification_sinks.lock()
@@ -703,7 +615,7 @@ impl<B, E, Block> Client<B, E, Block> where
 			let notification = BlockImportNotification::<Block> {
 				hash,
 				origin,
-				header,
+				header: import_headers.into_post(),
 				is_new_best,
 				tags,
 			};
@@ -979,6 +891,84 @@ impl<B, E, Block> Client<B, E, Block> where
 	}
 }
 
+
+impl<B, E, Block> consensus::BlockImport<Block> for Client<B, E, Block> where
+	B: backend::Backend<Block, Blake2Hasher>,
+	E: CallExecutor<Block, Blake2Hasher> + Clone,
+	Block: BlockT,
+{
+	type Error = Error;
+
+	/// Import a checked and validated block
+	fn import_block(
+		&self,
+		import_block: ImportBlock<Block>,
+		new_authorities: Option<Vec<AuthorityId>>,
+	) -> Result<ImportResult, Self::Error> {
+		use runtime_primitives::traits::Digest;
+
+		let ImportBlock {
+			origin,
+			header,
+			external_justification,
+			post_runtime_digests,
+			body,
+			finalized,
+			..
+		} = import_block;
+		let parent_hash = header.parent_hash().clone();
+
+		match self.backend.blockchain().status(BlockId::Hash(parent_hash))? {
+			blockchain::BlockStatus::InChain => {},
+			blockchain::BlockStatus::Unknown => return Ok(ImportResult::UnknownParent),
+		}
+
+		let import_headers = if post_runtime_digests.is_empty() {
+			PrePostHeader::Same(header)
+		} else {
+			let mut post_header = header.clone();
+			for item in post_runtime_digests {
+				post_header.digest_mut().push(item);
+			}
+			PrePostHeader::Different(header, post_header)
+		};
+
+		let hash = import_headers.post().hash();
+		let _import_lock = self.import_lock.lock();
+		let height: u64 = import_headers.post().number().as_();
+		*self.importing_block.write() = Some(hash);
+
+		let result = self.execute_and_import_block(
+			origin,
+			hash,
+			import_headers,
+			external_justification,
+			body,
+			new_authorities,
+			finalized,
+		);
+
+		*self.importing_block.write() = None;
+		telemetry!("block.import";
+			"height" => height,
+			"best" => ?hash,
+			"origin" => ?origin
+		);
+		result.map_err(|e| e.into())
+	}
+}
+
+impl<B, E, Block> consensus::Authorities<Block> for Client<B, E, Block> where
+	B: backend::Backend<Block, Blake2Hasher>,
+	E: CallExecutor<Block, Blake2Hasher> + Clone,
+	Block: BlockT,
+{
+	type Error = Error;
+	fn authorities(&self, at: &BlockId<Block>) -> Result<Vec<AuthorityId>, Self::Error> {
+		self.authorities_at(at).map_err(|e| e.into())
+	}
+}
+
 impl<B, E, Block> CurrentHeight for Client<B, E, Block> where
 	B: backend::Backend<Block, Blake2Hasher>,
 	E: CallExecutor<Block, Blake2Hasher> + Clone,
@@ -1135,26 +1125,6 @@ impl<B, E, Block> api::BlockBuilder<Block> for Client<B, E, Block> where
 	}
 }
 
-impl<B, E, Block> api::OldTxQueue<Block> for Client<B, E, Block> where
-	B: backend::Backend<Block, Blake2Hasher>,
-	E: CallExecutor<Block, Blake2Hasher>,
-	Block: BlockT,
-{
-	type Error = Error;
-
-	fn account_nonce<AccountId: Encode + Decode, Index: Encode + Decode>(
-		&self, at: &BlockId<Block>, account: &AccountId
-	) -> Result<Index, Self::Error> {
-		self.call_api_at(at, "account_nonce", &(account))
-	}
-
-	fn lookup_address<Address: Encode + Decode, AccountId: Encode + Decode>(
-		&self, at: &BlockId<Block>, address: &Address
-	) -> Result<Option<AccountId>, Self::Error> {
-		self.call_api_at(at, "lookup_address", &(address))
-	}
-}
-
 impl<B, E, Block> api::TaggedTransactionQueue<Block> for Client<B, E, Block> where
 	B: backend::Backend<Block, Blake2Hasher>,
 	E: CallExecutor<Block, Blake2Hasher>,
@@ -1169,30 +1139,6 @@ impl<B, E, Block> api::TaggedTransactionQueue<Block> for Client<B, E, Block> whe
 	}
 }
 
-impl<B, E, Block> api::Miscellaneous<Block> for Client<B, E, Block> where
-	B: backend::Backend<Block, Blake2Hasher>,
-	E: CallExecutor<Block, Blake2Hasher>,
-	Block: BlockT,
-{
-	type Error = Error;
-
-	fn validator_count(&self, at: &BlockId<Block>) -> Result<u32, Self::Error> {
-		self.call_api_at(at, "validator_count", &())
-	}
-
-	fn validators<AccountId: Encode + Decode>(
-		&self, at: &BlockId<Block>
-	) -> Result<Vec<AccountId>, Self::Error> {
-		self.call_api_at(at, "validators", &())
-	}
-
-	fn timestamp<Moment: Encode + Decode>(
-		&self, at: &BlockId<Block>
-	) -> Result<Moment, Self::Error> {
-		self.call_api_at(at, "timestamp", &())
-	}
-}
-
 #[cfg(test)]
 pub(crate) mod tests {
 	use std::collections::HashMap;
@@ -1202,7 +1148,7 @@ pub(crate) mod tests {
 	use runtime_primitives::traits::{Digest as DigestT, DigestItem as DigestItemT};
 	use runtime_primitives::generic::DigestItem;
 	use test_client::{self, TestClient};
-	use test_client::client::BlockOrigin;
+	use consensus::BlockOrigin;
 	use test_client::client::backend::Backend as TestBackend;
 	use test_client::BlockBuilderExt;
 	use test_client::runtime::{self, Block, Transfer};

substrate/core/client/src/error.rs

@@ -16,11 +16,17 @@
 
 //! Substrate client possible errors.
 
+#![allow(missing_docs)]
+
 use std;
 use state_machine;
 use runtime_primitives::ApplyError;
+use consensus;
 
 error_chain! {
+	links {
+		Consensus(consensus::Error, consensus::ErrorKind);
+	}
 	errors {
 		/// Backend error.
 		Backend(s: String) {

substrate/core/client/src/lib.rs

@@ -26,6 +26,7 @@ extern crate parity_codec as codec;
 extern crate substrate_primitives as primitives;
 extern crate sr_primitives as runtime_primitives;
 extern crate substrate_state_machine as state_machine;
+extern crate substrate_consensus_common as consensus;
 #[cfg(test)] extern crate substrate_keyring as keyring;
 #[cfg(test)] extern crate substrate_test_client as test_client;
 #[macro_use] extern crate substrate_telemetry;
@@ -63,8 +64,8 @@ pub use call_executor::{CallResult, CallExecutor, LocalCallExecutor};
 pub use client::{
 	new_with_backend,
 	new_in_mem,
-	BlockBody, BlockStatus, BlockOrigin, ImportNotifications, FinalityNotifications, BlockchainEvents,
-	Client, ClientInfo, ChainHead, ImportResult, ImportBlock,
+	BlockBody, BlockStatus, ImportNotifications, FinalityNotifications, BlockchainEvents,
+	Client, ClientInfo, ChainHead,
 };
 pub use notifications::{StorageEventStream, StorageChangeSet};
 pub use state_machine::ExecutionStrategy;

substrate/core/client/src/light/fetcher.rs

@@ -270,7 +270,8 @@ pub mod tests {
 	use error::Error as ClientError;
 	use test_client::{self, TestClient};
 	use test_client::runtime::{self, Hash, Block, Header};
-	use test_client::client::BlockOrigin;
+	use consensus::BlockOrigin;
+
 	use in_mem::{Blockchain as InMemoryBlockchain};
 	use light::fetcher::{Fetcher, FetchChecker, LightDataChecker,
 		RemoteCallRequest, RemoteHeaderRequest};