Consensus Engines Implementation: Aura (#911)

* Generalize BlockImport

 - move ImportBlock, BlockOrigin, ImportResult into shared sr-primitives
 - let Consensus provide  and  traits again
 - update consensus traits to latest development
 - implement traits on client::Client, test_client::TestClient
 - update RHD to use the new import_block API

* Move ImportBlock into consensus-common
* Send import notification in aura tests
* Integrating aura into service
* Make Signatures more generic
* Aura Block Production with the given key
* run aura on the thread pool
* start at exact step start in aura
* Add needed wasm blob, in leiu of better solutions.
* Make API ids consistent with traits and bring upstream for sharing.
* Add decrease_free_balance to Balances module
* Encode `Metadata` once instead of two times
* Bitops include xor
* Upgrade key module.
* Default pages to somewhat bigger.
* Introduce upgrade key into node
* Add `Created` event

substrate/core/consensus/common/Cargo.toml

@@ -4,5 +4,12 @@ version = "0.1.0"
 authors = ["Parity Technologies <admin@parity.io>"]
 description = "Common utilities for substrate consensus"
 
-[dev-dependencies]
-substrate-primitives = { path= "../../primitives"}
+[dependencies]
+substrate-primitives = { path= "../../primitives" }
+error-chain = "0.12"
+futures = "0.1"
+sr-version = { path = "../../sr-version" }
+sr-primitives = { path = "../../sr-primitives" }
+tokio = "0.1.7"
+parity-codec = "2.1"
+parity-codec-derive = "2.0"

substrate/core/consensus/common/src/block_import.rs

@@ -0,0 +1,104 @@
+
+use primitives::AuthorityId;
+use runtime_primitives::traits::{Block as BlockT, DigestItemFor};
+use runtime_primitives::Justification;
+
+/// Block import result.
+#[derive(Debug)]
+pub enum ImportResult {
+	/// Added to the import queue.
+	Queued,
+	/// Already in the import queue.
+	AlreadyQueued,
+	/// Already in the blockchain.
+	AlreadyInChain,
+	/// Block or parent is known to be bad.
+	KnownBad,
+	/// Block parent is not in the chain.
+	UnknownParent,
+}
+
+/// Block data origin.
+#[derive(Debug, PartialEq, Eq, Clone, Copy)]
+pub enum BlockOrigin {
+	/// Genesis block built into the client.
+	Genesis,
+	/// Block is part of the initial sync with the network.
+	NetworkInitialSync,
+	/// Block was broadcasted on the network.
+	NetworkBroadcast,
+	/// Block that was received from the network and validated in the consensus process.
+	ConsensusBroadcast,
+	/// Block that was collated by this node.
+	Own,
+	/// Block was imported from a file.
+	File,
+}
+
+/// Data required to import a Block
+pub struct ImportBlock<Block: BlockT> {
+	/// Origin of the Block
+	pub origin: BlockOrigin,
+	/// The header, without consensus post-digests applied. This should be in the same
+	/// state as it comes out of the runtime.
+	///
+	/// Consensus engines which alter the header (by adding post-runtime digests)
+	/// should strip those off in the initial verification process and pass them
+	/// via the `post_runtime_digests` field. During block authorship, they should
+	/// not be pushed to the header directly.
+	///
+	/// The reason for this distinction is so the header can be directly
+	/// re-executed in a runtime that checks digest equivalence -- the
+	/// post-runtime digests are pushed back on after.
+	pub header: Block::Header,
+	/// Justification provided for this block from the outside:.
+	pub external_justification: Justification,
+	/// Digest items that have been added after the runtime for external
+	/// work, like a consensus signature.
+	pub post_runtime_digests: Vec<DigestItemFor<Block>>,
+	/// Block's body
+	pub body: Option<Vec<Block::Extrinsic>>,
+	/// Is this block finalized already?
+	/// `true` implies instant finality.
+	pub finalized: bool,
+	/// Auxiliary consensus data produced by the block.
+	/// Contains a list of key-value pairs. If values are `None`, the keys
+	/// will be deleted.
+	pub auxiliary: Vec<(Vec<u8>, Option<Vec<u8>>)>,
+}
+
+impl<Block: BlockT> ImportBlock<Block> {
+	/// Deconstruct the justified header into parts.
+	pub fn into_inner(self)
+		-> (
+			BlockOrigin,
+			<Block as BlockT>::Header,
+			Justification,
+			Vec<DigestItemFor<Block>>,
+			Option<Vec<<Block as BlockT>::Extrinsic>>,
+			bool,
+			Vec<(Vec<u8>, Option<Vec<u8>>)>,
+		) {
+		(
+			self.origin,
+			self.header,
+			self.external_justification,
+			self.post_runtime_digests,
+			self.body,
+			self.finalized,
+			self.auxiliary,
+		)
+	}
+}
+
+
+
+/// Block import trait.
+pub trait BlockImport<B: BlockT> {
+	type Error: ::std::error::Error + Send + 'static;
+	/// Import a Block alongside the new authorities valid form this block forward
+	fn import_block(&self,
+		block: ImportBlock<B>,
+		new_authorities: Option<Vec<AuthorityId>>
+	) -> Result<ImportResult, Self::Error>;
+}

substrate/core/consensus/common/src/error.rs

@@ -0,0 +1,88 @@
+// Copyright 2017-2018 Parity Technologies (UK) Ltd.
+// This file is part of Substrate.
+
+// Substrate is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// Substrate is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with Substrate.  If not, see <http://www.gnu.org/licenses/>.
+
+//! Error types in Consensus
+use runtime_version::RuntimeVersion;
+
+error_chain! {
+	errors {
+		/// Missing state at block with given descriptor.
+		StateUnavailable(b: String) {
+			description("State missing at given block."),
+			display("State unavailable at block {}", b),
+		}
+
+		/// I/O terminated unexpectedly
+		IoTerminated {
+			description("I/O terminated unexpectedly."),
+			display("I/O terminated unexpectedly."),
+		}
+
+		/// Unable to schedule wakeup.
+		FaultyTimer(e: ::tokio::timer::Error) {
+			description("Timer error"),
+			display("Timer error: {}", e),
+		}
+
+		/// Unable to propose a block.
+		CannotPropose {
+			description("Unable to create block proposal."),
+			display("Unable to create block proposal."),
+		}
+
+		/// Error checking signature
+		InvalidSignature(s: ::primitives::ed25519::Signature, a: ::primitives::AuthorityId) {
+			description("Message signature is invalid"),
+			display("Message signature {:?} by {:?} is invalid.", s, a),
+		}
+
+		/// Account is not an authority.
+		InvalidAuthority(a: ::primitives::AuthorityId) {
+			description("Message sender is not a valid authority"),
+			display("Message sender {:?} is not a valid authority.", a),
+		}
+
+		/// Authoring interface does not match the runtime.
+		IncompatibleAuthoringRuntime(native: RuntimeVersion, on_chain: RuntimeVersion) {
+			description("Authoring for current runtime is not supported"),
+			display("Authoring for current runtime is not supported. Native ({}) cannot author for on-chain ({}).", native, on_chain),
+		}
+
+		/// Authoring interface does not match the runtime.
+		RuntimeVersionMissing {
+			description("Current runtime has no version"),
+			display("Authoring for current runtime is not supported since it has no version."),
+		}
+
+		/// Authoring interface does not match the runtime.
+		NativeRuntimeMissing {
+			description("This build has no native runtime"),
+			display("Authoring in current build is not supported since it has no runtime."),
+		}
+
+		/// Justification requirements not met.
+		InvalidJustification {
+			description("Invalid justification"),
+			display("Invalid justification."),
+		}
+
+		/// Some other error.
+		Other(e: Box<::std::error::Error + Send>) {
+			description("Other error")
+			display("Other error: {}", e.description())
+		}
+	}
+}

substrate/core/consensus/common/src/evaluation.rs

@@ -0,0 +1,82 @@
+// Copyright 2018 Parity Technologies (UK) Ltd.
+// This file is part of Substrate.
+
+// Substrate is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// Substrate is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with Substrate.  If not, see <http://www.gnu.org/licenses/>.
+
+//! Block evaluation and evaluation errors.
+
+use super::MAX_TRANSACTIONS_SIZE;
+
+use codec::Encode;
+use runtime_primitives::traits::{Block as BlockT, Header as HeaderT, As};
+
+type BlockNumber = u64;
+
+error_chain! {
+	errors {
+		BadProposalFormat {
+			description("Proposal provided not a block."),
+			display("Proposal provided not a block."),
+		}
+		WrongParentHash(expected: String, got: String) {
+			description("Proposal had wrong parent hash."),
+			display("Proposal had wrong parent hash. Expected {:?}, got {:?}", expected, got),
+		}
+		WrongNumber(expected: BlockNumber, got: BlockNumber) {
+			description("Proposal had wrong number."),
+			display("Proposal had wrong number. Expected {}, got {}", expected, got),
+		}
+		ProposalTooLarge(size: usize) {
+			description("Proposal exceeded the maximum size."),
+			display(
+				"Proposal exceeded the maximum size of {} by {} bytes.",
+				MAX_TRANSACTIONS_SIZE, size.saturating_sub(MAX_TRANSACTIONS_SIZE)
+			),
+		}
+	}
+}
+
+/// Attempt to evaluate a substrate block as a node block, returning error
+/// upon any initial validity checks failing.
+pub fn evaluate_initial<Block: BlockT>(
+	proposal: &Block,
+	parent_hash: &<Block as BlockT>::Hash,
+	parent_number: <<Block as BlockT>::Header as HeaderT>::Number,
+) -> Result<()> {
+
+	let encoded = Encode::encode(proposal);
+	let proposal = Block::decode(&mut &encoded[..])
+		.ok_or_else(|| ErrorKind::BadProposalFormat)?;
+
+	let transactions_size = proposal.extrinsics().iter().fold(0, |a, tx| {
+		a + Encode::encode(tx).len()
+	});
+
+	if transactions_size > MAX_TRANSACTIONS_SIZE {
+		bail!(ErrorKind::ProposalTooLarge(transactions_size))
+	}
+
+	if *parent_hash != *proposal.header().parent_hash() {
+		bail!(ErrorKind::WrongParentHash(
+			format!("{:?}", *parent_hash),
+			format!("{:?}", proposal.header().parent_hash())
+		));
+	}
+
+	if parent_number.as_() + 1 != proposal.header().number().as_() {
+		bail!(ErrorKind::WrongNumber(parent_number.as_() + 1, proposal.header().number().as_()));
+	}
+
+	Ok(())
+}

substrate/core/consensus/common/src/lib.rs

@@ -15,22 +15,118 @@
 // along with Substrate Consensus Common.  If not, see <http://www.gnu.org/licenses/>.
 
 // tag::description[]
-//! Tracks offline validators.
+//! Consensus basics and common features
 // end::description[]
 
+// This provides "unused" building blocks to other crates
 #![allow(dead_code)]
 
-#![cfg(feature="rhd")]
+// our error-chain could potentially blow up otherwise
+#![recursion_limit="128"]
 
 extern crate substrate_primitives as primitives;
+extern crate futures;
+extern crate sr_version as runtime_version;
+extern crate sr_primitives as runtime_primitives;
+extern crate tokio;
 
-use primitives::{generic::BlockId, Justification};
-use primitives::traits::{Block, Header};
+extern crate parity_codec as codec;
+#[macro_use]
+extern crate parity_codec_derive;
 
-/// Block import trait.
-pub trait BlockImport<B: Block> {
-	/// Import a block alongside its corresponding justification.
-	fn import_block(&self, block: B, justification: Justification, authorities: &[AuthorityId]) -> bool;
+#[macro_use]
+extern crate error_chain;
+
+use std::sync::Arc;
+
+use primitives::{ed25519, AuthorityId};
+use runtime_primitives::generic::BlockId;
+use runtime_primitives::traits::Block;
+use futures::prelude::*;
+
+pub mod offline_tracker;
+pub mod error;
+mod block_import;
+pub mod evaluation;
+
+// block size limit.
+const MAX_TRANSACTIONS_SIZE: usize = 4 * 1024 * 1024;
+
+pub use self::error::{Error, ErrorKind};
+pub use block_import::{BlockImport, ImportBlock, BlockOrigin, ImportResult};
+
+/// Trait for getting the authorities at a given block.
+pub trait Authorities<B: Block> {
+	type Error: ::std::error::Error + Send + 'static;	/// Get the authorities at the given block.
+	fn authorities(&self, at: &BlockId<B>) -> Result<Vec<AuthorityId>, Self::Error>;
 }
 
-pub mod offline_tracker;
+/// Environment producer for a Consensus instance. Creates proposer instance and communication streams.
+pub trait Environment<B: Block> {
+	/// The proposer type this creates.
+	type Proposer: Proposer<B>;
+	/// Error which can occur upon creation.
+	type Error: From<Error>;
+
+	/// Initialize the proposal logic on top of a specific header. Provide
+	/// the authorities at that header, and a local key to sign any additional
+	/// consensus messages with as well.
+	fn init(&self, parent_header: &B::Header, authorities: &[AuthorityId], sign_with: Arc<ed25519::Pair>)
+		-> Result<Self::Proposer, Self::Error>;
+}
+
+/// Logic for a proposer.
+///
+/// This will encapsulate creation and evaluation of proposals at a specific
+/// block.
+pub trait Proposer<B: Block> {
+	/// Error type which can occur when proposing or evaluating.
+	type Error: From<Error> + ::std::fmt::Debug + 'static;
+	/// Future that resolves to a committed proposal.
+	type Create: IntoFuture<Item=B,Error=Self::Error>;
+	/// Create a proposal.
+	fn propose(&self) -> Self::Create;
+}
+
+/// Inherent data to include in a block.
+#[derive(Encode, Decode)]
+pub struct InherentData {
+	/// Current timestamp.
+	pub timestamp: u64,
+	/// Indices of offline validators.
+	pub offline_indices: Vec<u32>,
+}
+
+impl InherentData {
+	/// Create a new `InherentData` instance.
+	pub fn new(timestamp: u64, offline_indices: Vec<u32>) -> Self {
+		Self {
+			timestamp,
+			offline_indices
+		}
+	}
+}
+
+/// An oracle for when major synchronization work is being undertaken.
+///
+/// Generally, consensus authoring work isn't undertaken while well behind
+/// the head of the chain.
+pub trait SyncOracle {
+	/// Whether the synchronization service is undergoing major sync.
+	/// Returns true if so.
+	fn is_major_syncing(&self) -> bool;
+}
+
+/// A synchronization oracle for when there is no network.
+#[derive(Clone, Copy, Debug)]
+pub struct NoNetwork;
+
+impl SyncOracle for NoNetwork {
+	fn is_major_syncing(&self) -> bool { false }
+}
+
+impl<T: SyncOracle> SyncOracle for Arc<T> {
+	fn is_major_syncing(&self) -> bool {
+		T::is_major_syncing(&*self)
+	}
+}

substrate/core/consensus/common/src/offline_tracker.rs

@@ -16,7 +16,7 @@
 
 //! Tracks offline validators.
 
-use node_primitives::AccountId;
+use primitives::AuthorityId;
 
 use std::collections::HashMap;
 use std::time::{Instant, Duration};
@@ -56,7 +56,7 @@ impl Observed {
 
 /// Tracks offline validators and can issue a report for those offline.
 pub struct OfflineTracker {
-	observed: HashMap<AccountId, Observed>,
+	observed: HashMap<AuthorityId, Observed>,
 }
 
 impl OfflineTracker {
@@ -66,7 +66,7 @@ impl OfflineTracker {
 	}
 
 	/// Note new consensus is starting with the given set of validators.
-	pub fn note_new_block(&mut self, validators: &[AccountId]) {
+	pub fn note_new_block(&mut self, validators: &[AuthorityId]) {
 		use std::collections::HashSet;
 
 		let set: HashSet<_> = validators.iter().cloned().collect();
@@ -74,14 +74,14 @@ impl OfflineTracker {
 	}
 
 	/// Note that a round has ended.
-	pub fn note_round_end(&mut self, validator: AccountId, was_online: bool) {
+	pub fn note_round_end(&mut self, validator: AuthorityId, was_online: bool) {
 		self.observed.entry(validator)
 			.or_insert_with(Observed::new)
 			.note_round_end(was_online);
 	}
 
 	/// Generate a vector of indices for offline account IDs.
-	pub fn reports(&self, validators: &[AccountId]) -> Vec<u32> {
+	pub fn reports(&self, validators: &[AuthorityId]) -> Vec<u32> {
 		validators.iter()
 			.enumerate()
 			.filter_map(|(i, v)| if self.is_online(v) {
@@ -93,7 +93,7 @@ impl OfflineTracker {
 	}
 
 	/// Whether reports on a validator set are consistent with our view of things.
-	pub fn check_consistency(&self, validators: &[AccountId], reports: &[u32]) -> bool {
+	pub fn check_consistency(&self, validators: &[AuthorityId], reports: &[u32]) -> bool {
 		reports.iter().cloned().all(|r| {
 			let v = match validators.get(r as usize) {
 				Some(v) => v,
@@ -106,7 +106,7 @@ impl OfflineTracker {
 		})
 	}
 
-	fn is_online(&self, v: &AccountId) -> bool {
+	fn is_online(&self, v: &AuthorityId) -> bool {
 		self.observed.get(v).map(Observed::is_active).unwrap_or(true)
 	}
 }