Generalize the Consensus Infrastructure (#883)

* Split out Consensus
* Supply ImportQueue through network-service
  - simplify ImportQueue.import_blocks
  - remove Deadlock on import_block
  - Adding Verifier-Trait
  - Implement import_queue provisioning in service; allow cli to import
* Allow to actually customize import queue
* Consensus Gossip: Cache Message hash per Topic

substrate/core/service/src/chain_ops.rs

@@ -22,8 +22,10 @@ use serde_json;
 
 use client::BlockOrigin;
 use runtime_primitives::generic::{SignedBlock, BlockId};
-use runtime_primitives::traits::{As};
-use components::{ServiceFactory, FactoryFullConfiguration, FactoryBlockNumber, RuntimeGenesis};
+use runtime_primitives::traits::{As, Block, Header};
+use network::import_queue::{ImportQueue, BlockData};
+use network::message;
+use components::{self, Components, ServiceFactory, FactoryFullConfiguration, FactoryBlockNumber, RuntimeGenesis};
 use new_client;
 use codec::{Decode, Encode};
 use error;
@@ -33,7 +35,7 @@ use chain_spec::ChainSpec;
 pub fn export_blocks<F, E, W>(config: FactoryFullConfiguration<F>, exit: E, mut output: W, from: FactoryBlockNumber<F>, to: Option<FactoryBlockNumber<F>>, json: bool) -> error::Result<()>
 	where F: ServiceFactory, E: Future<Item=(),Error=()> + Send + 'static, W: Write,
 {
-	let client = new_client::<F>(config)?;
+	let client = new_client::<F>(&config)?;
 	let mut block = from;
 
 	let last = match to {
@@ -85,7 +87,8 @@ pub fn export_blocks<F, E, W>(config: FactoryFullConfiguration<F>, exit: E, mut
 pub fn import_blocks<F, E, R>(config: FactoryFullConfiguration<F>, exit: E, mut input: R) -> error::Result<()>
 	where F: ServiceFactory, E: Future<Item=(),Error=()> + Send + 'static, R: Read,
 {
-	let client = new_client::<F>(config)?;
+	let client = new_client::<F>(&config)?;
+	let queue = components::FullComponents::<F>::build_import_queue(&config, client.clone())?;
 
 	let (exit_send, exit_recv) = std::sync::mpsc::channel();
 	::std::thread::spawn(move || {
@@ -95,28 +98,35 @@ pub fn import_blocks<F, E, R>(config: FactoryFullConfiguration<F>, exit: E, mut
 
 	let count: u32 = Decode::decode(&mut input).ok_or("Error reading file")?;
 	info!("Importing {} blocks", count);
-	let mut block = 0;
-	for _ in 0 .. count {
+	let mut block_count = 0; 
+	for b in 0 .. count {
 		if exit_recv.try_recv().is_ok() {
 			break;
 		}
-		match SignedBlock::decode(&mut input) {
-			Some(block) => {
-				// TODO: non-instant finality.
-				let header = client.check_justification(block.block.header, block.justification.into())?;
-				client.import_block(BlockOrigin::File, header, Some(block.block.extrinsics), true)?;
-			},
-			None => {
-				warn!("Error reading block data.");
-				break;
-			}
+		if let Some(signed) = SignedBlock::<<F::Block as Block>::Header, <F::Block as Block>::Extrinsic>::decode(&mut input) {
+			let header = signed.block.header;
+			let hash = header.hash();
+			let block  = message::BlockData::<F::Block> {
+				hash: hash,
+				justification: Some(signed.justification),
+				header: Some(header),
+				body: Some(signed.block.extrinsics),
+				receipt: None,
+				message_queue: None
+			};
+			// import queue handles verification and importing it into the client
+			queue.import_blocks(BlockOrigin::File, vec![BlockData::<F::Block> { block, origin: None }]);
+		} else {
+			warn!("Error reading block data at {}.", b);
+			break;
 		}
-		block += 1;
-		if block % 1000 == 0 {
-			info!("#{}", block);
+
+		block_count = b;
+		if b % 1000 == 0 {
+			info!("#{}", b);
 		}
 	}
-	info!("Imported {} blocks. Best: #{}", block, client.info()?.chain.best_number);
+	info!("Imported {} blocks. Best: #{}", block_count, client.info()?.chain.best_number);
 
 	Ok(())
 }
@@ -125,7 +135,7 @@ pub fn import_blocks<F, E, R>(config: FactoryFullConfiguration<F>, exit: E, mut
 pub fn revert_chain<F>(config: FactoryFullConfiguration<F>, blocks: FactoryBlockNumber<F>) -> error::Result<()>
 	where F: ServiceFactory,
 {
-	let client = new_client::<F>(config)?;
+	let client = new_client::<F>(&config)?;
 	let reverted = client.revert(blocks)?;
 	let info = client.info()?.chain;
 	info!("Reverted {} blocks. Best: #{} ({})", reverted, info.best_number, info.best_hash);

substrate/core/service/src/chain_spec.rs

@@ -87,6 +87,7 @@ struct ChainSpecFile {
 	pub boot_nodes: Vec<String>,
 	pub telemetry_url: Option<String>,
 	pub protocol_id: Option<String>,
+	pub consensus_engine: Option<String>,
 }
 
 /// A configuration of a chain. Can be used to build a genesis block.
@@ -125,6 +126,10 @@ impl<G: RuntimeGenesis> ChainSpec<G> {
 		self.spec.protocol_id.as_ref().map(String::as_str)
 	}
 
+	pub fn consensus_engine(&self) -> Option<&str> {
+		self.spec.consensus_engine.as_ref().map(String::as_str)
+	}
+
 	/// Parse json content into a `ChainSpec`
 	pub fn from_embedded(json: &'static [u8]) -> Result<Self, String> {
 		let spec = json::from_slice(json).map_err(|e| format!("Error parsing spec file: {}", e))?;
@@ -152,6 +157,7 @@ impl<G: RuntimeGenesis> ChainSpec<G> {
 		boot_nodes: Vec<String>,
 		telemetry_url: Option<&str>,
 		protocol_id: Option<&str>,
+		consensus_engine: Option<&str>,
 	) -> Self
 	{
 		let spec = ChainSpecFile {
@@ -160,6 +166,7 @@ impl<G: RuntimeGenesis> ChainSpec<G> {
 			boot_nodes: boot_nodes,
 			telemetry_url: telemetry_url.map(str::to_owned),
 			protocol_id: protocol_id.map(str::to_owned),
+			consensus_engine: consensus_engine.map(str::to_owned),
 		};
 		ChainSpec {
 			spec,

substrate/core/service/src/components.rs

@@ -139,6 +139,8 @@ pub trait ServiceFactory: 'static + Sized {
 	type FullService: Deref<Target = Service<FullComponents<Self>>> + Send + Sync + 'static;
 	/// Extended light service type.
 	type LightService: Deref<Target = Service<LightComponents<Self>>> + Send + Sync + 'static;
+	/// ImportQueue
+	type ImportQueue: network::import_queue::ImportQueue<Self::Block> + 'static;
 
 	//TODO: replace these with a constructor trait. that TransactionPool implements.
 	/// Extrinsic pool constructor for the full client.
@@ -158,6 +160,36 @@ pub trait ServiceFactory: 'static + Sized {
 	/// Build light service.
 	fn new_light(config: FactoryFullConfiguration<Self>, executor: TaskExecutor)
 		-> Result<Self::LightService, error::Error>;
+
+	/// ImportQueue for a full client
+	fn build_full_import_queue(
+		config: &FactoryFullConfiguration<Self>,
+		_client: Arc<FullClient<Self>>
+	) -> Result<Self::ImportQueue, error::Error> {
+		if let Some(name) = config.chain_spec.consensus_engine() {
+			match name {
+				_ => Err(format!("Chain Specification defines unknown consensus engine '{}'", name).into())
+			}
+
+		} else {
+			Err("Chain Specification doesn't contain any consensus_engine name".into())
+		}
+	}
+
+	/// ImportQueue for a light client
+	fn build_light_import_queue(
+		config: &FactoryFullConfiguration<Self>,
+		_client: Arc<LightClient<Self>>
+	) -> Result<Self::ImportQueue, error::Error> {
+		if let Some(name) = config.chain_spec.consensus_engine() {
+			match name {
+				_ => Err(format!("Chain Specification defines unknown consensus engine '{}'", name).into())
+			}
+
+		} else {
+			Err("Chain Specification doesn't contain any consensus_engine name".into())
+		}
+	}
 }
 
 /// A collection of types and function to generalise over full / light client type.
@@ -187,6 +219,12 @@ pub trait Components: 'static {
 	/// Create extrinsic pool.
 	fn build_transaction_pool(config: TransactionPoolOptions, client: Arc<ComponentClient<Self>>)
 		-> Result<TransactionPool<Self::TransactionPoolApi>, error::Error>;
+
+	/// instance of import queue for clients
+	fn build_import_queue(
+		config: &FactoryFullConfiguration<Self::Factory>,
+		client: Arc<ComponentClient<Self>>
+	) -> Result<<Self::Factory as ServiceFactory>::ImportQueue, error::Error>;
 }
 
 /// A struct that implement `Components` for the full client.
@@ -228,6 +266,13 @@ impl<Factory: ServiceFactory> Components for FullComponents<Factory> {
 	{
 		Factory::build_full_transaction_pool(config, client)
 	}
+
+	fn build_import_queue(
+		config: &FactoryFullConfiguration<Self::Factory>,
+		client: Arc<ComponentClient<Self>>
+	) -> Result<<Self::Factory as ServiceFactory>::ImportQueue, error::Error> {
+		Factory::build_full_import_queue(config, client)
+	}
 }
 
 /// A struct that implement `Components` for the light client.
@@ -270,4 +315,11 @@ impl<Factory: ServiceFactory> Components for LightComponents<Factory> {
 	{
 		Factory::build_light_transaction_pool(config, client)
 	}
+
+	fn build_import_queue(
+		config: &FactoryFullConfiguration<Self::Factory>,
+		client: Arc<ComponentClient<Self>>
+	) -> Result<<Self::Factory as ServiceFactory>::ImportQueue, error::Error> {
+		Factory::build_light_import_queue(config, client)
+	}
 }

substrate/core/service/src/lib.rs

@@ -102,12 +102,12 @@ pub struct Service<Components: components::Components> {
 }
 
 /// Creates bare client without any networking.
-pub fn new_client<Factory: components::ServiceFactory>(config: FactoryFullConfiguration<Factory>)
+pub fn new_client<Factory: components::ServiceFactory>(config: &FactoryFullConfiguration<Factory>)
 	-> Result<Arc<ComponentClient<components::FullComponents<Factory>>>, error::Error>
 {
 	let executor = NativeExecutor::new();
 	let (client, _) = components::FullComponents::<Factory>::build_client(
-		&config,
+		config,
 		executor,
 	)?;
 	Ok(client)
@@ -149,6 +149,7 @@ impl<Components> Service<Components>
 		};
 
 		let (client, on_demand) = Components::build_client(&config, executor)?;
+		let import_queue = Components::build_import_queue(&config, client.clone())?;
 		let best_header = client.best_block_header()?;
 
 		let version = config.full_version();
@@ -185,7 +186,7 @@ impl<Components> Service<Components>
 		let id_len = protocol_id_full.len().min(protocol_id.len());
 		&mut protocol_id[0..id_len].copy_from_slice(&protocol_id_full[0..id_len]);
 
-		let network = network::Service::new(network_params, protocol_id)?;
+		let network = network::Service::new(network_params, protocol_id, import_queue)?;
 		on_demand.map(|on_demand| on_demand.set_service_link(Arc::downgrade(&network)));
 
 		{