Minimal switch of substrate-node to GRANDPA /Aura (#1128)

* add beginnings of SRML grandpa library

* get srml-grandpa compiling

* tests for srml-grandpa

* add optional session integration to grandpa SRML

* start integration into node runtime

* Allow extracting pending change from header digest

* Make it compile on wasm

* make tests compile again

* Move Authority Key fetching into service, simplify service factory construction

* Generalize Authority Consensus Setup system

* Add Authority Setup Docs

* Allow CLI params to be extensible

 - move params to structopts
 - split parsing and default command execution
 - add custom config to node
 - extended parsing of custom config
 - extending params via structop's flatten

* Minor fixes on cli extension params:
 - added docs
 - re-add actual app name, rather than node-name
 - make strategy and subcommand optional

* better cli params

* synchronize GRANDPA and normal node authorities

* Implement grandpa::network for gossip consensus

* run_grandpa in Node

* Fix missed merge error

* Integrate grandpa import queue

* more specific type def

* link up linkhalf and import block

* make grandpa future send

* get compiling

* Fix new params convention and license header

* get it running

* rebuild node runtime WASM

* change logging level

* Update node/cli/src/params.rs

Co-Authored-By: rphmeier <rphmeier@gmail.com>

* Update node/cli/src/params.rs

Co-Authored-By: rphmeier <rphmeier@gmail.com>

* Update node/cli/src/lib.rs

Co-Authored-By: rphmeier <rphmeier@gmail.com>

* Update node/runtime/src/lib.rs

Co-Authored-By: rphmeier <rphmeier@gmail.com>

* Update node/cli/src/lib.rs

Co-Authored-By: rphmeier <rphmeier@gmail.com>

* Clean up and Fixme for mutable config

* Move GrandpaService Integration into grandpa, feature gated but on per default

* Fixing grandpa runtime module test

* Update wasm runtime hashes for tests

* GRANDPA: use post-header hash when logging scheduled changes

* add an extra bit of logging to authorities

* fixing missing constrain

* remove old code

* move `NewAuthorities` to an event in srml-grandpa

* fix node-executor tests to use grandpa log

* Remove GossipConsensus from tests, use newly provided sync-feature, fixes tests

* Update to latest wasm runtimes

* address grumbles

* address grumbles

* only derive deserialize when using std

* Clean up use of Deserialize

substrate/node/cli/src/chain_spec.rs

@@ -20,7 +20,7 @@ use primitives::{AuthorityId, ed25519};
 use node_primitives::AccountId;
 use node_runtime::{ConsensusConfig, CouncilSeatsConfig, CouncilVotingConfig, DemocracyConfig,
 	SessionConfig, StakingConfig, TimestampConfig, BalancesConfig, TreasuryConfig,
-	UpgradeKeyConfig, ContractConfig, Permill, Perbill};
+	UpgradeKeyConfig, ContractConfig, GrandpaConfig, Permill, Perbill};
 pub use node_runtime::GenesisConfig;
 use substrate_service;
 
@@ -140,6 +140,10 @@ fn staging_testnet_config_genesis() -> GenesisConfig {
 			key: endowed_accounts[0].clone(),
 			_genesis_phantom_data: Default::default(),
 		}),
+		grandpa: Some(GrandpaConfig {
+			authorities: initial_authorities.clone().into_iter().map(|k| (k, 1)).collect(),
+			_genesis_phantom_data: Default::default(),
+		})
 	}
 }
 
@@ -269,6 +273,10 @@ pub fn testnet_genesis(
 			key: upgrade_key,
 			_genesis_phantom_data: Default::default(),
 		}),
+		grandpa: Some(GrandpaConfig {
+			authorities: initial_authorities.clone().into_iter().map(|k| (k, 1)).collect(),
+			_genesis_phantom_data: Default::default(),
+		})
 	}
 }
 

substrate/node/cli/src/cli.yml

@@ -1,12 +0,0 @@
-name: substrate-node
-author: "Parity Team <admin@parity.io>"
-about: Substrate Node Rust Implementation
-args:
-  - log:
-      short: l
-      value_name: LOG_PATTERN
-      help: Sets a custom logging
-      takes_value: true
-subcommands:
-  - validator:
-      about: Run validator node

substrate/node/cli/src/lib.rs

@@ -34,6 +34,7 @@ extern crate substrate_transaction_pool as transaction_pool;
 extern crate substrate_network as network;
 extern crate substrate_consensus_aura as consensus;
 extern crate substrate_client as client;
+extern crate substrate_finality_grandpa as grandpa;
 extern crate node_primitives;
 #[macro_use]
 extern crate substrate_service;
@@ -42,14 +43,19 @@ extern crate substrate_keystore;
 
 #[macro_use]
 extern crate log;
+#[macro_use]
+extern crate structopt;
 
 pub use cli::error;
 pub mod chain_spec;
 mod service;
+mod params;
 
 use tokio::runtime::Runtime;
 pub use cli::{VersionInfo, IntoExit};
 use substrate_service::{ServiceFactory, Roles as ServiceRoles};
+use params::{Params as NodeParams};
+use structopt::StructOpt;
 use std::ops::Deref;
 
 /// The chain specification option.
@@ -100,9 +106,37 @@ pub fn run<I, T, E>(args: I, exit: E, version: cli::VersionInfo) -> error::Resul
 	T: Into<std::ffi::OsString> + Clone,
 	E: IntoExit,
 {
-	match cli::prepare_execution::<service::Factory, _, _, _, _>(args, exit, version, load_spec, "substrate-node")? {
+	let full_version = substrate_service::config::full_version_from_strs(
+		version.version,
+		version.commit
+	);
+
+	let matches = match NodeParams::clap()
+		.name(version.executable_name)
+		.author(version.author)
+		.about(version.description)
+		.version(&(full_version + "\n")[..])
+		.get_matches_from_safe(args) {
+			Ok(m) => m,
+			Err(e) => e.exit(),
+		};
+
+	let (spec, mut config) = cli::parse_matches::<service::Factory, _>(load_spec, version, "substrate-node", &matches)?;
+
+	if matches.is_present("grandpa_authority_only") {
+		config.custom.grandpa_authority = true;
+		config.custom.grandpa_authority_only = true;
+		// Authority Setup is only called if validator is set as true
+		config.roles = ServiceRoles::AUTHORITY;
+	} else if matches.is_present("grandpa_authority") {
+		config.custom.grandpa_authority = true;
+		// Authority Setup is only called if validator is set as true
+		config.roles = ServiceRoles::AUTHORITY;
+	}
+
+	match cli::execute_default::<service::Factory, _>(spec, exit, &matches)? {
 		cli::Action::ExecutedInternally => (),
-		cli::Action::RunService((config, exit)) => {
+		cli::Action::RunService(exit) => {
 			info!("Substrate Node");
 			info!("  version {}", config.full_version());
 			info!("  by Parity Technologies, 2017, 2018");

substrate/node/cli/src/params.rs

@@ -0,0 +1,33 @@
+// Copyright 2018 Parity Technologies (UK) Ltd.
+// This file is part of Substrate.
+
+// Substrate is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// Substrate is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with Substrate.  If not, see <http://www.gnu.org/licenses/>.
+
+use structopt::StructOpt;
+use cli::CoreParams;
+
+/// Extend params for Node
+#[derive(Debug, StructOpt)]
+pub struct Params {
+	/// Should run as a GRANDPA authority node
+	#[structopt(long = "grandpa-authority", help = "Run Node as a GRANDPA authority, implies --validator")]
+	grandpa_authority: bool,
+
+	/// Should run as a GRANDPA authority node only
+	#[structopt(long = "grandpa-authority-only", help = "Run Node as a GRANDPA authority only, don't as a usual validator, implies --grandpa-authority")]
+	grandpa_authority_only: bool,
+
+	#[structopt(flatten)]
+	core: CoreParams
+}

substrate/node/cli/src/service.rs

@@ -24,12 +24,14 @@ use node_runtime::{GenesisConfig, ClientWithApi};
 use node_primitives::Block;
 use substrate_service::{
 	FactoryFullConfiguration, LightComponents, FullComponents, FullBackend,
-	FullClient, LightClient, LightBackend, FullExecutor, LightExecutor,
-	Roles, TaskExecutor,
+	FullClient, LightClient, LightBackend, FullExecutor, LightExecutor, TaskExecutor
 };
 use node_executor;
 use consensus::{import_queue, start_aura, Config as AuraConfig, AuraImportQueue, NothingExtra};
+use primitives::ed25519::Pair;
 use client;
+use std::time::Duration;
+use grandpa;
 
 const AURA_SLOT_DURATION: u64 = 6;
 
@@ -38,6 +40,29 @@ construct_simple_protocol! {
 	pub struct NodeProtocol where Block = Block { }
 }
 
+/// Node specific configuration
+pub struct NodeConfig<F: substrate_service::ServiceFactory> {
+	/// should run as a grandpa authority
+	pub grandpa_authority: bool,
+	/// should run as a grandpa authority only, don't validate as usual
+	pub grandpa_authority_only: bool,
+	/// grandpa connection to import block
+
+	// FIXME: rather than putting this on the config, let's have an actual intermediate setup state
+	// https://github.com/paritytech/substrate/issues/1134
+	pub grandpa_link_half: Option<grandpa::LinkHalfForService<F>>,
+}
+
+impl<F> Default for NodeConfig<F> where F: substrate_service::ServiceFactory {
+	fn default() -> NodeConfig<F> {
+		NodeConfig {
+			grandpa_authority: false,
+			grandpa_authority_only: false,
+			grandpa_link_half: None
+		}
+	}
+}
+
 construct_service_factory! {
 	struct Factory {
 		Block = Block,
@@ -49,49 +74,62 @@ construct_service_factory! {
 		LightTransactionPoolApi = transaction_pool::ChainApi<client::Client<LightBackend<Self>, LightExecutor<Self>, Block, ClientWithApi>, Block>
 			{ |config, client| Ok(TransactionPool::new(config, transaction_pool::ChainApi::new(client))) },
 		Genesis = GenesisConfig,
-		Configuration = (),
+		Configuration = NodeConfig<Self>,
 		FullService = FullComponents<Self>
-			{ |config: FactoryFullConfiguration<Self>, executor: TaskExecutor| {
-				let is_auth = config.roles == Roles::AUTHORITY;
-				FullComponents::<Factory>::new(config, executor.clone()).map(move |service|{
-					if is_auth {
-						if let Ok(Some(Ok(key))) = service.keystore().contents()
-							.map(|keys| keys.get(0).map(|k| service.keystore().load(k, "")))
-						{
-							info!("Using authority key {}", key.public());
-							let task = start_aura(
-								AuraConfig {
-									local_key:  Some(Arc::new(key)),
-									slot_duration: AURA_SLOT_DURATION,
-								},
-								service.client(),
-								service.proposer(),
-								service.network(),
-							);
+			{ |config: FactoryFullConfiguration<Self>, executor: TaskExecutor|
+				FullComponents::<Factory>::new(config, executor) },
+		AuthoritySetup = {
+			|service: Self::FullService, executor: TaskExecutor, key: Arc<Pair>| {
+				if service.config.custom.grandpa_authority {
+					info!("Running Grandpa session as Authority {}", key.public());
+					let link_half = service.config().custom.grandpa_link_half.as_ref().take()
+						.expect("Link Half is present for Full Services or setup failed before. qed");
+					let grandpa_fut = grandpa::run_grandpa(
+						grandpa::Config {
+							gossip_duration: Duration::new(4, 0), // FIXME: make this available through chainspec?
+							local_key: Some(key.clone()),
+							name: Some(service.config().name.clone())
+						},
+						(*link_half).clone(),
+						grandpa::NetworkBridge::new(service.network())
+					)?;
 
-							executor.spawn(task);
-						}
-					}
-
-					service
-				})
+					executor.spawn(grandpa_fut);
+				}
+				if !service.config.custom.grandpa_authority_only {
+					info!("Using authority key {}", key.public());
+					executor.spawn(start_aura(
+						AuraConfig {
+							local_key: Some(key),
+							slot_duration: AURA_SLOT_DURATION,
+						},
+						service.client(),
+						service.proposer(),
+						service.network(),
+					));
+				}
+				Ok(service)
 			}
 		},
 		LightService = LightComponents<Self>
 			{ |config, executor| <LightComponents<Factory>>::new(config, executor) },
-		FullImportQueue = AuraImportQueue<Self::Block, FullClient<Self>, NothingExtra>
-			{ |config, client| Ok(import_queue(
-				AuraConfig {
-					local_key: None,
-					slot_duration: 5
-				},
-				client,
-				NothingExtra,
-			))
-			},
+		FullImportQueue = AuraImportQueue<Self::Block, grandpa::BlockImportForService<Self>, NothingExtra>
+			{ |config: &mut FactoryFullConfiguration<Self> , client: Arc<FullClient<Self>>| {
+				let (block_import, link_half) = grandpa::block_import::<_, _, _, ClientWithApi, FullClient<Self>>(client.clone(), client)?;
+				config.custom.grandpa_link_half = Some(link_half);
+
+				Ok(import_queue(
+					AuraConfig {
+						local_key: None,
+						slot_duration: 5
+					},
+					Arc::new(block_import),
+					NothingExtra,
+				))
+			}},
 		LightImportQueue = AuraImportQueue<Self::Block, LightClient<Self>, NothingExtra>
-			{ |config, client| Ok(import_queue(
-				AuraConfig {
+			{ |ref mut config, client| Ok(
+				import_queue(AuraConfig {
 					local_key: None,
 					slot_duration: 5
 				},