BFT gossip (#106)

* CLI options and keystore integration

* Replace multiqueue with future::mpsc

* BFT gossip

* Revert to app_dirs

* generate_from_seed commented

polkadot/cli/Cargo.toml

@@ -12,7 +12,7 @@ log = "0.3"
 hex-literal = "0.1"
 triehash = "0.1"
 ed25519 = { path = "../../substrate/ed25519" }
-app_dirs = "1.1"
+app_dirs = "1.2"
 substrate-client = { path = "../../substrate/client" }
 substrate-codec = { path = "../../substrate/codec" }
 substrate-runtime-io = { path = "../../substrate/runtime-io" }

polkadot/cli/src/cli.yml

@@ -4,15 +4,48 @@ about: Polkadot Node Rust Implementation
 args:
   - log:
       short: l
+      long: log
       value_name: LOG_PATTERN
       help: Sets a custom logging filter
       takes_value: true
-  - keystore-path:
-      value_name: KEYSTORE_PATH
-      help: specify custom keystore path
+  - base-path:
+      long: base-path
+      short: d
+      value_name: PATH
+      help: Specify custom base path
+      takes_value: true
+  - keystore-path:
+      long: keystore-path
+      value_name: PATH
+      help: Specify custom keystore path
+      takes_value: true
+  - key:
+      long: key
+      value_name: STRING
+      help: Specify additional key seed
       takes_value: true
-subcommands:
   - collator:
-      about: Run collator node
+      long: collator
+      help: Enable collator mode
+      takes_value: false
   - validator:
-      about: Run validator node
+      long: validator
+      help: Enable validator mode
+      takes_value: false
+  - port:
+      long: port
+      value_name: PORT
+      help: Specify p2p protocol TCP port
+      takes_value: true
+  - rpc-port:
+      long: rpc-port
+      value_name: PORT
+      help: Specify RPC server TCP port
+      takes_value: true
+  - bootnodes:
+      long: bootnodes
+      value_name: URL
+      help: Specify a list of bootnodes
+      takes_value: true
+      multiple: true
+subcommands:

polkadot/cli/src/lib.rs

@@ -42,6 +42,7 @@ extern crate log;
 pub mod error;
 
 use std::path::{Path, PathBuf};
+use std::net::SocketAddr;
 
 /// Parse command line arguments and start the node.
 ///
@@ -56,7 +57,15 @@ pub fn run<I, T>(args: I) -> error::Result<()> where
 	T: Into<std::ffi::OsString> + Clone,
 {
 	let yaml = load_yaml!("./cli.yml");
-	let matches = clap::App::from_yaml(yaml).version(crate_version!()).get_matches_from_safe(args)?;
+	let matches = match clap::App::from_yaml(yaml).version(crate_version!()).get_matches_from_safe(args) {
+		Ok(m) => m,
+		Err(ref e) if e.kind == clap::ErrorKind::VersionDisplayed => return Ok(()),
+		Err(ref e) if e.kind == clap::ErrorKind::HelpDisplayed || e.kind == clap::ErrorKind::VersionDisplayed => {
+			let _ = clap::App::from_yaml(yaml).print_long_help();
+			return Ok(());
+		}
+		Err(e) => return Err(e.into()),
+	};
 
 	// TODO [ToDr] Split parameters parsing from actual execution.
 	let log_pattern = matches.value_of("log").unwrap_or("");
@@ -64,38 +73,68 @@ pub fn run<I, T>(args: I) -> error::Result<()> where
 
 	let mut config = service::Configuration::default();
 
+	let base_path = matches.value_of("base-path")
+		.map(|x| Path::new(x).to_owned())
+		.unwrap_or_else(default_base_path);
+
 	config.keystore_path = matches.value_of("keystore")
 		.map(|x| Path::new(x).to_owned())
-		.unwrap_or_else(default_keystore_path)
+		.unwrap_or_else(|| keystore_path(&base_path))
 		.to_string_lossy()
 		.into();
 
 	let mut role = service::Role::FULL;
-	if let Some(_) = matches.subcommand_matches("collator") {
+	if matches.is_present("collator") {
 		info!("Starting collator.");
 		role = service::Role::COLLATOR;
 	}
-	else if let Some(_) = matches.subcommand_matches("validator") {
+	else if matches.is_present("validator") {
 		info!("Starting validator.");
 		role = service::Role::VALIDATOR;
 	}
 
 	config.roles = role;
+	config.network.boot_nodes = matches
+		.values_of("bootnodes")
+		.map_or(Default::default(), |v| v.map(|n| n.to_owned()).collect());
+	config.network.config_path = Some(network_path(&base_path).to_string_lossy().into());
+	config.network.net_config_path = config.network.config_path.clone();
+
+	let port = match matches.value_of("port") {
+		Some(port) => port.parse().expect("Invalid p2p port value specified."),
+		None => 30333,
+	};
+	config.network.listen_address = Some(SocketAddr::new("0.0.0.0".parse().unwrap(), port));
+
+	config.keys = matches.values_of("key").unwrap_or_default().map(str::to_owned).collect();
 
 	let service = service::Service::new(config)?;
 
-	let address = "127.0.0.1:9933".parse().unwrap();
+	let mut address: SocketAddr = "127.0.0.1:9933".parse().unwrap();
+	if let Some(port) = matches.value_of("rpc-port") {
+		let rpc_port: u16 = port.parse().expect("Invalid RPC port value specified.");
+		address.set_port(rpc_port);
+	}
 	let handler = rpc::rpc_handler(service.client());
 	let server = rpc::start_http(&address, handler)?;
 
 	server.wait();
-	println!("No command given.\n");
-	let _ = clap::App::from_yaml(yaml).print_long_help();
-
 	Ok(())
 }
 
-fn default_keystore_path() -> PathBuf {
+fn keystore_path(base_path: &Path) -> PathBuf {
+	let mut path = base_path.to_owned();
+	path.push("keystore");
+	path
+}
+
+fn network_path(base_path: &Path) -> PathBuf {
+	let mut path = base_path.to_owned();
+	path.push("network");
+	path
+}
+
+fn default_base_path() -> PathBuf {
 	use app_dirs::{AppInfo, AppDataType};
 
 	let app_info = AppInfo {
@@ -103,13 +142,11 @@ fn default_keystore_path() -> PathBuf {
 		author: "Parity Technologies",
 	};
 
-	app_dirs::get_app_dir(
+	app_dirs::get_app_root(
 		AppDataType::UserData,
 		&app_info,
-		"keystore",
 	).expect("app directories exist on all supported platforms; qed")
 }
-
 fn init_logger(pattern: &str) {
 	let mut builder = env_logger::LogBuilder::new();
 	// Disable info logging by default for some modules:

polkadot/consensus/src/service.rs

@@ -26,7 +26,6 @@ use parking_lot::Mutex;
 use substrate_network as net;
 use tokio_core::reactor;
 use client::BlockchainEvents;
-use substrate_keyring::Keyring;
 use primitives::{Hash, AuthorityId};
 use primitives::block::{Id as BlockId, HeaderHash, Header};
 use polkadot_primitives::parachain::{BlockData, Extrinsic, CandidateReceipt};
@@ -136,14 +135,18 @@ struct Network(Arc<net::ConsensusService>);
 
 impl Service {
 	/// Create and start a new instance.
-	pub fn new<C>(client: Arc<C>, network: Arc<net::ConsensusService>, transaction_pool: Arc<Mutex<TransactionPool>>, best_header: &Header) -> Service
+	pub fn new<C>(
+		client: Arc<C>,
+		network: Arc<net::ConsensusService>,
+		transaction_pool: Arc<Mutex<TransactionPool>>,
+		key: ed25519::Pair,
+		best_header: &Header) -> Service
 		where C: BlockchainEvents + bft::BlockImport + bft::Authorities + PolkadotApi + Send + Sync + 'static
 	{
-
 		let best_header = best_header.clone();
 		let thread = thread::spawn(move || {
 			let mut core = reactor::Core::new().expect("tokio::Core could not be created");
-			let key = Arc::new(Keyring::One.into());
+			let key = Arc::new(key);
 			let factory = ProposerFactory {
 				client: client.clone(),
 				transaction_pool: transaction_pool.clone(),

polkadot/keystore/src/lib.rs

@@ -33,6 +33,7 @@ extern crate error_chain;
 #[cfg(test)]
 extern crate tempdir;
 
+use std::collections::HashMap;
 use std::path::PathBuf;
 use std::fs::{self, File};
 use std::io::{self, Write};
@@ -120,16 +121,19 @@ impl EncryptedKey {
 	}
 }
 
+type Seed = [u8; 32];
+
 /// Key store.
 pub struct Store {
 	path: PathBuf,
+	additional: HashMap<Public, Seed>,
 }
 
 impl Store {
 	/// Create a new store at the given path.
 	pub fn open(path: PathBuf) -> Result<Self> {
 		fs::create_dir_all(&path)?;
-		Ok(Store { path })
+		Ok(Store { path, additional: HashMap::new() })
 	}
 
 	/// Generate a new key, placing it into the store.
@@ -145,8 +149,24 @@ impl Store {
 		Ok(pair)
 	}
 
+	/// Create a new key from seed. Do not place it into the store.
+	/// Only the first 32 bytes of the sead are used. This is meant to be used for testing only.
+	// TODO: Remove this
+	pub fn generate_from_seed(&mut self, seed: &str) -> Result<Pair> {
+		let mut s: [u8; 32] = [' ' as u8; 32];
+		let len = ::std::cmp::min(32, seed.len());
+		&mut s[..len].copy_from_slice(&seed.as_bytes()[..len]);
+		let pair = Pair::from_seed(&s);
+		self.additional.insert(pair.public(), s);
+		Ok(pair)
+	}
+
 	/// Load a key file with given public key.
 	pub fn load(&self, public: &Public, password: &str) -> Result<Pair> {
+		if let Some(ref seed) = self.additional.get(public) {
+			let pair = Pair::from_seed(seed);
+			return Ok(pair);
+		}
 		let path = self.key_file_path(public);
 		let file = File::open(path)?;
 
@@ -158,7 +178,7 @@ impl Store {
 
 	/// Get public keys of all stored keys.
 	pub fn contents(&self) -> Result<Vec<Public>> {
-		let mut public_keys = Vec::new();
+		let mut public_keys: Vec<Public> = self.additional.keys().cloned().collect();
 		for entry in fs::read_dir(&self.path)? {
 			let entry = entry?;
 			let path = entry.path();

polkadot/service/src/config.rs

@@ -30,7 +30,8 @@ pub struct Configuration {
 	pub network: NetworkConfiguration,
 	/// Path to key files.
 	pub keystore_path: String,
-	// TODO: add more network, client, tx pool configuration options
+	/// Additional key seeds.
+	pub keys: Vec<String>,
 }
 
 impl Default for Configuration {
@@ -40,6 +41,7 @@ impl Default for Configuration {
 			transaction_pool: Default::default(),
 			network: Default::default(),
 			keystore_path: Default::default(),
+			keys: Default::default(),
 		}
 	}
 }

polkadot/service/src/error.rs

@@ -18,18 +18,15 @@
 
 use client;
 use network;
+use keystore;
 
 error_chain! {
 	links {
 		Client(client::error::Error, client::error::ErrorKind) #[doc="Client error"];
 		Network(network::error::Error, network::error::ErrorKind) #[doc="Network error"];
+		Keystore(keystore::Error, keystore::ErrorKind) #[doc="Keystore error"];
 	}
 
 	errors {
-		/// Key store errors
-		Keystore(e: ::keystore::Error) {
-			description("Keystore error"),
-			display("Keystore error: {:?}", e),
-		}
 	}
 }

polkadot/service/src/lib.rs

@@ -56,7 +56,6 @@ use transaction_pool::TransactionPool;
 use substrate_keyring::Keyring;
 use substrate_executor::NativeExecutor;
 use polkadot_executor::Executor as LocalDispatch;
-use polkadot_primitives::AccountId;
 use keystore::Store as Keystore;
 use polkadot_api::PolkadotApi;
 use polkadot_runtime::genesismap::{additional_storage_with_genesis, GenesisConfig};
@@ -126,12 +125,27 @@ impl Service {
 		// Create client
 		let executor = polkadot_executor::Executor::new();
 		let mut storage = Default::default();
-		let key: AccountId = Keyring::One.into();
+
+		let mut keystore = Keystore::open(config.keystore_path.into())?;
+		for seed in &config.keys {
+			keystore.generate_from_seed(seed)?;
+		}
+
+		if keystore.contents()?.is_empty() {
+			let key = keystore.generate("")?;
+			info!("Generated a new keypair: {:?}", key.public());
+		}
 
 		let genesis_config = GenesisConfig {
-			validators: vec![key.clone()],
-			authorities: vec![key.clone()],
-			balances: vec![(Keyring::One.into(), 1u64 << 63), (Keyring::Two.into(), 1u64 << 63)].into_iter().collect(),
+			validators: vec![Keyring::Alice.into(), Keyring::Bob.into(), Keyring::Charlie.into()],
+			authorities: vec![Keyring::Alice.into(), Keyring::Bob.into(), Keyring::Charlie.into()],
+			balances: vec![
+				(Keyring::One.into(), 1u64 << 63),
+				(Keyring::Two.into(), 1u64 << 63),
+				(Keyring::Alice.into(), 1u64 << 63),
+				(Keyring::Bob.into(), 1u64 << 63),
+				(Keyring::Charlie.into(), 1u64 << 63),
+			].into_iter().collect(),
 			block_time: 5,			// 5 second block time.
 			session_length: 720,	// that's 1 hour per session.
 			sessions_per_era: 24,	// 24 hours per era.
@@ -145,7 +159,6 @@ impl Service {
 			(primitives::block::Header::decode(&mut block.header.encode().as_ref()).expect("to_vec() always gives a valid serialisation; qed"), storage.into_iter().collect())
 		};
 
-		let _keystore = Keystore::open(config.keystore_path.into()).map_err(::error::ErrorKind::Keystore)?;
 		let client = Arc::new(client::new_in_mem(executor, prepare_genesis)?);
 		let best_header = client.header(&BlockId::Hash(client.info()?.chain.best_hash))?.expect("Best header always exists; qed");
 		info!("Starting Polkadot. Best block is #{}", best_header.number);
@@ -166,7 +179,10 @@ impl Service {
 
 		// Spin consensus service if configured
 		let consensus_service = if config.roles & Role::VALIDATOR == Role::VALIDATOR {
-			Some(consensus::Service::new(client.clone(), network.clone(), transaction_pool, &best_header))
+			// Load the first available key. Code above makes sure it exisis.
+			let key = keystore.load(&keystore.contents()?[0], "")?;
+			info!("Using authority key {:?}", key.public());
+			Some(consensus::Service::new(client.clone(), network.clone(), transaction_pool, key, &best_header))
 		} else {
 			None
 		};