Node template folders restructuring (#4811)

* Restructure node-template so it is clear that node, runtime, and pallets are separated
* Separating to mock and tests
* restructuring runtime to top-level
* updated release script
* updated Cargo.lock

substrate/bin/node-template/node/src/chain_spec.rs

@@ -0,0 +1,153 @@
+use sp_core::{Pair, Public, sr25519};
+use node_template_runtime::{
+	AccountId, AuraConfig, BalancesConfig, GenesisConfig, GrandpaConfig,
+	SudoConfig, IndicesConfig, SystemConfig, WASM_BINARY, Signature
+};
+use sp_consensus_aura::sr25519::{AuthorityId as AuraId};
+use grandpa_primitives::{AuthorityId as GrandpaId};
+use sc_service;
+use sp_runtime::traits::{Verify, IdentifyAccount};
+
+// Note this is the URL for the telemetry server
+//const STAGING_TELEMETRY_URL: &str = "wss://telemetry.polkadot.io/submit/";
+
+/// Specialized `ChainSpec`. This is a specialization of the general Substrate ChainSpec type.
+pub type ChainSpec = sc_service::ChainSpec<GenesisConfig>;
+
+/// The chain specification option. This is expected to come in from the CLI and
+/// is little more than one of a number of alternatives which can easily be converted
+/// from a string (`--chain=...`) into a `ChainSpec`.
+#[derive(Clone, Debug)]
+pub enum Alternative {
+	/// Whatever the current runtime is, with just Alice as an auth.
+	Development,
+	/// Whatever the current runtime is, with simple Alice/Bob auths.
+	LocalTestnet,
+}
+
+/// Helper function to generate a crypto pair from seed
+pub fn get_from_seed<TPublic: Public>(seed: &str) -> <TPublic::Pair as Pair>::Public {
+	TPublic::Pair::from_string(&format!("//{}", seed), None)
+		.expect("static values are valid; qed")
+		.public()
+}
+
+type AccountPublic = <Signature as Verify>::Signer;
+
+/// Helper function to generate an account ID from seed
+pub fn get_account_id_from_seed<TPublic: Public>(seed: &str) -> AccountId where
+	AccountPublic: From<<TPublic::Pair as Pair>::Public>
+{
+	AccountPublic::from(get_from_seed::<TPublic>(seed)).into_account()
+}
+
+/// Helper function to generate an authority key for Aura
+pub fn get_authority_keys_from_seed(s: &str) -> (AuraId, GrandpaId) {
+	(
+		get_from_seed::<AuraId>(s),
+		get_from_seed::<GrandpaId>(s),
+	)
+}
+
+impl Alternative {
+	/// Get an actual chain config from one of the alternatives.
+	pub(crate) fn load(self) -> Result<ChainSpec, String> {
+		Ok(match self {
+			Alternative::Development => ChainSpec::from_genesis(
+				"Development",
+				"dev",
+				|| testnet_genesis(
+					vec![
+						get_authority_keys_from_seed("Alice"),
+					],
+					get_account_id_from_seed::<sr25519::Public>("Alice"),
+					vec![
+						get_account_id_from_seed::<sr25519::Public>("Alice"),
+						get_account_id_from_seed::<sr25519::Public>("Bob"),
+						get_account_id_from_seed::<sr25519::Public>("Alice//stash"),
+						get_account_id_from_seed::<sr25519::Public>("Bob//stash"),
+					],
+					true,
+				),
+				vec![],
+				None,
+				None,
+				None,
+				None
+			),
+			Alternative::LocalTestnet => ChainSpec::from_genesis(
+				"Local Testnet",
+				"local_testnet",
+				|| testnet_genesis(
+					vec![
+						get_authority_keys_from_seed("Alice"),
+						get_authority_keys_from_seed("Bob"),
+					],
+					get_account_id_from_seed::<sr25519::Public>("Alice"),
+					vec![
+						get_account_id_from_seed::<sr25519::Public>("Alice"),
+						get_account_id_from_seed::<sr25519::Public>("Bob"),
+						get_account_id_from_seed::<sr25519::Public>("Charlie"),
+						get_account_id_from_seed::<sr25519::Public>("Dave"),
+						get_account_id_from_seed::<sr25519::Public>("Eve"),
+						get_account_id_from_seed::<sr25519::Public>("Ferdie"),
+						get_account_id_from_seed::<sr25519::Public>("Alice//stash"),
+						get_account_id_from_seed::<sr25519::Public>("Bob//stash"),
+						get_account_id_from_seed::<sr25519::Public>("Charlie//stash"),
+						get_account_id_from_seed::<sr25519::Public>("Dave//stash"),
+						get_account_id_from_seed::<sr25519::Public>("Eve//stash"),
+						get_account_id_from_seed::<sr25519::Public>("Ferdie//stash"),
+					],
+					true,
+				),
+				vec![],
+				None,
+				None,
+				None,
+				None
+			),
+		})
+	}
+
+	pub(crate) fn from(s: &str) -> Option<Self> {
+		match s {
+			"dev" => Some(Alternative::Development),
+			"" | "local" => Some(Alternative::LocalTestnet),
+			_ => None,
+		}
+	}
+}
+
+fn testnet_genesis(initial_authorities: Vec<(AuraId, GrandpaId)>,
+	root_key: AccountId,
+	endowed_accounts: Vec<AccountId>,
+	_enable_println: bool) -> GenesisConfig {
+	GenesisConfig {
+		system: Some(SystemConfig {
+			code: WASM_BINARY.to_vec(),
+			changes_trie_config: Default::default(),
+		}),
+		indices: Some(IndicesConfig {
+			ids: endowed_accounts.clone(),
+		}),
+		balances: Some(BalancesConfig {
+			balances: endowed_accounts.iter().cloned().map(|k|(k, 1 << 60)).collect(),
+		}),
+		sudo: Some(SudoConfig {
+			key: root_key,
+		}),
+		aura: Some(AuraConfig {
+			authorities: initial_authorities.iter().map(|x| (x.0.clone())).collect(),
+		}),
+		grandpa: Some(GrandpaConfig {
+			authorities: initial_authorities.iter().map(|x| (x.1.clone(), 1)).collect(),
+		}),
+	}
+}
+
+pub fn load_spec(id: &str) -> Result<Option<ChainSpec>, String> {
+	Ok(match Alternative::from(id) {
+		Some(spec) => Some(spec.load()?),
+		None => None,
+	})
+}

substrate/bin/node-template/node/src/cli.rs

@@ -0,0 +1,11 @@
+use sc_cli::{RunCmd, Subcommand};
+use structopt::StructOpt;
+
+#[derive(Debug, StructOpt)]
+pub struct Cli {
+	#[structopt(subcommand)]
+	pub subcommand: Option<Subcommand>,
+
+	#[structopt(flatten)]
+	pub run: RunCmd,
+}

substrate/bin/node-template/node/src/command.rs

@@ -0,0 +1,48 @@
+// Copyright 2017-2020 Parity Technologies (UK) Ltd.
+// This file is part of Substrate.
+
+// Substrate is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// Substrate is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with Substrate.  If not, see <http://www.gnu.org/licenses/>.
+
+use sp_consensus_aura::sr25519::{AuthorityPair as AuraPair};
+use sc_cli::{VersionInfo, error};
+use crate::service;
+use crate::chain_spec;
+use crate::cli::Cli;
+
+/// Parse and run command line arguments
+pub fn run(version: VersionInfo) -> error::Result<()>
+{
+	let opt = sc_cli::from_args::<Cli>(&version);
+
+	let mut config = sc_service::Configuration::default();
+	config.impl_name = "node-template";
+
+	match opt.subcommand {
+		Some(subcommand) => sc_cli::run_subcommand(
+			config,
+			subcommand,
+			chain_spec::load_spec,
+			|config: _| Ok(new_full_start!(config).0),
+			&version,
+		),
+		None => sc_cli::run(
+			config,
+			opt.run,
+			service::new_light,
+			service::new_full,
+			chain_spec::load_spec,
+			&version,
+		)
+	}
+}

substrate/bin/node-template/node/src/main.rs

@@ -0,0 +1,25 @@
+//! Substrate Node Template CLI library.
+#![warn(missing_docs)]
+
+mod chain_spec;
+#[macro_use]
+mod service;
+mod cli;
+mod command;
+
+pub use sc_cli::{VersionInfo, error};
+
+fn main() -> Result<(), error::Error> {
+	let version = VersionInfo {
+		name: "Substrate Node",
+		commit: env!("VERGEN_SHA_SHORT"),
+		version: env!("CARGO_PKG_VERSION"),
+		executable_name: "node-template",
+		author: "Anonymous",
+		description: "Template Node",
+		support_url: "support.anonymous.an",
+		copyright_start_year: 2017,
+	};
+
+	command::run(version)
+}

substrate/bin/node-template/node/src/service.rs

@@ -0,0 +1,242 @@
+//! Service and ServiceFactory implementation. Specialized wrapper over substrate service.
+
+use std::sync::Arc;
+use std::time::Duration;
+use sc_client::LongestChain;
+use node_template_runtime::{self, GenesisConfig, opaque::Block, RuntimeApi};
+use sc_service::{error::{Error as ServiceError}, AbstractService, Configuration, ServiceBuilder};
+use sp_inherents::InherentDataProviders;
+use sc_network::{construct_simple_protocol};
+use sc_executor::native_executor_instance;
+pub use sc_executor::NativeExecutor;
+use sp_consensus_aura::sr25519::{AuthorityPair as AuraPair};
+use grandpa::{self, FinalityProofProvider as GrandpaFinalityProofProvider};
+
+// Our native executor instance.
+native_executor_instance!(
+	pub Executor,
+	node_template_runtime::api::dispatch,
+	node_template_runtime::native_version,
+);
+
+construct_simple_protocol! {
+	/// Demo protocol attachment for substrate.
+	pub struct NodeProtocol where Block = Block { }
+}
+
+/// Starts a `ServiceBuilder` for a full service.
+///
+/// Use this macro if you don't actually need the full service, but just the builder in order to
+/// be able to perform chain operations.
+macro_rules! new_full_start {
+	($config:expr) => {{
+		let mut import_setup = None;
+		let inherent_data_providers = sp_inherents::InherentDataProviders::new();
+
+		let builder = sc_service::ServiceBuilder::new_full::<
+			node_template_runtime::opaque::Block, node_template_runtime::RuntimeApi, crate::service::Executor
+		>($config)?
+			.with_select_chain(|_config, backend| {
+				Ok(sc_client::LongestChain::new(backend.clone()))
+			})?
+			.with_transaction_pool(|config, client, _fetcher| {
+				let pool_api = sc_transaction_pool::FullChainApi::new(client.clone());
+				let pool = sc_transaction_pool::BasicPool::new(config, std::sync::Arc::new(pool_api));
+				Ok(pool)
+			})?
+			.with_import_queue(|_config, client, mut select_chain, transaction_pool| {
+				let select_chain = select_chain.take()
+					.ok_or_else(|| sc_service::Error::SelectChainRequired)?;
+
+				let (grandpa_block_import, grandpa_link) =
+					grandpa::block_import::<_, _, _, node_template_runtime::RuntimeApi, _>(
+						client.clone(), &*client, select_chain
+					)?;
+
+				let aura_block_import = sc_consensus_aura::AuraBlockImport::<_, _, _, AuraPair>::new(
+					grandpa_block_import.clone(), client.clone(),
+				);
+
+				let import_queue = sc_consensus_aura::import_queue::<_, _, _, AuraPair, _>(
+					sc_consensus_aura::SlotDuration::get_or_compute(&*client)?,
+					aura_block_import,
+					Some(Box::new(grandpa_block_import.clone())),
+					None,
+					client,
+					inherent_data_providers.clone(),
+					Some(transaction_pool),
+				)?;
+
+				import_setup = Some((grandpa_block_import, grandpa_link));
+
+				Ok(import_queue)
+			})?;
+
+		(builder, import_setup, inherent_data_providers)
+	}}
+}
+
+/// Builds a new service for a full client.
+pub fn new_full(config: Configuration<GenesisConfig>)
+	-> Result<impl AbstractService, ServiceError>
+{
+	let is_authority = config.roles.is_authority();
+	let force_authoring = config.force_authoring;
+	let name = config.name.clone();
+	let disable_grandpa = config.disable_grandpa;
+
+	// sentry nodes announce themselves as authorities to the network
+	// and should run the same protocols authorities do, but it should
+	// never actively participate in any consensus process.
+	let participates_in_consensus = is_authority && !config.sentry_mode;
+
+	let (builder, mut import_setup, inherent_data_providers) = new_full_start!(config);
+
+	let (block_import, grandpa_link) =
+		import_setup.take()
+			.expect("Link Half and Block Import are present for Full Services or setup failed before. qed");
+
+	let service = builder.with_network_protocol(|_| Ok(NodeProtocol::new()))?
+		.with_finality_proof_provider(|client, backend|
+			Ok(Arc::new(GrandpaFinalityProofProvider::new(backend, client)) as _)
+		)?
+		.build()?;
+
+	if participates_in_consensus {
+		let proposer = sc_basic_authorship::ProposerFactory {
+			client: service.client(),
+			transaction_pool: service.transaction_pool(),
+		};
+
+		let client = service.client();
+		let select_chain = service.select_chain()
+			.ok_or(ServiceError::SelectChainRequired)?;
+
+		let can_author_with =
+			sp_consensus::CanAuthorWithNativeVersion::new(client.executor().clone());
+
+		let aura = sc_consensus_aura::start_aura::<_, _, _, _, _, AuraPair, _, _, _>(
+			sc_consensus_aura::SlotDuration::get_or_compute(&*client)?,
+			client,
+			select_chain,
+			block_import,
+			proposer,
+			service.network(),
+			inherent_data_providers.clone(),
+			force_authoring,
+			service.keystore(),
+			can_author_with,
+		)?;
+
+		// the AURA authoring task is considered essential, i.e. if it
+		// fails we take down the service with it.
+		service.spawn_essential_task("aura", aura);
+	}
+
+	// if the node isn't actively participating in consensus then it doesn't
+	// need a keystore, regardless of which protocol we use below.
+	let keystore = if participates_in_consensus {
+		Some(service.keystore())
+	} else {
+		None
+	};
+
+	let grandpa_config = grandpa::Config {
+		// FIXME #1578 make this available through chainspec
+		gossip_duration: Duration::from_millis(333),
+		justification_period: 512,
+		name: Some(name),
+		observer_enabled: true,
+		keystore,
+		is_authority,
+	};
+
+	match (is_authority, disable_grandpa) {
+		(false, false) => {
+			// start the lightweight GRANDPA observer
+			service.spawn_task("grandpa-observer", grandpa::run_grandpa_observer(
+				grandpa_config,
+				grandpa_link,
+				service.network(),
+				service.on_exit(),
+				service.spawn_task_handle(),
+			)?);
+		},
+		(true, false) => {
+			// start the full GRANDPA voter
+			let voter_config = grandpa::GrandpaParams {
+				config: grandpa_config,
+				link: grandpa_link,
+				network: service.network(),
+				inherent_data_providers: inherent_data_providers.clone(),
+				on_exit: service.on_exit(),
+				telemetry_on_connect: Some(service.telemetry_on_connect_stream()),
+				voting_rule: grandpa::VotingRulesBuilder::default().build(),
+				executor: service.spawn_task_handle(),
+			};
+
+			// the GRANDPA voter task is considered infallible, i.e.
+			// if it fails we take down the service with it.
+			service.spawn_essential_task("grandpa", grandpa::run_grandpa_voter(voter_config)?);
+		},
+		(_, true) => {
+			grandpa::setup_disabled_grandpa(
+				service.client(),
+				&inherent_data_providers,
+				service.network(),
+			)?;
+		},
+	}
+
+	Ok(service)
+}
+
+/// Builds a new service for a light client.
+pub fn new_light(config: Configuration<GenesisConfig>)
+	-> Result<impl AbstractService, ServiceError>
+{
+	let inherent_data_providers = InherentDataProviders::new();
+
+	ServiceBuilder::new_light::<Block, RuntimeApi, Executor>(config)?
+		.with_select_chain(|_config, backend| {
+			Ok(LongestChain::new(backend.clone()))
+		})?
+		.with_transaction_pool(|config, client, fetcher| {
+			let fetcher = fetcher
+				.ok_or_else(|| "Trying to start light transaction pool without active fetcher")?;
+
+			let pool_api = sc_transaction_pool::LightChainApi::new(client.clone(), fetcher.clone());
+			let pool = sc_transaction_pool::BasicPool::with_revalidation_type(
+				config, Arc::new(pool_api), sc_transaction_pool::RevalidationType::Light,
+			);
+			Ok(pool)
+		})?
+		.with_import_queue_and_fprb(|_config, client, backend, fetcher, _select_chain, _tx_pool| {
+			let fetch_checker = fetcher
+				.map(|fetcher| fetcher.checker().clone())
+				.ok_or_else(|| "Trying to start light import queue without active fetch checker")?;
+			let grandpa_block_import = grandpa::light_block_import::<_, _, _, RuntimeApi>(
+				client.clone(), backend, &*client.clone(), Arc::new(fetch_checker),
+			)?;
+			let finality_proof_import = grandpa_block_import.clone();
+			let finality_proof_request_builder =
+				finality_proof_import.create_finality_proof_request_builder();
+
+			let import_queue = sc_consensus_aura::import_queue::<_, _, _, AuraPair, ()>(
+				sc_consensus_aura::SlotDuration::get_or_compute(&*client)?,
+				grandpa_block_import,
+				None,
+				Some(Box::new(finality_proof_import)),
+				client,
+				inherent_data_providers.clone(),
+				None,
+			)?;
+
+			Ok((import_queue, finality_proof_request_builder))
+		})?
+		.with_network_protocol(|_| Ok(NodeProtocol::new()))?
+		.with_finality_proof_provider(|client, backend|
+			Ok(Arc::new(GrandpaFinalityProofProvider::new(backend, client)) as _)
+		)?
+		.build()
+}