Add a feature to create automatically a random temporary directory for base path & remove Clone (#6221)

* Initial commit

Forked at: 4adac40c07
Parent branch: origin/master

* Add a feature to create automatically a temporary directory for base path

* doc fix and todos

* use parking_lot instead

* use refcell instead since we stay in the main thread

* remove Clone derives

* add test

* solving dependency issue

* clarifying doc

* conflict argument with base-path

* WIP

Forked at: 4adac40c07
Parent branch: origin/master

* revert dep deletion

* fixing test and making base_path optional

* hold basepath while the service is running

* fixes

* Update client/cli/src/params/shared_params.rs

Co-authored-by: Bastian Köcher <bkchr@users.noreply.github.com>

* Update client/service/Cargo.toml

Co-authored-by: Bastian Köcher <bkchr@users.noreply.github.com>

* Update client/cli/src/commands/mod.rs

Co-authored-by: Bastian Köcher <bkchr@users.noreply.github.com>

* Update client/service/src/config.rs

Co-authored-by: Bastian Köcher <bkchr@users.noreply.github.com>

* WIP

Forked at: 4adac40c07
Parent branch: origin/master

* improve doc

Co-authored-by: Bastian Köcher <bkchr@users.noreply.github.com>

substrate/client/cli/src/commands/build_spec_cmd.rs

@@ -27,7 +27,7 @@ use structopt::StructOpt;
 use std::io::Write;
 
 /// The `build-spec` command used to build a specification.
-#[derive(Debug, StructOpt, Clone)]
+#[derive(Debug, StructOpt)]
 pub struct BuildSpecCmd {
 	/// Force raw genesis storage output.
 	#[structopt(long = "raw")]

substrate/client/cli/src/commands/check_block_cmd.rs

@@ -25,7 +25,7 @@ use std::{fmt::Debug, str::FromStr};
 use structopt::StructOpt;
 
 /// The `check-block` command used to validate blocks.
-#[derive(Debug, StructOpt, Clone)]
+#[derive(Debug, StructOpt)]
 pub struct CheckBlockCmd {
 	/// Block hash or number
 	#[structopt(value_name = "HASH or NUMBER")]

substrate/client/cli/src/commands/export_blocks_cmd.rs

@@ -31,7 +31,7 @@ use std::path::PathBuf;
 use structopt::StructOpt;
 
 /// The `export-blocks` command used to export blocks.
-#[derive(Debug, StructOpt, Clone)]
+#[derive(Debug, StructOpt)]
 pub struct ExportBlocksCmd {
 	/// Output file name or stdout if unspecified.
 	#[structopt(parse(from_os_str))]

substrate/client/cli/src/commands/export_state_cmd.rs

@@ -27,7 +27,7 @@ use structopt::StructOpt;
 
 /// The `export-state` command used to export the state of a given block into
 /// a chain spec.
-#[derive(Debug, StructOpt, Clone)]
+#[derive(Debug, StructOpt)]
 pub struct ExportStateCmd {
 	/// Block hash or number.
 	#[structopt(value_name = "HASH or NUMBER")]
@@ -59,7 +59,7 @@ impl ExportStateCmd {
 	{
 		info!("Exporting raw state...");
 		let mut input_spec = config.chain_spec.cloned_box();
-		let block_id = self.input.clone().map(|b| b.parse()).transpose()?;
+		let block_id = self.input.as_ref().map(|b| b.parse()).transpose()?;
 		let raw_state = builder(config)?.export_raw_state(block_id)?;
 		input_spec.set_storage(raw_state);
 

substrate/client/cli/src/commands/import_blocks_cmd.rs

@@ -29,7 +29,7 @@ use std::path::PathBuf;
 use structopt::StructOpt;
 
 /// The `import-blocks` command used to import blocks.
-#[derive(Debug, StructOpt, Clone)]
+#[derive(Debug, StructOpt)]
 pub struct ImportBlocksCmd {
 	/// Input file or stdin if unspecified.
 	#[structopt(parse(from_os_str))]

substrate/client/cli/src/commands/mod.rs

@@ -27,11 +27,11 @@ mod run_cmd;
 pub use self::build_spec_cmd::BuildSpecCmd;
 pub use self::check_block_cmd::CheckBlockCmd;
 pub use self::export_blocks_cmd::ExportBlocksCmd;
+pub use self::export_state_cmd::ExportStateCmd;
 pub use self::import_blocks_cmd::ImportBlocksCmd;
 pub use self::purge_chain_cmd::PurgeChainCmd;
 pub use self::revert_cmd::RevertCmd;
 pub use self::run_cmd::RunCmd;
-pub use self::export_state_cmd::ExportStateCmd;
 use std::fmt::Debug;
 use structopt::StructOpt;
 
@@ -40,7 +40,7 @@ use structopt::StructOpt;
 /// The core commands are split into multiple subcommands and `Run` is the default subcommand. From
 /// the CLI user perspective, it is not visible that `Run` is a subcommand. So, all parameters of
 /// `Run` are exported as main executable parameters.
-#[derive(Debug, Clone, StructOpt)]
+#[derive(Debug, StructOpt)]
 pub enum Subcommand {
 	/// Build a spec.json file, outputs to stdout.
 	BuildSpec(BuildSpecCmd),
@@ -162,7 +162,7 @@ macro_rules! substrate_cli_subcommands {
 				}
 			}
 
-			fn base_path(&self) -> $crate::Result<::std::option::Option<::std::path::PathBuf>> {
+			fn base_path(&self) -> $crate::Result<::std::option::Option<sc_service::config::BasePath>> {
 				match self {
 					$($enum::$variant(cmd) => cmd.base_path()),*
 				}
@@ -409,4 +409,3 @@ macro_rules! substrate_cli_subcommands {
 substrate_cli_subcommands!(
 	Subcommand => BuildSpec, ExportBlocks, ImportBlocks, CheckBlock, Revert, PurgeChain, ExportState
 );
-

substrate/client/cli/src/commands/purge_chain_cmd.rs

@@ -26,7 +26,7 @@ use std::io::{self, Write};
 use structopt::StructOpt;
 
 /// The `purge-chain` command used to remove the whole chain.
-#[derive(Debug, StructOpt, Clone)]
+#[derive(Debug, StructOpt)]
 pub struct PurgeChainCmd {
 	/// Skip interactive prompt by answering yes automatically.
 	#[structopt(short = "y")]

substrate/client/cli/src/commands/revert_cmd.rs

@@ -25,7 +25,7 @@ use std::fmt::Debug;
 use structopt::StructOpt;
 
 /// The `revert` command used revert the chain to a previous state.
-#[derive(Debug, StructOpt, Clone)]
+#[derive(Debug, StructOpt)]
 pub struct RevertCmd {
 	/// Number of blocks to revert.
 	#[structopt(default_value = "256")]

substrate/client/cli/src/commands/run_cmd.rs

@@ -21,13 +21,13 @@ use crate::error::{Error, Result};
 use crate::params::ImportParams;
 use crate::params::KeystoreParams;
 use crate::params::NetworkParams;
+use crate::params::OffchainWorkerParams;
 use crate::params::SharedParams;
 use crate::params::TransactionPoolParams;
-use crate::params::OffchainWorkerParams;
 use crate::CliConfiguration;
 use regex::Regex;
 use sc_service::{
-	config::{MultiaddrWithPeerId, PrometheusConfig, TransactionPoolOptions},
+	config::{BasePath, MultiaddrWithPeerId, PrometheusConfig, TransactionPoolOptions},
 	ChainSpec, Role,
 };
 use sc_telemetry::TelemetryEndpoints;
@@ -35,7 +35,7 @@ use std::net::{IpAddr, Ipv4Addr, SocketAddr};
 use structopt::StructOpt;
 
 /// The `run` command used to run a node.
-#[derive(Debug, StructOpt, Clone)]
+#[derive(Debug, StructOpt)]
 pub struct RunCmd {
 	/// Enable validator mode.
 	///
@@ -250,6 +250,16 @@ pub struct RunCmd {
 		conflicts_with_all = &[ "sentry", "public-addr" ]
 	)]
 	pub sentry_nodes: Vec<MultiaddrWithPeerId>,
+
+	/// Run a temporary node.
+	///
+	/// A temporary directory will be created to store the configuration and will be deleted
+	/// at the end of the process.
+	///
+	/// Note: the directory is random per process execution. This directory is used as base path
+	/// which includes: database, node key and keystore.
+	#[structopt(long, conflicts_with = "base-path")]
+	pub tmp: bool,
 }
 
 impl RunCmd {
@@ -446,6 +456,14 @@ impl CliConfiguration for RunCmd {
 	fn max_runtime_instances(&self) -> Result<Option<usize>> {
 		Ok(self.max_runtime_instances.map(|x| x.min(256)))
 	}
+
+	fn base_path(&self) -> Result<Option<BasePath>> {
+		Ok(if self.tmp {
+			Some(BasePath::new_temp_dir()?)
+		} else {
+			self.shared_params().base_path()
+		})
+	}
 }
 
 /// Check whether a node name is considered as valid.

substrate/client/cli/src/config.rs

@@ -27,9 +27,9 @@ use crate::{
 use names::{Generator, Name};
 use sc_client_api::execution_extensions::ExecutionStrategies;
 use sc_service::config::{
-	Configuration, DatabaseConfig, ExtTransport, KeystoreConfig, NetworkConfiguration,
-	NodeKeyConfig, OffchainWorkerConfig, PrometheusConfig, PruningMode, Role, RpcMethods,
-	TaskType, TelemetryEndpoints, TransactionPoolOptions, WasmExecutionMethod,
+	BasePath, Configuration, DatabaseConfig, ExtTransport, KeystoreConfig, NetworkConfiguration,
+	NodeKeyConfig, OffchainWorkerConfig, PrometheusConfig, PruningMode, Role, RpcMethods, TaskType,
+	TelemetryEndpoints, TransactionPoolOptions, WasmExecutionMethod,
 };
 use sc_service::{ChainSpec, TracingReceiver};
 use std::future::Future;
@@ -87,7 +87,7 @@ pub trait CliConfiguration: Sized {
 	/// Get the base path of the configuration (if any)
 	///
 	/// By default this is retrieved from `SharedParams`.
-	fn base_path(&self) -> Result<Option<PathBuf>> {
+	fn base_path(&self) -> Result<Option<BasePath>> {
 		Ok(self.shared_params().base_path())
 	}
 
@@ -402,14 +402,12 @@ pub trait CliConfiguration: Sized {
 		let is_dev = self.is_dev()?;
 		let chain_id = self.chain_id(is_dev)?;
 		let chain_spec = cli.load_spec(chain_id.as_str())?;
-		let config_dir = self
+		let base_path = self
 			.base_path()?
-			.unwrap_or_else(|| {
-				directories::ProjectDirs::from("", "", C::executable_name())
-					.expect("app directories exist on all supported platforms; qed")
-					.data_local_dir()
-					.into()
-			})
+			.unwrap_or_else(|| BasePath::from_project("", "", C::executable_name()));
+		let config_dir = base_path
+			.path()
+			.to_path_buf()
 			.join("chains")
 			.join(chain_spec.id());
 		let net_config_dir = config_dir.join(DEFAULT_NETWORK_CONFIG_PATH);
@@ -464,6 +462,7 @@ pub trait CliConfiguration: Sized {
 			max_runtime_instances,
 			announce_block: self.announce_block()?,
 			role,
+			base_path: Some(base_path),
 		})
 	}
 
@@ -507,5 +506,5 @@ pub fn generate_node_name() -> String {
 		if count < NODE_NAME_MAX_LENGTH {
 			return node_name;
 		}
-	};
+	}
 }

substrate/client/cli/src/params/database_params.rs

@@ -20,7 +20,7 @@ use crate::arg_enums::Database;
 use structopt::StructOpt;
 
 /// Parameters for block import.
-#[derive(Debug, StructOpt, Clone)]
+#[derive(Debug, StructOpt)]
 pub struct DatabaseParams {
 	/// Select database backend to use.
 	#[structopt(

substrate/client/cli/src/params/import_params.rs

@@ -27,7 +27,7 @@ use sc_client_api::execution_extensions::ExecutionStrategies;
 use structopt::StructOpt;
 
 /// Parameters for block import.
-#[derive(Debug, StructOpt, Clone)]
+#[derive(Debug, StructOpt)]
 pub struct ImportParams {
 	#[allow(missing_docs)]
 	#[structopt(flatten)]
@@ -130,7 +130,7 @@ impl ImportParams {
 }
 
 /// Execution strategies parameters.
-#[derive(Debug, StructOpt, Clone)]
+#[derive(Debug, StructOpt)]
 pub struct ExecutionStrategiesParams {
 	/// The means of execution used when calling into the runtime while syncing blocks.
 	#[structopt(

substrate/client/cli/src/params/keystore_params.rs

@@ -26,7 +26,7 @@ use structopt::StructOpt;
 const DEFAULT_KEYSTORE_CONFIG_PATH: &'static str = "keystore";
 
 /// Parameters of the keystore
-#[derive(Debug, StructOpt, Clone)]
+#[derive(Debug, StructOpt)]
 pub struct KeystoreParams {
 	/// Specify custom keystore path.
 	#[structopt(long = "keystore-path", value_name = "PATH", parse(from_os_str))]

substrate/client/cli/src/params/mod.rs

@@ -39,7 +39,7 @@ pub use crate::params::shared_params::*;
 pub use crate::params::transaction_pool_params::*;
 
 /// Wrapper type of `String` that holds an unsigned integer of arbitrary size, formatted as a decimal.
-#[derive(Debug, Clone)]
+#[derive(Debug)]
 pub struct BlockNumber(String);
 
 impl FromStr for BlockNumber {
@@ -72,7 +72,7 @@ impl BlockNumber {
 }
 
 /// Wrapper type that is either a `Hash` or the number of a `Block`.
-#[derive(Debug, Clone)]
+#[derive(Debug)]
 pub struct BlockNumberOrHash(String);
 
 impl FromStr for BlockNumberOrHash {

substrate/client/cli/src/params/network_params.rs

@@ -26,7 +26,7 @@ use std::path::PathBuf;
 use structopt::StructOpt;
 
 /// Parameters used to create the network configuration.
-#[derive(Debug, StructOpt, Clone)]
+#[derive(Debug, StructOpt)]
 pub struct NetworkParams {
 	/// Specify a list of bootnodes.
 	#[structopt(long = "bootnodes", value_name = "ADDR")]

substrate/client/cli/src/params/node_key_params.rs

@@ -31,7 +31,7 @@ const NODE_KEY_ED25519_FILE: &str = "secret_ed25519";
 
 /// Parameters used to create the `NodeKeyConfig`, which determines the keypair
 /// used for libp2p networking.
-#[derive(Debug, StructOpt, Clone)]
+#[derive(Debug, StructOpt)]
 pub struct NodeKeyParams {
 	/// The secret key to use for libp2p networking.
 	///

substrate/client/cli/src/params/offchain_worker_params.rs

@@ -32,7 +32,7 @@ use crate::OffchainWorkerEnabled;
 
 
 /// Offchain worker related parameters.
-#[derive(Debug, StructOpt, Clone)]
+#[derive(Debug, StructOpt)]
 pub struct OffchainWorkerParams {
 	/// Should execute offchain workers on every block.
 	///

substrate/client/cli/src/params/pruning_params.rs

@@ -21,7 +21,7 @@ use sc_service::{PruningMode, Role};
 use structopt::StructOpt;
 
 /// Parameters to define the pruning mode
-#[derive(Debug, StructOpt, Clone)]
+#[derive(Debug, StructOpt)]
 pub struct PruningParams {
 	/// Specify the state pruning mode, a number of blocks to keep or 'archive'.
 	///

substrate/client/cli/src/params/shared_params.rs

@@ -16,11 +16,12 @@
 // You should have received a copy of the GNU General Public License
 // along with this program. If not, see <https://www.gnu.org/licenses/>.
 
+use sc_service::config::BasePath;
 use std::path::PathBuf;
 use structopt::StructOpt;
 
 /// Shared parameters used by all `CoreParams`.
-#[derive(Debug, StructOpt, Clone)]
+#[derive(Debug, StructOpt)]
 pub struct SharedParams {
 	/// Specify the chain specification (one of dev, local, or staging).
 	#[structopt(long, value_name = "CHAIN_SPEC")]
@@ -31,12 +32,7 @@ pub struct SharedParams {
 	pub dev: bool,
 
 	/// Specify custom base path.
-	#[structopt(
-		long,
-		short = "d",
-		value_name = "PATH",
-		parse(from_os_str)
-	)]
+	#[structopt(long, short = "d", value_name = "PATH", parse(from_os_str))]
 	pub base_path: Option<PathBuf>,
 
 	/// Sets a custom logging filter. Syntax is <target>=<level>, e.g. -lsync=debug.
@@ -49,8 +45,8 @@ pub struct SharedParams {
 
 impl SharedParams {
 	/// Specify custom base path.
-	pub fn base_path(&self) -> Option<PathBuf> {
-		self.base_path.clone()
+	pub fn base_path(&self) -> Option<BasePath> {
+		self.base_path.clone().map(Into::into)
 	}
 
 	/// Specify the development chain.

substrate/client/cli/src/params/transaction_pool_params.rs

@@ -20,7 +20,7 @@ use sc_service::config::TransactionPoolOptions;
 use structopt::StructOpt;
 
 /// Parameters used to create the pool configuration.
-#[derive(Debug, StructOpt, Clone)]
+#[derive(Debug, StructOpt)]
 pub struct TransactionPoolParams {
 	/// Maximum number of transactions in the transaction pool.
 	#[structopt(long = "pool-limit", value_name = "COUNT", default_value = "8192")]

substrate/client/cli/src/runner.rs

@@ -271,6 +271,10 @@ impl<C: SubstrateCli> Runner<C> {
 		// and drop the runtime first.
 		let _telemetry = service.telemetry();
 
+		// we hold a reference to the base path so if the base path is a temporary directory it will
+		// not be deleted before the tokio runtime finish to clean up
+		let _base_path = service.base_path();
+
 		{
 			let f = service.fuse();
 			self.tokio_runtime