DatabaseSource::Auto (#9500)

* implement "auto" database backend in client/db, in progress, #9201

* move fn supports_ref_counting from DatabaseSource enum to Database trait to make it work correctly for all types of dbs

* update kvdb_rocksdb to 0.13 and use it's new config feature  to properly auto start existing database

* tests for auto database reopening

* introduce OpenDbError to cleanup opening database error handling and handle case when database is not enabled at the compile time

* cargo fmt strings again

* cargo fmt strings again

* rename DataSettingsSrc to fix test compilation

* fix the call to the new kvdb-rocksdb interdace in tests to fix compilation

* simplify OpenDbError and make it compile even when paritydb and rocksdb are disabled

* cargo fmt

* fix compilation without flag with-parity-db

* fix unused var compilation warning

* support different paths for rocksdb and paritydb in DatabaseSouce::Auto

* support "auto" database option in substrate cli

* enable Lz4 compression for some of the parity-db colums as per review suggestion

* applied review suggestions

substrate/client/cli/src/arg_enums.rs

@@ -197,6 +197,9 @@ pub enum Database {
 	RocksDb,
 	/// ParityDb. <https://github.com/paritytech/parity-db/>
 	ParityDb,
+	/// Detect whether there is an existing database. Use it, if there is, if not, create new
+	/// instance of paritydb
+	Auto,
 }
 
 impl std::str::FromStr for Database {
@@ -207,6 +210,8 @@ impl std::str::FromStr for Database {
 			Ok(Self::RocksDb)
 		} else if s.eq_ignore_ascii_case("paritydb-experimental") {
 			Ok(Self::ParityDb)
+		} else if s.eq_ignore_ascii_case("auto") {
+			Ok(Self::Auto)
 		} else {
 			Err(format!("Unknown variant `{}`, known variants: {:?}", s, Self::variants()))
 		}
@@ -216,7 +221,7 @@ impl std::str::FromStr for Database {
 impl Database {
 	/// Returns all the variants of this enum to be shown in the cli.
 	pub fn variants() -> &'static [&'static str] {
-		&["rocksdb", "paritydb-experimental"]
+		&["rocksdb", "paritydb-experimental", "auto"]
 	}
 }
 

substrate/client/cli/src/commands/export_blocks_cmd.rs

@@ -23,7 +23,7 @@ use crate::{
 };
 use log::info;
 use sc_client_api::{BlockBackend, UsageProvider};
-use sc_service::{chain_ops::export_blocks, config::DatabaseConfig};
+use sc_service::{chain_ops::export_blocks, config::DatabaseSource};
 use sp_runtime::traits::{Block as BlockT, Header as HeaderT};
 use std::{fmt::Debug, fs, io, path::PathBuf, str::FromStr, sync::Arc};
 use structopt::StructOpt;
@@ -69,14 +69,14 @@ impl ExportBlocksCmd {
 	pub async fn run<B, C>(
 		&self,
 		client: Arc<C>,
-		database_config: DatabaseConfig,
+		database_config: DatabaseSource,
 	) -> error::Result<()>
 	where
 		B: BlockT,
 		C: BlockBackend<B> + UsageProvider<B> + 'static,
 		<<B::Header as HeaderT>::Number as FromStr>::Err: Debug,
 	{
-		if let DatabaseConfig::RocksDb { ref path, .. } = database_config {
+		if let DatabaseSource::RocksDb { ref path, .. } = database_config {
 			info!("DB path: {}", path.display());
 		}
 

substrate/client/cli/src/commands/purge_chain_cmd.rs

@@ -21,7 +21,7 @@ use crate::{
 	params::{DatabaseParams, SharedParams},
 	CliConfiguration,
 };
-use sc_service::DatabaseConfig;
+use sc_service::DatabaseSource;
 use std::{
 	fmt::Debug,
 	fs,
@@ -47,7 +47,7 @@ pub struct PurgeChainCmd {
 
 impl PurgeChainCmd {
 	/// Run the purge command
-	pub fn run(&self, database_config: DatabaseConfig) -> error::Result<()> {
+	pub fn run(&self, database_config: DatabaseSource) -> error::Result<()> {
 		let db_path = database_config.path().ok_or_else(|| {
 			error::Error::Input("Cannot purge custom database implementation".into())
 		})?;

substrate/client/cli/src/config.rs

@@ -27,7 +27,7 @@ use names::{Generator, Name};
 use sc_client_api::execution_extensions::ExecutionStrategies;
 use sc_service::{
 	config::{
-		BasePath, Configuration, DatabaseConfig, ExtTransport, KeystoreConfig,
+		BasePath, Configuration, DatabaseSource, ExtTransport, KeystoreConfig,
 		NetworkConfiguration, NodeKeyConfig, OffchainWorkerConfig, PrometheusConfig, PruningMode,
 		Role, RpcMethods, TaskExecutor, TelemetryEndpoints, TransactionPoolOptions,
 		WasmExecutionMethod,
@@ -220,10 +220,13 @@ pub trait CliConfiguration<DCV: DefaultConfigurationValues = ()>: Sized {
 		base_path: &PathBuf,
 		cache_size: usize,
 		database: Database,
-	) -> Result<DatabaseConfig> {
+	) -> Result<DatabaseSource> {
+		let rocksdb_path = base_path.join("db");
+		let paritydb_path = base_path.join("paritydb");
 		Ok(match database {
-			Database::RocksDb => DatabaseConfig::RocksDb { path: base_path.join("db"), cache_size },
-			Database::ParityDb => DatabaseConfig::ParityDb { path: base_path.join("paritydb") },
+			Database::RocksDb => DatabaseSource::RocksDb { path: rocksdb_path, cache_size },
+			Database::ParityDb => DatabaseSource::ParityDb { path: rocksdb_path },
+			Database::Auto => DatabaseSource::Auto { paritydb_path, rocksdb_path, cache_size },
 		})
 	}