Storage chains part 1 (#7868)

* CLI options and DB upgrade

* Transaction storage

* Block pruning

* Block pruning test

* Style

* Naming

* Apply suggestions from code review

Co-authored-by: Bastian Köcher <bkchr@users.noreply.github.com>

* Apply suggestions from code review

Co-authored-by: Bastian Köcher <bkchr@users.noreply.github.com>

* Style

Co-authored-by: Bastian Köcher <bkchr@users.noreply.github.com>

substrate/client/cli/src/config.rs

@@ -32,7 +32,7 @@ use sc_service::config::{
 	NodeKeyConfig, OffchainWorkerConfig, PrometheusConfig, PruningMode, Role, RpcMethods,
 	TaskExecutor, TelemetryEndpoints, TransactionPoolOptions, WasmExecutionMethod,
 };
-use sc_service::{ChainSpec, TracingReceiver};
+use sc_service::{ChainSpec, TracingReceiver, KeepBlocks, TransactionStorageMode };
 use std::net::SocketAddr;
 use std::path::PathBuf;
 
@@ -203,6 +203,13 @@ pub trait CliConfiguration<DCV: DefaultConfigurationValues = ()>: Sized {
 			.unwrap_or_default())
 	}
 
+	/// Get the database transaction storage scheme.
+	fn database_transaction_storage(&self) -> Result<TransactionStorageMode> {
+		Ok(self.database_params()
+			.map(|x| x.transaction_storage())
+			.unwrap_or(TransactionStorageMode::BlockBody))
+	}
+
 	/// Get the database backend variant.
 	///
 	/// By default this is retrieved from `DatabaseParams` if it is available. Otherwise its `None`.
@@ -244,16 +251,26 @@ pub trait CliConfiguration<DCV: DefaultConfigurationValues = ()>: Sized {
 		Ok(Default::default())
 	}
 
-	/// Get the pruning mode.
+	/// Get the state pruning mode.
 	///
 	/// By default this is retrieved from `PruningMode` if it is available. Otherwise its
 	/// `PruningMode::default()`.
-	fn pruning(&self, unsafe_pruning: bool, role: &Role) -> Result<PruningMode> {
+	fn state_pruning(&self, unsafe_pruning: bool, role: &Role) -> Result<PruningMode> {
 		self.pruning_params()
-			.map(|x| x.pruning(unsafe_pruning, role))
+			.map(|x| x.state_pruning(unsafe_pruning, role))
 			.unwrap_or_else(|| Ok(Default::default()))
 	}
 
+	/// Get the block pruning mode.
+	///
+	/// By default this is retrieved from `block_pruning` if it is available. Otherwise its
+	/// `KeepBlocks::All`.
+	fn keep_blocks(&self) -> Result<KeepBlocks> {
+		self.pruning_params()
+			.map(|x| x.keep_blocks())
+			.unwrap_or_else(|| Ok(KeepBlocks::All))
+	}
+
 	/// Get the chain ID (string).
 	///
 	/// By default this is retrieved from `SharedParams`.
@@ -493,7 +510,9 @@ pub trait CliConfiguration<DCV: DefaultConfigurationValues = ()>: Sized {
 			database: self.database_config(&config_dir, database_cache_size, database)?,
 			state_cache_size: self.state_cache_size()?,
 			state_cache_child_ratio: self.state_cache_child_ratio()?,
-			pruning: self.pruning(unsafe_pruning, &role)?,
+			state_pruning: self.state_pruning(unsafe_pruning, &role)?,
+			keep_blocks: self.keep_blocks()?,
+			transaction_storage: self.database_transaction_storage()?,
 			wasm_method: self.wasm_method()?,
 			wasm_runtime_overrides: self.wasm_runtime_overrides(),
 			execution_strategies: self.execution_strategies(is_dev, is_validator)?,

substrate/client/cli/src/params/database_params.rs

@@ -18,6 +18,7 @@
 
 use crate::arg_enums::Database;
 use structopt::StructOpt;
+use sc_service::TransactionStorageMode;
 
 /// Parameters for block import.
 #[derive(Debug, StructOpt)]
@@ -34,6 +35,15 @@ pub struct DatabaseParams {
 	/// Limit the memory the database cache can use.
 	#[structopt(long = "db-cache", value_name = "MiB")]
 	pub database_cache_size: Option<usize>,
+
+	/// Enable storage chain mode
+	///
+	/// This changes the storage format for blocks bodys. 
+	/// If this is enabled, each transaction is stored separately in the
+	/// transaction database column and is only referenced by hash
+	/// in the block body column.
+	#[structopt(long)]
+	pub storage_chain: bool,
 }
 
 impl DatabaseParams {
@@ -46,4 +56,13 @@ impl DatabaseParams {
 	pub fn database_cache_size(&self) -> Option<usize> {
 		self.database_cache_size
 	}
+
+	/// Transaction storage scheme.
+	pub fn transaction_storage(&self) -> TransactionStorageMode {
+		if self.storage_chain {
+			TransactionStorageMode::StorageChain
+		} else {
+			TransactionStorageMode::BlockBody
+		}
+	}
 }

substrate/client/cli/src/params/pruning_params.rs

@@ -17,7 +17,7 @@
 // along with this program. If not, see <https://www.gnu.org/licenses/>.
 
 use crate::error;
-use sc_service::{PruningMode, Role};
+use sc_service::{PruningMode, Role, KeepBlocks};
 use structopt::StructOpt;
 
 /// Parameters to define the pruning mode
@@ -30,11 +30,16 @@ pub struct PruningParams {
 	/// 256 blocks.
 	#[structopt(long = "pruning", value_name = "PRUNING_MODE")]
 	pub pruning: Option<String>,
+	/// Specify the number of finalized blocks to keep in the database.
+	///
+	/// Default is to keep all blocks.
+	#[structopt(long, value_name = "COUNT")]
+	pub keep_blocks: Option<u32>,
 }
 
 impl PruningParams {
 	/// Get the pruning value from the parameters
-	pub fn pruning(&self, unsafe_pruning: bool, role: &Role) -> error::Result<PruningMode> {
+	pub fn state_pruning(&self, unsafe_pruning: bool, role: &Role) -> error::Result<PruningMode> {
 		// by default we disable pruning if the node is an authority (i.e.
 		// `ArchiveAll`), otherwise we keep state for the last 256 blocks. if the
 		// node is an authority and pruning is enabled explicitly, then we error
@@ -58,4 +63,12 @@ impl PruningParams {
 			}
 		})
 	}
+
+	/// Get the block pruning value from the parameters
+	pub fn keep_blocks(&self) -> error::Result<KeepBlocks> {
+		Ok(match self.keep_blocks {
+			Some(n) => KeepBlocks::Some(n),
+			None => KeepBlocks::All,
+		})
+	}
 }