New database trait (#5549)

* Introduce trait * The trait * Generic * Basic impls. * Remove unneeded bounds * Minor changes * Switch over to the new DB trait * Integrated parity-db and added CLI for db selection * Default impl. * Fix logs. * Started integrating subdb * Apply suggestions from code review Co-Authored-By: Cecile Tonglet <cecile@parity.io> * Apply suggestions from code review Co-Authored-By: Nikolay Volf <nikvolf@gmail.com> * Enable subdb * Bump parity-db * Fixed CLI macro * Fixed browser build * Fixed features * Sort out features * Use parity-db from crates.io * Typo Co-authored-by: arkpar <arkady.paronyan@gmail.com> Co-authored-by: Cecile Tonglet <cecile@parity.io> Co-authored-by: Nikolay Volf <nikvolf@gmail.com>
2026-06-12 17:01:09 +00:00 · 2020-04-15 14:38:39 +02:00
parent 3426d662f7
commit 91af5b6fcc
43 changed files with 1036 additions and 579 deletions
@@ -18,21 +18,19 @@
 //! full and light storages.

 use std::sync::Arc;
-use std::{io, convert::TryInto};
+use std::convert::TryInto;

-use kvdb::{KeyValueDB, DBTransaction};
-#[cfg(any(feature = "kvdb-rocksdb", test))]
-use kvdb_rocksdb::{Database, DatabaseConfig};
 use log::debug;

 use codec::Decode;
 use sp_trie::DBValue;
+use sp_database::Transaction;
 use sp_runtime::generic::BlockId;
 use sp_runtime::traits::{
 	Block as BlockT, Header as HeaderT, Zero,
 	UniqueSaturatedFrom, UniqueSaturatedInto,
 };
-use crate::{DatabaseSettings, DatabaseSettingsSrc};
+use crate::{DatabaseSettings, DatabaseSettingsSrc, Database, DbHash};

 /// Number of columns in the db. Must be the same for both full && light dbs.
 /// Otherwise RocksDb will fail to open database && check its type.
@@ -136,35 +134,35 @@ pub fn lookup_key_to_number<N>(key: &[u8]) -> sp_blockchain::Result<N> where

 /// Delete number to hash mapping in DB transaction.
 pub fn remove_number_to_key_mapping<N: TryInto<u32>>(
-	transaction: &mut DBTransaction,
+	transaction: &mut Transaction<DbHash>,
 	key_lookup_col: u32,
 	number: N,
 ) -> sp_blockchain::Result<()> {
-	transaction.delete(key_lookup_col, number_index_key(number)?.as_ref());
+	transaction.remove(key_lookup_col, number_index_key(number)?.as_ref());
 	Ok(())
 }

 /// Remove key mappings.
 pub fn remove_key_mappings<N: TryInto<u32>, H: AsRef<[u8]>>(
-	transaction: &mut DBTransaction,
+	transaction: &mut Transaction<DbHash>,
 	key_lookup_col: u32,
 	number: N,
 	hash: H,
 ) -> sp_blockchain::Result<()> {
 	remove_number_to_key_mapping(transaction, key_lookup_col, number)?;
-	transaction.delete(key_lookup_col, hash.as_ref());
+	transaction.remove(key_lookup_col, hash.as_ref());
 	Ok(())
 }

 /// Place a number mapping into the database. This maps number to current perceived
 /// block hash at that position.
 pub fn insert_number_to_key_mapping<N: TryInto<u32> + Clone, H: AsRef<[u8]>>(
-	transaction: &mut DBTransaction,
+	transaction: &mut Transaction<DbHash>,
 	key_lookup_col: u32,
 	number: N,
 	hash: H,
 ) -> sp_blockchain::Result<()> {
-	transaction.put_vec(
+	transaction.set_from_vec(
 		key_lookup_col,
 		number_index_key(number.clone())?.as_ref(),
 		number_and_hash_to_lookup_key(number, hash)?,
@@ -174,12 +172,12 @@ pub fn insert_number_to_key_mapping<N: TryInto<u32> + Clone, H: AsRef<[u8]>>(

 /// Insert a hash to key mapping in the database.
 pub fn insert_hash_to_key_mapping<N: TryInto<u32>, H: AsRef<[u8]> + Clone>(
-	transaction: &mut DBTransaction,
+	transaction: &mut Transaction<DbHash>,
 	key_lookup_col: u32,
 	number: N,
 	hash: H,
 ) -> sp_blockchain::Result<()> {
-	transaction.put_vec(
+	transaction.set_from_vec(
 		key_lookup_col,
 		hash.clone().as_ref(),
 		number_and_hash_to_lookup_key(number, hash)?,
@@ -191,42 +189,35 @@ pub fn insert_hash_to_key_mapping<N: TryInto<u32>, H: AsRef<[u8]> + Clone>(
 /// block lookup key is the DB-key header, block and justification are stored under.
 /// looks up lookup key by hash from DB as necessary.
 pub fn block_id_to_lookup_key<Block>(
-	db: &dyn KeyValueDB,
+	db: &dyn Database<DbHash>,
 	key_lookup_col: u32,
 	id: BlockId<Block>
 ) -> Result<Option<Vec<u8>>, sp_blockchain::Error> where
 	Block: BlockT,
 	::sp_runtime::traits::NumberFor<Block>: UniqueSaturatedFrom<u64> + UniqueSaturatedInto<u64>,
 {
-	let res = match id {
+	Ok(match id {
 		BlockId::Number(n) => db.get(
 			key_lookup_col,
 			number_index_key(n)?.as_ref(),
 		),
-		BlockId::Hash(h) => db.get(key_lookup_col, h.as_ref()),
-	};
-
-	res.map_err(db_err)
+		BlockId::Hash(h) => db.get(key_lookup_col, h.as_ref())
+	})
 }

-/// Maps database error to client error
-pub fn db_err(err: io::Error) -> sp_blockchain::Error {
-	sp_blockchain::Error::Backend(format!("{}", err))
-}
-
-/// Open RocksDB database.
+/// Opens the configured database.
 pub fn open_database<Block: BlockT>(
 	config: &DatabaseSettings,
 	db_type: DatabaseType,
-) -> sp_blockchain::Result<Arc<dyn KeyValueDB>> {
-	let db: Arc<dyn KeyValueDB> = match &config.source {
+) -> sp_blockchain::Result<Arc<dyn Database<DbHash>>> {
+	let db: Arc<dyn Database<DbHash>> = match &config.source {
 		#[cfg(any(feature = "kvdb-rocksdb", test))]
-		DatabaseSettingsSrc::Path { path, cache_size } => {
+		DatabaseSettingsSrc::RocksDb { path, cache_size } => {
 			// first upgrade database to required version
 			crate::upgrade::upgrade_db::<Block>(&path, db_type)?;

 			// and now open database assuming that it has the latest version
-			let mut db_config = DatabaseConfig::with_columns(NUM_COLUMNS);
+			let mut db_config = kvdb_rocksdb::DatabaseConfig::with_columns(NUM_COLUMNS);
 			let state_col_budget = (*cache_size as f64 * 0.9) as usize;
 			let other_col_budget = (cache_size - state_col_budget) / (NUM_COLUMNS as usize - 1);
 			let mut memory_budget = std::collections::HashMap::new();
@@ -245,21 +236,32 @@ pub fn open_database<Block: BlockT>(

 			log::trace!(
 				target: "db",
-				"Open database at {}, state column budget: {} MiB, others({}) column cache: {} MiB",
+				"Open RocksDB database at {}, state column budget: {} MiB, others({}) column cache: {} MiB",
 				path,
 				state_col_budget,
 				NUM_COLUMNS,
 				other_col_budget,
 			);

-			Arc::new(Database::open(&db_config, &path).map_err(db_err)?)
+			let db = kvdb_rocksdb::Database::open(&db_config, &path)
+				.map_err(|err| sp_blockchain::Error::Backend(format!("{}", err)))?;
+			sp_database::as_database(db)
 		},
-		#[cfg(not(any(feature = "kvdb-rocksdb", test)))]
-		DatabaseSettingsSrc::Path { .. } => {
-			let msg = "Try to open RocksDB database with RocksDB disabled".into();
-			return Err(sp_blockchain::Error::Backend(msg));
+		#[cfg(feature = "subdb")]
+		DatabaseSettingsSrc::SubDb { path } => {
+			crate::subdb::open(&path, NUM_COLUMNS)
+				.map_err(|e| sp_blockchain::Error::Backend(format!("{:?}", e)))?
+		},
+		#[cfg(feature = "parity-db")]
+		DatabaseSettingsSrc::ParityDb { path } => {
+			crate::parity_db::open(&path, NUM_COLUMNS)
+				.map_err(|e| sp_blockchain::Error::Backend(format!("{:?}", e)))?
 		},
 		DatabaseSettingsSrc::Custom(db) => db.clone(),
+		_ => {
+			let msg = "Trying to open a unsupported database".into();
+			return Err(sp_blockchain::Error::Backend(msg));
+		},
 	};

 	check_database_type(&*db, db_type)?;
@@ -268,8 +270,8 @@ pub fn open_database<Block: BlockT>(
 }

 /// Check database type.
-pub fn check_database_type(db: &dyn KeyValueDB, db_type: DatabaseType) -> sp_blockchain::Result<()> {
-	match db.get(COLUMN_META, meta_keys::TYPE).map_err(db_err)? {
+pub fn check_database_type(db: &dyn Database<DbHash>, db_type: DatabaseType) -> sp_blockchain::Result<()> {
+	match db.get(COLUMN_META, meta_keys::TYPE) {
 		Some(stored_type) => {
 			if db_type.as_str().as_bytes() != &*stored_type {
 				return Err(sp_blockchain::Error::Backend(
@@ -277,9 +279,9 @@ pub fn check_database_type(db: &dyn KeyValueDB, db_type: DatabaseType) -> sp_blo
 			}
 		},
 		None => {
-			let mut transaction = DBTransaction::new();
-			transaction.put(COLUMN_META, meta_keys::TYPE, db_type.as_str().as_bytes());
-			db.write(transaction).map_err(db_err)?;
+			let mut transaction = Transaction::new();
+			transaction.set(COLUMN_META, meta_keys::TYPE, db_type.as_str().as_bytes());
+			db.commit(transaction)
 		},
 	}

@@ -288,7 +290,7 @@ pub fn check_database_type(db: &dyn KeyValueDB, db_type: DatabaseType) -> sp_blo

 /// Read database column entry for the given block.
 pub fn read_db<Block>(
-	db: &dyn KeyValueDB,
+	db: &dyn Database<DbHash>,
 	col_index: u32,
 	col: u32,
 	id: BlockId<Block>
@@ -297,14 +299,14 @@ pub fn read_db<Block>(
 		Block: BlockT,
 {
 	block_id_to_lookup_key(db, col_index, id).and_then(|key| match key {
-		Some(key) => db.get(col, key.as_ref()).map_err(db_err),
+		Some(key) => Ok(db.get(col, key.as_ref())),
 		None => Ok(None),
 	})
 }

 /// Read a header from the database.
 pub fn read_header<Block: BlockT>(
-	db: &dyn KeyValueDB,
+	db: &dyn Database<DbHash>,
 	col_index: u32,
 	col: u32,
 	id: BlockId<Block>,
@@ -322,7 +324,7 @@ pub fn read_header<Block: BlockT>(

 /// Required header from the database.
 pub fn require_header<Block: BlockT>(
-	db: &dyn KeyValueDB,
+	db: &dyn Database<DbHash>,
 	col_index: u32,
 	col: u32,
 	id: BlockId<Block>,
@@ -334,7 +336,7 @@ pub fn require_header<Block: BlockT>(
 }

 /// Read meta from the database.
-pub fn read_meta<Block>(db: &dyn KeyValueDB, col_header: u32) -> Result<
+pub fn read_meta<Block>(db: &dyn Database<DbHash>, col_header: u32) -> Result<
 	Meta<<<Block as BlockT>::Header as HeaderT>::Number, Block::Hash>,
 	sp_blockchain::Error,
 >
@@ -353,11 +355,10 @@ pub fn read_meta<Block>(db: &dyn KeyValueDB, col_header: u32) -> Result<
 	};

 	let load_meta_block = |desc, key| -> Result<_, sp_blockchain::Error> {
-		if let Some(Some(header)) = db.get(COLUMN_META, key).and_then(|id|
-			match id {
-				Some(id) => db.get(col_header, &id).map(|h| h.map(|b| Block::Header::decode(&mut &b[..]).ok())),
-				None => Ok(None),
-			}).map_err(db_err)?
+		if let Some(Some(header)) = match db.get(COLUMN_META, key) {
+				Some(id) => db.get(col_header, &id).map(|b| Block::Header::decode(&mut &b[..]).ok()),
+				None => None,
+			}
 		{
 			let hash = header.hash();
 			debug!("DB Opened blockchain db, fetched {} = {:?} ({})", desc, hash, header.number());
@@ -380,8 +381,8 @@ pub fn read_meta<Block>(db: &dyn KeyValueDB, col_header: u32) -> Result<
 }

 /// Read genesis hash from database.
-pub fn read_genesis_hash<Hash: Decode>(db: &dyn KeyValueDB) -> sp_blockchain::Result<Option<Hash>> {
-	match db.get(COLUMN_META, meta_keys::GENESIS_HASH).map_err(db_err)? {
+pub fn read_genesis_hash<Hash: Decode>(db: &dyn Database<DbHash>) -> sp_blockchain::Result<Option<Hash>> {
+	match db.get(COLUMN_META, meta_keys::GENESIS_HASH) {
 		Some(h) => match Decode::decode(&mut &h[..]) {
 			Ok(h) => Ok(Some(h)),
 			Err(err) => Err(sp_blockchain::Error::Backend(