Store the database in a role specific subdirectory (#9645)

* Store the database in a role specific subdirectory

This is a cleaned up version of #8658 fixing #6880

polkadot companion: paritytech/polkadot#2923

* Disable prometheus in tests

* Also change p2p port

* Fix migration logic

* Use different identification file for rocks and parity db

Add tests for paritydb migration

substrate/client/db/src/lib.rs

@@ -355,7 +355,7 @@ pub enum DatabaseSource {
 }
 
 impl DatabaseSource {
-	/// Return dabase path for databases that are on the disk.
+	/// Return path for databases that are stored on disk.
 	pub fn path(&self) -> Option<&Path> {
 		match self {
 			// as per https://github.com/paritytech/substrate/pull/9500#discussion_r684312550
@@ -367,6 +367,22 @@ impl DatabaseSource {
 			DatabaseSource::Custom(..) => None,
 		}
 	}
+
+	/// Set path for databases that are stored on disk.
+	pub fn set_path(&mut self, p: &Path) -> bool {
+		match self {
+			DatabaseSource::Auto { ref mut paritydb_path, .. } => {
+				*paritydb_path = p.into();
+				true
+			},
+			DatabaseSource::RocksDb { ref mut path, .. } |
+			DatabaseSource::ParityDb { ref mut path } => {
+				*path = p.into();
+				true
+			},
+			DatabaseSource::Custom(..) => false,
+		}
+	}
 }
 
 impl std::fmt::Display for DatabaseSource {

substrate/client/db/src/upgrade.rs

@@ -186,7 +186,7 @@ mod tests {
 		}
 	}
 
-	fn open_database(db_path: &Path) -> sp_blockchain::Result<()> {
+	fn open_database(db_path: &Path, db_type: DatabaseType) -> sp_blockchain::Result<()> {
 		crate::utils::open_database::<Block>(
 			&DatabaseSettings {
 				state_cache_size: 0,
@@ -196,7 +196,7 @@ mod tests {
 				keep_blocks: KeepBlocks::All,
 				transaction_storage: TransactionStorageMode::BlockBody,
 			},
-			DatabaseType::Full,
+			db_type,
 		)
 		.map(|_| ())
 	}
@@ -205,25 +205,28 @@ mod tests {
 	fn downgrade_never_happens() {
 		let db_dir = tempfile::TempDir::new().unwrap();
 		create_db(db_dir.path(), Some(CURRENT_VERSION + 1));
-		assert!(open_database(db_dir.path()).is_err());
+		assert!(open_database(db_dir.path(), DatabaseType::Full).is_err());
 	}
 
 	#[test]
 	fn open_empty_database_works() {
+		let db_type = DatabaseType::Full;
 		let db_dir = tempfile::TempDir::new().unwrap();
-		open_database(db_dir.path()).unwrap();
-		open_database(db_dir.path()).unwrap();
-		assert_eq!(current_version(db_dir.path()).unwrap(), CURRENT_VERSION);
+		let db_dir = db_dir.path().join(db_type.as_str());
+		open_database(&db_dir, db_type).unwrap();
+		open_database(&db_dir, db_type).unwrap();
+		assert_eq!(current_version(&db_dir).unwrap(), CURRENT_VERSION);
 	}
 
 	#[test]
 	fn upgrade_to_3_works() {
+		let db_type = DatabaseType::Full;
 		for version_from_file in &[None, Some(1), Some(2)] {
 			let db_dir = tempfile::TempDir::new().unwrap();
-			let db_path = db_dir.path();
-			create_db(db_path, *version_from_file);
-			open_database(db_path).unwrap();
-			assert_eq!(current_version(db_path).unwrap(), CURRENT_VERSION);
+			let db_path = db_dir.path().join(db_type.as_str());
+			create_db(&db_path, *version_from_file);
+			open_database(&db_path, db_type).unwrap();
+			assert_eq!(current_version(&db_path).unwrap(), CURRENT_VERSION);
 		}
 	}
 }

substrate/client/db/src/utils.rs

@@ -19,9 +19,9 @@
 //! Db-based backend utility structures and functions, used by both
 //! full and light storages.
 
-use std::{convert::TryInto, fmt, io, path::Path, sync::Arc};
+use std::{convert::TryInto, fmt, fs, io, path::Path, sync::Arc};
 
-use log::debug;
+use log::{debug, info};
 
 use crate::{Database, DatabaseSettings, DatabaseSource, DbHash};
 use codec::Decode;
@@ -213,7 +213,21 @@ pub fn open_database<Block: BlockT>(
 	config: &DatabaseSettings,
 	db_type: DatabaseType,
 ) -> sp_blockchain::Result<Arc<dyn Database<DbHash>>> {
-	let db: Arc<dyn Database<DbHash>> = match &config.source {
+	// Maybe migrate (copy) the database to a type specific subdirectory to make it
+	// possible that light and full databases coexist
+	// NOTE: This function can be removed in a few releases
+	maybe_migrate_to_type_subdir::<Block>(&config.source, db_type).map_err(|e| {
+		sp_blockchain::Error::Backend(format!("Error in migration to role subdirectory: {}", e))
+	})?;
+
+	open_database_at::<Block>(&config.source, db_type)
+}
+
+fn open_database_at<Block: BlockT>(
+	source: &DatabaseSource,
+	db_type: DatabaseType,
+) -> sp_blockchain::Result<Arc<dyn Database<DbHash>>> {
+	let db: Arc<dyn Database<DbHash>> = match &source {
 		DatabaseSource::ParityDb { path } => open_parity_db::<Block>(&path, db_type, true)?,
 		DatabaseSource::RocksDb { path, cache_size } =>
 			open_kvdb_rocksdb::<Block>(&path, db_type, true, *cache_size)?,
@@ -394,6 +408,46 @@ pub fn check_database_type(
 	Ok(())
 }
 
+fn maybe_migrate_to_type_subdir<Block: BlockT>(
+	source: &DatabaseSource,
+	db_type: DatabaseType,
+) -> io::Result<()> {
+	if let Some(p) = source.path() {
+		let mut basedir = p.to_path_buf();
+		basedir.pop();
+
+		// Do we have to migrate to a database-type-based subdirectory layout:
+		// See if there's a file identifying a rocksdb or paritydb folder in the parent dir and
+		// the target path ends in a role specific directory
+		if (basedir.join("db_version").exists() || basedir.join("metadata").exists()) &&
+			(p.ends_with(DatabaseType::Full.as_str()) ||
+				p.ends_with(DatabaseType::Light.as_str()))
+		{
+			// Try to open the database to check if the current `DatabaseType` matches the type of
+			// database stored in the target directory and close the database on success.
+			let mut old_source = source.clone();
+			old_source.set_path(&basedir);
+			open_database_at::<Block>(&old_source, db_type)
+				.map_err(|e| io::Error::new(io::ErrorKind::InvalidData, e))?;
+
+			info!(
+				"Migrating database to a database-type-based subdirectory: '{:?}' -> '{:?}'",
+				basedir,
+				basedir.join(db_type.as_str())
+			);
+			let mut tmp_dir = basedir.clone();
+			tmp_dir.pop();
+			tmp_dir.push("tmp");
+
+			fs::rename(&basedir, &tmp_dir)?;
+			fs::create_dir_all(&p)?;
+			fs::rename(tmp_dir, &p)?;
+		}
+	}
+
+	Ok(())
+}
+
 /// Read database column entry for the given block.
 pub fn read_db<Block>(
 	db: &dyn Database<DbHash>,
@@ -570,8 +624,95 @@ mod tests {
 	use codec::Input;
 	use sc_state_db::PruningMode;
 	use sp_runtime::testing::{Block as RawBlock, ExtrinsicWrapper};
+	use std::path::PathBuf;
 	type Block = RawBlock<ExtrinsicWrapper<u32>>;
 
+	#[cfg(any(feature = "with-kvdb-rocksdb", test))]
+	#[test]
+	fn database_type_subdir_migration() {
+		type Block = RawBlock<ExtrinsicWrapper<u64>>;
+
+		fn check_dir_for_db_type(
+			db_type: DatabaseType,
+			mut source: DatabaseSource,
+			db_check_file: &str,
+		) {
+			let base_path = tempfile::TempDir::new().unwrap();
+			let old_db_path = base_path.path().join("chains/dev/db");
+
+			source.set_path(&old_db_path);
+			let settings = db_settings(source.clone());
+
+			{
+				let db_res = open_database::<Block>(&settings, db_type);
+				assert!(db_res.is_ok(), "New database should be created.");
+				assert!(old_db_path.join(db_check_file).exists());
+				assert!(!old_db_path.join(db_type.as_str()).join("db_version").exists());
+			}
+
+			source.set_path(&old_db_path.join(db_type.as_str()));
+			let settings = db_settings(source);
+			let db_res = open_database::<Block>(&settings, db_type);
+			assert!(db_res.is_ok(), "Reopening the db with the same role should work");
+			// check if the database dir had been migrated
+			assert!(!old_db_path.join(db_check_file).exists());
+			assert!(old_db_path.join(db_type.as_str()).join(db_check_file).exists());
+		}
+
+		check_dir_for_db_type(
+			DatabaseType::Light,
+			DatabaseSource::RocksDb { path: PathBuf::new(), cache_size: 128 },
+			"db_version",
+		);
+		check_dir_for_db_type(
+			DatabaseType::Full,
+			DatabaseSource::RocksDb { path: PathBuf::new(), cache_size: 128 },
+			"db_version",
+		);
+
+		#[cfg(feature = "with-parity-db")]
+		check_dir_for_db_type(
+			DatabaseType::Light,
+			DatabaseSource::ParityDb { path: PathBuf::new() },
+			"metadata",
+		);
+		#[cfg(feature = "with-parity-db")]
+		check_dir_for_db_type(
+			DatabaseType::Full,
+			DatabaseSource::ParityDb { path: PathBuf::new() },
+			"metadata",
+		);
+
+		// check failure on reopening with wrong role
+		{
+			let base_path = tempfile::TempDir::new().unwrap();
+			let old_db_path = base_path.path().join("chains/dev/db");
+
+			let source = DatabaseSource::RocksDb { path: old_db_path.clone(), cache_size: 128 };
+			let settings = db_settings(source);
+			{
+				let db_res = open_database::<Block>(&settings, DatabaseType::Full);
+				assert!(db_res.is_ok(), "New database should be created.");
+
+				// check if the database dir had been migrated
+				assert!(old_db_path.join("db_version").exists());
+				assert!(!old_db_path.join("light/db_version").exists());
+				assert!(!old_db_path.join("full/db_version").exists());
+			}
+			let source = DatabaseSource::RocksDb {
+				path: old_db_path.join(DatabaseType::Light.as_str()),
+				cache_size: 128,
+			};
+			let settings = db_settings(source);
+			let db_res = open_database::<Block>(&settings, DatabaseType::Light);
+			assert!(db_res.is_err(), "Opening a light database in full role should fail");
+			// assert nothing was changed
+			assert!(old_db_path.join("db_version").exists());
+			assert!(!old_db_path.join("light/db_version").exists());
+			assert!(!old_db_path.join("full/db_version").exists());
+		}
+	}
+
 	#[test]
 	fn number_index_key_doesnt_panic() {
 		let id = BlockId::<Block>::Number(72340207214430721);