parachain db versioning (#2829)

* move parachains_db.rs to a module

* parachain versioning

* fixes

* more fixes

* Update node/service/src/parachains_db/upgrade.rs

* fix web-wasm check

* suggested style fixes

Co-authored-by: Bastian Köcher <bkchr@users.noreply.github.com>

* Update node/service/src/parachains_db/upgrade.rs

* compilation fixes

Co-authored-by: Bastian Köcher <bkchr@users.noreply.github.com>

polkadot/node/service/src/parachains_db/mod.rs

@@ -0,0 +1,109 @@
+// Copyright 2021 Parity Technologies (UK) Ltd.
+// This file is part of Polkadot.
+
+// Polkadot is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// Polkadot is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+//! A RocksDB instance for storing parachain data; availability data, and approvals.
+
+#[cfg(feature = "real-overseer")]
+use {
+	std::io,
+	std::path::PathBuf,
+	std::sync::Arc,
+
+	kvdb::KeyValueDB,
+};
+
+mod upgrade;
+
+mod columns {
+	#[cfg(feature = "real-overseer")]
+	pub const NUM_COLUMNS: u32 = 3;
+
+
+	pub const COL_AVAILABILITY_DATA: u32 = 0;
+	pub const COL_AVAILABILITY_META: u32 = 1;
+	pub const COL_APPROVAL_DATA: u32 = 2;
+}
+
+/// Columns used by different subsystems.
+#[derive(Debug, Clone)]
+pub struct ColumnsConfig {
+	/// The column used by the av-store for data.
+	pub col_availability_data: u32,
+	/// The column used by the av-store for meta information.
+	pub col_availability_meta: u32,
+	/// The column used by approval voting for data.
+	pub col_approval_data: u32,
+}
+
+/// The real columns used by the parachains DB.
+pub const REAL_COLUMNS: ColumnsConfig = ColumnsConfig {
+	col_availability_data: columns::COL_AVAILABILITY_DATA,
+	col_availability_meta: columns::COL_AVAILABILITY_META,
+	col_approval_data: columns::COL_APPROVAL_DATA,
+};
+
+/// The cache size for each column, in megabytes.
+#[derive(Debug, Clone)]
+pub struct CacheSizes {
+	/// Cache used by availability data.
+	pub availability_data: usize,
+	/// Cache used by availability meta.
+	pub availability_meta: usize,
+	/// Cache used by approval data.
+	pub approval_data: usize,
+}
+
+impl Default for CacheSizes {
+	fn default() -> Self {
+		CacheSizes {
+			availability_data: 25,
+			availability_meta: 1,
+			approval_data: 5,
+		}
+	}
+}
+
+#[cfg(feature = "real-overseer")]
+fn other_io_error(err: String) -> io::Error {
+	io::Error::new(io::ErrorKind::Other, err)
+}
+
+/// Open the database on disk, creating it if it doesn't exist.
+#[cfg(feature = "real-overseer")]
+pub fn open_creating(
+	root: PathBuf,
+	cache_sizes: CacheSizes,
+) -> io::Result<Arc<dyn KeyValueDB>> {
+	use kvdb_rocksdb::{DatabaseConfig, Database};
+
+	let path = root.join("parachains").join("db");
+
+	let mut db_config = DatabaseConfig::with_columns(columns::NUM_COLUMNS);
+
+	let _ = db_config.memory_budget
+		.insert(columns::COL_AVAILABILITY_DATA, cache_sizes.availability_data);
+	let _ = db_config.memory_budget
+		.insert(columns::COL_AVAILABILITY_META, cache_sizes.availability_meta);
+	let _ = db_config.memory_budget
+		.insert(columns::COL_APPROVAL_DATA, cache_sizes.approval_data);
+
+	let path_str = path.to_str().ok_or_else(|| other_io_error(
+		format!("Bad database path: {:?}", path),
+	))?;
+
+	std::fs::create_dir_all(&path_str)?;
+	upgrade::try_upgrade_db(&path)?;
+	let db = Database::open(&db_config, &path_str)?;
+
+	Ok(Arc::new(db))
+}

polkadot/node/service/src/parachains_db/upgrade.rs

@@ -0,0 +1,92 @@
+// Copyright 2021 Parity Technologies (UK) Ltd.
+// This file is part of Polkadot.
+
+// Polkadot is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// Polkadot is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+//! Migration code for the parachain's DB.
+
+
+#![cfg(feature = "real-overseer")]
+
+use std::fs;
+use std::io;
+use std::path::{Path, PathBuf};
+use std::str::FromStr;
+
+type Version = u32;
+
+/// Version file name.
+const VERSION_FILE_NAME: &'static str = "parachain_db_version";
+
+/// Current db version.
+const CURRENT_VERSION: Version = 0;
+
+#[derive(thiserror::Error, Debug)]
+pub enum Error {
+	#[error("I/O error when reading/writing the version")]
+	Io(#[from] io::Error),
+	#[error("The version file format is incorrect")]
+	CorruptedVersionFile,
+	#[error("Future version (expected {current:?}, found {got:?})")]
+	FutureVersion {
+		current: Version,
+		got: Version,
+	},
+}
+
+impl From<Error> for io::Error {
+	fn from(me: Error) -> io::Error {
+		match me {
+			Error::Io(e) => e,
+			_ => super::other_io_error(me.to_string()),
+		}
+	}
+}
+
+/// Try upgrading parachain's database to the current version.
+pub fn try_upgrade_db(db_path: &Path) -> Result<(), Error> {
+	let is_empty = db_path.read_dir().map_or(true, |mut d| d.next().is_none());
+	if !is_empty {
+		match current_version(db_path)? {
+			CURRENT_VERSION => (),
+			v => return Err(Error::FutureVersion {
+				current: CURRENT_VERSION,
+				got: v,
+			}),
+		}
+	}
+
+	update_version(db_path)
+}
+
+/// Reads current database version from the file at given path.
+/// If the file does not exist, assumes version 0.
+fn current_version(path: &Path) -> Result<Version, Error> {
+	match fs::read_to_string(version_file_path(path)) {
+		Err(ref err) if err.kind() == io::ErrorKind::NotFound => Ok(0),
+		Err(err) => Err(err.into()),
+		Ok(content) => u32::from_str(&content).map_err(|_| Error::CorruptedVersionFile),
+	}
+}
+
+/// Writes current database version to the file.
+/// Creates a new file if the version file does not exist yet.
+fn update_version(path: &Path) -> Result<(), Error> {
+	fs::create_dir_all(path)?;
+	fs::write(version_file_path(path), CURRENT_VERSION.to_string()).map_err(Into::into)
+}
+
+/// Returns the version file path.
+fn version_file_path(path: &Path) -> PathBuf {
+	let mut file_path = path.to_owned();
+	file_path.push(VERSION_FILE_NAME);
+	file_path
+}