Remove the --unsafe-pruning CLI-argument (step 1) (#10995)

* sc-client-db: utils::open_database(...) — return OpenDbError so that the caller could tell the `OpenDbError::DoesNotExist` clearly * sc-client-db: utils::open_database(..) — accept the `create: bool` argument * sc-client-db: pruning — optional argument in the DatabaseSettings * sc-state-db: Split `Error<E>` into separate `Error<E>` and `StateDbError` * StateDb::open: choose the pruning-mode depending on the requested and stored values * sc-state-db: test for different combinations of stored and requested pruning-modes * CLI-argument: mark the unsafe-pruning as deprecated * Fix tests * tests: do not specify --pruning when running the substrate over the existing storage * fix types for benches * cargo fmt * Check whether the pruning-mode and sync-mode are compatible * cargo fmt * parity-db: 0.3.11 -> 0.3.12 * sc-state-db: MetaDb::set_meta — a better doc-test * cargo fmt * make MetaDb read-only again! * Remove the stray newline (and run the CI once again please) * Last nitpicks * A more comprehensive error message
2026-07-20 03:25:41 +00:00 · 2022-05-06 13:07:44 +03:00
parent 994f8076b1
commit 729cba9d9e
23 changed files with 546 additions and 338 deletions
@@ -115,9 +115,13 @@ pub trait NodeDb {
 }

 /// Error type.
-pub enum Error<E: fmt::Debug> {
+pub enum Error<E> {
 	/// Database backend error.
 	Db(E),
+	StateDb(StateDbError),
+}
+
+pub enum StateDbError {
 	/// `Codec` decoding error.
 	Decoding(codec::Error),
 	/// Trying to canonicalize invalid block.
@@ -127,11 +131,19 @@ pub enum Error<E: fmt::Debug> {
 	/// Trying to insert block with unknown parent.
 	InvalidParent,
 	/// Invalid pruning mode specified. Contains expected mode.
-	InvalidPruningMode(String),
+	IncompatiblePruningModes { stored: PruningMode, requested: PruningMode },
 	/// Too many unfinalized sibling blocks inserted.
 	TooManySiblingBlocks,
 	/// Trying to insert existing block.
 	BlockAlreadyExists,
+	/// Invalid metadata
+	Metadata(String),
+}
+
+impl<E> From<StateDbError> for Error<E> {
+	fn from(inner: StateDbError) -> Self {
+		Self::StateDb(inner)
+	}
 }

 /// Pinning error type.
@@ -142,21 +154,34 @@ pub enum PinError {

 impl<E: fmt::Debug> From<codec::Error> for Error<E> {
 	fn from(x: codec::Error) -> Self {
-		Error::Decoding(x)
+		StateDbError::Decoding(x).into()
 	}
 }

 impl<E: fmt::Debug> fmt::Debug for Error<E> {
 	fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
 		match self {
-			Error::Db(e) => e.fmt(f),
-			Error::Decoding(e) => write!(f, "Error decoding sliceable value: {}", e),
-			Error::InvalidBlock => write!(f, "Trying to canonicalize invalid block"),
-			Error::InvalidBlockNumber => write!(f, "Trying to insert block with invalid number"),
-			Error::InvalidParent => write!(f, "Trying to insert block with unknown parent"),
-			Error::InvalidPruningMode(e) => write!(f, "Expected pruning mode: {}", e),
-			Error::TooManySiblingBlocks => write!(f, "Too many sibling blocks inserted"),
-			Error::BlockAlreadyExists => write!(f, "Block already exists"),
+			Self::Db(e) => e.fmt(f),
+			Self::StateDb(e) => e.fmt(f),
+		}
+	}
+}
+
+impl fmt::Debug for StateDbError {
+	fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
+		match self {
+			Self::Decoding(e) => write!(f, "Error decoding sliceable value: {}", e),
+			Self::InvalidBlock => write!(f, "Trying to canonicalize invalid block"),
+			Self::InvalidBlockNumber => write!(f, "Trying to insert block with invalid number"),
+			Self::InvalidParent => write!(f, "Trying to insert block with unknown parent"),
+			Self::IncompatiblePruningModes { stored, requested } => write!(
+				f,
+				"Incompatible pruning modes [stored: {:?}; requested: {:?}]",
+				stored, requested
+			),
+			Self::TooManySiblingBlocks => write!(f, "Too many sibling blocks inserted"),
+			Self::BlockAlreadyExists => write!(f, "Block already exists"),
+			Self::Metadata(message) => write!(f, "Invalid metadata: {}", message),
 		}
 	}
 }
@@ -180,7 +205,7 @@ pub struct CommitSet<H: Hash> {
 }

 /// Pruning constraints. If none are specified pruning is
-#[derive(Default, Debug, Clone, Eq, PartialEq)]
+#[derive(Debug, Clone, Eq, PartialEq)]
 pub struct Constraints {
 	/// Maximum blocks. Defaults to 0 when unspecified, effectively keeping only non-canonical
 	/// states.
@@ -222,11 +247,25 @@ impl PruningMode {
 			PruningMode::Constrained(_) => PRUNING_MODE_CONSTRAINED,
 		}
 	}
+	pub fn from_id(id: &[u8]) -> Option<Self> {
+		match id {
+			PRUNING_MODE_ARCHIVE => Some(Self::ArchiveAll),
+			PRUNING_MODE_ARCHIVE_CANON => Some(Self::ArchiveCanonical),
+			PRUNING_MODE_CONSTRAINED => Some(Self::Constrained(Default::default())),
+			_ => None,
+		}
+	}
 }

 impl Default for PruningMode {
 	fn default() -> Self {
-		PruningMode::keep_blocks(256)
+		PruningMode::Constrained(Default::default())
+	}
+}
+
+impl Default for Constraints {
+	fn default() -> Self {
+		Self { max_blocks: Some(256), max_mem: None }
 	}
 }

@@ -251,9 +290,6 @@ impl<BlockHash: Hash + MallocSizeOf, Key: Hash + MallocSizeOf> StateDbSync<Block
 	) -> Result<StateDbSync<BlockHash, Key>, Error<D::Error>> {
 		trace!(target: "state-db", "StateDb settings: {:?}. Ref-counting: {}", mode, ref_counting);

-		// Check that settings match
-		Self::check_meta(&mode, db)?;
-
 		let non_canonical: NonCanonicalOverlay<BlockHash, Key> = NonCanonicalOverlay::new(db)?;
 		let pruning: Option<RefWindow<BlockHash, Key>> = match mode {
 			PruningMode::Constrained(Constraints { max_mem: Some(_), .. }) => unimplemented!(),
@@ -264,19 +300,6 @@ impl<BlockHash: Hash + MallocSizeOf, Key: Hash + MallocSizeOf> StateDbSync<Block
 		Ok(StateDbSync { mode, non_canonical, pruning, pinned: Default::default() })
 	}

-	fn check_meta<D: MetaDb>(mode: &PruningMode, db: &D) -> Result<(), Error<D::Error>> {
-		let db_mode = db.get_meta(&to_meta_key(PRUNING_MODE, &())).map_err(Error::Db)?;
-		trace!(target: "state-db",
-			"DB pruning mode: {:?}",
-			db_mode.as_ref().map(|v| std::str::from_utf8(v))
-		);
-		match &db_mode {
-			Some(v) if v.as_slice() == mode.id() => Ok(()),
-			Some(v) => Err(Error::InvalidPruningMode(String::from_utf8_lossy(v).into())),
-			None => Ok(()),
-		}
-	}
-
 	fn insert_block<E: fmt::Debug>(
 		&mut self,
 		hash: &BlockHash,
@@ -284,25 +307,16 @@ impl<BlockHash: Hash + MallocSizeOf, Key: Hash + MallocSizeOf> StateDbSync<Block
 		parent_hash: &BlockHash,
 		mut changeset: ChangeSet<Key>,
 	) -> Result<CommitSet<Key>, Error<E>> {
-		let mut meta = ChangeSet::default();
-		if number == 0 {
-			// Save pruning mode when writing first block.
-			meta.inserted.push((to_meta_key(PRUNING_MODE, &()), self.mode.id().into()));
-		}
-
 		match self.mode {
 			PruningMode::ArchiveAll => {
 				changeset.deleted.clear();
 				// write changes immediately
-				Ok(CommitSet { data: changeset, meta })
-			},
-			PruningMode::Constrained(_) | PruningMode::ArchiveCanonical => {
-				let commit = self.non_canonical.insert(hash, number, parent_hash, changeset);
-				commit.map(|mut c| {
-					c.meta.inserted.extend(meta.inserted);
-					c
-				})
+				Ok(CommitSet { data: changeset, meta: Default::default() })
 			},
+			PruningMode::Constrained(_) | PruningMode::ArchiveCanonical => self
+				.non_canonical
+				.insert(hash, number, parent_hash, changeset)
+				.map_err(Into::into),
 		}
 	}

@@ -319,7 +333,7 @@ impl<BlockHash: Hash + MallocSizeOf, Key: Hash + MallocSizeOf> StateDbSync<Block
 				if self.mode == PruningMode::ArchiveCanonical {
 					commit.data.deleted.clear();
 				},
-			Err(e) => return Err(e),
+			Err(e) => return Err(e.into()),
 		};
 		if let Some(ref mut pruning) = self.pruning {
 			pruning.note_canonical(hash, &mut commit);
@@ -480,13 +494,56 @@ pub struct StateDb<BlockHash: Hash, Key: Hash> {
 }

 impl<BlockHash: Hash + MallocSizeOf, Key: Hash + MallocSizeOf> StateDb<BlockHash, Key> {
-	/// Creates a new instance. Does not expect any metadata in the database.
-	pub fn new<D: MetaDb>(
-		mode: PruningMode,
-		ref_counting: bool,
+	/// Create an instance of [`StateDb`].
+	pub fn open<D>(
 		db: &D,
-	) -> Result<StateDb<BlockHash, Key>, Error<D::Error>> {
-		Ok(StateDb { db: RwLock::new(StateDbSync::new(mode, ref_counting, db)?) })
+		requested_mode: Option<PruningMode>,
+		ref_counting: bool,
+		should_init: bool,
+	) -> Result<(CommitSet<Key>, StateDb<BlockHash, Key>), Error<D::Error>>
+	where
+		D: MetaDb,
+	{
+		let stored_mode = fetch_stored_pruning_mode(db)?;
+
+		let selected_mode = match (should_init, stored_mode, requested_mode) {
+			(true, stored_mode, requested_mode) => {
+				assert!(stored_mode.is_none(), "The storage has just been initialized. No meta-data is expected to be found in it.");
+				requested_mode.unwrap_or_default()
+			},
+
+			(false, None, _) =>
+				return Err(StateDbError::Metadata(
+					"An existing StateDb does not have PRUNING_MODE stored in its meta-data".into(),
+				)
+				.into()),
+
+			(false, Some(stored), None) => stored,
+
+			(false, Some(stored), Some(requested)) => choose_pruning_mode(stored, requested)?,
+		};
+
+		let db_init_commit_set = if should_init {
+			let mut cs: CommitSet<Key> = Default::default();
+
+			let key = to_meta_key(PRUNING_MODE, &());
+			let value = selected_mode.id().to_owned();
+
+			cs.meta.inserted.push((key, value));
+
+			cs
+		} else {
+			Default::default()
+		};
+
+		let state_db =
+			StateDb { db: RwLock::new(StateDbSync::new(selected_mode, ref_counting, db)?) };
+
+		Ok((db_init_commit_set, state_db))
+	}
+
+	pub fn pruning_mode(&self) -> PruningMode {
+		self.db.read().mode.clone()
 	}

 	/// Add a new non-canonical block.
@@ -571,18 +628,51 @@ impl<BlockHash: Hash + MallocSizeOf, Key: Hash + MallocSizeOf> StateDb<BlockHash
 	}
 }

+fn fetch_stored_pruning_mode<D: MetaDb>(db: &D) -> Result<Option<PruningMode>, Error<D::Error>> {
+	let meta_key_mode = to_meta_key(PRUNING_MODE, &());
+	if let Some(stored_mode) = db.get_meta(&meta_key_mode).map_err(Error::Db)? {
+		if let Some(mode) = PruningMode::from_id(&stored_mode) {
+			Ok(Some(mode))
+		} else {
+			Err(StateDbError::Metadata(format!(
+				"Invalid value stored for PRUNING_MODE: {:02x?}",
+				stored_mode
+			))
+			.into())
+		}
+	} else {
+		Ok(None)
+	}
+}
+
+fn choose_pruning_mode(
+	stored: PruningMode,
+	requested: PruningMode,
+) -> Result<PruningMode, StateDbError> {
+	match (stored, requested) {
+		(PruningMode::ArchiveAll, PruningMode::ArchiveAll) => Ok(PruningMode::ArchiveAll),
+		(PruningMode::ArchiveCanonical, PruningMode::ArchiveCanonical) =>
+			Ok(PruningMode::ArchiveCanonical),
+		(PruningMode::Constrained(_), PruningMode::Constrained(requested)) =>
+			Ok(PruningMode::Constrained(requested)),
+		(stored, requested) => Err(StateDbError::IncompatiblePruningModes { requested, stored }),
+	}
+}
+
 #[cfg(test)]
 mod tests {
 	use crate::{
 		test::{make_changeset, make_db, TestDb},
-		Constraints, PruningMode, StateDb,
+		Constraints, Error, PruningMode, StateDb, StateDbError,
 	};
 	use sp_core::H256;
 	use std::io;

 	fn make_test_db(settings: PruningMode) -> (TestDb, StateDb<H256, H256>) {
 		let mut db = make_db(&[91, 921, 922, 93, 94]);
-		let state_db = StateDb::new(settings, false, &db).unwrap();
+		let (state_db_init, state_db) =
+			StateDb::open(&mut db, Some(settings), false, true).unwrap();
+		db.commit(&state_db_init);

 		db.commit(
 			&state_db
@@ -697,7 +787,9 @@ mod tests {
 	#[test]
 	fn detects_incompatible_mode() {
 		let mut db = make_db(&[]);
-		let state_db = StateDb::new(PruningMode::ArchiveAll, false, &db).unwrap();
+		let (state_db_init, state_db) =
+			StateDb::open(&mut db, Some(PruningMode::ArchiveAll), false, true).unwrap();
+		db.commit(&state_db_init);
 		db.commit(
 			&state_db
 				.insert_block::<io::Error>(
@@ -709,7 +801,85 @@ mod tests {
 				.unwrap(),
 		);
 		let new_mode = PruningMode::Constrained(Constraints { max_blocks: Some(2), max_mem: None });
-		let state_db: Result<StateDb<H256, H256>, _> = StateDb::new(new_mode, false, &db);
-		assert!(state_db.is_err());
+		let state_db_open_result: Result<(_, StateDb<H256, H256>), _> =
+			StateDb::open(&mut db, Some(new_mode), false, false);
+		assert!(state_db_open_result.is_err());
+	}
+
+	fn check_stored_and_requested_mode_compatibility(
+		mode_when_created: Option<PruningMode>,
+		mode_when_reopened: Option<PruningMode>,
+		expected_effective_mode_when_reopenned: Result<PruningMode, ()>,
+	) {
+		let mut db = make_db(&[]);
+		let (state_db_init, state_db) =
+			StateDb::<H256, H256>::open(&mut db, mode_when_created, false, true).unwrap();
+		db.commit(&state_db_init);
+		std::mem::drop(state_db);
+
+		let state_db_reopen_result =
+			StateDb::<H256, H256>::open(&mut db, mode_when_reopened, false, false);
+		if let Ok(expected_mode) = expected_effective_mode_when_reopenned {
+			let (state_db_init, state_db_reopened) = state_db_reopen_result.unwrap();
+			db.commit(&state_db_init);
+			assert_eq!(state_db_reopened.pruning_mode(), expected_mode,)
+		} else {
+			assert!(matches!(
+				state_db_reopen_result,
+				Err(Error::StateDb(StateDbError::IncompatiblePruningModes { .. }))
+			));
+		}
+	}
+
+	#[test]
+	fn pruning_mode_compatibility() {
+		for (created, reopened, expected) in [
+			(None, None, Ok(PruningMode::keep_blocks(256))),
+			(None, Some(PruningMode::keep_blocks(256)), Ok(PruningMode::keep_blocks(256))),
+			(None, Some(PruningMode::keep_blocks(128)), Ok(PruningMode::keep_blocks(128))),
+			(None, Some(PruningMode::keep_blocks(512)), Ok(PruningMode::keep_blocks(512))),
+			(None, Some(PruningMode::ArchiveAll), Err(())),
+			(None, Some(PruningMode::ArchiveCanonical), Err(())),
+			(Some(PruningMode::keep_blocks(256)), None, Ok(PruningMode::keep_blocks(256))),
+			(
+				Some(PruningMode::keep_blocks(256)),
+				Some(PruningMode::keep_blocks(256)),
+				Ok(PruningMode::keep_blocks(256)),
+			),
+			(
+				Some(PruningMode::keep_blocks(256)),
+				Some(PruningMode::keep_blocks(128)),
+				Ok(PruningMode::keep_blocks(128)),
+			),
+			(
+				Some(PruningMode::keep_blocks(256)),
+				Some(PruningMode::keep_blocks(512)),
+				Ok(PruningMode::keep_blocks(512)),
+			),
+			(Some(PruningMode::keep_blocks(256)), Some(PruningMode::ArchiveAll), Err(())),
+			(Some(PruningMode::keep_blocks(256)), Some(PruningMode::ArchiveCanonical), Err(())),
+			(Some(PruningMode::ArchiveAll), None, Ok(PruningMode::ArchiveAll)),
+			(Some(PruningMode::ArchiveAll), Some(PruningMode::keep_blocks(256)), Err(())),
+			(Some(PruningMode::ArchiveAll), Some(PruningMode::keep_blocks(128)), Err(())),
+			(Some(PruningMode::ArchiveAll), Some(PruningMode::keep_blocks(512)), Err(())),
+			(
+				Some(PruningMode::ArchiveAll),
+				Some(PruningMode::ArchiveAll),
+				Ok(PruningMode::ArchiveAll),
+			),
+			(Some(PruningMode::ArchiveAll), Some(PruningMode::ArchiveCanonical), Err(())),
+			(Some(PruningMode::ArchiveCanonical), None, Ok(PruningMode::ArchiveCanonical)),
+			(Some(PruningMode::ArchiveCanonical), Some(PruningMode::keep_blocks(256)), Err(())),
+			(Some(PruningMode::ArchiveCanonical), Some(PruningMode::keep_blocks(128)), Err(())),
+			(Some(PruningMode::ArchiveCanonical), Some(PruningMode::keep_blocks(512)), Err(())),
+			(Some(PruningMode::ArchiveCanonical), Some(PruningMode::ArchiveAll), Err(())),
+			(
+				Some(PruningMode::ArchiveCanonical),
+				Some(PruningMode::ArchiveCanonical),
+				Ok(PruningMode::ArchiveCanonical),
+			),
+		] {
+			check_stored_and_requested_mode_compatibility(created, reopened, expected);
+		}
 	}
 }