Remove the --unsafe-pruning CLI-argument (step 1) (#10995)

* sc-client-db: utils::open_database(...) — return OpenDbError so that the caller could tell the `OpenDbError::DoesNotExist` clearly

* sc-client-db: utils::open_database(..) — accept the `create: bool` argument

* sc-client-db: pruning — optional argument in the DatabaseSettings

* sc-state-db: Split `Error<E>` into separate `Error<E>` and `StateDbError`

* StateDb::open: choose the pruning-mode depending on the requested and stored values

* sc-state-db: test for different combinations of stored and requested pruning-modes

* CLI-argument: mark the unsafe-pruning as deprecated

* Fix tests

* tests: do not specify --pruning when running the substrate over the existing storage

* fix types for benches

* cargo fmt

* Check whether the pruning-mode and sync-mode are compatible

* cargo fmt

* parity-db: 0.3.11 -> 0.3.12

* sc-state-db: MetaDb::set_meta — a better doc-test

* cargo fmt

* make MetaDb read-only again!

* Remove the stray newline (and run the CI once again please)

* Last nitpicks

* A more comprehensive error message
This commit is contained in:
Roman Gafiyatullin
2022-05-06 13:07:44 +03:00
committed by GitHub
parent 994f8076b1
commit 729cba9d9e
23 changed files with 546 additions and 338 deletions
+225 -55
View File
@@ -115,9 +115,13 @@ pub trait NodeDb {
}
/// Error type.
pub enum Error<E: fmt::Debug> {
pub enum Error<E> {
/// Database backend error.
Db(E),
StateDb(StateDbError),
}
pub enum StateDbError {
/// `Codec` decoding error.
Decoding(codec::Error),
/// Trying to canonicalize invalid block.
@@ -127,11 +131,19 @@ pub enum Error<E: fmt::Debug> {
/// Trying to insert block with unknown parent.
InvalidParent,
/// Invalid pruning mode specified. Contains expected mode.
InvalidPruningMode(String),
IncompatiblePruningModes { stored: PruningMode, requested: PruningMode },
/// Too many unfinalized sibling blocks inserted.
TooManySiblingBlocks,
/// Trying to insert existing block.
BlockAlreadyExists,
/// Invalid metadata
Metadata(String),
}
impl<E> From<StateDbError> for Error<E> {
fn from(inner: StateDbError) -> Self {
Self::StateDb(inner)
}
}
/// Pinning error type.
@@ -142,21 +154,34 @@ pub enum PinError {
impl<E: fmt::Debug> From<codec::Error> for Error<E> {
fn from(x: codec::Error) -> Self {
Error::Decoding(x)
StateDbError::Decoding(x).into()
}
}
impl<E: fmt::Debug> fmt::Debug for Error<E> {
fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
match self {
Error::Db(e) => e.fmt(f),
Error::Decoding(e) => write!(f, "Error decoding sliceable value: {}", e),
Error::InvalidBlock => write!(f, "Trying to canonicalize invalid block"),
Error::InvalidBlockNumber => write!(f, "Trying to insert block with invalid number"),
Error::InvalidParent => write!(f, "Trying to insert block with unknown parent"),
Error::InvalidPruningMode(e) => write!(f, "Expected pruning mode: {}", e),
Error::TooManySiblingBlocks => write!(f, "Too many sibling blocks inserted"),
Error::BlockAlreadyExists => write!(f, "Block already exists"),
Self::Db(e) => e.fmt(f),
Self::StateDb(e) => e.fmt(f),
}
}
}
impl fmt::Debug for StateDbError {
fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
match self {
Self::Decoding(e) => write!(f, "Error decoding sliceable value: {}", e),
Self::InvalidBlock => write!(f, "Trying to canonicalize invalid block"),
Self::InvalidBlockNumber => write!(f, "Trying to insert block with invalid number"),
Self::InvalidParent => write!(f, "Trying to insert block with unknown parent"),
Self::IncompatiblePruningModes { stored, requested } => write!(
f,
"Incompatible pruning modes [stored: {:?}; requested: {:?}]",
stored, requested
),
Self::TooManySiblingBlocks => write!(f, "Too many sibling blocks inserted"),
Self::BlockAlreadyExists => write!(f, "Block already exists"),
Self::Metadata(message) => write!(f, "Invalid metadata: {}", message),
}
}
}
@@ -180,7 +205,7 @@ pub struct CommitSet<H: Hash> {
}
/// Pruning constraints. If none are specified pruning is
#[derive(Default, Debug, Clone, Eq, PartialEq)]
#[derive(Debug, Clone, Eq, PartialEq)]
pub struct Constraints {
/// Maximum blocks. Defaults to 0 when unspecified, effectively keeping only non-canonical
/// states.
@@ -222,11 +247,25 @@ impl PruningMode {
PruningMode::Constrained(_) => PRUNING_MODE_CONSTRAINED,
}
}
pub fn from_id(id: &[u8]) -> Option<Self> {
match id {
PRUNING_MODE_ARCHIVE => Some(Self::ArchiveAll),
PRUNING_MODE_ARCHIVE_CANON => Some(Self::ArchiveCanonical),
PRUNING_MODE_CONSTRAINED => Some(Self::Constrained(Default::default())),
_ => None,
}
}
}
impl Default for PruningMode {
fn default() -> Self {
PruningMode::keep_blocks(256)
PruningMode::Constrained(Default::default())
}
}
impl Default for Constraints {
fn default() -> Self {
Self { max_blocks: Some(256), max_mem: None }
}
}
@@ -251,9 +290,6 @@ impl<BlockHash: Hash + MallocSizeOf, Key: Hash + MallocSizeOf> StateDbSync<Block
) -> Result<StateDbSync<BlockHash, Key>, Error<D::Error>> {
trace!(target: "state-db", "StateDb settings: {:?}. Ref-counting: {}", mode, ref_counting);
// Check that settings match
Self::check_meta(&mode, db)?;
let non_canonical: NonCanonicalOverlay<BlockHash, Key> = NonCanonicalOverlay::new(db)?;
let pruning: Option<RefWindow<BlockHash, Key>> = match mode {
PruningMode::Constrained(Constraints { max_mem: Some(_), .. }) => unimplemented!(),
@@ -264,19 +300,6 @@ impl<BlockHash: Hash + MallocSizeOf, Key: Hash + MallocSizeOf> StateDbSync<Block
Ok(StateDbSync { mode, non_canonical, pruning, pinned: Default::default() })
}
fn check_meta<D: MetaDb>(mode: &PruningMode, db: &D) -> Result<(), Error<D::Error>> {
let db_mode = db.get_meta(&to_meta_key(PRUNING_MODE, &())).map_err(Error::Db)?;
trace!(target: "state-db",
"DB pruning mode: {:?}",
db_mode.as_ref().map(|v| std::str::from_utf8(v))
);
match &db_mode {
Some(v) if v.as_slice() == mode.id() => Ok(()),
Some(v) => Err(Error::InvalidPruningMode(String::from_utf8_lossy(v).into())),
None => Ok(()),
}
}
fn insert_block<E: fmt::Debug>(
&mut self,
hash: &BlockHash,
@@ -284,25 +307,16 @@ impl<BlockHash: Hash + MallocSizeOf, Key: Hash + MallocSizeOf> StateDbSync<Block
parent_hash: &BlockHash,
mut changeset: ChangeSet<Key>,
) -> Result<CommitSet<Key>, Error<E>> {
let mut meta = ChangeSet::default();
if number == 0 {
// Save pruning mode when writing first block.
meta.inserted.push((to_meta_key(PRUNING_MODE, &()), self.mode.id().into()));
}
match self.mode {
PruningMode::ArchiveAll => {
changeset.deleted.clear();
// write changes immediately
Ok(CommitSet { data: changeset, meta })
},
PruningMode::Constrained(_) | PruningMode::ArchiveCanonical => {
let commit = self.non_canonical.insert(hash, number, parent_hash, changeset);
commit.map(|mut c| {
c.meta.inserted.extend(meta.inserted);
c
})
Ok(CommitSet { data: changeset, meta: Default::default() })
},
PruningMode::Constrained(_) | PruningMode::ArchiveCanonical => self
.non_canonical
.insert(hash, number, parent_hash, changeset)
.map_err(Into::into),
}
}
@@ -319,7 +333,7 @@ impl<BlockHash: Hash + MallocSizeOf, Key: Hash + MallocSizeOf> StateDbSync<Block
if self.mode == PruningMode::ArchiveCanonical {
commit.data.deleted.clear();
},
Err(e) => return Err(e),
Err(e) => return Err(e.into()),
};
if let Some(ref mut pruning) = self.pruning {
pruning.note_canonical(hash, &mut commit);
@@ -480,13 +494,56 @@ pub struct StateDb<BlockHash: Hash, Key: Hash> {
}
impl<BlockHash: Hash + MallocSizeOf, Key: Hash + MallocSizeOf> StateDb<BlockHash, Key> {
/// Creates a new instance. Does not expect any metadata in the database.
pub fn new<D: MetaDb>(
mode: PruningMode,
ref_counting: bool,
/// Create an instance of [`StateDb`].
pub fn open<D>(
db: &D,
) -> Result<StateDb<BlockHash, Key>, Error<D::Error>> {
Ok(StateDb { db: RwLock::new(StateDbSync::new(mode, ref_counting, db)?) })
requested_mode: Option<PruningMode>,
ref_counting: bool,
should_init: bool,
) -> Result<(CommitSet<Key>, StateDb<BlockHash, Key>), Error<D::Error>>
where
D: MetaDb,
{
let stored_mode = fetch_stored_pruning_mode(db)?;
let selected_mode = match (should_init, stored_mode, requested_mode) {
(true, stored_mode, requested_mode) => {
assert!(stored_mode.is_none(), "The storage has just been initialized. No meta-data is expected to be found in it.");
requested_mode.unwrap_or_default()
},
(false, None, _) =>
return Err(StateDbError::Metadata(
"An existing StateDb does not have PRUNING_MODE stored in its meta-data".into(),
)
.into()),
(false, Some(stored), None) => stored,
(false, Some(stored), Some(requested)) => choose_pruning_mode(stored, requested)?,
};
let db_init_commit_set = if should_init {
let mut cs: CommitSet<Key> = Default::default();
let key = to_meta_key(PRUNING_MODE, &());
let value = selected_mode.id().to_owned();
cs.meta.inserted.push((key, value));
cs
} else {
Default::default()
};
let state_db =
StateDb { db: RwLock::new(StateDbSync::new(selected_mode, ref_counting, db)?) };
Ok((db_init_commit_set, state_db))
}
pub fn pruning_mode(&self) -> PruningMode {
self.db.read().mode.clone()
}
/// Add a new non-canonical block.
@@ -571,18 +628,51 @@ impl<BlockHash: Hash + MallocSizeOf, Key: Hash + MallocSizeOf> StateDb<BlockHash
}
}
fn fetch_stored_pruning_mode<D: MetaDb>(db: &D) -> Result<Option<PruningMode>, Error<D::Error>> {
let meta_key_mode = to_meta_key(PRUNING_MODE, &());
if let Some(stored_mode) = db.get_meta(&meta_key_mode).map_err(Error::Db)? {
if let Some(mode) = PruningMode::from_id(&stored_mode) {
Ok(Some(mode))
} else {
Err(StateDbError::Metadata(format!(
"Invalid value stored for PRUNING_MODE: {:02x?}",
stored_mode
))
.into())
}
} else {
Ok(None)
}
}
fn choose_pruning_mode(
stored: PruningMode,
requested: PruningMode,
) -> Result<PruningMode, StateDbError> {
match (stored, requested) {
(PruningMode::ArchiveAll, PruningMode::ArchiveAll) => Ok(PruningMode::ArchiveAll),
(PruningMode::ArchiveCanonical, PruningMode::ArchiveCanonical) =>
Ok(PruningMode::ArchiveCanonical),
(PruningMode::Constrained(_), PruningMode::Constrained(requested)) =>
Ok(PruningMode::Constrained(requested)),
(stored, requested) => Err(StateDbError::IncompatiblePruningModes { requested, stored }),
}
}
#[cfg(test)]
mod tests {
use crate::{
test::{make_changeset, make_db, TestDb},
Constraints, PruningMode, StateDb,
Constraints, Error, PruningMode, StateDb, StateDbError,
};
use sp_core::H256;
use std::io;
fn make_test_db(settings: PruningMode) -> (TestDb, StateDb<H256, H256>) {
let mut db = make_db(&[91, 921, 922, 93, 94]);
let state_db = StateDb::new(settings, false, &db).unwrap();
let (state_db_init, state_db) =
StateDb::open(&mut db, Some(settings), false, true).unwrap();
db.commit(&state_db_init);
db.commit(
&state_db
@@ -697,7 +787,9 @@ mod tests {
#[test]
fn detects_incompatible_mode() {
let mut db = make_db(&[]);
let state_db = StateDb::new(PruningMode::ArchiveAll, false, &db).unwrap();
let (state_db_init, state_db) =
StateDb::open(&mut db, Some(PruningMode::ArchiveAll), false, true).unwrap();
db.commit(&state_db_init);
db.commit(
&state_db
.insert_block::<io::Error>(
@@ -709,7 +801,85 @@ mod tests {
.unwrap(),
);
let new_mode = PruningMode::Constrained(Constraints { max_blocks: Some(2), max_mem: None });
let state_db: Result<StateDb<H256, H256>, _> = StateDb::new(new_mode, false, &db);
assert!(state_db.is_err());
let state_db_open_result: Result<(_, StateDb<H256, H256>), _> =
StateDb::open(&mut db, Some(new_mode), false, false);
assert!(state_db_open_result.is_err());
}
fn check_stored_and_requested_mode_compatibility(
mode_when_created: Option<PruningMode>,
mode_when_reopened: Option<PruningMode>,
expected_effective_mode_when_reopenned: Result<PruningMode, ()>,
) {
let mut db = make_db(&[]);
let (state_db_init, state_db) =
StateDb::<H256, H256>::open(&mut db, mode_when_created, false, true).unwrap();
db.commit(&state_db_init);
std::mem::drop(state_db);
let state_db_reopen_result =
StateDb::<H256, H256>::open(&mut db, mode_when_reopened, false, false);
if let Ok(expected_mode) = expected_effective_mode_when_reopenned {
let (state_db_init, state_db_reopened) = state_db_reopen_result.unwrap();
db.commit(&state_db_init);
assert_eq!(state_db_reopened.pruning_mode(), expected_mode,)
} else {
assert!(matches!(
state_db_reopen_result,
Err(Error::StateDb(StateDbError::IncompatiblePruningModes { .. }))
));
}
}
#[test]
fn pruning_mode_compatibility() {
for (created, reopened, expected) in [
(None, None, Ok(PruningMode::keep_blocks(256))),
(None, Some(PruningMode::keep_blocks(256)), Ok(PruningMode::keep_blocks(256))),
(None, Some(PruningMode::keep_blocks(128)), Ok(PruningMode::keep_blocks(128))),
(None, Some(PruningMode::keep_blocks(512)), Ok(PruningMode::keep_blocks(512))),
(None, Some(PruningMode::ArchiveAll), Err(())),
(None, Some(PruningMode::ArchiveCanonical), Err(())),
(Some(PruningMode::keep_blocks(256)), None, Ok(PruningMode::keep_blocks(256))),
(
Some(PruningMode::keep_blocks(256)),
Some(PruningMode::keep_blocks(256)),
Ok(PruningMode::keep_blocks(256)),
),
(
Some(PruningMode::keep_blocks(256)),
Some(PruningMode::keep_blocks(128)),
Ok(PruningMode::keep_blocks(128)),
),
(
Some(PruningMode::keep_blocks(256)),
Some(PruningMode::keep_blocks(512)),
Ok(PruningMode::keep_blocks(512)),
),
(Some(PruningMode::keep_blocks(256)), Some(PruningMode::ArchiveAll), Err(())),
(Some(PruningMode::keep_blocks(256)), Some(PruningMode::ArchiveCanonical), Err(())),
(Some(PruningMode::ArchiveAll), None, Ok(PruningMode::ArchiveAll)),
(Some(PruningMode::ArchiveAll), Some(PruningMode::keep_blocks(256)), Err(())),
(Some(PruningMode::ArchiveAll), Some(PruningMode::keep_blocks(128)), Err(())),
(Some(PruningMode::ArchiveAll), Some(PruningMode::keep_blocks(512)), Err(())),
(
Some(PruningMode::ArchiveAll),
Some(PruningMode::ArchiveAll),
Ok(PruningMode::ArchiveAll),
),
(Some(PruningMode::ArchiveAll), Some(PruningMode::ArchiveCanonical), Err(())),
(Some(PruningMode::ArchiveCanonical), None, Ok(PruningMode::ArchiveCanonical)),
(Some(PruningMode::ArchiveCanonical), Some(PruningMode::keep_blocks(256)), Err(())),
(Some(PruningMode::ArchiveCanonical), Some(PruningMode::keep_blocks(128)), Err(())),
(Some(PruningMode::ArchiveCanonical), Some(PruningMode::keep_blocks(512)), Err(())),
(Some(PruningMode::ArchiveCanonical), Some(PruningMode::ArchiveAll), Err(())),
(
Some(PruningMode::ArchiveCanonical),
Some(PruningMode::ArchiveCanonical),
Ok(PruningMode::ArchiveCanonical),
),
] {
check_stored_and_requested_mode_compatibility(created, reopened, expected);
}
}
}
+96 -130
View File
@@ -22,13 +22,10 @@
//! All pending changes are kept in memory until next call to `apply_pending` or
//! `revert_pending`
use super::{to_meta_key, ChangeSet, CommitSet, DBValue, Error, Hash, MetaDb};
use super::{to_meta_key, ChangeSet, CommitSet, DBValue, Error, Hash, MetaDb, StateDbError};
use codec::{Decode, Encode};
use log::trace;
use std::{
collections::{hash_map::Entry, HashMap, VecDeque},
fmt,
};
use std::collections::{hash_map::Entry, HashMap, VecDeque};
const NON_CANONICAL_JOURNAL: &[u8] = b"noncanonical_journal";
const LAST_CANONICAL: &[u8] = b"last_canonical";
@@ -242,13 +239,13 @@ impl<BlockHash: Hash, Key: Hash> NonCanonicalOverlay<BlockHash, Key> {
/// Insert a new block into the overlay. If inserted on the second level or lover expects parent
/// to be present in the window.
pub fn insert<E: fmt::Debug>(
pub fn insert(
&mut self,
hash: &BlockHash,
number: u64,
parent_hash: &BlockHash,
changeset: ChangeSet<Key>,
) -> Result<CommitSet<Key>, Error<E>> {
) -> Result<CommitSet<Key>, StateDbError> {
let mut commit = CommitSet::default();
let front_block_number = self.front_block_number();
if self.levels.is_empty() && self.last_canonicalized.is_none() && number > 0 {
@@ -267,7 +264,7 @@ impl<BlockHash: Hash, Key: Hash> NonCanonicalOverlay<BlockHash, Key> {
front_block_number,
front_block_number + self.levels.len() as u64,
);
return Err(Error::InvalidBlockNumber)
return Err(StateDbError::InvalidBlockNumber)
}
// check for valid parent if inserting on second level or higher
if number == front_block_number {
@@ -276,10 +273,10 @@ impl<BlockHash: Hash, Key: Hash> NonCanonicalOverlay<BlockHash, Key> {
.as_ref()
.map_or(false, |&(ref h, n)| h == parent_hash && n == number - 1)
{
return Err(Error::InvalidParent)
return Err(StateDbError::InvalidParent)
}
} else if !self.parents.contains_key(parent_hash) {
return Err(Error::InvalidParent)
return Err(StateDbError::InvalidParent)
}
}
let level = if self.levels.is_empty() ||
@@ -293,10 +290,10 @@ impl<BlockHash: Hash, Key: Hash> NonCanonicalOverlay<BlockHash, Key> {
};
if level.blocks.len() >= MAX_BLOCKS_PER_LEVEL as usize {
return Err(Error::TooManySiblingBlocks)
return Err(StateDbError::TooManySiblingBlocks)
}
if level.blocks.iter().any(|b| b.hash == *hash) {
return Err(Error::BlockAlreadyExists)
return Err(StateDbError::BlockAlreadyExists)
}
let index = level.available_index();
@@ -380,21 +377,21 @@ impl<BlockHash: Hash, Key: Hash> NonCanonicalOverlay<BlockHash, Key> {
/// Select a top-level root and canonicalized it. Discards all sibling subtrees and the root.
/// Returns a set of changes that need to be added to the DB.
pub fn canonicalize<E: fmt::Debug>(
pub fn canonicalize(
&mut self,
hash: &BlockHash,
commit: &mut CommitSet<Key>,
) -> Result<(), Error<E>> {
) -> Result<(), StateDbError> {
trace!(target: "state-db", "Canonicalizing {:?}", hash);
let level = self
.levels
.get(self.pending_canonicalizations.len())
.ok_or(Error::InvalidBlock)?;
.ok_or(StateDbError::InvalidBlock)?;
let index = level
.blocks
.iter()
.position(|overlay| overlay.hash == *hash)
.ok_or(Error::InvalidBlock)?;
.ok_or(StateDbError::InvalidBlock)?;
let mut discarded_journals = Vec::new();
let mut discarded_blocks = Vec::new();
@@ -640,10 +637,9 @@ mod tests {
use super::{to_journal_key, NonCanonicalOverlay};
use crate::{
test::{make_changeset, make_db},
ChangeSet, CommitSet, Error, MetaDb,
ChangeSet, CommitSet, MetaDb, StateDbError,
};
use sp_core::H256;
use std::io;
fn contains(overlay: &NonCanonicalOverlay<H256, H256>, key: u64) -> bool {
overlay.get(&H256::from_low_u64_be(key)) ==
@@ -665,7 +661,7 @@ mod tests {
let db = make_db(&[]);
let mut overlay = NonCanonicalOverlay::<H256, H256>::new(&db).unwrap();
let mut commit = CommitSet::default();
overlay.canonicalize::<io::Error>(&H256::default(), &mut commit).unwrap();
overlay.canonicalize(&H256::default(), &mut commit).unwrap();
}
#[test]
@@ -675,10 +671,8 @@ mod tests {
let h1 = H256::random();
let h2 = H256::random();
let mut overlay = NonCanonicalOverlay::<H256, H256>::new(&db).unwrap();
overlay
.insert::<io::Error>(&h1, 2, &H256::default(), ChangeSet::default())
.unwrap();
overlay.insert::<io::Error>(&h2, 1, &h1, ChangeSet::default()).unwrap();
overlay.insert(&h1, 2, &H256::default(), ChangeSet::default()).unwrap();
overlay.insert(&h2, 1, &h1, ChangeSet::default()).unwrap();
}
#[test]
@@ -688,10 +682,8 @@ mod tests {
let h2 = H256::random();
let db = make_db(&[]);
let mut overlay = NonCanonicalOverlay::<H256, H256>::new(&db).unwrap();
overlay
.insert::<io::Error>(&h1, 1, &H256::default(), ChangeSet::default())
.unwrap();
overlay.insert::<io::Error>(&h2, 3, &h1, ChangeSet::default()).unwrap();
overlay.insert(&h1, 1, &H256::default(), ChangeSet::default()).unwrap();
overlay.insert(&h2, 3, &h1, ChangeSet::default()).unwrap();
}
#[test]
@@ -701,12 +693,8 @@ mod tests {
let h1 = H256::random();
let h2 = H256::random();
let mut overlay = NonCanonicalOverlay::<H256, H256>::new(&db).unwrap();
overlay
.insert::<io::Error>(&h1, 1, &H256::default(), ChangeSet::default())
.unwrap();
overlay
.insert::<io::Error>(&h2, 2, &H256::default(), ChangeSet::default())
.unwrap();
overlay.insert(&h1, 1, &H256::default(), ChangeSet::default()).unwrap();
overlay.insert(&h2, 2, &H256::default(), ChangeSet::default()).unwrap();
}
#[test]
@@ -714,12 +702,10 @@ mod tests {
let db = make_db(&[]);
let h1 = H256::random();
let mut overlay = NonCanonicalOverlay::<H256, H256>::new(&db).unwrap();
overlay
.insert::<io::Error>(&h1, 2, &H256::default(), ChangeSet::default())
.unwrap();
overlay.insert(&h1, 2, &H256::default(), ChangeSet::default()).unwrap();
assert!(matches!(
overlay.insert::<io::Error>(&h1, 2, &H256::default(), ChangeSet::default()),
Err(Error::BlockAlreadyExists)
overlay.insert(&h1, 2, &H256::default(), ChangeSet::default()),
Err(StateDbError::BlockAlreadyExists)
));
}
@@ -730,11 +716,9 @@ mod tests {
let h2 = H256::random();
let db = make_db(&[]);
let mut overlay = NonCanonicalOverlay::<H256, H256>::new(&db).unwrap();
overlay
.insert::<io::Error>(&h1, 1, &H256::default(), ChangeSet::default())
.unwrap();
overlay.insert(&h1, 1, &H256::default(), ChangeSet::default()).unwrap();
let mut commit = CommitSet::default();
overlay.canonicalize::<io::Error>(&h2, &mut commit).unwrap();
overlay.canonicalize(&h2, &mut commit).unwrap();
}
#[test]
@@ -743,16 +727,14 @@ mod tests {
let mut db = make_db(&[1, 2]);
let mut overlay = NonCanonicalOverlay::<H256, H256>::new(&db).unwrap();
let changeset = make_changeset(&[3, 4], &[2]);
let insertion = overlay
.insert::<io::Error>(&h1, 1, &H256::default(), changeset.clone())
.unwrap();
let insertion = overlay.insert(&h1, 1, &H256::default(), changeset.clone()).unwrap();
assert_eq!(insertion.data.inserted.len(), 0);
assert_eq!(insertion.data.deleted.len(), 0);
assert_eq!(insertion.meta.inserted.len(), 2);
assert_eq!(insertion.meta.deleted.len(), 0);
db.commit(&insertion);
let mut finalization = CommitSet::default();
overlay.canonicalize::<io::Error>(&h1, &mut finalization).unwrap();
overlay.canonicalize(&h1, &mut finalization).unwrap();
assert_eq!(finalization.data.inserted.len(), changeset.inserted.len());
assert_eq!(finalization.data.deleted.len(), changeset.deleted.len());
assert_eq!(finalization.meta.inserted.len(), 1);
@@ -769,10 +751,10 @@ mod tests {
let mut overlay = NonCanonicalOverlay::<H256, H256>::new(&db).unwrap();
db.commit(
&overlay
.insert::<io::Error>(&h1, 10, &H256::default(), make_changeset(&[3, 4], &[2]))
.insert(&h1, 10, &H256::default(), make_changeset(&[3, 4], &[2]))
.unwrap(),
);
db.commit(&overlay.insert::<io::Error>(&h2, 11, &h1, make_changeset(&[5], &[3])).unwrap());
db.commit(&overlay.insert(&h2, 11, &h1, make_changeset(&[5], &[3])).unwrap());
assert_eq!(db.meta.len(), 3);
let overlay2 = NonCanonicalOverlay::<H256, H256>::new(&db).unwrap();
@@ -789,12 +771,12 @@ mod tests {
let mut overlay = NonCanonicalOverlay::<H256, H256>::new(&db).unwrap();
db.commit(
&overlay
.insert::<io::Error>(&h1, 10, &H256::default(), make_changeset(&[3, 4], &[2]))
.insert(&h1, 10, &H256::default(), make_changeset(&[3, 4], &[2]))
.unwrap(),
);
db.commit(&overlay.insert::<io::Error>(&h2, 11, &h1, make_changeset(&[5], &[3])).unwrap());
db.commit(&overlay.insert(&h2, 11, &h1, make_changeset(&[5], &[3])).unwrap());
let mut commit = CommitSet::default();
overlay.canonicalize::<io::Error>(&h1, &mut commit).unwrap();
overlay.canonicalize(&h1, &mut commit).unwrap();
db.commit(&commit);
overlay.apply_pending();
assert_eq!(overlay.levels.len(), 1);
@@ -813,15 +795,15 @@ mod tests {
let mut overlay = NonCanonicalOverlay::<H256, H256>::new(&db).unwrap();
let changeset1 = make_changeset(&[5, 6], &[2]);
let changeset2 = make_changeset(&[7, 8], &[5, 3]);
db.commit(&overlay.insert::<io::Error>(&h1, 1, &H256::default(), changeset1).unwrap());
db.commit(&overlay.insert(&h1, 1, &H256::default(), changeset1).unwrap());
assert!(contains(&overlay, 5));
db.commit(&overlay.insert::<io::Error>(&h2, 2, &h1, changeset2).unwrap());
db.commit(&overlay.insert(&h2, 2, &h1, changeset2).unwrap());
assert!(contains(&overlay, 7));
assert!(contains(&overlay, 5));
assert_eq!(overlay.levels.len(), 2);
assert_eq!(overlay.parents.len(), 2);
let mut commit = CommitSet::default();
overlay.canonicalize::<io::Error>(&h1, &mut commit).unwrap();
overlay.canonicalize(&h1, &mut commit).unwrap();
db.commit(&commit);
assert!(contains(&overlay, 5));
assert_eq!(overlay.levels.len(), 2);
@@ -832,7 +814,7 @@ mod tests {
assert!(!contains(&overlay, 5));
assert!(contains(&overlay, 7));
let mut commit = CommitSet::default();
overlay.canonicalize::<io::Error>(&h2, &mut commit).unwrap();
overlay.canonicalize(&h2, &mut commit).unwrap();
db.commit(&commit);
overlay.apply_pending();
assert_eq!(overlay.levels.len(), 0);
@@ -847,11 +829,11 @@ mod tests {
let (h_2, c_2) = (H256::random(), make_changeset(&[1], &[]));
let mut overlay = NonCanonicalOverlay::<H256, H256>::new(&db).unwrap();
db.commit(&overlay.insert::<io::Error>(&h_1, 1, &H256::default(), c_1).unwrap());
db.commit(&overlay.insert::<io::Error>(&h_2, 1, &H256::default(), c_2).unwrap());
db.commit(&overlay.insert(&h_1, 1, &H256::default(), c_1).unwrap());
db.commit(&overlay.insert(&h_2, 1, &H256::default(), c_2).unwrap());
assert!(contains(&overlay, 1));
let mut commit = CommitSet::default();
overlay.canonicalize::<io::Error>(&h_1, &mut commit).unwrap();
overlay.canonicalize(&h_1, &mut commit).unwrap();
db.commit(&commit);
assert!(contains(&overlay, 1));
overlay.apply_pending();
@@ -866,18 +848,14 @@ mod tests {
let mut db = make_db(&[]);
let mut overlay = NonCanonicalOverlay::<H256, H256>::new(&db).unwrap();
let changeset = make_changeset(&[], &[]);
db.commit(
&overlay
.insert::<io::Error>(&h1, 1, &H256::default(), changeset.clone())
.unwrap(),
);
db.commit(&overlay.insert::<io::Error>(&h2, 2, &h1, changeset.clone()).unwrap());
db.commit(&overlay.insert(&h1, 1, &H256::default(), changeset.clone()).unwrap());
db.commit(&overlay.insert(&h2, 2, &h1, changeset.clone()).unwrap());
overlay.apply_pending();
let mut commit = CommitSet::default();
overlay.canonicalize::<io::Error>(&h1, &mut commit).unwrap();
overlay.canonicalize::<io::Error>(&h2, &mut commit).unwrap();
overlay.canonicalize(&h1, &mut commit).unwrap();
overlay.canonicalize(&h2, &mut commit).unwrap();
db.commit(&commit);
db.commit(&overlay.insert::<io::Error>(&h3, 3, &h2, changeset.clone()).unwrap());
db.commit(&overlay.insert(&h3, 3, &h2, changeset.clone()).unwrap());
overlay.apply_pending();
assert_eq!(overlay.levels.len(), 1);
}
@@ -912,21 +890,21 @@ mod tests {
let (h_2_1_1, c_2_1_1) = (H256::random(), make_changeset(&[211], &[]));
let mut overlay = NonCanonicalOverlay::<H256, H256>::new(&db).unwrap();
db.commit(&overlay.insert::<io::Error>(&h_1, 1, &H256::default(), c_1).unwrap());
db.commit(&overlay.insert(&h_1, 1, &H256::default(), c_1).unwrap());
db.commit(&overlay.insert::<io::Error>(&h_1_1, 2, &h_1, c_1_1).unwrap());
db.commit(&overlay.insert::<io::Error>(&h_1_2, 2, &h_1, c_1_2).unwrap());
db.commit(&overlay.insert(&h_1_1, 2, &h_1, c_1_1).unwrap());
db.commit(&overlay.insert(&h_1_2, 2, &h_1, c_1_2).unwrap());
db.commit(&overlay.insert::<io::Error>(&h_2, 1, &H256::default(), c_2).unwrap());
db.commit(&overlay.insert(&h_2, 1, &H256::default(), c_2).unwrap());
db.commit(&overlay.insert::<io::Error>(&h_2_1, 2, &h_2, c_2_1).unwrap());
db.commit(&overlay.insert::<io::Error>(&h_2_2, 2, &h_2, c_2_2).unwrap());
db.commit(&overlay.insert(&h_2_1, 2, &h_2, c_2_1).unwrap());
db.commit(&overlay.insert(&h_2_2, 2, &h_2, c_2_2).unwrap());
db.commit(&overlay.insert::<io::Error>(&h_1_1_1, 3, &h_1_1, c_1_1_1).unwrap());
db.commit(&overlay.insert::<io::Error>(&h_1_2_1, 3, &h_1_2, c_1_2_1).unwrap());
db.commit(&overlay.insert::<io::Error>(&h_1_2_2, 3, &h_1_2, c_1_2_2).unwrap());
db.commit(&overlay.insert::<io::Error>(&h_1_2_3, 3, &h_1_2, c_1_2_3).unwrap());
db.commit(&overlay.insert::<io::Error>(&h_2_1_1, 3, &h_2_1, c_2_1_1).unwrap());
db.commit(&overlay.insert(&h_1_1_1, 3, &h_1_1, c_1_1_1).unwrap());
db.commit(&overlay.insert(&h_1_2_1, 3, &h_1_2, c_1_2_1).unwrap());
db.commit(&overlay.insert(&h_1_2_2, 3, &h_1_2, c_1_2_2).unwrap());
db.commit(&overlay.insert(&h_1_2_3, 3, &h_1_2, c_1_2_3).unwrap());
db.commit(&overlay.insert(&h_2_1_1, 3, &h_2_1, c_2_1_1).unwrap());
assert!(contains(&overlay, 2));
assert!(contains(&overlay, 11));
@@ -946,7 +924,7 @@ mod tests {
// canonicalize 1. 2 and all its children should be discarded
let mut commit = CommitSet::default();
overlay.canonicalize::<io::Error>(&h_1, &mut commit).unwrap();
overlay.canonicalize(&h_1, &mut commit).unwrap();
db.commit(&commit);
overlay.apply_pending();
assert_eq!(overlay.levels.len(), 2);
@@ -967,7 +945,7 @@ mod tests {
// canonicalize 1_2. 1_1 and all its children should be discarded
let mut commit = CommitSet::default();
overlay.canonicalize::<io::Error>(&h_1_2, &mut commit).unwrap();
overlay.canonicalize(&h_1_2, &mut commit).unwrap();
db.commit(&commit);
overlay.apply_pending();
assert_eq!(overlay.levels.len(), 1);
@@ -984,7 +962,7 @@ mod tests {
// canonicalize 1_2_2
let mut commit = CommitSet::default();
overlay.canonicalize::<io::Error>(&h_1_2_2, &mut commit).unwrap();
overlay.canonicalize(&h_1_2_2, &mut commit).unwrap();
db.commit(&commit);
overlay.apply_pending();
assert_eq!(overlay.levels.len(), 0);
@@ -1002,8 +980,8 @@ mod tests {
assert!(overlay.revert_one().is_none());
let changeset1 = make_changeset(&[5, 6], &[2]);
let changeset2 = make_changeset(&[7, 8], &[5, 3]);
db.commit(&overlay.insert::<io::Error>(&h1, 1, &H256::default(), changeset1).unwrap());
db.commit(&overlay.insert::<io::Error>(&h2, 2, &h1, changeset2).unwrap());
db.commit(&overlay.insert(&h1, 1, &H256::default(), changeset1).unwrap());
db.commit(&overlay.insert(&h2, 2, &h1, changeset2).unwrap());
assert!(contains(&overlay, 7));
db.commit(&overlay.revert_one().unwrap());
assert_eq!(overlay.parents.len(), 1);
@@ -1025,10 +1003,10 @@ mod tests {
let changeset1 = make_changeset(&[5, 6], &[2]);
let changeset2 = make_changeset(&[7, 8], &[5, 3]);
let changeset3 = make_changeset(&[9], &[]);
overlay.insert::<io::Error>(&h1, 1, &H256::default(), changeset1).unwrap();
overlay.insert(&h1, 1, &H256::default(), changeset1).unwrap();
assert!(contains(&overlay, 5));
overlay.insert::<io::Error>(&h2_1, 2, &h1, changeset2).unwrap();
overlay.insert::<io::Error>(&h2_2, 2, &h1, changeset3).unwrap();
overlay.insert(&h2_1, 2, &h1, changeset2).unwrap();
overlay.insert(&h2_2, 2, &h1, changeset3).unwrap();
assert!(contains(&overlay, 7));
assert!(contains(&overlay, 5));
assert!(contains(&overlay, 9));
@@ -1052,14 +1030,14 @@ mod tests {
let (h_2, c_2) = (H256::random(), make_changeset(&[2], &[]));
let mut overlay = NonCanonicalOverlay::<H256, H256>::new(&db).unwrap();
db.commit(&overlay.insert::<io::Error>(&h_1, 1, &H256::default(), c_1).unwrap());
db.commit(&overlay.insert::<io::Error>(&h_2, 1, &H256::default(), c_2).unwrap());
db.commit(&overlay.insert(&h_1, 1, &H256::default(), c_1).unwrap());
db.commit(&overlay.insert(&h_2, 1, &H256::default(), c_2).unwrap());
overlay.apply_pending();
overlay.pin(&h_1);
let mut commit = CommitSet::default();
overlay.canonicalize::<io::Error>(&h_2, &mut commit).unwrap();
overlay.canonicalize(&h_2, &mut commit).unwrap();
db.commit(&commit);
overlay.apply_pending();
assert!(contains(&overlay, 1));
@@ -1082,15 +1060,15 @@ mod tests {
let (h_3, c_3) = (H256::random(), make_changeset(&[], &[]));
let mut overlay = NonCanonicalOverlay::<H256, H256>::new(&db).unwrap();
db.commit(&overlay.insert::<io::Error>(&h_1, 1, &H256::default(), c_1).unwrap());
db.commit(&overlay.insert::<io::Error>(&h_2, 1, &H256::default(), c_2).unwrap());
db.commit(&overlay.insert::<io::Error>(&h_3, 1, &H256::default(), c_3).unwrap());
db.commit(&overlay.insert(&h_1, 1, &H256::default(), c_1).unwrap());
db.commit(&overlay.insert(&h_2, 1, &H256::default(), c_2).unwrap());
db.commit(&overlay.insert(&h_3, 1, &H256::default(), c_3).unwrap());
overlay.apply_pending();
overlay.pin(&h_1);
let mut commit = CommitSet::default();
overlay.canonicalize::<io::Error>(&h_3, &mut commit).unwrap();
overlay.canonicalize(&h_3, &mut commit).unwrap();
db.commit(&commit);
overlay.apply_pending(); // 1_2 should be discarded, 1_1 is pinned
@@ -1112,15 +1090,15 @@ mod tests {
let (h_21, c_21) = (H256::random(), make_changeset(&[], &[]));
let mut overlay = NonCanonicalOverlay::<H256, H256>::new(&db).unwrap();
db.commit(&overlay.insert::<io::Error>(&h_11, 1, &H256::default(), c_11).unwrap());
db.commit(&overlay.insert::<io::Error>(&h_12, 1, &H256::default(), c_12).unwrap());
db.commit(&overlay.insert::<io::Error>(&h_21, 2, &h_11, c_21).unwrap());
db.commit(&overlay.insert(&h_11, 1, &H256::default(), c_11).unwrap());
db.commit(&overlay.insert(&h_12, 1, &H256::default(), c_12).unwrap());
db.commit(&overlay.insert(&h_21, 2, &h_11, c_21).unwrap());
overlay.apply_pending();
overlay.pin(&h_21);
let mut commit = CommitSet::default();
overlay.canonicalize::<io::Error>(&h_12, &mut commit).unwrap();
overlay.canonicalize(&h_12, &mut commit).unwrap();
db.commit(&commit);
overlay.apply_pending(); // 1_1 and 2_1 should be both pinned
@@ -1141,18 +1119,14 @@ mod tests {
let h21 = H256::random();
let mut db = make_db(&[]);
let mut overlay = NonCanonicalOverlay::<H256, H256>::new(&db).unwrap();
db.commit(
&overlay
.insert::<io::Error>(&root, 10, &H256::default(), make_changeset(&[], &[]))
.unwrap(),
);
db.commit(&overlay.insert::<io::Error>(&h1, 11, &root, make_changeset(&[1], &[])).unwrap());
db.commit(&overlay.insert::<io::Error>(&h2, 11, &root, make_changeset(&[2], &[])).unwrap());
db.commit(&overlay.insert::<io::Error>(&h11, 12, &h1, make_changeset(&[11], &[])).unwrap());
db.commit(&overlay.insert::<io::Error>(&h21, 12, &h2, make_changeset(&[21], &[])).unwrap());
db.commit(&overlay.insert(&root, 10, &H256::default(), make_changeset(&[], &[])).unwrap());
db.commit(&overlay.insert(&h1, 11, &root, make_changeset(&[1], &[])).unwrap());
db.commit(&overlay.insert(&h2, 11, &root, make_changeset(&[2], &[])).unwrap());
db.commit(&overlay.insert(&h11, 12, &h1, make_changeset(&[11], &[])).unwrap());
db.commit(&overlay.insert(&h21, 12, &h2, make_changeset(&[21], &[])).unwrap());
let mut commit = CommitSet::default();
overlay.canonicalize::<io::Error>(&root, &mut commit).unwrap();
overlay.canonicalize::<io::Error>(&h2, &mut commit).unwrap(); // h11 should stay in the DB
overlay.canonicalize(&root, &mut commit).unwrap();
overlay.canonicalize(&h2, &mut commit).unwrap(); // h11 should stay in the DB
db.commit(&commit);
overlay.apply_pending();
assert_eq!(overlay.levels.len(), 1);
@@ -1166,7 +1140,7 @@ mod tests {
assert!(contains(&overlay, 21));
let mut commit = CommitSet::default();
overlay.canonicalize::<io::Error>(&h21, &mut commit).unwrap(); // h11 should stay in the DB
overlay.canonicalize(&h21, &mut commit).unwrap(); // h11 should stay in the DB
db.commit(&commit);
overlay.apply_pending();
assert!(!contains(&overlay, 21));
@@ -1183,25 +1157,21 @@ mod tests {
let h21 = H256::random();
let mut db = make_db(&[]);
let mut overlay = NonCanonicalOverlay::<H256, H256>::new(&db).unwrap();
db.commit(
&overlay
.insert::<io::Error>(&root, 10, &H256::default(), make_changeset(&[], &[]))
.unwrap(),
);
db.commit(&overlay.insert::<io::Error>(&h1, 11, &root, make_changeset(&[1], &[])).unwrap());
db.commit(&overlay.insert::<io::Error>(&h2, 11, &root, make_changeset(&[2], &[])).unwrap());
db.commit(&overlay.insert::<io::Error>(&h11, 12, &h1, make_changeset(&[11], &[])).unwrap());
db.commit(&overlay.insert::<io::Error>(&h21, 12, &h2, make_changeset(&[21], &[])).unwrap());
db.commit(&overlay.insert(&root, 10, &H256::default(), make_changeset(&[], &[])).unwrap());
db.commit(&overlay.insert(&h1, 11, &root, make_changeset(&[1], &[])).unwrap());
db.commit(&overlay.insert(&h2, 11, &root, make_changeset(&[2], &[])).unwrap());
db.commit(&overlay.insert(&h11, 12, &h1, make_changeset(&[11], &[])).unwrap());
db.commit(&overlay.insert(&h21, 12, &h2, make_changeset(&[21], &[])).unwrap());
let mut commit = CommitSet::default();
overlay.canonicalize::<io::Error>(&root, &mut commit).unwrap();
overlay.canonicalize::<io::Error>(&h2, &mut commit).unwrap(); // h11 should stay in the DB
overlay.canonicalize(&root, &mut commit).unwrap();
overlay.canonicalize(&h2, &mut commit).unwrap(); // h11 should stay in the DB
db.commit(&commit);
overlay.apply_pending();
// add another block at top level. It should reuse journal index 0 of previously discarded
// block
let h22 = H256::random();
db.commit(&overlay.insert::<io::Error>(&h22, 12, &h2, make_changeset(&[22], &[])).unwrap());
db.commit(&overlay.insert(&h22, 12, &h2, make_changeset(&[22], &[])).unwrap());
assert_eq!(overlay.levels[0].blocks[0].journal_index, 1);
assert_eq!(overlay.levels[0].blocks[1].journal_index, 0);
@@ -1221,15 +1191,11 @@ mod tests {
let h21 = H256::random();
let mut db = make_db(&[]);
let mut overlay = NonCanonicalOverlay::<H256, H256>::new(&db).unwrap();
db.commit(
&overlay
.insert::<io::Error>(&root, 10, &H256::default(), make_changeset(&[], &[]))
.unwrap(),
);
db.commit(&overlay.insert::<io::Error>(&h1, 11, &root, make_changeset(&[1], &[])).unwrap());
db.commit(&overlay.insert::<io::Error>(&h2, 11, &root, make_changeset(&[2], &[])).unwrap());
db.commit(&overlay.insert::<io::Error>(&h11, 12, &h1, make_changeset(&[11], &[])).unwrap());
db.commit(&overlay.insert::<io::Error>(&h21, 12, &h2, make_changeset(&[21], &[])).unwrap());
db.commit(&overlay.insert(&root, 10, &H256::default(), make_changeset(&[], &[])).unwrap());
db.commit(&overlay.insert(&h1, 11, &root, make_changeset(&[1], &[])).unwrap());
db.commit(&overlay.insert(&h2, 11, &root, make_changeset(&[2], &[])).unwrap());
db.commit(&overlay.insert(&h11, 12, &h1, make_changeset(&[11], &[])).unwrap());
db.commit(&overlay.insert(&h21, 12, &h2, make_changeset(&[21], &[])).unwrap());
assert!(overlay.remove(&h1).is_none());
assert!(overlay.remove(&h2).is_none());
assert_eq!(overlay.levels.len(), 3);