feat: Rebrand Polkadot/Substrate references to PezkuwiChain

This commit systematically rebrands various references from Parity Technologies'
Polkadot/Substrate ecosystem to PezkuwiChain within the kurdistan-sdk.

Key changes include:
- Updated external repository URLs (zombienet-sdk, parity-db, parity-scale-codec, wasm-instrument) to point to pezkuwichain forks.
- Modified internal documentation and code comments to reflect PezkuwiChain naming and structure.
- Replaced direct references to  with  or specific paths within the  for XCM, Pezkuwi, and other modules.
- Cleaned up deprecated  issue and PR references in various  and  files, particularly in  and  modules.
- Adjusted image and logo URLs in documentation to point to PezkuwiChain assets.
- Removed or rephrased comments related to external Polkadot/Substrate PRs and issues.

This is a significant step towards fully customizing the SDK for the PezkuwiChain ecosystem.

bizinikiwi/client/state-db/Cargo.toml

@@ -0,0 +1,22 @@
+[package]
+name = "pezsc-state-db"
+version = "0.30.0"
+authors.workspace = true
+edition.workspace = true
+license = "GPL-3.0-or-later WITH Classpath-exception-2.0"
+homepage.workspace = true
+repository.workspace = true
+description = "State database maintenance. Handles canonicalization and pruning in the database."
+readme = "README.md"
+
+[lints]
+workspace = true
+
+[package.metadata.docs.rs]
+targets = ["x86_64-unknown-linux-gnu"]
+
+[dependencies]
+codec = { features = ["derive"], workspace = true, default-features = true }
+log = { workspace = true, default-features = true }
+parking_lot = { workspace = true, default-features = true }
+pezsp-core = { workspace = true, default-features = true }

bizinikiwi/client/state-db/README.md

@@ -0,0 +1,16 @@
+State database maintenance. Handles canonicalization and pruning in the database. The input to
+this module is a `ChangeSet` which is basically a list of key-value pairs (trie nodes) that
+were added or deleted during block execution.
+
+# Canonicalization
+Canonicalization window tracks a tree of blocks identified by header hash. The in-memory
+overlay allows to get any node that was inserted in any of the blocks within the window.
+The tree is journaled to the backing database and rebuilt on startup.
+Canonicalization function selects one root from the top of the tree and discards all other roots and
+their subtrees.
+
+# Pruning
+See `RefWindow` for pruning algorithm details. `StateDb` prunes on each canonicalization until pruning
+constraints are satisfied.
+
+License: GPL-3.0-or-later WITH Classpath-exception-2.0

bizinikiwi/client/state-db/src/lib.rs

@@ -0,0 +1,949 @@
+// This file is part of Bizinikiwi.
+
+// Copyright (C) Parity Technologies (UK) Ltd.
+// SPDX-License-Identifier: GPL-3.0-or-later WITH Classpath-exception-2.0
+
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with this program. If not, see <https://www.gnu.org/licenses/>.
+
+//! State database maintenance. Handles canonicalization and pruning in the database.
+//!
+//! # Canonicalization.
+//! Canonicalization window tracks a tree of blocks identified by header hash. The in-memory
+//! overlay allows to get any trie node that was inserted in any of the blocks within the window.
+//! The overlay is journaled to the backing database and rebuilt on startup.
+//! There's a limit of 32 blocks that may have the same block number in the canonicalization window.
+//!
+//! Canonicalization function selects one root from the top of the tree and discards all other roots
+//! and their subtrees. Upon canonicalization all trie nodes that were inserted in the block are
+//! added to the backing DB and block tracking is moved to the pruning window, where no forks are
+//! allowed.
+//!
+//! # Canonicalization vs Finality
+//! Database engine uses a notion of canonicality, rather then finality. A canonical block may not
+//! be yet finalized from the perspective of the consensus engine, but it still can't be reverted in
+//! the database. Most of the time during normal operation last canonical block is the same as last
+//! finalized. However if finality stall for a long duration for some reason, there's only a certain
+//! number of blocks that can fit in the non-canonical overlay, so canonicalization of an
+//! unfinalized block may be forced.
+//!
+//! # Pruning.
+//! See `RefWindow` for pruning algorithm details. `StateDb` prunes on each canonicalization until
+//! pruning constraints are satisfied.
+
+mod noncanonical;
+mod pruning;
+#[cfg(test)]
+mod test;
+
+use codec::Codec;
+use log::trace;
+use noncanonical::NonCanonicalOverlay;
+use parking_lot::RwLock;
+use pruning::{HaveBlock, RefWindow};
+use std::{
+	collections::{hash_map::Entry, HashMap},
+	fmt,
+};
+
+const LOG_TARGET: &str = "state-db";
+const LOG_TARGET_PIN: &str = "state-db::pin";
+const PRUNING_MODE: &[u8] = b"mode";
+const PRUNING_MODE_ARCHIVE: &[u8] = b"archive";
+const PRUNING_MODE_ARCHIVE_CANON: &[u8] = b"archive_canonical";
+const PRUNING_MODE_CONSTRAINED: &[u8] = b"constrained";
+pub(crate) const DEFAULT_MAX_BLOCK_CONSTRAINT: u32 = 256;
+
+/// Database value type.
+pub type DBValue = Vec<u8>;
+
+/// Basic set of requirements for the Block hash and node key types.
+pub trait Hash:
+	Send
+	+ Sync
+	+ Sized
+	+ Eq
+	+ PartialEq
+	+ Clone
+	+ Default
+	+ fmt::Debug
+	+ Codec
+	+ std::hash::Hash
+	+ 'static
+{
+}
+impl<
+		T: Send
+			+ Sync
+			+ Sized
+			+ Eq
+			+ PartialEq
+			+ Clone
+			+ Default
+			+ fmt::Debug
+			+ Codec
+			+ std::hash::Hash
+			+ 'static,
+	> Hash for T
+{
+}
+
+/// Backend database trait. Read-only.
+pub trait MetaDb {
+	type Error: fmt::Debug;
+
+	/// Get meta value, such as the journal.
+	fn get_meta(&self, key: &[u8]) -> Result<Option<DBValue>, Self::Error>;
+}
+
+/// Backend database trait. Read-only.
+pub trait NodeDb {
+	type Key: ?Sized;
+	type Error: fmt::Debug;
+
+	/// Get state trie node.
+	fn get(&self, key: &Self::Key) -> Result<Option<DBValue>, Self::Error>;
+}
+
+/// Error type.
+#[derive(Eq, PartialEq)]
+pub enum Error<E> {
+	/// Database backend error.
+	Db(E),
+	StateDb(StateDbError),
+}
+
+#[derive(Eq, PartialEq)]
+pub enum StateDbError {
+	/// `Codec` decoding error.
+	Decoding(codec::Error),
+	/// Trying to canonicalize invalid block.
+	InvalidBlock,
+	/// Trying to insert block with invalid number.
+	InvalidBlockNumber,
+	/// Trying to insert block with unknown parent.
+	InvalidParent,
+	/// Invalid pruning mode specified. Contains expected mode.
+	IncompatiblePruningModes { stored: PruningMode, requested: PruningMode },
+	/// Too many unfinalized sibling blocks inserted.
+	TooManySiblingBlocks { number: u64 },
+	/// Trying to insert existing block.
+	BlockAlreadyExists,
+	/// Invalid metadata
+	Metadata(String),
+	/// Trying to get a block record from db while it is not commit to db yet
+	BlockUnavailable,
+	/// Block record is missing from the pruning window
+	BlockMissing,
+}
+
+impl<E> From<StateDbError> for Error<E> {
+	fn from(inner: StateDbError) -> Self {
+		Self::StateDb(inner)
+	}
+}
+
+/// Pinning error type.
+#[derive(Debug)]
+pub enum PinError {
+	/// Trying to pin invalid block.
+	InvalidBlock,
+}
+
+impl<E: fmt::Debug> From<codec::Error> for Error<E> {
+	fn from(x: codec::Error) -> Self {
+		StateDbError::Decoding(x).into()
+	}
+}
+
+impl<E: fmt::Debug> fmt::Debug for Error<E> {
+	fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
+		match self {
+			Self::Db(e) => e.fmt(f),
+			Self::StateDb(e) => e.fmt(f),
+		}
+	}
+}
+
+impl fmt::Debug for StateDbError {
+	fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
+		match self {
+			Self::Decoding(e) => write!(f, "Error decoding sliceable value: {}", e),
+			Self::InvalidBlock => write!(f, "Trying to canonicalize invalid block"),
+			Self::InvalidBlockNumber => write!(f, "Trying to insert block with invalid number"),
+			Self::InvalidParent => write!(f, "Trying to insert block with unknown parent"),
+			Self::IncompatiblePruningModes { stored, requested } => write!(
+				f,
+				"Incompatible pruning modes [stored: {:?}; requested: {:?}]",
+				stored, requested
+			),
+			Self::TooManySiblingBlocks { number } => {
+				write!(f, "Too many sibling blocks at #{number} inserted")
+			},
+			Self::BlockAlreadyExists => write!(f, "Block already exists"),
+			Self::Metadata(message) => write!(f, "Invalid metadata: {}", message),
+			Self::BlockUnavailable => {
+				write!(f, "Trying to get a block record from db while it is not commit to db yet")
+			},
+			Self::BlockMissing => write!(f, "Block record is missing from the pruning window"),
+		}
+	}
+}
+
+/// A set of state node changes.
+#[derive(Default, Debug, Clone)]
+pub struct ChangeSet<H: Hash> {
+	/// Inserted nodes.
+	pub inserted: Vec<(H, DBValue)>,
+	/// Deleted nodes.
+	pub deleted: Vec<H>,
+}
+
+/// A set of changes to the backing database.
+#[derive(Default, Debug, Clone)]
+pub struct CommitSet<H: Hash> {
+	/// State node changes.
+	pub data: ChangeSet<H>,
+	/// Metadata changes.
+	pub meta: ChangeSet<Vec<u8>>,
+}
+
+/// Pruning constraints. If none are specified pruning is
+#[derive(Debug, Clone, Eq, PartialEq)]
+pub struct Constraints {
+	/// Maximum blocks. Defaults to 0 when unspecified, effectively keeping only non-canonical
+	/// states.
+	pub max_blocks: Option<u32>,
+}
+
+/// Pruning mode.
+#[derive(Debug, Clone, Eq, PartialEq)]
+pub enum PruningMode {
+	/// Maintain a pruning window.
+	Constrained(Constraints),
+	/// No pruning. Canonicalization is a no-op.
+	ArchiveAll,
+	/// Canonicalization discards non-canonical nodes. All the canonical nodes are kept in the DB.
+	ArchiveCanonical,
+}
+
+impl PruningMode {
+	/// Create a mode that keeps given number of blocks.
+	pub fn blocks_pruning(n: u32) -> PruningMode {
+		PruningMode::Constrained(Constraints { max_blocks: Some(n) })
+	}
+
+	/// Is this an archive (either ArchiveAll or ArchiveCanonical) pruning mode?
+	pub fn is_archive(&self) -> bool {
+		match *self {
+			PruningMode::ArchiveAll | PruningMode::ArchiveCanonical => true,
+			PruningMode::Constrained(_) => false,
+		}
+	}
+
+	/// Returns the pruning mode
+	pub fn id(&self) -> &[u8] {
+		match self {
+			PruningMode::ArchiveAll => PRUNING_MODE_ARCHIVE,
+			PruningMode::ArchiveCanonical => PRUNING_MODE_ARCHIVE_CANON,
+			PruningMode::Constrained(_) => PRUNING_MODE_CONSTRAINED,
+		}
+	}
+
+	pub fn from_id(id: &[u8]) -> Option<Self> {
+		match id {
+			PRUNING_MODE_ARCHIVE => Some(Self::ArchiveAll),
+			PRUNING_MODE_ARCHIVE_CANON => Some(Self::ArchiveCanonical),
+			PRUNING_MODE_CONSTRAINED => Some(Self::Constrained(Default::default())),
+			_ => None,
+		}
+	}
+}
+
+impl Default for PruningMode {
+	fn default() -> Self {
+		PruningMode::Constrained(Default::default())
+	}
+}
+
+impl Default for Constraints {
+	fn default() -> Self {
+		Self { max_blocks: Some(DEFAULT_MAX_BLOCK_CONSTRAINT) }
+	}
+}
+
+fn to_meta_key<S: Codec>(suffix: &[u8], data: &S) -> Vec<u8> {
+	let mut buffer = data.encode();
+	buffer.extend(suffix);
+	buffer
+}
+
+/// Status information about the last canonicalized block.
+#[derive(Clone, Debug, PartialEq, Eq)]
+pub enum LastCanonicalized {
+	/// Not yet have canonicalized any block.
+	None,
+	/// The given block number is the last canonicalized block.
+	Block(u64),
+	/// No canonicalization is happening (pruning mode is archive all).
+	NotCanonicalizing,
+}
+
+pub struct StateDbSync<BlockHash: Hash, Key: Hash, D: MetaDb> {
+	mode: PruningMode,
+	non_canonical: NonCanonicalOverlay<BlockHash, Key>,
+	pruning: Option<RefWindow<BlockHash, Key, D>>,
+	pinned: HashMap<BlockHash, u32>,
+	ref_counting: bool,
+}
+
+impl<BlockHash: Hash, Key: Hash, D: MetaDb> StateDbSync<BlockHash, Key, D> {
+	fn new(
+		mode: PruningMode,
+		ref_counting: bool,
+		db: D,
+	) -> Result<StateDbSync<BlockHash, Key, D>, Error<D::Error>> {
+		trace!(target: LOG_TARGET, "StateDb settings: {:?}. Ref-counting: {}", mode, ref_counting);
+
+		let non_canonical: NonCanonicalOverlay<BlockHash, Key> = NonCanonicalOverlay::new(&db)?;
+		let pruning: Option<RefWindow<BlockHash, Key, D>> = match mode {
+			PruningMode::Constrained(Constraints { max_blocks }) =>
+				Some(RefWindow::new(db, max_blocks.unwrap_or(0), ref_counting)?),
+			PruningMode::ArchiveAll | PruningMode::ArchiveCanonical => None,
+		};
+
+		Ok(StateDbSync { mode, non_canonical, pruning, pinned: Default::default(), ref_counting })
+	}
+
+	fn insert_block(
+		&mut self,
+		hash: &BlockHash,
+		number: u64,
+		parent_hash: &BlockHash,
+		mut changeset: ChangeSet<Key>,
+	) -> Result<CommitSet<Key>, Error<D::Error>> {
+		match self.mode {
+			PruningMode::ArchiveAll => {
+				changeset.deleted.clear();
+				// write changes immediately
+				Ok(CommitSet { data: changeset, meta: Default::default() })
+			},
+			PruningMode::Constrained(_) | PruningMode::ArchiveCanonical => self
+				.non_canonical
+				.insert(hash, number, parent_hash, changeset)
+				.map_err(Into::into),
+		}
+	}
+
+	fn canonicalize_block(&mut self, hash: &BlockHash) -> Result<CommitSet<Key>, Error<D::Error>> {
+		// NOTE: it is important that the change to `LAST_CANONICAL` (emit from
+		// `non_canonical.canonicalize`) and the insert of the new pruning journal (emit from
+		// `pruning.note_canonical`) are collected into the same `CommitSet` and are committed to
+		// the database atomically to keep their consistency when restarting the node
+		let mut commit = CommitSet::default();
+		if self.mode == PruningMode::ArchiveAll {
+			return Ok(commit);
+		}
+		let number = self.non_canonical.canonicalize(hash, &mut commit)?;
+		if self.mode == PruningMode::ArchiveCanonical {
+			commit.data.deleted.clear();
+		}
+		if let Some(ref mut pruning) = self.pruning {
+			pruning.note_canonical(hash, number, &mut commit)?;
+		}
+		self.prune(&mut commit)?;
+		Ok(commit)
+	}
+
+	/// Returns the block number of the last canonicalized block.
+	fn last_canonicalized(&self) -> LastCanonicalized {
+		if self.mode == PruningMode::ArchiveAll {
+			LastCanonicalized::NotCanonicalizing
+		} else {
+			self.non_canonical
+				.last_canonicalized_block_number()
+				.map(LastCanonicalized::Block)
+				.unwrap_or_else(|| LastCanonicalized::None)
+		}
+	}
+
+	fn is_pruned(&self, hash: &BlockHash, number: u64) -> IsPruned {
+		match self.mode {
+			PruningMode::ArchiveAll => IsPruned::NotPruned,
+			PruningMode::ArchiveCanonical | PruningMode::Constrained(_) => {
+				if self
+					.non_canonical
+					.last_canonicalized_block_number()
+					.map(|c| number > c)
+					.unwrap_or(true)
+				{
+					if self.non_canonical.have_block(hash) {
+						IsPruned::NotPruned
+					} else {
+						IsPruned::Pruned
+					}
+				} else {
+					match self.pruning.as_ref() {
+						// We don't know for sure.
+						None => IsPruned::MaybePruned,
+						Some(pruning) => match pruning.have_block(hash, number) {
+							HaveBlock::No => IsPruned::Pruned,
+							HaveBlock::Yes => IsPruned::NotPruned,
+							HaveBlock::Maybe => IsPruned::MaybePruned,
+						},
+					}
+				}
+			},
+		}
+	}
+
+	fn prune(&mut self, commit: &mut CommitSet<Key>) -> Result<(), Error<D::Error>> {
+		if let (&mut Some(ref mut pruning), PruningMode::Constrained(constraints)) =
+			(&mut self.pruning, &self.mode)
+		{
+			loop {
+				if pruning.window_size() <= constraints.max_blocks.unwrap_or(0) as u64 {
+					break;
+				}
+
+				let pinned = &self.pinned;
+				match pruning.next_hash() {
+					// the block record is temporary unavailable, break and try next time
+					Err(Error::StateDb(StateDbError::BlockUnavailable)) => break,
+					res =>
+						if res?.map_or(false, |h| pinned.contains_key(&h)) {
+							break;
+						},
+				}
+				match pruning.prune_one(commit) {
+					// this branch should not reach as previous `next_hash` don't return error
+					// keeping it for robustness
+					Err(Error::StateDb(StateDbError::BlockUnavailable)) => break,
+					res => res?,
+				}
+			}
+		}
+		Ok(())
+	}
+
+	/// Revert all non-canonical blocks with the best block number.
+	/// Returns a database commit or `None` if not possible.
+	/// For archive an empty commit set is returned.
+	fn revert_one(&mut self) -> Option<CommitSet<Key>> {
+		match self.mode {
+			PruningMode::ArchiveAll => Some(CommitSet::default()),
+			PruningMode::ArchiveCanonical | PruningMode::Constrained(_) =>
+				self.non_canonical.revert_one(),
+		}
+	}
+
+	fn remove(&mut self, hash: &BlockHash) -> Option<CommitSet<Key>> {
+		match self.mode {
+			PruningMode::ArchiveAll => Some(CommitSet::default()),
+			PruningMode::ArchiveCanonical | PruningMode::Constrained(_) =>
+				self.non_canonical.remove(hash),
+		}
+	}
+
+	fn pin<F>(&mut self, hash: &BlockHash, number: u64, hint: F) -> Result<(), PinError>
+	where
+		F: Fn() -> bool,
+	{
+		match self.mode {
+			PruningMode::ArchiveAll => Ok(()),
+			PruningMode::ArchiveCanonical | PruningMode::Constrained(_) => {
+				let have_block = self.non_canonical.have_block(hash) ||
+					self.pruning.as_ref().map_or_else(
+						|| hint(),
+						|pruning| match pruning.have_block(hash, number) {
+							HaveBlock::No => false,
+							HaveBlock::Yes => true,
+							HaveBlock::Maybe => hint(),
+						},
+					);
+				if have_block {
+					let refs = self.pinned.entry(hash.clone()).or_default();
+					if *refs == 0 {
+						trace!(target: LOG_TARGET_PIN, "Pinned block: {:?}", hash);
+						self.non_canonical.pin(hash);
+					}
+					*refs += 1;
+					Ok(())
+				} else {
+					Err(PinError::InvalidBlock)
+				}
+			},
+		}
+	}
+
+	fn unpin(&mut self, hash: &BlockHash) {
+		match self.pinned.entry(hash.clone()) {
+			Entry::Occupied(mut entry) => {
+				*entry.get_mut() -= 1;
+				if *entry.get() == 0 {
+					trace!(target: LOG_TARGET_PIN, "Unpinned block: {:?}", hash);
+					entry.remove();
+					self.non_canonical.unpin(hash);
+				} else {
+					trace!(target: LOG_TARGET_PIN, "Releasing reference for {:?}", hash);
+				}
+			},
+			Entry::Vacant(_) => {},
+		}
+	}
+
+	fn sync(&mut self) {
+		self.non_canonical.sync();
+	}
+
+	pub fn get<DB: NodeDb, Q: ?Sized>(
+		&self,
+		key: &Q,
+		db: &DB,
+	) -> Result<Option<DBValue>, Error<DB::Error>>
+	where
+		Q: AsRef<DB::Key>,
+		Key: std::borrow::Borrow<Q>,
+		Q: std::hash::Hash + Eq,
+	{
+		if let Some(value) = self.non_canonical.get(key) {
+			return Ok(Some(value));
+		}
+		db.get(key.as_ref()).map_err(Error::Db)
+	}
+}
+
+/// State DB maintenance. See module description.
+/// Can be shared across threads.
+pub struct StateDb<BlockHash: Hash, Key: Hash, D: MetaDb> {
+	db: RwLock<StateDbSync<BlockHash, Key, D>>,
+}
+
+impl<BlockHash: Hash, Key: Hash, D: MetaDb> StateDb<BlockHash, Key, D> {
+	/// Create an instance of [`StateDb`].
+	pub fn open(
+		db: D,
+		requested_mode: Option<PruningMode>,
+		ref_counting: bool,
+		should_init: bool,
+	) -> Result<(CommitSet<Key>, StateDb<BlockHash, Key, D>), Error<D::Error>> {
+		let stored_mode = fetch_stored_pruning_mode(&db)?;
+
+		let selected_mode = match (should_init, stored_mode, requested_mode) {
+			(true, stored_mode, requested_mode) => {
+				assert!(stored_mode.is_none(), "The storage has just been initialized. No meta-data is expected to be found in it.");
+				requested_mode.unwrap_or_default()
+			},
+
+			(false, None, _) =>
+				return Err(StateDbError::Metadata(
+					"An existing StateDb does not have PRUNING_MODE stored in its meta-data".into(),
+				)
+				.into()),
+
+			(false, Some(stored), None) => stored,
+
+			(false, Some(stored), Some(requested)) => choose_pruning_mode(stored, requested)?,
+		};
+
+		let db_init_commit_set = if should_init {
+			let mut cs: CommitSet<Key> = Default::default();
+
+			let key = to_meta_key(PRUNING_MODE, &());
+			let value = selected_mode.id().to_owned();
+
+			cs.meta.inserted.push((key, value));
+
+			cs
+		} else {
+			Default::default()
+		};
+
+		let state_db =
+			StateDb { db: RwLock::new(StateDbSync::new(selected_mode, ref_counting, db)?) };
+
+		Ok((db_init_commit_set, state_db))
+	}
+
+	pub fn pruning_mode(&self) -> PruningMode {
+		self.db.read().mode.clone()
+	}
+
+	/// Add a new non-canonical block.
+	pub fn insert_block(
+		&self,
+		hash: &BlockHash,
+		number: u64,
+		parent_hash: &BlockHash,
+		changeset: ChangeSet<Key>,
+	) -> Result<CommitSet<Key>, Error<D::Error>> {
+		self.db.write().insert_block(hash, number, parent_hash, changeset)
+	}
+
+	/// Finalize a previously inserted block.
+	pub fn canonicalize_block(&self, hash: &BlockHash) -> Result<CommitSet<Key>, Error<D::Error>> {
+		self.db.write().canonicalize_block(hash)
+	}
+
+	/// Prevents pruning of specified block and its descendants.
+	/// `hint` used for further checking if the given block exists
+	pub fn pin<F>(&self, hash: &BlockHash, number: u64, hint: F) -> Result<(), PinError>
+	where
+		F: Fn() -> bool,
+	{
+		self.db.write().pin(hash, number, hint)
+	}
+
+	/// Allows pruning of specified block.
+	pub fn unpin(&self, hash: &BlockHash) {
+		self.db.write().unpin(hash)
+	}
+
+	/// Confirm that all changes made to commit sets are on disk. Allows for temporarily pinned
+	/// blocks to be released.
+	pub fn sync(&self) {
+		self.db.write().sync()
+	}
+
+	/// Get a value from non-canonical/pruning overlay or the backing DB.
+	pub fn get<DB: NodeDb, Q: ?Sized>(
+		&self,
+		key: &Q,
+		db: &DB,
+	) -> Result<Option<DBValue>, Error<DB::Error>>
+	where
+		Q: AsRef<DB::Key>,
+		Key: std::borrow::Borrow<Q>,
+		Q: std::hash::Hash + Eq,
+	{
+		self.db.read().get(key, db)
+	}
+
+	/// Revert all non-canonical blocks with the best block number.
+	/// Returns a database commit or `None` if not possible.
+	/// For archive an empty commit set is returned.
+	pub fn revert_one(&self) -> Option<CommitSet<Key>> {
+		self.db.write().revert_one()
+	}
+
+	/// Remove specified non-canonical block.
+	/// Returns a database commit or `None` if not possible.
+	pub fn remove(&self, hash: &BlockHash) -> Option<CommitSet<Key>> {
+		self.db.write().remove(hash)
+	}
+
+	/// Returns last canonicalized block.
+	pub fn last_canonicalized(&self) -> LastCanonicalized {
+		self.db.read().last_canonicalized()
+	}
+
+	/// Check if block is pruned away.
+	pub fn is_pruned(&self, hash: &BlockHash, number: u64) -> IsPruned {
+		self.db.read().is_pruned(hash, number)
+	}
+
+	/// Reset in-memory changes to the last disk-backed state.
+	pub fn reset(&self, db: D) -> Result<(), Error<D::Error>> {
+		let mut state_db = self.db.write();
+		*state_db = StateDbSync::new(state_db.mode.clone(), state_db.ref_counting, db)?;
+		Ok(())
+	}
+}
+
+/// The result return by `StateDb::is_pruned`
+#[derive(Debug, PartialEq, Eq)]
+pub enum IsPruned {
+	/// Definitely pruned
+	Pruned,
+	/// Definitely not pruned
+	NotPruned,
+	/// May or may not pruned, need further checking
+	MaybePruned,
+}
+
+fn fetch_stored_pruning_mode<D: MetaDb>(db: &D) -> Result<Option<PruningMode>, Error<D::Error>> {
+	let meta_key_mode = to_meta_key(PRUNING_MODE, &());
+	if let Some(stored_mode) = db.get_meta(&meta_key_mode).map_err(Error::Db)? {
+		if let Some(mode) = PruningMode::from_id(&stored_mode) {
+			Ok(Some(mode))
+		} else {
+			Err(StateDbError::Metadata(format!(
+				"Invalid value stored for PRUNING_MODE: {:02x?}",
+				stored_mode
+			))
+			.into())
+		}
+	} else {
+		Ok(None)
+	}
+}
+
+fn choose_pruning_mode(
+	stored: PruningMode,
+	requested: PruningMode,
+) -> Result<PruningMode, StateDbError> {
+	match (stored, requested) {
+		(PruningMode::ArchiveAll, PruningMode::ArchiveAll) => Ok(PruningMode::ArchiveAll),
+		(PruningMode::ArchiveCanonical, PruningMode::ArchiveCanonical) =>
+			Ok(PruningMode::ArchiveCanonical),
+		(PruningMode::Constrained(_), PruningMode::Constrained(requested)) =>
+			Ok(PruningMode::Constrained(requested)),
+		(stored, requested) => Err(StateDbError::IncompatiblePruningModes { requested, stored }),
+	}
+}
+
+#[cfg(test)]
+mod tests {
+	use crate::{
+		test::{make_changeset, make_db, TestDb},
+		Constraints, Error, IsPruned, PruningMode, StateDb, StateDbError,
+	};
+	use pezsp_core::H256;
+
+	fn make_test_db(settings: PruningMode) -> (TestDb, StateDb<H256, H256, TestDb>) {
+		let mut db = make_db(&[91, 921, 922, 93, 94]);
+		let (state_db_init, state_db) =
+			StateDb::open(db.clone(), Some(settings), false, true).unwrap();
+		db.commit(&state_db_init);
+
+		db.commit(
+			&state_db
+				.insert_block(
+					&H256::from_low_u64_be(1),
+					1,
+					&H256::from_low_u64_be(0),
+					make_changeset(&[1], &[91]),
+				)
+				.unwrap(),
+		);
+		db.commit(
+			&state_db
+				.insert_block(
+					&H256::from_low_u64_be(21),
+					2,
+					&H256::from_low_u64_be(1),
+					make_changeset(&[21], &[921, 1]),
+				)
+				.unwrap(),
+		);
+		db.commit(
+			&state_db
+				.insert_block(
+					&H256::from_low_u64_be(22),
+					2,
+					&H256::from_low_u64_be(1),
+					make_changeset(&[22], &[922]),
+				)
+				.unwrap(),
+		);
+		db.commit(
+			&state_db
+				.insert_block(
+					&H256::from_low_u64_be(3),
+					3,
+					&H256::from_low_u64_be(21),
+					make_changeset(&[3], &[93]),
+				)
+				.unwrap(),
+		);
+		db.commit(&state_db.canonicalize_block(&H256::from_low_u64_be(1)).unwrap());
+		db.commit(
+			&state_db
+				.insert_block(
+					&H256::from_low_u64_be(4),
+					4,
+					&H256::from_low_u64_be(3),
+					make_changeset(&[4], &[94]),
+				)
+				.unwrap(),
+		);
+		db.commit(&state_db.canonicalize_block(&H256::from_low_u64_be(21)).unwrap());
+		db.commit(&state_db.canonicalize_block(&H256::from_low_u64_be(3)).unwrap());
+
+		(db, state_db)
+	}
+
+	#[test]
+	fn full_archive_keeps_everything() {
+		let (db, sdb) = make_test_db(PruningMode::ArchiveAll);
+		assert!(db.data_eq(&make_db(&[1, 21, 22, 3, 4, 91, 921, 922, 93, 94])));
+		assert_eq!(sdb.is_pruned(&H256::from_low_u64_be(0), 0), IsPruned::NotPruned);
+	}
+
+	#[test]
+	fn canonical_archive_keeps_canonical() {
+		let (db, _) = make_test_db(PruningMode::ArchiveCanonical);
+		assert!(db.data_eq(&make_db(&[1, 21, 3, 91, 921, 922, 93, 94])));
+	}
+
+	#[test]
+	fn block_record_unavailable() {
+		let (mut db, state_db) =
+			make_test_db(PruningMode::Constrained(Constraints { max_blocks: Some(1) }));
+		// import 2 blocks
+		for i in &[5, 6] {
+			db.commit(
+				&state_db
+					.insert_block(
+						&H256::from_low_u64_be(*i),
+						*i,
+						&H256::from_low_u64_be(*i - 1),
+						make_changeset(&[], &[]),
+					)
+					.unwrap(),
+			);
+		}
+		// canonicalize block 4 but not commit it to db
+		let c1 = state_db.canonicalize_block(&H256::from_low_u64_be(4)).unwrap();
+		assert_eq!(state_db.is_pruned(&H256::from_low_u64_be(3), 3), IsPruned::Pruned);
+
+		// canonicalize block 5 but not commit it to db, block 4 is not pruned due to it is not
+		// commit to db yet (unavailable), return `MaybePruned` here because `apply_pending` is not
+		// called and block 3 is still in cache
+		let c2 = state_db.canonicalize_block(&H256::from_low_u64_be(5)).unwrap();
+		assert_eq!(state_db.is_pruned(&H256::from_low_u64_be(4), 4), IsPruned::MaybePruned);
+
+		// commit block 4 and 5 to db, and import a new block will prune both block 4 and 5
+		db.commit(&c1);
+		db.commit(&c2);
+		db.commit(&state_db.canonicalize_block(&H256::from_low_u64_be(6)).unwrap());
+		assert_eq!(state_db.is_pruned(&H256::from_low_u64_be(4), 4), IsPruned::Pruned);
+		assert_eq!(state_db.is_pruned(&H256::from_low_u64_be(5), 5), IsPruned::Pruned);
+	}
+
+	#[test]
+	fn prune_window_0() {
+		let (db, _) = make_test_db(PruningMode::Constrained(Constraints { max_blocks: Some(0) }));
+		assert!(db.data_eq(&make_db(&[21, 3, 922, 94])));
+	}
+
+	#[test]
+	fn prune_window_1() {
+		let (db, sdb) = make_test_db(PruningMode::Constrained(Constraints { max_blocks: Some(1) }));
+		assert_eq!(sdb.is_pruned(&H256::from_low_u64_be(0), 0), IsPruned::Pruned);
+		assert_eq!(sdb.is_pruned(&H256::from_low_u64_be(1), 1), IsPruned::Pruned);
+		assert_eq!(sdb.is_pruned(&H256::from_low_u64_be(21), 2), IsPruned::Pruned);
+		assert_eq!(sdb.is_pruned(&H256::from_low_u64_be(22), 2), IsPruned::Pruned);
+		assert!(db.data_eq(&make_db(&[21, 3, 922, 93, 94])));
+	}
+
+	#[test]
+	fn prune_window_2() {
+		let (db, sdb) = make_test_db(PruningMode::Constrained(Constraints { max_blocks: Some(2) }));
+		assert_eq!(sdb.is_pruned(&H256::from_low_u64_be(0), 0), IsPruned::Pruned);
+		assert_eq!(sdb.is_pruned(&H256::from_low_u64_be(1), 1), IsPruned::Pruned);
+		assert_eq!(sdb.is_pruned(&H256::from_low_u64_be(21), 2), IsPruned::NotPruned);
+		assert_eq!(sdb.is_pruned(&H256::from_low_u64_be(22), 2), IsPruned::Pruned);
+		assert!(db.data_eq(&make_db(&[1, 21, 3, 921, 922, 93, 94])));
+	}
+
+	#[test]
+	fn detects_incompatible_mode() {
+		let mut db = make_db(&[]);
+		let (state_db_init, state_db) =
+			StateDb::open(db.clone(), Some(PruningMode::ArchiveAll), false, true).unwrap();
+		db.commit(&state_db_init);
+		db.commit(
+			&state_db
+				.insert_block(
+					&H256::from_low_u64_be(0),
+					0,
+					&H256::from_low_u64_be(0),
+					make_changeset(&[], &[]),
+				)
+				.unwrap(),
+		);
+		let new_mode = PruningMode::Constrained(Constraints { max_blocks: Some(2) });
+		let state_db_open_result: Result<(_, StateDb<H256, H256, TestDb>), _> =
+			StateDb::open(db.clone(), Some(new_mode), false, false);
+		assert!(state_db_open_result.is_err());
+	}
+
+	fn check_stored_and_requested_mode_compatibility(
+		mode_when_created: Option<PruningMode>,
+		mode_when_reopened: Option<PruningMode>,
+		expected_effective_mode_when_reopened: Result<PruningMode, ()>,
+	) {
+		let mut db = make_db(&[]);
+		let (state_db_init, state_db) =
+			StateDb::<H256, H256, TestDb>::open(db.clone(), mode_when_created, false, true)
+				.unwrap();
+		db.commit(&state_db_init);
+		std::mem::drop(state_db);
+
+		let state_db_reopen_result =
+			StateDb::<H256, H256, TestDb>::open(db.clone(), mode_when_reopened, false, false);
+		if let Ok(expected_mode) = expected_effective_mode_when_reopened {
+			let (state_db_init, state_db_reopened) = state_db_reopen_result.unwrap();
+			db.commit(&state_db_init);
+			assert_eq!(state_db_reopened.pruning_mode(), expected_mode,)
+		} else {
+			assert!(matches!(
+				state_db_reopen_result,
+				Err(Error::StateDb(StateDbError::IncompatiblePruningModes { .. }))
+			));
+		}
+	}
+
+	#[test]
+	fn pruning_mode_compatibility() {
+		for (created, reopened, expected) in [
+			(None, None, Ok(PruningMode::blocks_pruning(256))),
+			(None, Some(PruningMode::blocks_pruning(256)), Ok(PruningMode::blocks_pruning(256))),
+			(None, Some(PruningMode::blocks_pruning(128)), Ok(PruningMode::blocks_pruning(128))),
+			(None, Some(PruningMode::blocks_pruning(512)), Ok(PruningMode::blocks_pruning(512))),
+			(None, Some(PruningMode::ArchiveAll), Err(())),
+			(None, Some(PruningMode::ArchiveCanonical), Err(())),
+			(Some(PruningMode::blocks_pruning(256)), None, Ok(PruningMode::blocks_pruning(256))),
+			(
+				Some(PruningMode::blocks_pruning(256)),
+				Some(PruningMode::blocks_pruning(256)),
+				Ok(PruningMode::blocks_pruning(256)),
+			),
+			(
+				Some(PruningMode::blocks_pruning(256)),
+				Some(PruningMode::blocks_pruning(128)),
+				Ok(PruningMode::blocks_pruning(128)),
+			),
+			(
+				Some(PruningMode::blocks_pruning(256)),
+				Some(PruningMode::blocks_pruning(512)),
+				Ok(PruningMode::blocks_pruning(512)),
+			),
+			(Some(PruningMode::blocks_pruning(256)), Some(PruningMode::ArchiveAll), Err(())),
+			(Some(PruningMode::blocks_pruning(256)), Some(PruningMode::ArchiveCanonical), Err(())),
+			(Some(PruningMode::ArchiveAll), None, Ok(PruningMode::ArchiveAll)),
+			(Some(PruningMode::ArchiveAll), Some(PruningMode::blocks_pruning(256)), Err(())),
+			(Some(PruningMode::ArchiveAll), Some(PruningMode::blocks_pruning(128)), Err(())),
+			(Some(PruningMode::ArchiveAll), Some(PruningMode::blocks_pruning(512)), Err(())),
+			(
+				Some(PruningMode::ArchiveAll),
+				Some(PruningMode::ArchiveAll),
+				Ok(PruningMode::ArchiveAll),
+			),
+			(Some(PruningMode::ArchiveAll), Some(PruningMode::ArchiveCanonical), Err(())),
+			(Some(PruningMode::ArchiveCanonical), None, Ok(PruningMode::ArchiveCanonical)),
+			(Some(PruningMode::ArchiveCanonical), Some(PruningMode::blocks_pruning(256)), Err(())),
+			(Some(PruningMode::ArchiveCanonical), Some(PruningMode::blocks_pruning(128)), Err(())),
+			(Some(PruningMode::ArchiveCanonical), Some(PruningMode::blocks_pruning(512)), Err(())),
+			(Some(PruningMode::ArchiveCanonical), Some(PruningMode::ArchiveAll), Err(())),
+			(
+				Some(PruningMode::ArchiveCanonical),
+				Some(PruningMode::ArchiveCanonical),
+				Ok(PruningMode::ArchiveCanonical),
+			),
+		] {
+			check_stored_and_requested_mode_compatibility(created, reopened, expected);
+		}
+	}
+}

bizinikiwi/client/state-db/src/pruning.rs

@@ -0,0 +1,910 @@
+// This file is part of Bizinikiwi.
+
+// Copyright (C) Parity Technologies (UK) Ltd.
+// SPDX-License-Identifier: GPL-3.0-or-later WITH Classpath-exception-2.0
+
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with this program. If not, see <https://www.gnu.org/licenses/>.
+
+//! Pruning window.
+//!
+//! For each block we maintain a list of nodes pending deletion.
+//! There is also a global index of node key to block number.
+//! If a node is re-inserted into the window it gets removed from
+//! the death list.
+//! The changes are journaled in the DB.
+
+use crate::{
+	noncanonical::LAST_CANONICAL, to_meta_key, CommitSet, Error, Hash, MetaDb, StateDbError,
+	DEFAULT_MAX_BLOCK_CONSTRAINT, LOG_TARGET,
+};
+use codec::{Decode, Encode};
+use log::trace;
+use std::collections::{HashMap, HashSet, VecDeque};
+
+pub(crate) const LAST_PRUNED: &[u8] = b"last_pruned";
+const PRUNING_JOURNAL: &[u8] = b"pruning_journal";
+
+/// See module documentation.
+pub struct RefWindow<BlockHash: Hash, Key: Hash, D: MetaDb> {
+	/// A queue of blocks keep tracking keys that should be deleted for each block in the
+	/// pruning window.
+	queue: DeathRowQueue<BlockHash, Key, D>,
+	/// Block number that is next to be pruned.
+	base: u64,
+}
+
+/// `DeathRowQueue` used to keep track of blocks in the pruning window, there are two flavors:
+/// - `Mem`, used when the backend database do not supports reference counting, keep all
+/// 	blocks in memory, and keep track of re-inserted keys to not delete them when pruning
+/// - `DbBacked`, used when the backend database supports reference counting, only keep
+/// 	a few number of blocks in memory and load more blocks on demand
+enum DeathRowQueue<BlockHash: Hash, Key: Hash, D: MetaDb> {
+	Mem {
+		/// A queue of keys that should be deleted for each block in the pruning window.
+		death_rows: VecDeque<DeathRow<BlockHash, Key>>,
+		/// An index that maps each key from `death_rows` to block number.
+		death_index: HashMap<Key, u64>,
+	},
+	DbBacked {
+		// The backend database
+		db: D,
+		/// A queue of keys that should be deleted for each block in the pruning window.
+		/// Only caching the first few blocks of the pruning window, blocks inside are
+		/// successive and ordered by block number
+		cache: VecDeque<DeathRow<BlockHash, Key>>,
+		/// A soft limit of the cache's size
+		cache_capacity: usize,
+		/// Last block number added to the window
+		last: Option<u64>,
+	},
+}
+
+impl<BlockHash: Hash, Key: Hash, D: MetaDb> DeathRowQueue<BlockHash, Key, D> {
+	/// Return a `DeathRowQueue` that all blocks are keep in memory
+	fn new_mem(db: &D, base: u64) -> Result<DeathRowQueue<BlockHash, Key, D>, Error<D::Error>> {
+		let mut block = base;
+		let mut queue = DeathRowQueue::<BlockHash, Key, D>::Mem {
+			death_rows: VecDeque::new(),
+			death_index: HashMap::new(),
+		};
+		// read the journal
+		trace!(
+			target: LOG_TARGET,
+			"Reading pruning journal for the memory queue. Pending #{}",
+			base,
+		);
+		loop {
+			let journal_key = to_journal_key(block);
+			match db.get_meta(&journal_key).map_err(Error::Db)? {
+				Some(record) => {
+					let record: JournalRecord<BlockHash, Key> =
+						Decode::decode(&mut record.as_slice())?;
+					trace!(
+						target: LOG_TARGET,
+						"Pruning journal entry {} ({} inserted, {} deleted)",
+						block,
+						record.inserted.len(),
+						record.deleted.len(),
+					);
+					queue.import(base, block, record);
+				},
+				None => break,
+			}
+			block += 1;
+		}
+		Ok(queue)
+	}
+
+	/// Return a `DeathRowQueue` that backed by an database, and only keep a few number
+	/// of blocks in memory
+	fn new_db_backed(
+		db: D,
+		base: u64,
+		last: Option<u64>,
+		window_size: u32,
+	) -> Result<DeathRowQueue<BlockHash, Key, D>, Error<D::Error>> {
+		// limit the cache capacity from 1 to `DEFAULT_MAX_BLOCK_CONSTRAINT`
+		let cache_capacity = window_size.clamp(1, DEFAULT_MAX_BLOCK_CONSTRAINT) as usize;
+		let mut cache = VecDeque::with_capacity(cache_capacity);
+		trace!(
+			target: LOG_TARGET,
+			"Reading pruning journal for the database-backed queue. Pending #{}",
+			base
+		);
+		DeathRowQueue::load_batch_from_db(&db, &mut cache, base, cache_capacity)?;
+		Ok(DeathRowQueue::DbBacked { db, cache, cache_capacity, last })
+	}
+
+	/// import a new block to the back of the queue
+	fn import(&mut self, base: u64, num: u64, journal_record: JournalRecord<BlockHash, Key>) {
+		let JournalRecord { hash, inserted, deleted } = journal_record;
+		trace!(target: LOG_TARGET, "Importing {}, base={}", num, base);
+		match self {
+			DeathRowQueue::DbBacked { cache, cache_capacity, last, .. } => {
+				// If the new block continues cached range and there is space, load it directly into
+				// cache.
+				if num == base + cache.len() as u64 && cache.len() < *cache_capacity {
+					trace!(target: LOG_TARGET, "Adding to DB backed cache {:?} (#{})", hash, num);
+					cache.push_back(DeathRow { hash, deleted: deleted.into_iter().collect() });
+				}
+				*last = Some(num);
+			},
+			DeathRowQueue::Mem { death_rows, death_index } => {
+				// remove all re-inserted keys from death rows
+				for k in inserted {
+					if let Some(block) = death_index.remove(&k) {
+						death_rows[(block - base) as usize].deleted.remove(&k);
+					}
+				}
+				// add new keys
+				let imported_block = base + death_rows.len() as u64;
+				for k in deleted.iter() {
+					death_index.insert(k.clone(), imported_block);
+				}
+				death_rows.push_back(DeathRow { hash, deleted: deleted.into_iter().collect() });
+			},
+		}
+	}
+
+	/// Pop out one block from the front of the queue, `base` is the block number
+	/// of the first block of the queue
+	fn pop_front(
+		&mut self,
+		base: u64,
+	) -> Result<Option<DeathRow<BlockHash, Key>>, Error<D::Error>> {
+		match self {
+			DeathRowQueue::DbBacked { db, cache, cache_capacity, .. } => {
+				if cache.is_empty() {
+					DeathRowQueue::load_batch_from_db(db, cache, base, *cache_capacity)?;
+				}
+				Ok(cache.pop_front())
+			},
+			DeathRowQueue::Mem { death_rows, death_index } => match death_rows.pop_front() {
+				Some(row) => {
+					for k in row.deleted.iter() {
+						death_index.remove(k);
+					}
+					Ok(Some(row))
+				},
+				None => Ok(None),
+			},
+		}
+	}
+
+	/// Load a batch of blocks from the backend database into `cache`, starting from `base` and up
+	/// to `base + cache_capacity`
+	fn load_batch_from_db(
+		db: &D,
+		cache: &mut VecDeque<DeathRow<BlockHash, Key>>,
+		base: u64,
+		cache_capacity: usize,
+	) -> Result<(), Error<D::Error>> {
+		let start = base + cache.len() as u64;
+		let batch_size = cache_capacity;
+		for i in 0..batch_size as u64 {
+			match load_death_row_from_db::<BlockHash, Key, D>(db, start + i)? {
+				Some(row) => {
+					cache.push_back(row);
+				},
+				None => break,
+			}
+		}
+		Ok(())
+	}
+
+	/// Check if the block at the given `index` of the queue exist
+	/// it is the caller's responsibility to ensure `index` won't be out of bounds
+	fn have_block(&self, hash: &BlockHash, index: usize) -> HaveBlock {
+		match self {
+			DeathRowQueue::DbBacked { cache, .. } => {
+				if cache.len() > index {
+					(cache[index].hash == *hash).into()
+				} else {
+					// The block is not in the cache but it still may exist on disk.
+					HaveBlock::Maybe
+				}
+			},
+			DeathRowQueue::Mem { death_rows, .. } => (death_rows[index].hash == *hash).into(),
+		}
+	}
+
+	/// Return the number of block in the pruning window
+	fn len(&self, base: u64) -> u64 {
+		match self {
+			DeathRowQueue::DbBacked { last, .. } => last.map_or(0, |l| l + 1 - base),
+			DeathRowQueue::Mem { death_rows, .. } => death_rows.len() as u64,
+		}
+	}
+
+	#[cfg(test)]
+	fn get_mem_queue_state(
+		&self,
+	) -> Option<(&VecDeque<DeathRow<BlockHash, Key>>, &HashMap<Key, u64>)> {
+		match self {
+			DeathRowQueue::DbBacked { .. } => None,
+			DeathRowQueue::Mem { death_rows, death_index } => Some((death_rows, death_index)),
+		}
+	}
+
+	#[cfg(test)]
+	fn get_db_backed_queue_state(
+		&self,
+	) -> Option<(&VecDeque<DeathRow<BlockHash, Key>>, Option<u64>)> {
+		match self {
+			DeathRowQueue::DbBacked { cache, last, .. } => Some((cache, *last)),
+			DeathRowQueue::Mem { .. } => None,
+		}
+	}
+}
+
+fn load_death_row_from_db<BlockHash: Hash, Key: Hash, D: MetaDb>(
+	db: &D,
+	block: u64,
+) -> Result<Option<DeathRow<BlockHash, Key>>, Error<D::Error>> {
+	let journal_key = to_journal_key(block);
+	match db.get_meta(&journal_key).map_err(Error::Db)? {
+		Some(record) => {
+			let JournalRecord { hash, deleted, .. } = Decode::decode(&mut record.as_slice())?;
+			Ok(Some(DeathRow { hash, deleted: deleted.into_iter().collect() }))
+		},
+		None => Ok(None),
+	}
+}
+
+#[derive(Clone, Debug, PartialEq, Eq)]
+struct DeathRow<BlockHash: Hash, Key: Hash> {
+	hash: BlockHash,
+	deleted: HashSet<Key>,
+}
+
+#[derive(Encode, Decode, Default)]
+struct JournalRecord<BlockHash: Hash, Key: Hash> {
+	hash: BlockHash,
+	inserted: Vec<Key>,
+	deleted: Vec<Key>,
+}
+
+fn to_journal_key(block: u64) -> Vec<u8> {
+	to_meta_key(PRUNING_JOURNAL, &block)
+}
+
+/// The result return by `RefWindow::have_block`
+#[derive(Debug, PartialEq, Eq)]
+pub enum HaveBlock {
+	/// Definitely don't have this block.
+	No,
+	/// May or may not have this block, need further checking
+	Maybe,
+	/// Definitely has this block
+	Yes,
+}
+
+impl From<bool> for HaveBlock {
+	fn from(have: bool) -> Self {
+		if have {
+			HaveBlock::Yes
+		} else {
+			HaveBlock::No
+		}
+	}
+}
+
+impl<BlockHash: Hash, Key: Hash, D: MetaDb> RefWindow<BlockHash, Key, D> {
+	pub fn new(
+		db: D,
+		window_size: u32,
+		count_insertions: bool,
+	) -> Result<RefWindow<BlockHash, Key, D>, Error<D::Error>> {
+		// the block number of the first block in the queue or the next block number if the queue is
+		// empty
+		let base = match db.get_meta(&to_meta_key(LAST_PRUNED, &())).map_err(Error::Db)? {
+			Some(buffer) => u64::decode(&mut buffer.as_slice())? + 1,
+			None => 0,
+		};
+		// the block number of the last block in the queue
+		let last_canonicalized_number =
+			match db.get_meta(&to_meta_key(LAST_CANONICAL, &())).map_err(Error::Db)? {
+				Some(buffer) => Some(<(BlockHash, u64)>::decode(&mut buffer.as_slice())?.1),
+				None => None,
+			};
+
+		let queue = if count_insertions {
+			// Highly scientific crafted number for deciding when to print the warning!
+			//
+			// Rocksdb doesn't support refcounting and requires that we load the entire pruning
+			// window into the memory.
+			if window_size > 1000 {
+				log::warn!(
+					target: LOG_TARGET,
+					"Large pruning window of {window_size} detected! THIS CAN LEAD TO HIGH MEMORY USAGE AND CRASHES. \
+					Reduce the pruning window or switch your database to paritydb."
+				);
+			}
+
+			DeathRowQueue::new_mem(&db, base)?
+		} else {
+			let last = match last_canonicalized_number {
+				Some(last_canonicalized_number) => {
+					debug_assert!(last_canonicalized_number + 1 >= base);
+					Some(last_canonicalized_number)
+				},
+				// None means `LAST_CANONICAL` is never been wrote, since the pruning journals are
+				// in the same `CommitSet` as `LAST_CANONICAL`, it means no pruning journal have
+				// ever been committed to the db, thus set `unload` to zero
+				None => None,
+			};
+			DeathRowQueue::new_db_backed(db, base, last, window_size)?
+		};
+
+		Ok(RefWindow { queue, base })
+	}
+
+	pub fn window_size(&self) -> u64 {
+		self.queue.len(self.base) as u64
+	}
+
+	/// Get the hash of the next pruning block
+	pub fn next_hash(&mut self) -> Result<Option<BlockHash>, Error<D::Error>> {
+		let res = match &mut self.queue {
+			DeathRowQueue::DbBacked { db, cache, cache_capacity, .. } => {
+				if cache.is_empty() {
+					DeathRowQueue::load_batch_from_db(db, cache, self.base, *cache_capacity)?;
+				}
+				cache.front().map(|r| r.hash.clone())
+			},
+			DeathRowQueue::Mem { death_rows, .. } => death_rows.front().map(|r| r.hash.clone()),
+		};
+		Ok(res)
+	}
+
+	fn is_empty(&self) -> bool {
+		self.window_size() == 0
+	}
+
+	// Check if a block is in the pruning window and not be pruned yet
+	pub fn have_block(&self, hash: &BlockHash, number: u64) -> HaveBlock {
+		// if the queue is empty or the block number exceed the pruning window, we definitely
+		// do not have this block
+		if self.is_empty() || number < self.base || number >= self.base + self.window_size() {
+			return HaveBlock::No;
+		}
+		self.queue.have_block(hash, (number - self.base) as usize)
+	}
+
+	/// Prune next block. Expects at least one block in the window. Adds changes to `commit`.
+	pub fn prune_one(&mut self, commit: &mut CommitSet<Key>) -> Result<(), Error<D::Error>> {
+		if let Some(pruned) = self.queue.pop_front(self.base)? {
+			trace!(target: LOG_TARGET, "Pruning {:?} ({} deleted)", pruned.hash, pruned.deleted.len());
+			let index = self.base;
+			commit.data.deleted.extend(pruned.deleted.into_iter());
+			commit.meta.inserted.push((to_meta_key(LAST_PRUNED, &()), index.encode()));
+			commit.meta.deleted.push(to_journal_key(self.base));
+			self.base += 1;
+			Ok(())
+		} else {
+			trace!(target: LOG_TARGET, "Trying to prune when there's nothing to prune");
+			Err(Error::StateDb(StateDbError::BlockUnavailable))
+		}
+	}
+
+	/// Add a change set to the window. Creates a journal record and pushes it to `commit`
+	pub fn note_canonical(
+		&mut self,
+		hash: &BlockHash,
+		number: u64,
+		commit: &mut CommitSet<Key>,
+	) -> Result<(), Error<D::Error>> {
+		if self.base == 0 && self.is_empty() && number > 0 {
+			// This branch is taken if the node imports the target block of a warp sync.
+			// assume that the block was canonicalized
+			self.base = number;
+			// The parent of the block was the last block that got pruned.
+			commit
+				.meta
+				.inserted
+				.push((to_meta_key(LAST_PRUNED, &()), (number - 1).encode()));
+		} else if (self.base + self.window_size()) != number {
+			return Err(Error::StateDb(StateDbError::InvalidBlockNumber));
+		}
+		trace!(
+			target: LOG_TARGET,
+			"Adding to pruning window: {:?} ({} inserted, {} deleted)",
+			hash,
+			commit.data.inserted.len(),
+			commit.data.deleted.len(),
+		);
+		let inserted = if matches!(self.queue, DeathRowQueue::Mem { .. }) {
+			commit.data.inserted.iter().map(|(k, _)| k.clone()).collect()
+		} else {
+			Default::default()
+		};
+		let deleted = std::mem::take(&mut commit.data.deleted);
+		let journal_record = JournalRecord { hash: hash.clone(), inserted, deleted };
+		commit.meta.inserted.push((to_journal_key(number), journal_record.encode()));
+		self.queue.import(self.base, number, journal_record);
+		Ok(())
+	}
+}
+
+#[cfg(test)]
+mod tests {
+	use super::{to_journal_key, DeathRowQueue, HaveBlock, JournalRecord, RefWindow, LAST_PRUNED};
+	use crate::{
+		noncanonical::LAST_CANONICAL,
+		test::{make_commit, make_db, TestDb},
+		to_meta_key, CommitSet, Error, Hash, StateDbError, DEFAULT_MAX_BLOCK_CONSTRAINT,
+	};
+	use codec::Encode;
+	use pezsp_core::H256;
+
+	fn check_journal(pruning: &RefWindow<H256, H256, TestDb>, db: &TestDb) {
+		let count_insertions = matches!(pruning.queue, DeathRowQueue::Mem { .. });
+		let restored: RefWindow<H256, H256, TestDb> =
+			RefWindow::new(db.clone(), DEFAULT_MAX_BLOCK_CONSTRAINT, count_insertions).unwrap();
+		assert_eq!(pruning.base, restored.base);
+		assert_eq!(pruning.queue.get_mem_queue_state(), restored.queue.get_mem_queue_state());
+	}
+
+	#[test]
+	fn created_from_empty_db() {
+		let db = make_db(&[]);
+		let pruning: RefWindow<H256, H256, TestDb> =
+			RefWindow::new(db, DEFAULT_MAX_BLOCK_CONSTRAINT, true).unwrap();
+		assert_eq!(pruning.base, 0);
+		let (death_rows, death_index) = pruning.queue.get_mem_queue_state().unwrap();
+		assert!(death_rows.is_empty());
+		assert!(death_index.is_empty());
+	}
+
+	#[test]
+	fn prune_empty() {
+		let db = make_db(&[]);
+		let mut pruning: RefWindow<H256, H256, TestDb> =
+			RefWindow::new(db, DEFAULT_MAX_BLOCK_CONSTRAINT, true).unwrap();
+		let mut commit = CommitSet::default();
+		assert_eq!(
+			Err(Error::StateDb(StateDbError::BlockUnavailable)),
+			pruning.prune_one(&mut commit)
+		);
+		assert_eq!(pruning.base, 0);
+		let (death_rows, death_index) = pruning.queue.get_mem_queue_state().unwrap();
+		assert!(death_rows.is_empty());
+		assert!(death_index.is_empty());
+	}
+
+	#[test]
+	fn prune_one() {
+		let mut db = make_db(&[1, 2, 3]);
+		let mut pruning: RefWindow<H256, H256, TestDb> =
+			RefWindow::new(db.clone(), DEFAULT_MAX_BLOCK_CONSTRAINT, true).unwrap();
+		let mut commit = make_commit(&[4, 5], &[1, 3]);
+		let hash = H256::random();
+		pruning.note_canonical(&hash, 0, &mut commit).unwrap();
+		db.commit(&commit);
+		assert_eq!(pruning.have_block(&hash, 0), HaveBlock::Yes);
+		assert_eq!(pruning.have_block(&hash, 0), HaveBlock::Yes);
+		assert!(commit.data.deleted.is_empty());
+		let (death_rows, death_index) = pruning.queue.get_mem_queue_state().unwrap();
+		assert_eq!(death_rows.len(), 1);
+		assert_eq!(death_index.len(), 2);
+		assert!(db.data_eq(&make_db(&[1, 2, 3, 4, 5])));
+		check_journal(&pruning, &db);
+
+		let mut commit = CommitSet::default();
+		pruning.prune_one(&mut commit).unwrap();
+		assert_eq!(pruning.have_block(&hash, 0), HaveBlock::No);
+		db.commit(&commit);
+		assert_eq!(pruning.have_block(&hash, 0), HaveBlock::No);
+		assert!(db.data_eq(&make_db(&[2, 4, 5])));
+		let (death_rows, death_index) = pruning.queue.get_mem_queue_state().unwrap();
+		assert!(death_rows.is_empty());
+		assert!(death_index.is_empty());
+		assert_eq!(pruning.base, 1);
+	}
+
+	#[test]
+	fn prune_two() {
+		let mut db = make_db(&[1, 2, 3]);
+		let mut pruning: RefWindow<H256, H256, TestDb> =
+			RefWindow::new(db.clone(), DEFAULT_MAX_BLOCK_CONSTRAINT, true).unwrap();
+		let mut commit = make_commit(&[4], &[1]);
+		pruning.note_canonical(&H256::random(), 0, &mut commit).unwrap();
+		db.commit(&commit);
+		let mut commit = make_commit(&[5], &[2]);
+		pruning.note_canonical(&H256::random(), 1, &mut commit).unwrap();
+		db.commit(&commit);
+		assert!(db.data_eq(&make_db(&[1, 2, 3, 4, 5])));
+
+		check_journal(&pruning, &db);
+
+		let mut commit = CommitSet::default();
+		pruning.prune_one(&mut commit).unwrap();
+		db.commit(&commit);
+		assert!(db.data_eq(&make_db(&[2, 3, 4, 5])));
+		let mut commit = CommitSet::default();
+		pruning.prune_one(&mut commit).unwrap();
+		db.commit(&commit);
+		assert!(db.data_eq(&make_db(&[3, 4, 5])));
+		assert_eq!(pruning.base, 2);
+	}
+
+	#[test]
+	fn prune_two_pending() {
+		let mut db = make_db(&[1, 2, 3]);
+		let mut pruning: RefWindow<H256, H256, TestDb> =
+			RefWindow::new(db.clone(), DEFAULT_MAX_BLOCK_CONSTRAINT, true).unwrap();
+		let mut commit = make_commit(&[4], &[1]);
+		pruning.note_canonical(&H256::random(), 0, &mut commit).unwrap();
+		db.commit(&commit);
+		let mut commit = make_commit(&[5], &[2]);
+		pruning.note_canonical(&H256::random(), 1, &mut commit).unwrap();
+		db.commit(&commit);
+		assert!(db.data_eq(&make_db(&[1, 2, 3, 4, 5])));
+		let mut commit = CommitSet::default();
+		pruning.prune_one(&mut commit).unwrap();
+		db.commit(&commit);
+		assert!(db.data_eq(&make_db(&[2, 3, 4, 5])));
+		let mut commit = CommitSet::default();
+		pruning.prune_one(&mut commit).unwrap();
+		db.commit(&commit);
+		assert!(db.data_eq(&make_db(&[3, 4, 5])));
+		assert_eq!(pruning.base, 2);
+	}
+
+	#[test]
+	fn reinserted_survives() {
+		let mut db = make_db(&[1, 2, 3]);
+		let mut pruning: RefWindow<H256, H256, TestDb> =
+			RefWindow::new(db.clone(), DEFAULT_MAX_BLOCK_CONSTRAINT, true).unwrap();
+		let mut commit = make_commit(&[], &[2]);
+		pruning.note_canonical(&H256::random(), 0, &mut commit).unwrap();
+		db.commit(&commit);
+		let mut commit = make_commit(&[2], &[]);
+		pruning.note_canonical(&H256::random(), 1, &mut commit).unwrap();
+		db.commit(&commit);
+		let mut commit = make_commit(&[], &[2]);
+		pruning.note_canonical(&H256::random(), 2, &mut commit).unwrap();
+		db.commit(&commit);
+		assert!(db.data_eq(&make_db(&[1, 2, 3])));
+
+		check_journal(&pruning, &db);
+
+		let mut commit = CommitSet::default();
+		pruning.prune_one(&mut commit).unwrap();
+		db.commit(&commit);
+		assert!(db.data_eq(&make_db(&[1, 2, 3])));
+		let mut commit = CommitSet::default();
+		pruning.prune_one(&mut commit).unwrap();
+		db.commit(&commit);
+		assert!(db.data_eq(&make_db(&[1, 2, 3])));
+		pruning.prune_one(&mut commit).unwrap();
+		db.commit(&commit);
+		assert!(db.data_eq(&make_db(&[1, 3])));
+		assert_eq!(pruning.base, 3);
+	}
+
+	#[test]
+	fn reinserted_survive_pending() {
+		let mut db = make_db(&[1, 2, 3]);
+		let mut pruning: RefWindow<H256, H256, TestDb> =
+			RefWindow::new(db.clone(), DEFAULT_MAX_BLOCK_CONSTRAINT, true).unwrap();
+		let mut commit = make_commit(&[], &[2]);
+		pruning.note_canonical(&H256::random(), 0, &mut commit).unwrap();
+		db.commit(&commit);
+		let mut commit = make_commit(&[2], &[]);
+		pruning.note_canonical(&H256::random(), 1, &mut commit).unwrap();
+		db.commit(&commit);
+		let mut commit = make_commit(&[], &[2]);
+		pruning.note_canonical(&H256::random(), 2, &mut commit).unwrap();
+		db.commit(&commit);
+		assert!(db.data_eq(&make_db(&[1, 2, 3])));
+
+		let mut commit = CommitSet::default();
+		pruning.prune_one(&mut commit).unwrap();
+		db.commit(&commit);
+		assert!(db.data_eq(&make_db(&[1, 2, 3])));
+		let mut commit = CommitSet::default();
+		pruning.prune_one(&mut commit).unwrap();
+		db.commit(&commit);
+		assert!(db.data_eq(&make_db(&[1, 2, 3])));
+		pruning.prune_one(&mut commit).unwrap();
+		db.commit(&commit);
+		assert!(db.data_eq(&make_db(&[1, 3])));
+		assert_eq!(pruning.base, 3);
+	}
+
+	#[test]
+	fn reinserted_ignores() {
+		let mut db = make_db(&[1, 2, 3]);
+		let mut pruning: RefWindow<H256, H256, TestDb> =
+			RefWindow::new(db.clone(), DEFAULT_MAX_BLOCK_CONSTRAINT, false).unwrap();
+		let mut commit = make_commit(&[], &[2]);
+		pruning.note_canonical(&H256::random(), 0, &mut commit).unwrap();
+		db.commit(&commit);
+		let mut commit = make_commit(&[2], &[]);
+		pruning.note_canonical(&H256::random(), 1, &mut commit).unwrap();
+		db.commit(&commit);
+		let mut commit = make_commit(&[], &[2]);
+		pruning.note_canonical(&H256::random(), 2, &mut commit).unwrap();
+		db.commit(&commit);
+		assert!(db.data_eq(&make_db(&[1, 2, 3])));
+
+		check_journal(&pruning, &db);
+
+		let mut commit = CommitSet::default();
+		pruning.prune_one(&mut commit).unwrap();
+		db.commit(&commit);
+		assert!(db.data_eq(&make_db(&[1, 3])));
+	}
+
+	fn push_last_canonicalized<H: Hash>(block: u64, commit: &mut CommitSet<H>) {
+		commit
+			.meta
+			.inserted
+			.push((to_meta_key(LAST_CANONICAL, &()), (block, block).encode()));
+	}
+
+	fn push_last_pruned<H: Hash>(block: u64, commit: &mut CommitSet<H>) {
+		commit.meta.inserted.push((to_meta_key(LAST_PRUNED, &()), block.encode()));
+	}
+
+	#[test]
+	fn init_db_backed_queue() {
+		let mut db = make_db(&[]);
+		let mut commit = CommitSet::default();
+
+		fn load_pruning_from_db(db: TestDb) -> (usize, u64) {
+			let pruning: RefWindow<u64, H256, TestDb> =
+				RefWindow::new(db, DEFAULT_MAX_BLOCK_CONSTRAINT, false).unwrap();
+			let (cache, _) = pruning.queue.get_db_backed_queue_state().unwrap();
+			(cache.len(), pruning.base)
+		}
+
+		fn push_record(block: u64, commit: &mut CommitSet<H256>) {
+			commit
+				.meta
+				.inserted
+				.push((to_journal_key(block), JournalRecord::<u64, H256>::default().encode()));
+		}
+
+		// empty database
+		let (loaded_blocks, base) = load_pruning_from_db(db.clone());
+		assert_eq!(loaded_blocks, 0);
+		assert_eq!(base, 0);
+
+		// canonicalized the genesis block but no pruning
+		push_last_canonicalized(0, &mut commit);
+		push_record(0, &mut commit);
+		db.commit(&commit);
+		let (loaded_blocks, base) = load_pruning_from_db(db.clone());
+		assert_eq!(loaded_blocks, 1);
+		assert_eq!(base, 0);
+
+		// pruned the genesis block
+		push_last_pruned(0, &mut commit);
+		db.commit(&commit);
+		let (loaded_blocks, base) = load_pruning_from_db(db.clone());
+		assert_eq!(loaded_blocks, 0);
+		assert_eq!(base, 1);
+
+		// canonicalize more blocks
+		push_last_canonicalized(10, &mut commit);
+		for i in 1..=10 {
+			push_record(i, &mut commit);
+		}
+		db.commit(&commit);
+		let (loaded_blocks, base) = load_pruning_from_db(db.clone());
+		assert_eq!(loaded_blocks, 10);
+		assert_eq!(base, 1);
+
+		// pruned all blocks
+		push_last_pruned(10, &mut commit);
+		db.commit(&commit);
+		let (loaded_blocks, base) = load_pruning_from_db(db.clone());
+		assert_eq!(loaded_blocks, 0);
+		assert_eq!(base, 11);
+	}
+
+	#[test]
+	fn db_backed_queue() {
+		let mut db = make_db(&[]);
+		let mut pruning: RefWindow<u64, H256, TestDb> =
+			RefWindow::new(db.clone(), DEFAULT_MAX_BLOCK_CONSTRAINT, false).unwrap();
+		let cache_capacity = DEFAULT_MAX_BLOCK_CONSTRAINT as usize;
+
+		// start as an empty queue
+		let (cache, last) = pruning.queue.get_db_backed_queue_state().unwrap();
+		assert_eq!(cache.len(), 0);
+		assert_eq!(last, None);
+
+		// import blocks
+		// queue size and content should match
+		for i in 0..(cache_capacity + 10) {
+			let mut commit = make_commit(&[], &[]);
+			pruning.note_canonical(&(i as u64), i as u64, &mut commit).unwrap();
+			push_last_canonicalized(i as u64, &mut commit);
+			db.commit(&commit);
+			// blocks will fill the cache first
+			let (cache, last) = pruning.queue.get_db_backed_queue_state().unwrap();
+			if i < cache_capacity {
+				assert_eq!(cache.len(), i + 1);
+			} else {
+				assert_eq!(cache.len(), cache_capacity);
+			}
+			assert_eq!(last, Some(i as u64));
+		}
+		assert_eq!(pruning.window_size(), cache_capacity as u64 + 10);
+		let (cache, last) = pruning.queue.get_db_backed_queue_state().unwrap();
+		assert_eq!(cache.len(), cache_capacity);
+		assert_eq!(last, Some(cache_capacity as u64 + 10 - 1));
+		for i in 0..cache_capacity {
+			assert_eq!(cache[i].hash, i as u64);
+		}
+
+		// import a new block to the end of the queue
+		// won't keep the new block in memory
+		let mut commit = CommitSet::default();
+		pruning
+			.note_canonical(&(cache_capacity as u64 + 10), cache_capacity as u64 + 10, &mut commit)
+			.unwrap();
+		assert_eq!(pruning.window_size(), cache_capacity as u64 + 11);
+		let (cache, _) = pruning.queue.get_db_backed_queue_state().unwrap();
+		assert_eq!(cache.len(), cache_capacity);
+
+		// revert the last add that no apply yet
+		// NOTE: do not commit the previous `CommitSet` to db
+		pruning = RefWindow::new(db.clone(), DEFAULT_MAX_BLOCK_CONSTRAINT, false).unwrap();
+		let cache_capacity = DEFAULT_MAX_BLOCK_CONSTRAINT as usize;
+		assert_eq!(pruning.window_size(), cache_capacity as u64 + 10);
+		let (cache, _) = pruning.queue.get_db_backed_queue_state().unwrap();
+		assert_eq!(cache.len(), cache_capacity);
+
+		// remove one block from the start of the queue
+		// block is removed from the head of cache
+		let mut commit = CommitSet::default();
+		pruning.prune_one(&mut commit).unwrap();
+		db.commit(&commit);
+		assert_eq!(pruning.window_size(), cache_capacity as u64 + 9);
+		let (cache, _) = pruning.queue.get_db_backed_queue_state().unwrap();
+		assert_eq!(cache.len(), cache_capacity - 1);
+		for i in 0..(cache_capacity - 1) {
+			assert_eq!(cache[i].hash, (i + 1) as u64);
+		}
+
+		// load a new queue from db
+		// `cache` is full again but the content of the queue should be the same
+		let pruning: RefWindow<u64, H256, TestDb> =
+			RefWindow::new(db, DEFAULT_MAX_BLOCK_CONSTRAINT, false).unwrap();
+		assert_eq!(pruning.window_size(), cache_capacity as u64 + 9);
+		let (cache, _) = pruning.queue.get_db_backed_queue_state().unwrap();
+		assert_eq!(cache.len(), cache_capacity);
+		for i in 0..cache_capacity {
+			assert_eq!(cache[i].hash, (i + 1) as u64);
+		}
+	}
+
+	#[test]
+	fn load_block_from_db() {
+		let mut db = make_db(&[]);
+		let mut pruning: RefWindow<u64, H256, TestDb> =
+			RefWindow::new(db.clone(), DEFAULT_MAX_BLOCK_CONSTRAINT, false).unwrap();
+		let cache_capacity = DEFAULT_MAX_BLOCK_CONSTRAINT as usize;
+
+		// import blocks
+		for i in 0..(cache_capacity as u64 * 2 + 10) {
+			let mut commit = make_commit(&[], &[]);
+			pruning.note_canonical(&i, i, &mut commit).unwrap();
+			push_last_canonicalized(i as u64, &mut commit);
+			db.commit(&commit);
+		}
+
+		// the following operations won't trigger loading block from db:
+		// - getting block in cache
+		// - getting block not in the queue
+		assert_eq!(pruning.next_hash().unwrap().unwrap(), 0);
+		let (cache, last) = pruning.queue.get_db_backed_queue_state().unwrap();
+		assert_eq!(cache.len(), cache_capacity);
+		assert_eq!(last, Some(cache_capacity as u64 * 2 + 10 - 1));
+
+		// clear all block loaded in cache
+		for _ in 0..cache_capacity * 2 {
+			let mut commit = CommitSet::default();
+			pruning.prune_one(&mut commit).unwrap();
+			db.commit(&commit);
+		}
+		let (cache, _) = pruning.queue.get_db_backed_queue_state().unwrap();
+		assert!(cache.is_empty());
+
+		// getting the hash of block that not in cache will also trigger loading
+		// the remaining blocks from db
+		assert_eq!(pruning.next_hash().unwrap().unwrap(), (cache_capacity * 2) as u64);
+		let (cache, _) = pruning.queue.get_db_backed_queue_state().unwrap();
+		assert_eq!(cache.len(), 10);
+
+		// load a new queue from db
+		// `cache` should be the same
+		let pruning: RefWindow<u64, H256, TestDb> =
+			RefWindow::new(db, DEFAULT_MAX_BLOCK_CONSTRAINT, false).unwrap();
+		assert_eq!(pruning.window_size(), 10);
+		let (cache, _) = pruning.queue.get_db_backed_queue_state().unwrap();
+		assert_eq!(cache.len(), 10);
+		for i in 0..10 {
+			assert_eq!(cache[i].hash, (cache_capacity * 2 + i) as u64);
+		}
+	}
+
+	#[test]
+	fn get_block_from_queue() {
+		let mut db = make_db(&[]);
+		let mut pruning: RefWindow<u64, H256, TestDb> =
+			RefWindow::new(db.clone(), DEFAULT_MAX_BLOCK_CONSTRAINT, false).unwrap();
+		let cache_capacity = DEFAULT_MAX_BLOCK_CONSTRAINT as u64;
+
+		// import blocks and commit to db
+		let mut commit = make_commit(&[], &[]);
+		for i in 0..(cache_capacity + 10) {
+			pruning.note_canonical(&i, i, &mut commit).unwrap();
+		}
+		db.commit(&commit);
+
+		// import a block but not commit to db yet
+		let mut pending_commit = make_commit(&[], &[]);
+		let index = cache_capacity + 10;
+		pruning.note_canonical(&index, index, &mut pending_commit).unwrap();
+
+		let mut commit = make_commit(&[], &[]);
+		// prune blocks that had committed to db
+		for i in 0..(cache_capacity + 10) {
+			assert_eq!(pruning.next_hash().unwrap(), Some(i));
+			pruning.prune_one(&mut commit).unwrap();
+		}
+		// return `None` for block that did not commit to db
+		assert_eq!(pruning.next_hash().unwrap(), None);
+		assert_eq!(
+			pruning.prune_one(&mut commit).unwrap_err(),
+			Error::StateDb(StateDbError::BlockUnavailable)
+		);
+		// commit block to db and no error return
+		db.commit(&pending_commit);
+		assert_eq!(pruning.next_hash().unwrap(), Some(index));
+		pruning.prune_one(&mut commit).unwrap();
+		db.commit(&commit);
+	}
+
+	/// Ensure that after warp syncing the state is stored correctly in the db. The warp sync target
+	/// block is imported with all its state at once. This test ensures that after a restart
+	/// `pruning` still knows that this block was imported.
+	#[test]
+	fn store_correct_state_after_warp_syncing() {
+		for count_insertions in [true, false] {
+			let mut db = make_db(&[]);
+			let mut pruning: RefWindow<u64, H256, TestDb> =
+				RefWindow::new(db.clone(), DEFAULT_MAX_BLOCK_CONSTRAINT, count_insertions).unwrap();
+			let block = 10000;
+
+			// import blocks
+			let mut commit = make_commit(&[], &[]);
+			pruning.note_canonical(&block, block, &mut commit).unwrap();
+			push_last_canonicalized(block, &mut commit);
+			db.commit(&commit);
+
+			// load a new queue from db
+			// `cache` should be the same
+			let pruning: RefWindow<u64, H256, TestDb> =
+				RefWindow::new(db, DEFAULT_MAX_BLOCK_CONSTRAINT, count_insertions).unwrap();
+
+			assert_eq!(HaveBlock::Yes, pruning.have_block(&block, block));
+		}
+	}
+}

bizinikiwi/client/state-db/src/test.rs

@@ -0,0 +1,98 @@
+// This file is part of Bizinikiwi.
+
+// Copyright (C) Parity Technologies (UK) Ltd.
+// SPDX-License-Identifier: GPL-3.0-or-later WITH Classpath-exception-2.0
+
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with this program. If not, see <https://www.gnu.org/licenses/>.
+
+//! Test utils
+
+use crate::{ChangeSet, CommitSet, DBValue, MetaDb, NodeDb};
+use pezsp_core::H256;
+use std::{
+	collections::HashMap,
+	sync::{Arc, RwLock},
+};
+
+#[derive(Default, Debug, Clone)]
+pub struct TestDb(Arc<RwLock<TestDbInner>>);
+
+#[derive(Default, Debug, Clone, PartialEq, Eq)]
+struct TestDbInner {
+	pub data: HashMap<H256, DBValue>,
+	pub meta: HashMap<Vec<u8>, DBValue>,
+}
+
+impl MetaDb for TestDb {
+	type Error = ();
+
+	fn get_meta(&self, key: &[u8]) -> Result<Option<DBValue>, ()> {
+		Ok(self.0.read().unwrap().meta.get(key).cloned())
+	}
+}
+
+impl NodeDb for TestDb {
+	type Error = ();
+	type Key = H256;
+
+	fn get(&self, key: &H256) -> Result<Option<DBValue>, ()> {
+		Ok(self.0.read().unwrap().data.get(key).cloned())
+	}
+}
+
+impl TestDb {
+	pub fn commit(&mut self, commit: &CommitSet<H256>) {
+		self.0.write().unwrap().data.extend(commit.data.inserted.iter().cloned());
+		self.0.write().unwrap().meta.extend(commit.meta.inserted.iter().cloned());
+		for k in commit.data.deleted.iter() {
+			self.0.write().unwrap().data.remove(k);
+		}
+		self.0.write().unwrap().meta.extend(commit.meta.inserted.iter().cloned());
+		for k in commit.meta.deleted.iter() {
+			self.0.write().unwrap().meta.remove(k);
+		}
+	}
+
+	pub fn data_eq(&self, other: &TestDb) -> bool {
+		self.0.read().unwrap().data == other.0.read().unwrap().data
+	}
+
+	pub fn meta_len(&self) -> usize {
+		self.0.read().unwrap().meta.len()
+	}
+}
+
+pub fn make_changeset(inserted: &[u64], deleted: &[u64]) -> ChangeSet<H256> {
+	ChangeSet {
+		inserted: inserted
+			.iter()
+			.map(|v| (H256::from_low_u64_be(*v), H256::from_low_u64_be(*v).as_bytes().to_vec()))
+			.collect(),
+		deleted: deleted.iter().map(|v| H256::from_low_u64_be(*v)).collect(),
+	}
+}
+
+pub fn make_commit(inserted: &[u64], deleted: &[u64]) -> CommitSet<H256> {
+	CommitSet { data: make_changeset(inserted, deleted), meta: ChangeSet::default() }
+}
+
+pub fn make_db(inserted: &[u64]) -> TestDb {
+	TestDb(Arc::new(RwLock::new(TestDbInner {
+		data: inserted
+			.iter()
+			.map(|v| (H256::from_low_u64_be(*v), H256::from_low_u64_be(*v).as_bytes().to_vec()))
+			.collect(),
+		meta: Default::default(),
+	})))
+}