pezkuwichain/pezkuwi-subxt
1
0
Fork 0
mirror of https://github.com/pezkuwichain/pezkuwi-subxt.git synced 2026-06-13 21:01:05 +00:00
Code Issues Packages Projects Releases Wiki Activity

Reorganising the repository - external renames and moves (#4074)

Browse Source 
* Adding first rough ouline of the repository structure

* Remove old CI stuff

* add title

* formatting fixes

* move node-exits job's script to scripts dir

* Move docs into subdir

* move to bin

* move maintainence scripts, configs and helpers into its own dir

* add .local to ignore

* move core->client

* start up 'test' area

* move test client

* move test runtime

* make test move compile

* Add dependencies rule enforcement.

* Fix indexing.

* Update docs to reflect latest changes

* Moving /srml->/paint

* update docs

* move client/sr-* -> primitives/

* clean old readme

* remove old broken code in rhd

* update lock

* Step 1.

* starting to untangle client

* Fix after merge.

* start splitting out client interfaces

* move children and blockchain interfaces

* Move trie and state-machine to primitives.

* Fix WASM builds.

* fixing broken imports

* more interface moves

* move backend and light to interfaces

* move CallExecutor

* move cli off client

* moving around more interfaces

* re-add consensus crates into the mix

* fix subkey path

* relieve client from executor

* starting to pull out client from grandpa

* move is_decendent_of out of client

* grandpa still depends on client directly

* lemme tests pass

* rename srml->paint

* Make it compile.

* rename interfaces->client-api

* Move keyring to primitives.

* fixup libp2p dep

* fix broken use

* allow dependency enforcement to fail

* move fork-tree

* Moving wasm-builder

* make env

* move build-script-utils

* fixup broken crate depdencies and names

* fix imports for authority discovery

* fix typo

* update cargo.lock

* fixing imports

* Fix paths and add missing crates

* re-add missing crates
This commit is contained in:
Benjamin Kampmann
2019-11-14 21:51:17 +01:00
committed by Bastian Köcher
parent becc3b0a4f
commit 60e5011c72
809 changed files with 7801 additions and 6464 deletions
Download Patch File Download Diff File Expand all files Collapse all files
substrate/client/state-db/src/lib.rs
+610
View File
@@ -0,0 +1,610 @@
// Copyright 2017-2019 Parity Technologies (UK) Ltd.
// This file is part of Substrate.
// Substrate is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
// Substrate is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
// You should have received a copy of the GNU General Public License
// along with Substrate. If not, see <http://www.gnu.org/licenses/>.
//! State database maintenance. Handles canonicalization and pruning in the database. The input to
//! this module is a `ChangeSet` which is basically a list of key-value pairs (trie nodes) that
//! were added or deleted during block execution.
//!
//! # Canonicalization.
//! Canonicalization window tracks a tree of blocks identified by header hash. The in-memory
//! overlay allows to get any node that was inserted in any of the blocks within the window.
//! The tree is journaled to the backing database and rebuilt on startup.
//! Canonicalization function selects one root from the top of the tree and discards all other roots and
//! their subtrees.
//!
//! # Pruning.
//! See `RefWindow` for pruning algorithm details. `StateDb` prunes on each canonicalization until pruning
//! constraints are satisfied.
mod noncanonical;
mod pruning;
#[cfg(test)] mod test;
use std::fmt;
use parking_lot::RwLock;
use codec::Codec;
use std::collections::{HashMap, hash_map::Entry};
use noncanonical::NonCanonicalOverlay;
use pruning::RefWindow;
use log::trace;
const PRUNING_MODE: &[u8] = b"mode";
const PRUNING_MODE_ARCHIVE: &[u8] = b"archive";
const PRUNING_MODE_ARCHIVE_CANON: &[u8] = b"archive_canonical";
const PRUNING_MODE_CONSTRAINED: &[u8] = b"constrained";
/// Database value type.
pub type DBValue = Vec<u8>;
/// Basic set of requirements for the Block hash and node key types.
pub trait Hash: Send + Sync + Sized + Eq + PartialEq + Clone + Default + fmt::Debug + Codec + std::hash::Hash + 'static {}
impl<T: Send + Sync + Sized + Eq + PartialEq + Clone + Default + fmt::Debug + Codec + std::hash::Hash + 'static> Hash for T {}
/// Backend database trait. Read-only.
pub trait MetaDb {
type Error: fmt::Debug;
/// Get meta value, such as the journal.
fn get_meta(&self, key: &[u8]) -> Result<Option<DBValue>, Self::Error>;
}
/// Backend database trait. Read-only.
pub trait NodeDb {
type Key: ?Sized;
type Error: fmt::Debug;
/// Get state trie node.
fn get(&self, key: &Self::Key) -> Result<Option<DBValue>, Self::Error>;
}
/// Error type.
pub enum Error<E: fmt::Debug> {
/// Database backend error.
Db(E),
/// `Codec` decoding error.
Decoding(codec::Error),
/// Trying to canonicalize invalid block.
InvalidBlock,
/// Trying to insert block with invalid number.
InvalidBlockNumber,
/// Trying to insert block with unknown parent.
InvalidParent,
/// Invalid pruning mode specified. Contains expected mode.
InvalidPruningMode(String),
}
/// Pinning error type.
pub enum PinError {
/// Trying to pin invalid block.
InvalidBlock,
}
impl<E: fmt::Debug> From<codec::Error> for Error<E> {
fn from(x: codec::Error) -> Self {
Error::Decoding(x)
}
}
impl<E: fmt::Debug> fmt::Debug for Error<E> {
fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
match self {
Error::Db(e) => e.fmt(f),
Error::Decoding(e) => write!(f, "Error decoding slicable value: {}", e.what()),
Error::InvalidBlock => write!(f, "Trying to canonicalize invalid block"),
Error::InvalidBlockNumber => write!(f, "Trying to insert block with invalid number"),
Error::InvalidParent => write!(f, "Trying to insert block with unknown parent"),
Error::InvalidPruningMode(e) => write!(f, "Expected pruning mode: {}", e),
}
}
}
/// A set of state node changes.
#[derive(Default, Debug, Clone)]
pub struct ChangeSet<H: Hash> {
/// Inserted nodes.
pub inserted: Vec<(H, DBValue)>,
/// Deleted nodes.
pub deleted: Vec<H>,
}
/// A set of changes to the backing database.
#[derive(Default, Debug, Clone)]
pub struct CommitSet<H: Hash> {
/// State node changes.
pub data: ChangeSet<H>,
/// Metadata changes.
pub meta: ChangeSet<Vec<u8>>,
}
/// Pruning constraints. If none are specified pruning is
#[derive(Default, Debug, Clone, Eq, PartialEq)]
pub struct Constraints {
/// Maximum blocks. Defaults to 0 when unspecified, effectively keeping only non-canonical states.
pub max_blocks: Option<u32>,
/// Maximum memory in the pruning overlay.
pub max_mem: Option<usize>,
}
/// Pruning mode.
#[derive(Debug, Clone, Eq, PartialEq)]
pub enum PruningMode {
/// Maintain a pruning window.
Constrained(Constraints),
/// No pruning. Canonicalization is a no-op.
ArchiveAll,
/// Canonicalization discards non-canonical nodes. All the canonical nodes are kept in the DB.
ArchiveCanonical,
}
impl PruningMode {
/// Create a mode that keeps given number of blocks.
pub fn keep_blocks(n: u32) -> PruningMode {
PruningMode::Constrained(Constraints {
max_blocks: Some(n),
max_mem: None,
})
}
/// Is this an archive (either ArchiveAll or ArchiveCanonical) pruning mode?
pub fn is_archive(&self) -> bool {
match *self {
PruningMode::ArchiveAll | PruningMode::ArchiveCanonical => true,
PruningMode::Constrained(_) => false
}
}
/// Is this an archive (either ArchiveAll or ArchiveCanonical) pruning mode?
pub fn id(&self) -> &[u8] {
match self {
PruningMode::ArchiveAll => PRUNING_MODE_ARCHIVE,
PruningMode::ArchiveCanonical => PRUNING_MODE_ARCHIVE_CANON,
PruningMode::Constrained(_) => PRUNING_MODE_CONSTRAINED,
}
}
}
impl Default for PruningMode {
fn default() -> Self {
PruningMode::keep_blocks(256)
}
}
fn to_meta_key<S: Codec>(suffix: &[u8], data: &S) -> Vec<u8> {
let mut buffer = data.encode();
buffer.extend(suffix);
buffer
}
struct StateDbSync<BlockHash: Hash, Key: Hash> {
mode: PruningMode,
non_canonical: NonCanonicalOverlay<BlockHash, Key>,
pruning: Option<RefWindow<BlockHash, Key>>,
pinned: HashMap<BlockHash, u32>,
}
impl<BlockHash: Hash, Key: Hash> StateDbSync<BlockHash, Key> {
pub fn new<D: MetaDb>(mode: PruningMode, db: &D) -> Result<StateDbSync<BlockHash, Key>, Error<D::Error>> {
trace!(target: "state-db", "StateDb settings: {:?}", mode);
// Check that settings match
Self::check_meta(&mode, db)?;
let non_canonical: NonCanonicalOverlay<BlockHash, Key> = NonCanonicalOverlay::new(db)?;
let pruning: Option<RefWindow<BlockHash, Key>> = match mode {
PruningMode::Constrained(Constraints {
max_mem: Some(_),
..
}) => unimplemented!(),
PruningMode::Constrained(_) => Some(RefWindow::new(db)?),
PruningMode::ArchiveAll | PruningMode::ArchiveCanonical => None,
};
Ok(StateDbSync {
mode,
non_canonical,
pruning,
pinned: Default::default(),
})
}
fn check_meta<D: MetaDb>(mode: &PruningMode, db: &D) -> Result<(), Error<D::Error>> {
let db_mode = db.get_meta(&to_meta_key(PRUNING_MODE, &())).map_err(Error::Db)?;
trace!(target: "state-db",
"DB pruning mode: {:?}",
db_mode.as_ref().map(|v| std::str::from_utf8(&v))
);
match &db_mode {
Some(v) if v.as_slice() == mode.id() => Ok(()),
Some(v) => Err(Error::InvalidPruningMode(String::from_utf8_lossy(v).into())),
None => Ok(()),
}
}
pub fn insert_block<E: fmt::Debug>(&mut self, hash: &BlockHash, number: u64, parent_hash: &BlockHash, mut changeset: ChangeSet<Key>) -> Result<CommitSet<Key>, Error<E>> {
let mut meta = ChangeSet::default();
if number == 0 {
// Save pruning mode when writing first block.
meta.inserted.push((to_meta_key(PRUNING_MODE, &()), self.mode.id().into()));
}
match self.mode {
PruningMode::ArchiveAll => {
changeset.deleted.clear();
// write changes immediately
Ok(CommitSet {
data: changeset,
meta: meta,
})
},
PruningMode::Constrained(_) | PruningMode::ArchiveCanonical => {
let commit = self.non_canonical.insert(hash, number, parent_hash, changeset);
commit.map(|mut c| {
c.meta.inserted.extend(meta.inserted);
c
})
}
}
}
pub fn canonicalize_block<E: fmt::Debug>(&mut self, hash: &BlockHash) -> Result<CommitSet<Key>, Error<E>> {
let mut commit = CommitSet::default();
if self.mode == PruningMode::ArchiveAll {
return Ok(commit)
}
match self.non_canonical.canonicalize(&hash, &mut commit) {
Ok(()) => {
if self.mode == PruningMode::ArchiveCanonical {
commit.data.deleted.clear();
}
}
Err(e) => return Err(e),
};
if let Some(ref mut pruning) = self.pruning {
pruning.note_canonical(&hash, &mut commit);
}
self.prune(&mut commit);
Ok(commit)
}
pub fn best_canonical(&self) -> Option<u64> {
return self.non_canonical.last_canonicalized_block_number()
}
pub fn is_pruned(&self, hash: &BlockHash, number: u64) -> bool {
match self.mode {
PruningMode::ArchiveAll => false,
PruningMode::ArchiveCanonical | PruningMode::Constrained(_) => {
if self.best_canonical().map(|c| number > c).unwrap_or(true) {
!self.non_canonical.have_block(hash)
} else {
self.pruning.as_ref().map_or(false, |pruning| number < pruning.pending() || !pruning.have_block(hash))
}
}
}
}
fn prune(&mut self, commit: &mut CommitSet<Key>) {
if let (&mut Some(ref mut pruning), &PruningMode::Constrained(ref constraints)) = (&mut self.pruning, &self.mode) {
loop {
if pruning.window_size() <= constraints.max_blocks.unwrap_or(0) as u64 {
break;
}
if constraints.max_mem.map_or(false, |m| pruning.mem_used() > m) {
break;
}
let pinned = &self.pinned;
if pruning.next_hash().map_or(false, |h| pinned.contains_key(&h)) {
break;
}
pruning.prune_one(commit);
}
}
}
/// Revert all non-canonical blocks with the best block number.
/// Returns a database commit or `None` if not possible.
/// For archive an empty commit set is returned.
pub fn revert_one(&mut self) -> Option<CommitSet<Key>> {
match self.mode {
PruningMode::ArchiveAll => {
Some(CommitSet::default())
},
PruningMode::ArchiveCanonical | PruningMode::Constrained(_) => {
self.non_canonical.revert_one()
},
}
}
pub fn pin(&mut self, hash: &BlockHash) -> Result<(), PinError> {
match self.mode {
PruningMode::ArchiveAll => Ok(()),
PruningMode::ArchiveCanonical | PruningMode::Constrained(_) => {
if self.non_canonical.have_block(hash) ||
self.pruning.as_ref().map_or(false, |pruning| pruning.have_block(hash))
{
let refs = self.pinned.entry(hash.clone()).or_default();
if *refs == 0 {
trace!(target: "state-db", "Pinned block: {:?}", hash);
self.non_canonical.pin(hash);
}
*refs += 1;
Ok(())
} else {
Err(PinError::InvalidBlock)
}
}
}
}
pub fn unpin(&mut self, hash: &BlockHash) {
match self.pinned.entry(hash.clone()) {
Entry::Occupied(mut entry) => {
*entry.get_mut() -= 1;
if *entry.get() == 0 {
trace!(target: "state-db", "Unpinned block: {:?}", hash);
entry.remove();
self.non_canonical.unpin(hash);
} else {
trace!(target: "state-db", "Releasing reference for {:?}", hash);
}
},
Entry::Vacant(_) => {},
}
}
pub fn get<D: NodeDb>(&self, key: &Key, db: &D) -> Result<Option<DBValue>, Error<D::Error>>
where Key: AsRef<D::Key>
{
if let Some(value) = self.non_canonical.get(key) {
return Ok(Some(value));
}
db.get(key.as_ref()).map_err(|e| Error::Db(e))
}
pub fn apply_pending(&mut self) {
self.non_canonical.apply_pending();
if let Some(pruning) = &mut self.pruning {
pruning.apply_pending();
}
trace!(target: "forks", "First available: {:?} ({}), Last canon: {:?} ({}), Best forks: {:?}",
self.pruning.as_ref().and_then(|p| p.next_hash()),
self.pruning.as_ref().map(|p| p.pending()).unwrap_or(0),
self.non_canonical.last_canonicalized_hash(),
self.non_canonical.last_canonicalized_block_number().unwrap_or(0),
self.non_canonical.top_level(),
);
}
pub fn revert_pending(&mut self) {
if let Some(pruning) = &mut self.pruning {
pruning.revert_pending();
}
self.non_canonical.revert_pending();
}
}
/// State DB maintenance. See module description.
/// Can be shared across threads.
pub struct StateDb<BlockHash: Hash, Key: Hash> {
db: RwLock<StateDbSync<BlockHash, Key>>,
}
impl<BlockHash: Hash, Key: Hash> StateDb<BlockHash, Key> {
/// Creates a new instance. Does not expect any metadata in the database.
pub fn new<D: MetaDb>(mode: PruningMode, db: &D) -> Result<StateDb<BlockHash, Key>, Error<D::Error>> {
Ok(StateDb {
db: RwLock::new(StateDbSync::new(mode, db)?)
})
}
/// Add a new non-canonical block.
pub fn insert_block<E: fmt::Debug>(&self, hash: &BlockHash, number: u64, parent_hash: &BlockHash, changeset: ChangeSet<Key>) -> Result<CommitSet<Key>, Error<E>> {
self.db.write().insert_block(hash, number, parent_hash, changeset)
}
/// Finalize a previously inserted block.
pub fn canonicalize_block<E: fmt::Debug>(&self, hash: &BlockHash) -> Result<CommitSet<Key>, Error<E>> {
self.db.write().canonicalize_block(hash)
}
/// Prevents pruning of specified block and its descendants.
pub fn pin(&self, hash: &BlockHash) -> Result<(), PinError> {
self.db.write().pin(hash)
}
/// Allows pruning of specified block.
pub fn unpin(&self, hash: &BlockHash) {
self.db.write().unpin(hash)
}
/// Get a value from non-canonical/pruning overlay or the backing DB.
pub fn get<D: NodeDb>(&self, key: &Key, db: &D) -> Result<Option<DBValue>, Error<D::Error>>
where Key: AsRef<D::Key>
{
self.db.read().get(key, db)
}
/// Revert all non-canonical blocks with the best block number.
/// Returns a database commit or `None` if not possible.
/// For archive an empty commit set is returned.
pub fn revert_one(&self) -> Option<CommitSet<Key>> {
self.db.write().revert_one()
}
/// Returns last finalized block number.
pub fn best_canonical(&self) -> Option<u64> {
return self.db.read().best_canonical()
}
/// Check if block is pruned away.
pub fn is_pruned(&self, hash: &BlockHash, number: u64) -> bool {
return self.db.read().is_pruned(hash, number)
}
/// Apply all pending changes
pub fn apply_pending(&self) {
self.db.write().apply_pending();
}
/// Revert all pending changes
pub fn revert_pending(&self) {
self.db.write().revert_pending();
}
}
#[cfg(test)]
mod tests {
use std::io;
use primitives::H256;
use crate::{StateDb, PruningMode, Constraints};
use crate::test::{make_db, make_changeset, TestDb};
fn make_test_db(settings: PruningMode) -> (TestDb, StateDb<H256, H256>) {
let mut db = make_db(&[91, 921, 922, 93, 94]);
let state_db = StateDb::new(settings, &db).unwrap();
db.commit(
&state_db
.insert_block::<io::Error>(
&H256::from_low_u64_be(1),
1,
&H256::from_low_u64_be(0),
make_changeset(&[1], &[91]),
)
.unwrap(),
);
db.commit(
&state_db
.insert_block::<io::Error>(
&H256::from_low_u64_be(21),
2,
&H256::from_low_u64_be(1),
make_changeset(&[21], &[921, 1]),
)
.unwrap(),
);
db.commit(
&state_db
.insert_block::<io::Error>(
&H256::from_low_u64_be(22),
2,
&H256::from_low_u64_be(1),
make_changeset(&[22], &[922]),
)
.unwrap(),
);
db.commit(
&state_db
.insert_block::<io::Error>(
&H256::from_low_u64_be(3),
3,
&H256::from_low_u64_be(21),
make_changeset(&[3], &[93]),
)
.unwrap(),
);
state_db.apply_pending();
db.commit(&state_db.canonicalize_block::<io::Error>(&H256::from_low_u64_be(1)).unwrap());
state_db.apply_pending();
db.commit(
&state_db
.insert_block::<io::Error>(
&H256::from_low_u64_be(4),
4,
&H256::from_low_u64_be(3),
make_changeset(&[4], &[94]),
)
.unwrap(),
);
state_db.apply_pending();
db.commit(&state_db.canonicalize_block::<io::Error>(&H256::from_low_u64_be(21)).unwrap());
state_db.apply_pending();
db.commit(&state_db.canonicalize_block::<io::Error>(&H256::from_low_u64_be(3)).unwrap());
state_db.apply_pending();
(db, state_db)
}
#[test]
fn full_archive_keeps_everything() {
let (db, sdb) = make_test_db(PruningMode::ArchiveAll);
assert!(db.data_eq(&make_db(&[1, 21, 22, 3, 4, 91, 921, 922, 93, 94])));
assert!(!sdb.is_pruned(&H256::from_low_u64_be(0), 0));
}
#[test]
fn canonical_archive_keeps_canonical() {
let (db, _) = make_test_db(PruningMode::ArchiveCanonical);
assert!(db.data_eq(&make_db(&[1, 21, 3, 91, 921, 922, 93, 94])));
}
#[test]
fn prune_window_0() {
let (db, _) = make_test_db(PruningMode::Constrained(Constraints {
max_blocks: Some(0),
max_mem: None,
}));
assert!(db.data_eq(&make_db(&[21, 3, 922, 94])));
}
#[test]
fn prune_window_1() {
let (db, sdb) = make_test_db(PruningMode::Constrained(Constraints {
max_blocks: Some(1),
max_mem: None,
}));
assert!(sdb.is_pruned(&H256::from_low_u64_be(0), 0));
assert!(sdb.is_pruned(&H256::from_low_u64_be(1), 1));
assert!(sdb.is_pruned(&H256::from_low_u64_be(21), 2));
assert!(sdb.is_pruned(&H256::from_low_u64_be(22), 2));
assert!(db.data_eq(&make_db(&[21, 3, 922, 93, 94])));
}
#[test]
fn prune_window_2() {
let (db, sdb) = make_test_db(PruningMode::Constrained(Constraints {
max_blocks: Some(2),
max_mem: None,
}));
assert!(sdb.is_pruned(&H256::from_low_u64_be(0), 0));
assert!(sdb.is_pruned(&H256::from_low_u64_be(1), 1));
assert!(!sdb.is_pruned(&H256::from_low_u64_be(21), 2));
assert!(sdb.is_pruned(&H256::from_low_u64_be(22), 2));
assert!(db.data_eq(&make_db(&[1, 21, 3, 921, 922, 93, 94])));
}
#[test]
fn detects_incompatible_mode() {
let mut db = make_db(&[]);
let state_db = StateDb::new(PruningMode::ArchiveAll, &db).unwrap();
db.commit(
&state_db
.insert_block::<io::Error>(
&H256::from_low_u64_be(0),
0,
&H256::from_low_u64_be(0),
make_changeset(&[], &[]),
)
.unwrap(),
);
let new_mode = PruningMode::Constrained(Constraints { max_blocks: Some(2), max_mem: None });
let state_db: Result<StateDb<H256, H256>, _> = StateDb::new(new_mode, &db);
assert!(state_db.is_err());
}
}
substrate/client/state-db/src/noncanonical.rs
+824
View File
@@ -0,0 +1,824 @@
// Copyright 2017-2019 Parity Technologies (UK) Ltd.
// This file is part of Substrate.
// Substrate is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
// Substrate is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
// You should have received a copy of the GNU General Public License
// along with Substrate. If not, see <http://www.gnu.org/licenses/>.
//! Canonicalization window.
//! Maintains trees of block overlays and allows discarding trees/roots
//! The overlays are added in `insert` and removed in `canonicalize`.
//! All pending changes are kept in memory until next call to `apply_pending` or
//! `revert_pending`
use std::fmt;
use std::collections::{HashMap, VecDeque, hash_map::Entry};
use super::{Error, DBValue, ChangeSet, CommitSet, MetaDb, Hash, to_meta_key};
use codec::{Encode, Decode};
use log::trace;
const NON_CANONICAL_JOURNAL: &[u8] = b"noncanonical_journal";
const LAST_CANONICAL: &[u8] = b"last_canonical";
/// See module documentation.
pub struct NonCanonicalOverlay<BlockHash: Hash, Key: Hash> {
last_canonicalized: Option<(BlockHash, u64)>,
levels: VecDeque<Vec<BlockOverlay<BlockHash, Key>>>,
parents: HashMap<BlockHash, BlockHash>,
pending_canonicalizations: Vec<BlockHash>,
pending_insertions: Vec<BlockHash>,
values: HashMap<Key, (u32, DBValue)>, //ref counted
pinned: HashMap<BlockHash, HashMap<Key, DBValue>>, //would be deleted but kept around because block is pinned
}
#[derive(Encode, Decode)]
struct JournalRecord<BlockHash: Hash, Key: Hash> {
hash: BlockHash,
parent_hash: BlockHash,
inserted: Vec<(Key, DBValue)>,
deleted: Vec<Key>,
}
fn to_journal_key(block: u64, index: u64) -> Vec<u8> {
to_meta_key(NON_CANONICAL_JOURNAL, &(block, index))
}
#[cfg_attr(test, derive(PartialEq, Debug))]
struct BlockOverlay<BlockHash: Hash, Key: Hash> {
hash: BlockHash,
journal_key: Vec<u8>,
inserted: Vec<Key>,
deleted: Vec<Key>,
}
fn insert_values<Key: Hash>(values: &mut HashMap<Key, (u32, DBValue)>, inserted: Vec<(Key, DBValue)>) {
for (k, v) in inserted {
debug_assert!(values.get(&k).map_or(true, |(_, value)| *value == v));
let (ref mut counter, _) = values.entry(k).or_insert_with(|| (0, v));
*counter += 1;
}
}
fn discard_values<Key: Hash>(
values: &mut HashMap<Key, (u32, DBValue)>,
inserted: Vec<Key>,
mut into: Option<&mut HashMap<Key, DBValue>>,
) {
for k in inserted {
match values.entry(k) {
Entry::Occupied(mut e) => {
let (ref mut counter, _) = e.get_mut();
*counter -= 1;
if *counter == 0 {
let (key, (_, value)) = e.remove_entry();
if let Some(ref mut into) = into {
into.insert(key, value);
}
}
},
Entry::Vacant(_) => {
debug_assert!(false, "Trying to discard missing value");
}
}
}
}
fn discard_descendants<BlockHash: Hash, Key: Hash>(
levels: &mut VecDeque<Vec<BlockOverlay<BlockHash, Key>>>,
mut values: &mut HashMap<Key, (u32, DBValue)>,
index: usize,
parents: &mut HashMap<BlockHash, BlockHash>,
pinned: &mut HashMap<BlockHash, HashMap<Key, DBValue>>,
hash: &BlockHash,
) {
let mut discarded = Vec::new();
if let Some(level) = levels.get_mut(index) {
*level = level.drain(..).filter_map(|overlay| {
let parent = parents.get(&overlay.hash).expect("there is a parent entry for each entry in levels; qed").clone();
if parent == *hash {
parents.remove(&overlay.hash);
discarded.push(overlay.hash);
discard_values(&mut values, overlay.inserted, pinned.get_mut(hash));
None
} else {
Some(overlay)
}
}).collect();
}
for hash in discarded {
discard_descendants(levels, values, index + 1, parents, pinned, &hash);
}
}
impl<BlockHash: Hash, Key: Hash> NonCanonicalOverlay<BlockHash, Key> {
/// Creates a new instance. Does not expect any metadata to be present in the DB.
pub fn new<D: MetaDb>(db: &D) -> Result<NonCanonicalOverlay<BlockHash, Key>, Error<D::Error>> {
let last_canonicalized = db.get_meta(&to_meta_key(LAST_CANONICAL, &()))
.map_err(|e| Error::Db(e))?;
let last_canonicalized = match last_canonicalized {
Some(buffer) => Some(<(BlockHash, u64)>::decode(&mut buffer.as_slice())?),
None => None,
};
let mut levels = VecDeque::new();
let mut parents = HashMap::new();
let mut values = HashMap::new();
if let Some((ref hash, mut block)) = last_canonicalized {
// read the journal
trace!(target: "state-db", "Reading uncanonicalized journal. Last canonicalized #{} ({:?})", block, hash);
let mut total: u64 = 0;
block += 1;
loop {
let mut index: u64 = 0;
let mut level = Vec::new();
loop {
let journal_key = to_journal_key(block, index);
match db.get_meta(&journal_key).map_err(|e| Error::Db(e))? {
Some(record) => {
let record: JournalRecord<BlockHash, Key> = Decode::decode(&mut record.as_slice())?;
let inserted = record.inserted.iter().map(|(k, _)| k.clone()).collect();
let overlay = BlockOverlay {
hash: record.hash.clone(),
journal_key,
inserted: inserted,
deleted: record.deleted,
};
insert_values(&mut values, record.inserted);
trace!(target: "state-db", "Uncanonicalized journal entry {}.{} ({} inserted, {} deleted)", block, index, overlay.inserted.len(), overlay.deleted.len());
level.push(overlay);
parents.insert(record.hash, record.parent_hash);
index += 1;
total += 1;
},
None => break,
}
}
if level.is_empty() {
break;
}
levels.push_back(level);
block += 1;
}
trace!(target: "state-db", "Finished reading uncanonicalized journal, {} entries", total);
}
Ok(NonCanonicalOverlay {
last_canonicalized,
levels,
parents,
pending_canonicalizations: Default::default(),
pending_insertions: Default::default(),
pinned: Default::default(),
values: values,
})
}
/// Insert a new block into the overlay. If inserted on the second level or lover expects parent to be present in the window.
pub fn insert<E: fmt::Debug>(&mut self, hash: &BlockHash, number: u64, parent_hash: &BlockHash, changeset: ChangeSet<Key>) -> Result<CommitSet<Key>, Error<E>> {
let mut commit = CommitSet::default();
let front_block_number = self.front_block_number();
if self.levels.is_empty() && self.last_canonicalized.is_none() && number > 0 {
// assume that parent was canonicalized
let last_canonicalized = (parent_hash.clone(), number - 1);
commit.meta.inserted.push((to_meta_key(LAST_CANONICAL, &()), last_canonicalized.encode()));
self.last_canonicalized = Some(last_canonicalized);
} else if self.last_canonicalized.is_some() {
if number < front_block_number || number >= front_block_number + self.levels.len() as u64 + 1 {
trace!(target: "state-db", "Failed to insert block {}, current is {} .. {})",
number,
front_block_number,
front_block_number + self.levels.len() as u64,
);
return Err(Error::InvalidBlockNumber);
}
// check for valid parent if inserting on second level or higher
if number == front_block_number {
if !self.last_canonicalized.as_ref().map_or(false, |&(ref h, n)| h == parent_hash && n == number - 1) {
return Err(Error::InvalidParent);
}
} else if !self.parents.contains_key(&parent_hash) {
return Err(Error::InvalidParent);
}
}
let level = if self.levels.is_empty() || number == front_block_number + self.levels.len() as u64 {
self.levels.push_back(Vec::new());
self.levels.back_mut().expect("can't be empty after insertion; qed")
} else {
self.levels.get_mut((number - front_block_number) as usize)
.expect("number is [front_block_number .. front_block_number + levels.len()) is asserted in precondition; qed")
};
let index = level.len() as u64;
let journal_key = to_journal_key(number, index);
let inserted = changeset.inserted.iter().map(|(k, _)| k.clone()).collect();
let overlay = BlockOverlay {
hash: hash.clone(),
journal_key: journal_key.clone(),
inserted: inserted,
deleted: changeset.deleted.clone(),
};
level.push(overlay);
self.parents.insert(hash.clone(), parent_hash.clone());
let journal_record = JournalRecord {
hash: hash.clone(),
parent_hash: parent_hash.clone(),
inserted: changeset.inserted,
deleted: changeset.deleted,
};
commit.meta.inserted.push((journal_key, journal_record.encode()));
trace!(target: "state-db", "Inserted uncanonicalized changeset {}.{} ({} inserted, {} deleted)", number, index, journal_record.inserted.len(), journal_record.deleted.len());
insert_values(&mut self.values, journal_record.inserted);
self.pending_insertions.push(hash.clone());
Ok(commit)
}
fn discard_journals(
&self,
level_index: usize,
discarded_journals: &mut Vec<Vec<u8>>,
discarded_blocks: &mut Vec<BlockHash>,
hash: &BlockHash
) {
if let Some(level) = self.levels.get(level_index) {
level.iter().for_each(|overlay| {
let parent = self.parents.get(&overlay.hash).expect("there is a parent entry for each entry in levels; qed").clone();
if parent == *hash {
discarded_journals.push(overlay.journal_key.clone());
discarded_blocks.push(overlay.hash.clone());
self.discard_journals(level_index + 1, discarded_journals, discarded_blocks, &overlay.hash);
}
});
}
}
fn front_block_number(&self) -> u64 {
self.last_canonicalized.as_ref().map(|&(_, n)| n + 1).unwrap_or(0)
}
pub fn last_canonicalized_block_number(&self) -> Option<u64> {
match self.last_canonicalized.as_ref().map(|&(_, n)| n) {
Some(n) => Some(n + self.pending_canonicalizations.len() as u64),
None if !self.pending_canonicalizations.is_empty() => Some(self.pending_canonicalizations.len() as u64),
_ => None,
}
}
pub fn last_canonicalized_hash(&self) -> Option<BlockHash> {
self.last_canonicalized.as_ref().map(|&(ref h, _)| h.clone())
}
pub fn top_level(&self) -> Vec<(BlockHash, u64)> {
let start = self.last_canonicalized_block_number().unwrap_or(0);
self.levels
.get(self.pending_canonicalizations.len())
.map(|level| level.iter().map(|r| (r.hash.clone(), start)).collect())
.unwrap_or_default()
}
/// Select a top-level root and canonicalized it. Discards all sibling subtrees and the root.
/// Returns a set of changes that need to be added to the DB.
pub fn canonicalize<E: fmt::Debug>(
&mut self,
hash: &BlockHash,
commit: &mut CommitSet<Key>,
) -> Result<(), Error<E>> {
trace!(target: "state-db", "Canonicalizing {:?}", hash);
let level = self.levels.get(self.pending_canonicalizations.len()).ok_or_else(|| Error::InvalidBlock)?;
let index = level
.iter()
.position(|overlay| overlay.hash == *hash)
.ok_or_else(|| Error::InvalidBlock)?;
let mut discarded_journals = Vec::new();
let mut discarded_blocks = Vec::new();
for (i, overlay) in level.iter().enumerate() {
if i != index {
self.discard_journals(
self.pending_canonicalizations.len() + 1,
&mut discarded_journals,
&mut discarded_blocks,
&overlay.hash
);
}
discarded_journals.push(overlay.journal_key.clone());
discarded_blocks.push(overlay.hash.clone());
}
// get the one we need to canonicalize
let overlay = &level[index];
commit.data.inserted.extend(overlay.inserted.iter()
.map(|k| (k.clone(), self.values.get(k).expect("For each key in overlays there's a value in values").1.clone())));
commit.data.deleted.extend(overlay.deleted.clone());
commit.meta.deleted.append(&mut discarded_journals);
let canonicalized = (hash.clone(), self.front_block_number() + self.pending_canonicalizations.len() as u64);
commit.meta.inserted.push((to_meta_key(LAST_CANONICAL, &()), canonicalized.encode()));
trace!(target: "state-db", "Discarding {} records", commit.meta.deleted.len());
self.pending_canonicalizations.push(hash.clone());
Ok(())
}
fn apply_canonicalizations(&mut self) {
let last = self.pending_canonicalizations.last().cloned();
let count = self.pending_canonicalizations.len() as u64;
for hash in self.pending_canonicalizations.drain(..) {
trace!(target: "state-db", "Post canonicalizing {:?}", hash);
let level = self.levels.pop_front().expect("Hash validity is checked in `canonicalize`");
let index = level
.iter()
.position(|overlay| overlay.hash == hash)
.expect("Hash validity is checked in `canonicalize`");
// discard unfinalized overlays and values
for (i, overlay) in level.into_iter().enumerate() {
self.parents.remove(&overlay.hash);
if i != index {
discard_descendants(
&mut self.levels,
&mut self.values,
0,
&mut self.parents,
&mut self.pinned,
&overlay.hash,
);
}
discard_values(&mut self.values, overlay.inserted, self.pinned.get_mut(&overlay.hash));
}
}
if let Some(hash) = last {
let last_canonicalized = (hash, self.last_canonicalized.as_ref().map(|(_, n)| n + count).unwrap_or(count - 1));
self.last_canonicalized = Some(last_canonicalized);
}
}
/// Get a value from the node overlay. This searches in every existing changeset.
pub fn get(&self, key: &Key) -> Option<DBValue> {
if let Some((_, value)) = self.values.get(&key) {
return Some(value.clone());
}
for pinned in self.pinned.values() {
if let Some(value) = pinned.get(&key) {
return Some(value.clone());
}
}
None
}
/// Check if the block is in the canonicalization queue.
pub fn have_block(&self, hash: &BlockHash) -> bool {
(self.parents.contains_key(hash) || self.pending_insertions.contains(hash))
&& !self.pending_canonicalizations.contains(hash)
}
/// Revert a single level. Returns commit set that deletes the journal or `None` if not possible.
pub fn revert_one(&mut self) -> Option<CommitSet<Key>> {
self.levels.pop_back().map(|level| {
let mut commit = CommitSet::default();
for overlay in level.into_iter() {
commit.meta.deleted.push(overlay.journal_key);
self.parents.remove(&overlay.hash);
discard_values(&mut self.values, overlay.inserted, None);
}
commit
})
}
fn revert_insertions(&mut self) {
self.pending_insertions.reverse();
for hash in self.pending_insertions.drain(..) {
self.parents.remove(&hash);
// find a level. When iterating insertions backwards the hash is always last in the level.
let level_index =
self.levels.iter().position(|level|
level.last().expect("Hash is added in `insert` in reverse order").hash == hash)
.expect("Hash is added in insert");
let overlay = self.levels[level_index].pop().expect("Empty levels are not allowed in self.levels");
discard_values(&mut self.values, overlay.inserted, None);
if self.levels[level_index].is_empty() {
debug_assert_eq!(level_index, self.levels.len() - 1);
self.levels.pop_back();
}
}
}
/// Apply all pending changes
pub fn apply_pending(&mut self) {
self.apply_canonicalizations();
self.pending_insertions.clear();
}
/// Revert all pending changes
pub fn revert_pending(&mut self) {
self.pending_canonicalizations.clear();
self.revert_insertions();
}
/// Pin state values in memory
pub fn pin(&mut self, hash: &BlockHash) {
self.pinned.insert(hash.clone(), HashMap::default());
}
/// Discard pinned state
pub fn unpin(&mut self, hash: &BlockHash) {
self.pinned.remove(hash);
}
}
#[cfg(test)]
mod tests {
use std::io;
use primitives::H256;
use super::{NonCanonicalOverlay, to_journal_key};
use crate::{ChangeSet, CommitSet};
use crate::test::{make_db, make_changeset};
fn contains(overlay: &NonCanonicalOverlay<H256, H256>, key: u64) -> bool {
overlay.get(&H256::from_low_u64_be(key)) == Some(H256::from_low_u64_be(key).as_bytes().to_vec())
}
#[test]
fn created_from_empty_db() {
let db = make_db(&[]);
let overlay: NonCanonicalOverlay<H256, H256> = NonCanonicalOverlay::new(&db).unwrap();
assert_eq!(overlay.last_canonicalized, None);
assert!(overlay.levels.is_empty());
assert!(overlay.parents.is_empty());
}
#[test]
#[should_panic]
fn canonicalize_empty_panics() {
let db = make_db(&[]);
let mut overlay = NonCanonicalOverlay::<H256, H256>::new(&db).unwrap();
let mut commit = CommitSet::default();
overlay.canonicalize::<io::Error>(&H256::default(), &mut commit).unwrap();
}
#[test]
#[should_panic]
fn insert_ahead_panics() {
let db = make_db(&[]);
let h1 = H256::random();
let h2 = H256::random();
let mut overlay = NonCanonicalOverlay::<H256, H256>::new(&db).unwrap();
overlay.insert::<io::Error>(&h1, 2, &H256::default(), ChangeSet::default()).unwrap();
overlay.insert::<io::Error>(&h2, 1, &h1, ChangeSet::default()).unwrap();
}
#[test]
#[should_panic]
fn insert_behind_panics() {
let h1 = H256::random();
let h2 = H256::random();
let db = make_db(&[]);
let mut overlay = NonCanonicalOverlay::<H256, H256>::new(&db).unwrap();
overlay.insert::<io::Error>(&h1, 1, &H256::default(), ChangeSet::default()).unwrap();
overlay.insert::<io::Error>(&h2, 3, &h1, ChangeSet::default()).unwrap();
}
#[test]
#[should_panic]
fn insert_unknown_parent_panics() {
let db = make_db(&[]);
let h1 = H256::random();
let h2 = H256::random();
let mut overlay = NonCanonicalOverlay::<H256, H256>::new(&db).unwrap();
overlay.insert::<io::Error>(&h1, 1, &H256::default(), ChangeSet::default()).unwrap();
overlay.insert::<io::Error>(&h2, 2, &H256::default(), ChangeSet::default()).unwrap();
}
#[test]
#[should_panic]
fn canonicalize_unknown_panics() {
let h1 = H256::random();
let h2 = H256::random();
let db = make_db(&[]);
let mut overlay = NonCanonicalOverlay::<H256, H256>::new(&db).unwrap();
overlay.insert::<io::Error>(&h1, 1, &H256::default(), ChangeSet::default()).unwrap();
let mut commit = CommitSet::default();
overlay.canonicalize::<io::Error>(&h2, &mut commit).unwrap();
}
#[test]
fn insert_canonicalize_one() {
let h1 = H256::random();
let mut db = make_db(&[1, 2]);
let mut overlay = NonCanonicalOverlay::<H256, H256>::new(&db).unwrap();
let changeset = make_changeset(&[3, 4], &[2]);
let insertion = overlay.insert::<io::Error>(&h1, 1, &H256::default(), changeset.clone()).unwrap();
assert_eq!(insertion.data.inserted.len(), 0);
assert_eq!(insertion.data.deleted.len(), 0);
assert_eq!(insertion.meta.inserted.len(), 2);
assert_eq!(insertion.meta.deleted.len(), 0);
db.commit(&insertion);
let mut finalization = CommitSet::default();
overlay.canonicalize::<io::Error>(&h1, &mut finalization).unwrap();
assert_eq!(finalization.data.inserted.len(), changeset.inserted.len());
assert_eq!(finalization.data.deleted.len(), changeset.deleted.len());
assert_eq!(finalization.meta.inserted.len(), 1);
assert_eq!(finalization.meta.deleted.len(), 1);
db.commit(&finalization);
assert!(db.data_eq(&make_db(&[1, 3, 4])));
}
#[test]
fn restore_from_journal() {
let h1 = H256::random();
let h2 = H256::random();
let mut db = make_db(&[1, 2]);
let mut overlay = NonCanonicalOverlay::<H256, H256>::new(&db).unwrap();
db.commit(&overlay.insert::<io::Error>(&h1, 10, &H256::default(), make_changeset(&[3, 4], &[2])).unwrap());
db.commit(&overlay.insert::<io::Error>(&h2, 11, &h1, make_changeset(&[5], &[3])).unwrap());
assert_eq!(db.meta.len(), 3);
let overlay2 = NonCanonicalOverlay::<H256, H256>::new(&db).unwrap();
assert_eq!(overlay.levels, overlay2.levels);
assert_eq!(overlay.parents, overlay2.parents);
assert_eq!(overlay.last_canonicalized, overlay2.last_canonicalized);
}
#[test]
fn restore_from_journal_after_canonicalize() {
let h1 = H256::random();
let h2 = H256::random();
let mut db = make_db(&[1, 2]);
let mut overlay = NonCanonicalOverlay::<H256, H256>::new(&db).unwrap();
db.commit(&overlay.insert::<io::Error>(&h1, 10, &H256::default(), make_changeset(&[3, 4], &[2])).unwrap());
db.commit(&overlay.insert::<io::Error>(&h2, 11, &h1, make_changeset(&[5], &[3])).unwrap());
let mut commit = CommitSet::default();
overlay.canonicalize::<io::Error>(&h1, &mut commit).unwrap();
db.commit(&commit);
overlay.apply_pending();
assert_eq!(overlay.levels.len(), 1);
let overlay2 = NonCanonicalOverlay::<H256, H256>::new(&db).unwrap();
assert_eq!(overlay.levels, overlay2.levels);
assert_eq!(overlay.parents, overlay2.parents);
assert_eq!(overlay.last_canonicalized, overlay2.last_canonicalized);
}
#[test]
fn insert_canonicalize_two() {
let h1 = H256::random();
let h2 = H256::random();
let mut db = make_db(&[1, 2, 3, 4]);
let mut overlay = NonCanonicalOverlay::<H256, H256>::new(&db).unwrap();
let changeset1 = make_changeset(&[5, 6], &[2]);
let changeset2 = make_changeset(&[7, 8], &[5, 3]);
db.commit(&overlay.insert::<io::Error>(&h1, 1, &H256::default(), changeset1).unwrap());
assert!(contains(&overlay, 5));
db.commit(&overlay.insert::<io::Error>(&h2, 2, &h1, changeset2).unwrap());
assert!(contains(&overlay, 7));
assert!(contains(&overlay, 5));
assert_eq!(overlay.levels.len(), 2);
assert_eq!(overlay.parents.len(), 2);
let mut commit = CommitSet::default();
overlay.canonicalize::<io::Error>(&h1, &mut commit).unwrap();
db.commit(&commit);
assert!(contains(&overlay, 5));
assert_eq!(overlay.levels.len(), 2);
assert_eq!(overlay.parents.len(), 2);
overlay.apply_pending();
assert_eq!(overlay.levels.len(), 1);
assert_eq!(overlay.parents.len(), 1);
assert!(!contains(&overlay, 5));
assert!(contains(&overlay, 7));
let mut commit = CommitSet::default();
overlay.canonicalize::<io::Error>(&h2, &mut commit).unwrap();
db.commit(&commit);
overlay.apply_pending();
assert_eq!(overlay.levels.len(), 0);
assert_eq!(overlay.parents.len(), 0);
assert!(db.data_eq(&make_db(&[1, 4, 6, 7, 8])));
}
#[test]
fn insert_same_key() {
let mut db = make_db(&[]);
let (h_1, c_1) = (H256::random(), make_changeset(&[1], &[]));
let (h_2, c_2) = (H256::random(), make_changeset(&[1], &[]));
let mut overlay = NonCanonicalOverlay::<H256, H256>::new(&db).unwrap();
db.commit(&overlay.insert::<io::Error>(&h_1, 1, &H256::default(), c_1).unwrap());
db.commit(&overlay.insert::<io::Error>(&h_2, 1, &H256::default(), c_2).unwrap());
assert!(contains(&overlay, 1));
let mut commit = CommitSet::default();
overlay.canonicalize::<io::Error>(&h_1, &mut commit).unwrap();
db.commit(&commit);
assert!(contains(&overlay, 1));
overlay.apply_pending();
assert!(!contains(&overlay, 1));
}
#[test]
fn insert_with_pending_canonicalization() {
let h1 = H256::random();
let h2 = H256::random();
let h3 = H256::random();
let mut db = make_db(&[]);
let mut overlay = NonCanonicalOverlay::<H256, H256>::new(&db).unwrap();
let changeset = make_changeset(&[], &[]);
db.commit(&overlay.insert::<io::Error>(&h1, 1, &H256::default(), changeset.clone()).unwrap());
db.commit(&overlay.insert::<io::Error>(&h2, 2, &h1, changeset.clone()).unwrap());
overlay.apply_pending();
let mut commit = CommitSet::default();
overlay.canonicalize::<io::Error>(&h1, &mut commit).unwrap();
overlay.canonicalize::<io::Error>(&h2, &mut commit).unwrap();
db.commit(&commit);
db.commit(&overlay.insert::<io::Error>(&h3, 3, &h2, changeset.clone()).unwrap());
overlay.apply_pending();
assert_eq!(overlay.levels.len(), 1);
}
#[test]
fn complex_tree() {
use crate::MetaDb;
let mut db = make_db(&[]);
// - 1 - 1_1 - 1_1_1
// \ 1_2 - 1_2_1
// \ 1_2_2
// \ 1_2_3
//
// - 2 - 2_1 - 2_1_1
// \ 2_2
//
// 1_2_2 is the winner
let (h_1, c_1) = (H256::random(), make_changeset(&[1], &[]));
let (h_2, c_2) = (H256::random(), make_changeset(&[2], &[]));
let (h_1_1, c_1_1) = (H256::random(), make_changeset(&[11], &[]));
let (h_1_2, c_1_2) = (H256::random(), make_changeset(&[12], &[]));
let (h_2_1, c_2_1) = (H256::random(), make_changeset(&[21], &[]));
let (h_2_2, c_2_2) = (H256::random(), make_changeset(&[22], &[]));
let (h_1_1_1, c_1_1_1) = (H256::random(), make_changeset(&[111], &[]));
let (h_1_2_1, c_1_2_1) = (H256::random(), make_changeset(&[121], &[]));
let (h_1_2_2, c_1_2_2) = (H256::random(), make_changeset(&[122], &[]));
let (h_1_2_3, c_1_2_3) = (H256::random(), make_changeset(&[123], &[]));
let (h_2_1_1, c_2_1_1) = (H256::random(), make_changeset(&[211], &[]));
let mut overlay = NonCanonicalOverlay::<H256, H256>::new(&db).unwrap();
db.commit(&overlay.insert::<io::Error>(&h_1, 1, &H256::default(), c_1).unwrap());
db.commit(&overlay.insert::<io::Error>(&h_1_1, 2, &h_1, c_1_1).unwrap());
db.commit(&overlay.insert::<io::Error>(&h_1_2, 2, &h_1, c_1_2).unwrap());
db.commit(&overlay.insert::<io::Error>(&h_2, 1, &H256::default(), c_2).unwrap());
db.commit(&overlay.insert::<io::Error>(&h_2_1, 2, &h_2, c_2_1).unwrap());
db.commit(&overlay.insert::<io::Error>(&h_2_2, 2, &h_2, c_2_2).unwrap());
db.commit(&overlay.insert::<io::Error>(&h_1_1_1, 3, &h_1_1, c_1_1_1).unwrap());
db.commit(&overlay.insert::<io::Error>(&h_1_2_1, 3, &h_1_2, c_1_2_1).unwrap());
db.commit(&overlay.insert::<io::Error>(&h_1_2_2, 3, &h_1_2, c_1_2_2).unwrap());
db.commit(&overlay.insert::<io::Error>(&h_1_2_3, 3, &h_1_2, c_1_2_3).unwrap());
db.commit(&overlay.insert::<io::Error>(&h_2_1_1, 3, &h_2_1, c_2_1_1).unwrap());
assert!(contains(&overlay, 2));
assert!(contains(&overlay, 11));
assert!(contains(&overlay, 21));
assert!(contains(&overlay, 111));
assert!(contains(&overlay, 122));
assert!(contains(&overlay, 211));
assert_eq!(overlay.levels.len(), 3);
assert_eq!(overlay.parents.len(), 11);
assert_eq!(overlay.last_canonicalized, Some((H256::default(), 0)));
// check if restoration from journal results in the same tree
let overlay2 = NonCanonicalOverlay::<H256, H256>::new(&db).unwrap();
assert_eq!(overlay.levels, overlay2.levels);
assert_eq!(overlay.parents, overlay2.parents);
assert_eq!(overlay.last_canonicalized, overlay2.last_canonicalized);
// canonicalize 1. 2 and all its children should be discarded
let mut commit = CommitSet::default();
overlay.canonicalize::<io::Error>(&h_1, &mut commit).unwrap();
db.commit(&commit);
overlay.apply_pending();
assert_eq!(overlay.levels.len(), 2);
assert_eq!(overlay.parents.len(), 6);
assert!(!contains(&overlay, 1));
assert!(!contains(&overlay, 2));
assert!(!contains(&overlay, 21));
assert!(!contains(&overlay, 22));
assert!(!contains(&overlay, 211));
assert!(contains(&overlay, 111));
assert!(!contains(&overlay, 211));
// check that journals are deleted
assert!(db.get_meta(&to_journal_key(1, 0)).unwrap().is_none());
assert!(db.get_meta(&to_journal_key(1, 1)).unwrap().is_none());
assert!(db.get_meta(&to_journal_key(2, 1)).unwrap().is_some());
assert!(db.get_meta(&to_journal_key(2, 2)).unwrap().is_none());
assert!(db.get_meta(&to_journal_key(2, 3)).unwrap().is_none());
// canonicalize 1_2. 1_1 and all its children should be discarded
let mut commit = CommitSet::default();
overlay.canonicalize::<io::Error>(&h_1_2, &mut commit).unwrap();
db.commit(&commit);
overlay.apply_pending();
assert_eq!(overlay.levels.len(), 1);
assert_eq!(overlay.parents.len(), 3);
assert!(!contains(&overlay, 11));
assert!(!contains(&overlay, 111));
assert!(contains(&overlay, 121));
assert!(contains(&overlay, 122));
assert!(contains(&overlay, 123));
assert!(overlay.have_block(&h_1_2_1));
assert!(!overlay.have_block(&h_1_2));
assert!(!overlay.have_block(&h_1_1));
assert!(!overlay.have_block(&h_1_1_1));
// canonicalize 1_2_2
let mut commit = CommitSet::default();
overlay.canonicalize::<io::Error>(&h_1_2_2, &mut commit).unwrap();
db.commit(&commit);
overlay.apply_pending();
assert_eq!(overlay.levels.len(), 0);
assert_eq!(overlay.parents.len(), 0);
assert!(db.data_eq(&make_db(&[1, 12, 122])));
assert_eq!(overlay.last_canonicalized, Some((h_1_2_2, 3)));
}
#[test]
fn insert_revert() {
let h1 = H256::random();
let h2 = H256::random();
let mut db = make_db(&[1, 2, 3, 4]);
let mut overlay = NonCanonicalOverlay::<H256, H256>::new(&db).unwrap();
assert!(overlay.revert_one().is_none());
let changeset1 = make_changeset(&[5, 6], &[2]);
let changeset2 = make_changeset(&[7, 8], &[5, 3]);
db.commit(&overlay.insert::<io::Error>(&h1, 1, &H256::default(), changeset1).unwrap());
db.commit(&overlay.insert::<io::Error>(&h2, 2, &h1, changeset2).unwrap());
assert!(contains(&overlay, 7));
db.commit(&overlay.revert_one().unwrap());
assert_eq!(overlay.parents.len(), 1);
assert!(contains(&overlay, 5));
assert!(!contains(&overlay, 7));
db.commit(&overlay.revert_one().unwrap());
assert_eq!(overlay.levels.len(), 0);
assert_eq!(overlay.parents.len(), 0);
assert!(overlay.revert_one().is_none());
}
#[test]
fn revert_pending_insertion() {
let h1 = H256::random();
let h2_1 = H256::random();
let h2_2 = H256::random();
let db = make_db(&[]);
let mut overlay = NonCanonicalOverlay::<H256, H256>::new(&db).unwrap();
let changeset1 = make_changeset(&[5, 6], &[2]);
let changeset2 = make_changeset(&[7, 8], &[5, 3]);
let changeset3 = make_changeset(&[9], &[]);
overlay.insert::<io::Error>(&h1, 1, &H256::default(), changeset1).unwrap();
assert!(contains(&overlay, 5));
overlay.insert::<io::Error>(&h2_1, 2, &h1, changeset2).unwrap();
overlay.insert::<io::Error>(&h2_2, 2, &h1, changeset3).unwrap();
assert!(contains(&overlay, 7));
assert!(contains(&overlay, 5));
assert!(contains(&overlay, 9));
assert_eq!(overlay.levels.len(), 2);
assert_eq!(overlay.parents.len(), 3);
overlay.revert_pending();
assert!(!contains(&overlay, 5));
assert_eq!(overlay.levels.len(), 0);
assert_eq!(overlay.parents.len(), 0);
}
#[test]
fn keeps_pinned() {
let mut db = make_db(&[]);
// - 1 - 1_1
// \ 1_2
let (h_1, c_1) = (H256::random(), make_changeset(&[1], &[]));
let (h_2, c_2) = (H256::random(), make_changeset(&[2], &[]));
let mut overlay = NonCanonicalOverlay::<H256, H256>::new(&db).unwrap();
db.commit(&overlay.insert::<io::Error>(&h_1, 1, &H256::default(), c_1).unwrap());
db.commit(&overlay.insert::<io::Error>(&h_2, 1, &H256::default(), c_2).unwrap());
overlay.pin(&h_1);
let mut commit = CommitSet::default();
overlay.canonicalize::<io::Error>(&h_2, &mut commit).unwrap();
db.commit(&commit);
overlay.apply_pending();
assert!(contains(&overlay, 1));
overlay.unpin(&h_1);
assert!(!contains(&overlay, 1));
}
}
substrate/client/state-db/src/pruning.rs
+379
View File
@@ -0,0 +1,379 @@
// Copyright 2017-2019 Parity Technologies (UK) Ltd.
// This file is part of Substrate.
// Substrate is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
// Substrate is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
// You should have received a copy of the GNU General Public License
// along with Substrate. If not, see <http://www.gnu.org/licenses/>.
//! Pruning window.
//!
//! For each block we maintain a list of nodes pending deletion.
//! There is also a global index of node key to block number.
//! If a node is re-inserted into the window it gets removed from
//! the death list.
//! The changes are journaled in the DB.
use std::collections::{HashMap, HashSet, VecDeque};
use codec::{Encode, Decode};
use crate::{CommitSet, Error, MetaDb, to_meta_key, Hash};
use log::{trace, warn};
const LAST_PRUNED: &[u8] = b"last_pruned";
const PRUNING_JOURNAL: &[u8] = b"pruning_journal";
/// See module documentation.
pub struct RefWindow<BlockHash: Hash, Key: Hash> {
/// A queue of keys that should be deleted for each block in the pruning window.
death_rows: VecDeque<DeathRow<BlockHash, Key>>,
/// An index that maps each key from `death_rows` to block number.
death_index: HashMap<Key, u64>,
/// Block number that corresponts to the front of `death_rows`
pending_number: u64,
/// Number of call of `note_canonical` after
/// last call `apply_pending` or `revert_pending`
pending_canonicalizations: usize,
/// Number of calls of `prune_one` after
/// last call `apply_pending` or `revert_pending`
pending_prunings: usize,
}
#[derive(Debug, PartialEq, Eq)]
struct DeathRow<BlockHash: Hash, Key: Hash> {
hash: BlockHash,
journal_key: Vec<u8>,
deleted: HashSet<Key>,
}
#[derive(Encode, Decode)]
struct JournalRecord<BlockHash: Hash, Key: Hash> {
hash: BlockHash,
inserted: Vec<Key>,
deleted: Vec<Key>,
}
fn to_journal_key(block: u64) -> Vec<u8> {
to_meta_key(PRUNING_JOURNAL, &block)
}
impl<BlockHash: Hash, Key: Hash> RefWindow<BlockHash, Key> {
pub fn new<D: MetaDb>(db: &D) -> Result<RefWindow<BlockHash, Key>, Error<D::Error>> {
let last_pruned = db.get_meta(&to_meta_key(LAST_PRUNED, &()))
.map_err(|e| Error::Db(e))?;
let pending_number: u64 = match last_pruned {
Some(buffer) => u64::decode(&mut buffer.as_slice())? + 1,
None => 0,
};
let mut block = pending_number;
let mut pruning = RefWindow {
death_rows: Default::default(),
death_index: Default::default(),
pending_number: pending_number,
pending_canonicalizations: 0,
pending_prunings: 0,
};
// read the journal
trace!(target: "state-db", "Reading pruning journal. Pending #{}", pending_number);
loop {
let journal_key = to_journal_key(block);
match db.get_meta(&journal_key).map_err(|e| Error::Db(e))? {
Some(record) => {
let record: JournalRecord<BlockHash, Key> = Decode::decode(&mut record.as_slice())?;
trace!(target: "state-db", "Pruning journal entry {} ({} inserted, {} deleted)", block, record.inserted.len(), record.deleted.len());
pruning.import(&record.hash, journal_key, record.inserted.into_iter(), record.deleted);
},
None => break,
}
block += 1;
}
Ok(pruning)
}
fn import<I: IntoIterator<Item=Key>>(&mut self, hash: &BlockHash, journal_key: Vec<u8>, inserted: I, deleted: Vec<Key>) {
// remove all re-inserted keys from death rows
for k in inserted {
if let Some(block) = self.death_index.remove(&k) {
self.death_rows[(block - self.pending_number) as usize].deleted.remove(&k);
}
}
// add new keys
let imported_block = self.pending_number + self.death_rows.len() as u64;
for k in deleted.iter() {
self.death_index.insert(k.clone(), imported_block);
}
self.death_rows.push_back(
DeathRow {
hash: hash.clone(),
deleted: deleted.into_iter().collect(),
journal_key: journal_key,
}
);
}
pub fn window_size(&self) -> u64 {
(self.death_rows.len() - self.pending_prunings) as u64
}
pub fn next_hash(&self) -> Option<BlockHash> {
self.death_rows.get(self.pending_prunings).map(|r| r.hash.clone())
}
pub fn mem_used(&self) -> usize {
0
}
pub fn pending(&self) -> u64 {
self.pending_number + self.pending_prunings as u64
}
pub fn have_block(&self, hash: &BlockHash) -> bool {
self.death_rows.iter().skip(self.pending_prunings).any(|r| r.hash == *hash)
}
/// Prune next block. Expects at least one block in the window. Adds changes to `commit`.
pub fn prune_one(&mut self, commit: &mut CommitSet<Key>) {
if let Some(pruned) = self.death_rows.get(self.pending_prunings) {
trace!(target: "state-db", "Pruning {:?} ({} deleted)", pruned.hash, pruned.deleted.len());
let index = self.pending_number + self.pending_prunings as u64;
commit.data.deleted.extend(pruned.deleted.iter().cloned());
commit.meta.inserted.push((to_meta_key(LAST_PRUNED, &()), index.encode()));
commit.meta.deleted.push(pruned.journal_key.clone());
self.pending_prunings += 1;
} else {
warn!(target: "state-db", "Trying to prune when there's nothing to prune");
}
}
/// Add a change set to the window. Creates a journal record and pushes it to `commit`
pub fn note_canonical(&mut self, hash: &BlockHash, commit: &mut CommitSet<Key>) {
trace!(target: "state-db", "Adding to pruning window: {:?} ({} inserted, {} deleted)", hash, commit.data.inserted.len(), commit.data.deleted.len());
let inserted = commit.data.inserted.iter().map(|(k, _)| k.clone()).collect();
let deleted = ::std::mem::replace(&mut commit.data.deleted, Vec::new());
let journal_record = JournalRecord {
hash: hash.clone(),
inserted,
deleted,
};
let block = self.pending_number + self.death_rows.len() as u64;
let journal_key = to_journal_key(block);
commit.meta.inserted.push((journal_key.clone(), journal_record.encode()));
self.import(&journal_record.hash, journal_key, journal_record.inserted.into_iter(), journal_record.deleted);
self.pending_canonicalizations += 1;
}
/// Apply all pending changes
pub fn apply_pending(&mut self) {
self.pending_canonicalizations = 0;
for _ in 0 .. self.pending_prunings {
let pruned = self.death_rows.pop_front().expect("pending_prunings is always < death_rows.len()");
trace!(target: "state-db", "Applying pruning {:?} ({} deleted)", pruned.hash, pruned.deleted.len());
for k in pruned.deleted.iter() {
self.death_index.remove(&k);
}
self.pending_number += 1;
}
self.pending_prunings = 0;
}
/// Revert all pending changes
pub fn revert_pending(&mut self) {
// Revert pending deletions.
// Note that pending insertions might cause some existing deletions to be removed from `death_index`
// We don't bother to track and revert that for now. This means that a few nodes might end up no being
// deleted in case transaction fails and `revert_pending` is called.
self.death_rows.truncate(self.death_rows.len() - self.pending_canonicalizations);
let new_max_block = self.death_rows.len() as u64 + self.pending_number;
self.death_index.retain(|_, block| *block < new_max_block);
self.pending_canonicalizations = 0;
self.pending_prunings = 0;
}
}
#[cfg(test)]
mod tests {
use super::RefWindow;
use primitives::H256;
use crate::CommitSet;
use crate::test::{make_db, make_commit, TestDb};
fn check_journal(pruning: &RefWindow<H256, H256>, db: &TestDb) {
let restored: RefWindow<H256, H256> = RefWindow::new(db).unwrap();
assert_eq!(pruning.pending_number, restored.pending_number);
assert_eq!(pruning.death_rows, restored.death_rows);
assert_eq!(pruning.death_index, restored.death_index);
}
#[test]
fn created_from_empty_db() {
let db = make_db(&[]);
let pruning: RefWindow<H256, H256> = RefWindow::new(&db).unwrap();
assert_eq!(pruning.pending_number, 0);
assert!(pruning.death_rows.is_empty());
assert!(pruning.death_index.is_empty());
}
#[test]
fn prune_empty() {
let db = make_db(&[]);
let mut pruning: RefWindow<H256, H256> = RefWindow::new(&db).unwrap();
let mut commit = CommitSet::default();
pruning.prune_one(&mut commit);
assert_eq!(pruning.pending_number, 0);
assert!(pruning.death_rows.is_empty());
assert!(pruning.death_index.is_empty());
assert!(pruning.pending_prunings == 0);
assert!(pruning.pending_canonicalizations == 0);
}
#[test]
fn prune_one() {
let mut db = make_db(&[1, 2, 3]);
let mut pruning: RefWindow<H256, H256> = RefWindow::new(&db).unwrap();
let mut commit = make_commit(&[4, 5], &[1, 3]);
let h = H256::random();
pruning.note_canonical(&h, &mut commit);
db.commit(&commit);
assert!(pruning.have_block(&h));
pruning.apply_pending();
assert!(pruning.have_block(&h));
assert!(commit.data.deleted.is_empty());
assert_eq!(pruning.death_rows.len(), 1);
assert_eq!(pruning.death_index.len(), 2);
assert!(db.data_eq(&make_db(&[1, 2, 3, 4, 5])));
check_journal(&pruning, &db);
let mut commit = CommitSet::default();
pruning.prune_one(&mut commit);
assert!(!pruning.have_block(&h));
db.commit(&commit);
pruning.apply_pending();
assert!(!pruning.have_block(&h));
assert!(db.data_eq(&make_db(&[2, 4, 5])));
assert!(pruning.death_rows.is_empty());
assert!(pruning.death_index.is_empty());
assert_eq!(pruning.pending_number, 1);
}
#[test]
fn prune_two() {
let mut db = make_db(&[1, 2, 3]);
let mut pruning: RefWindow<H256, H256> = RefWindow::new(&db).unwrap();
let mut commit = make_commit(&[4], &[1]);
pruning.note_canonical(&H256::random(), &mut commit);
db.commit(&commit);
let mut commit = make_commit(&[5], &[2]);
pruning.note_canonical(&H256::random(), &mut commit);
db.commit(&commit);
pruning.apply_pending();
assert!(db.data_eq(&make_db(&[1, 2, 3, 4, 5])));
check_journal(&pruning, &db);
let mut commit = CommitSet::default();
pruning.prune_one(&mut commit);
db.commit(&commit);
pruning.apply_pending();
assert!(db.data_eq(&make_db(&[2, 3, 4, 5])));
let mut commit = CommitSet::default();
pruning.prune_one(&mut commit);
db.commit(&commit);
pruning.apply_pending();
assert!(db.data_eq(&make_db(&[3, 4, 5])));
assert_eq!(pruning.pending_number, 2);
}
#[test]
fn prune_two_pending() {
let mut db = make_db(&[1, 2, 3]);
let mut pruning: RefWindow<H256, H256> = RefWindow::new(&db).unwrap();
let mut commit = make_commit(&[4], &[1]);
pruning.note_canonical(&H256::random(), &mut commit);
db.commit(&commit);
let mut commit = make_commit(&[5], &[2]);
pruning.note_canonical(&H256::random(), &mut commit);
db.commit(&commit);
assert!(db.data_eq(&make_db(&[1, 2, 3, 4, 5])));
let mut commit = CommitSet::default();
pruning.prune_one(&mut commit);
db.commit(&commit);
assert!(db.data_eq(&make_db(&[2, 3, 4, 5])));
let mut commit = CommitSet::default();
pruning.prune_one(&mut commit);
db.commit(&commit);
pruning.apply_pending();
assert!(db.data_eq(&make_db(&[3, 4, 5])));
assert_eq!(pruning.pending_number, 2);
}
#[test]
fn reinserted_survives() {
let mut db = make_db(&[1, 2, 3]);
let mut pruning: RefWindow<H256, H256> = RefWindow::new(&db).unwrap();
let mut commit = make_commit(&[], &[2]);
pruning.note_canonical(&H256::random(), &mut commit);
db.commit(&commit);
let mut commit = make_commit(&[2], &[]);
pruning.note_canonical(&H256::random(), &mut commit);
db.commit(&commit);
let mut commit = make_commit(&[], &[2]);
pruning.note_canonical(&H256::random(), &mut commit);
db.commit(&commit);
assert!(db.data_eq(&make_db(&[1, 2, 3])));
pruning.apply_pending();
check_journal(&pruning, &db);
let mut commit = CommitSet::default();
pruning.prune_one(&mut commit);
db.commit(&commit);
assert!(db.data_eq(&make_db(&[1, 2, 3])));
let mut commit = CommitSet::default();
pruning.prune_one(&mut commit);
db.commit(&commit);
assert!(db.data_eq(&make_db(&[1, 2, 3])));
pruning.prune_one(&mut commit);
db.commit(&commit);
assert!(db.data_eq(&make_db(&[1, 3])));
pruning.apply_pending();
assert_eq!(pruning.pending_number, 3);
}
#[test]
fn reinserted_survivew_pending() {
let mut db = make_db(&[1, 2, 3]);
let mut pruning: RefWindow<H256, H256> = RefWindow::new(&db).unwrap();
let mut commit = make_commit(&[], &[2]);
pruning.note_canonical(&H256::random(), &mut commit);
db.commit(&commit);
let mut commit = make_commit(&[2], &[]);
pruning.note_canonical(&H256::random(), &mut commit);
db.commit(&commit);
let mut commit = make_commit(&[], &[2]);
pruning.note_canonical(&H256::random(), &mut commit);
db.commit(&commit);
assert!(db.data_eq(&make_db(&[1, 2, 3])));
let mut commit = CommitSet::default();
pruning.prune_one(&mut commit);
db.commit(&commit);
assert!(db.data_eq(&make_db(&[1, 2, 3])));
let mut commit = CommitSet::default();
pruning.prune_one(&mut commit);
db.commit(&commit);
assert!(db.data_eq(&make_db(&[1, 2, 3])));
pruning.prune_one(&mut commit);
db.commit(&commit);
assert!(db.data_eq(&make_db(&[1, 3])));
pruning.apply_pending();
assert_eq!(pruning.pending_number, 3);
}
}
substrate/client/state-db/src/test.rs
+94
View File
@@ -0,0 +1,94 @@
// Copyright 2017-2019 Parity Technologies (UK) Ltd.
// This file is part of Substrate.
// Substrate is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
// Substrate is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
// You should have received a copy of the GNU General Public License
// along with Substrate. If not, see <http://www.gnu.org/licenses/>.
//! Test utils
use std::collections::HashMap;
use primitives::H256;
use crate::{DBValue, ChangeSet, CommitSet, MetaDb, NodeDb};
#[derive(Default, Debug, Clone, PartialEq, Eq)]
pub struct TestDb {
pub data: HashMap<H256, DBValue>,
pub meta: HashMap<Vec<u8>, DBValue>,
}
impl MetaDb for TestDb {
type Error = ();
fn get_meta(&self, key: &[u8]) -> Result<Option<DBValue>, ()> {
Ok(self.meta.get(key).cloned())
}
}
impl NodeDb for TestDb {
type Error = ();
type Key = H256;
fn get(&self, key: &H256) -> Result<Option<DBValue>, ()> {
Ok(self.data.get(key).cloned())
}
}
impl TestDb {
pub fn commit(&mut self, commit: &CommitSet<H256>) {
self.data.extend(commit.data.inserted.iter().cloned());
self.meta.extend(commit.meta.inserted.iter().cloned());
for k in commit.data.deleted.iter() {
self.data.remove(k);
}
self.meta.extend(commit.meta.inserted.iter().cloned());
for k in commit.meta.deleted.iter() {
self.meta.remove(k);
}
}
pub fn data_eq(&self, other: &TestDb) -> bool {
self.data == other.data
}
}
pub fn make_changeset(inserted: &[u64], deleted: &[u64]) -> ChangeSet<H256> {
ChangeSet {
inserted: inserted
.iter()
.map(|v| {
(H256::from_low_u64_be(*v), H256::from_low_u64_be(*v).as_bytes().to_vec())
})
.collect(),
deleted: deleted.iter().map(|v| H256::from_low_u64_be(*v)).collect(),
}
}
pub fn make_commit(inserted: &[u64], deleted: &[u64]) -> CommitSet<H256> {
CommitSet {
data: make_changeset(inserted, deleted),
meta: ChangeSet::default(),
}
}
pub fn make_db(inserted: &[u64]) -> TestDb {
TestDb {
data: inserted
.iter()
.map(|v| {
(H256::from_low_u64_be(*v), H256::from_low_u64_be(*v).as_bytes().to_vec())
})
.collect(),
meta: Default::default(),
}
}