pezkuwichain/pezkuwi-subxt
1
0
Fork 0
mirror of https://github.com/pezkuwichain/pezkuwi-subxt.git synced 2026-06-12 08:51:09 +00:00
Code Issues Packages Projects Releases Wiki Activity

Run cargo fmt on the whole code base (#9394)

Browse Source 
* Run cargo fmt on the whole code base

* Second run

* Add CI check

* Fix compilation

* More unnecessary braces

* Handle weights

* Use --all

* Use correct attributes...

* Fix UI tests

* AHHHHHHHHH

* 🤦

* Docs

* Fix compilation

* 🤷

* Please stop

* 🤦 x 2

* More

* make rustfmt.toml consistent with polkadot

Co-authored-by: André Silva <andrerfosilva@gmail.com>
This commit is contained in:
Bastian Köcher
2021-07-21 16:32:32 +02:00
committed by GitHub
parent d451c38c1c
commit 7b56ab15b4
1010 changed files with 53339 additions and 51208 deletions
Download Patch File Download Diff File Expand all files Collapse all files
substrate/client/state-db/src/lib.rs
+91 -76
View File
@@ -44,15 +44,17 @@ mod pruning;
#[cfg(test)]
mod test;
use std::fmt;
use parking_lot::RwLock;
use codec::Codec;
use std::collections::{HashMap, hash_map::Entry};
use noncanonical::NonCanonicalOverlay;
use pruning::RefWindow;
use log::trace;
use parity_util_mem::{MallocSizeOf, malloc_size};
use sc_client_api::{StateDbMemoryInfo, MemorySize};
use noncanonical::NonCanonicalOverlay;
use parity_util_mem::{malloc_size, MallocSizeOf};
use parking_lot::RwLock;
use pruning::RefWindow;
use sc_client_api::{MemorySize, StateDbMemoryInfo};
use std::{
collections::{hash_map::Entry, HashMap},
fmt,
};
const PRUNING_MODE: &[u8] = b"mode";
const PRUNING_MODE_ARCHIVE: &[u8] = b"archive";
@@ -63,8 +65,35 @@ const PRUNING_MODE_CONSTRAINED: &[u8] = b"constrained";
pub type DBValue = Vec<u8>;
/// Basic set of requirements for the Block hash and node key types.
pub trait Hash: Send + Sync + Sized + Eq + PartialEq + Clone + Default + fmt::Debug + Codec + std::hash::Hash + 'static {}
impl<T: Send + Sync + Sized + Eq + PartialEq + Clone + Default + fmt::Debug + Codec + std::hash::Hash + 'static> Hash for T {}
pub trait Hash:
Send
+ Sync
+ Sized
+ Eq
+ PartialEq
+ Clone
+ Default
+ fmt::Debug
+ Codec
+ std::hash::Hash
+ 'static
{
}
impl<
T: Send
+ Sync
+ Sized
+ Eq
+ PartialEq
+ Clone
+ Default
+ fmt::Debug
+ Codec
+ std::hash::Hash
+ 'static,
> Hash for T
{
}
/// Backend database trait. Read-only.
pub trait MetaDb {
@@ -168,17 +197,14 @@ pub enum PruningMode {
impl PruningMode {
/// Create a mode that keeps given number of blocks.
pub fn keep_blocks(n: u32) -> PruningMode {
PruningMode::Constrained(Constraints {
max_blocks: Some(n),
max_mem: None,
})
PruningMode::Constrained(Constraints { max_blocks: Some(n), max_mem: None })
}
/// Is this an archive (either ArchiveAll or ArchiveCanonical) pruning mode?
pub fn is_archive(&self) -> bool {
match *self {
PruningMode::ArchiveAll | PruningMode::ArchiveCanonical => true,
PruningMode::Constrained(_) => false
PruningMode::Constrained(_) => false,
}
}
@@ -224,20 +250,12 @@ impl<BlockHash: Hash + MallocSizeOf, Key: Hash + MallocSizeOf> StateDbSync<Block
let non_canonical: NonCanonicalOverlay<BlockHash, Key> = NonCanonicalOverlay::new(db)?;
let pruning: Option<RefWindow<BlockHash, Key>> = match mode {
PruningMode::Constrained(Constraints {
max_mem: Some(_),
..
}) => unimplemented!(),
PruningMode::Constrained(Constraints { max_mem: Some(_), .. }) => unimplemented!(),
PruningMode::Constrained(_) => Some(RefWindow::new(db, ref_counting)?),
PruningMode::ArchiveAll | PruningMode::ArchiveCanonical => None,
};
Ok(StateDbSync {
mode,
non_canonical,
pruning,
pinned: Default::default(),
})
Ok(StateDbSync { mode, non_canonical, pruning, pinned: Default::default() })
}
fn check_meta<D: MetaDb>(mode: &PruningMode, db: &D) -> Result<(), Error<D::Error>> {
@@ -270,10 +288,7 @@ impl<BlockHash: Hash + MallocSizeOf, Key: Hash + MallocSizeOf> StateDbSync<Block
PruningMode::ArchiveAll => {
changeset.deleted.clear();
// write changes immediately
Ok(CommitSet {
data: changeset,
meta,
})
Ok(CommitSet { data: changeset, meta })
},
PruningMode::Constrained(_) | PruningMode::ArchiveCanonical => {
let commit = self.non_canonical.insert(hash, number, parent_hash, changeset);
@@ -281,7 +296,7 @@ impl<BlockHash: Hash + MallocSizeOf, Key: Hash + MallocSizeOf> StateDbSync<Block
c.meta.inserted.extend(meta.inserted);
c
})
}
},
}
}
@@ -294,11 +309,10 @@ impl<BlockHash: Hash + MallocSizeOf, Key: Hash + MallocSizeOf> StateDbSync<Block
return Ok(commit)
}
match self.non_canonical.canonicalize(&hash, &mut commit) {
Ok(()) => {
Ok(()) =>
if self.mode == PruningMode::ArchiveCanonical {
commit.data.deleted.clear();
}
}
},
Err(e) => return Err(e),
};
if let Some(ref mut pruning) = self.pruning {
@@ -319,31 +333,30 @@ impl<BlockHash: Hash + MallocSizeOf, Key: Hash + MallocSizeOf> StateDbSync<Block
if self.best_canonical().map(|c| number > c).unwrap_or(true) {
!self.non_canonical.have_block(hash)
} else {
self.pruning
.as_ref()
.map_or(
false,
|pruning| number < pruning.pending() || !pruning.have_block(hash),
)
self.pruning.as_ref().map_or(false, |pruning| {
number < pruning.pending() || !pruning.have_block(hash)
})
}
}
},
}
}
fn prune(&mut self, commit: &mut CommitSet<Key>) {
if let (&mut Some(ref mut pruning), &PruningMode::Constrained(ref constraints)) = (&mut self.pruning, &self.mode) {
if let (&mut Some(ref mut pruning), &PruningMode::Constrained(ref constraints)) =
(&mut self.pruning, &self.mode)
{
loop {
if pruning.window_size() <= constraints.max_blocks.unwrap_or(0) as u64 {
break;
break
}
if constraints.max_mem.map_or(false, |m| pruning.mem_used() > m) {
break;
break
}
let pinned = &self.pinned;
if pruning.next_hash().map_or(false, |h| pinned.contains_key(&h)) {
break;
break
}
pruning.prune_one(commit);
}
@@ -355,23 +368,17 @@ impl<BlockHash: Hash + MallocSizeOf, Key: Hash + MallocSizeOf> StateDbSync<Block
/// For archive an empty commit set is returned.
fn revert_one(&mut self) -> Option<CommitSet<Key>> {
match self.mode {
PruningMode::ArchiveAll => {
Some(CommitSet::default())
},
PruningMode::ArchiveCanonical | PruningMode::Constrained(_) => {
self.non_canonical.revert_one()
},
PruningMode::ArchiveAll => Some(CommitSet::default()),
PruningMode::ArchiveCanonical | PruningMode::Constrained(_) =>
self.non_canonical.revert_one(),
}
}
fn remove(&mut self, hash: &BlockHash) -> Option<CommitSet<Key>> {
match self.mode {
PruningMode::ArchiveAll => {
Some(CommitSet::default())
},
PruningMode::ArchiveCanonical | PruningMode::Constrained(_) => {
self.non_canonical.remove(hash)
},
PruningMode::ArchiveAll => Some(CommitSet::default()),
PruningMode::ArchiveCanonical | PruningMode::Constrained(_) =>
self.non_canonical.remove(hash),
}
}
@@ -392,7 +399,7 @@ impl<BlockHash: Hash + MallocSizeOf, Key: Hash + MallocSizeOf> StateDbSync<Block
} else {
Err(PinError::InvalidBlock)
}
}
},
}
}
@@ -412,14 +419,18 @@ impl<BlockHash: Hash + MallocSizeOf, Key: Hash + MallocSizeOf> StateDbSync<Block
}
}
pub fn get<D: NodeDb, Q: ?Sized>(&self, key: &Q, db: &D) -> Result<Option<DBValue>, Error<D::Error>>
pub fn get<D: NodeDb, Q: ?Sized>(
&self,
key: &Q,
db: &D,
) -> Result<Option<DBValue>, Error<D::Error>>
where
Q: AsRef<D::Key>,
Key: std::borrow::Borrow<Q>,
Q: std::hash::Hash + Eq,
{
if let Some(value) = self.non_canonical.get(key) {
return Ok(Some(value));
return Ok(Some(value))
}
db.get(key.as_ref()).map_err(|e| Error::Db(e))
}
@@ -469,9 +480,7 @@ impl<BlockHash: Hash + MallocSizeOf, Key: Hash + MallocSizeOf> StateDb<BlockHash
ref_counting: bool,
db: &D,
) -> Result<StateDb<BlockHash, Key>, Error<D::Error>> {
Ok(StateDb {
db: RwLock::new(StateDbSync::new(mode, ref_counting, db)?)
})
Ok(StateDb { db: RwLock::new(StateDbSync::new(mode, ref_counting, db)?) })
}
/// Add a new non-canonical block.
@@ -504,11 +513,15 @@ impl<BlockHash: Hash + MallocSizeOf, Key: Hash + MallocSizeOf> StateDb<BlockHash
}
/// Get a value from non-canonical/pruning overlay or the backing DB.
pub fn get<D: NodeDb, Q: ?Sized>(&self, key: &Q, db: &D) -> Result<Option<DBValue>, Error<D::Error>>
where
Q: AsRef<D::Key>,
Key: std::borrow::Borrow<Q>,
Q: std::hash::Hash + Eq,
pub fn get<D: NodeDb, Q: ?Sized>(
&self,
key: &Q,
db: &D,
) -> Result<Option<DBValue>, Error<D::Error>>
where
Q: AsRef<D::Key>,
Key: std::borrow::Borrow<Q>,
Q: std::hash::Hash + Eq,
{
self.db.read().get(key, db)
}
@@ -554,10 +567,12 @@ impl<BlockHash: Hash + MallocSizeOf, Key: Hash + MallocSizeOf> StateDb<BlockHash
#[cfg(test)]
mod tests {
use std::io;
use crate::{
test::{make_changeset, make_db, TestDb},
Constraints, PruningMode, StateDb,
};
use sp_core::H256;
use crate::{StateDb, PruningMode, Constraints};
use crate::test::{make_db, make_changeset, TestDb};
use std::io;
fn make_test_db(settings: PruningMode) -> (TestDb, StateDb<H256, H256>) {
let mut db = make_db(&[91, 921, 922, 93, 94]);
@@ -679,13 +694,13 @@ mod tests {
let state_db = StateDb::new(PruningMode::ArchiveAll, false, &db).unwrap();
db.commit(
&state_db
.insert_block::<io::Error>(
&H256::from_low_u64_be(0),
0,
&H256::from_low_u64_be(0),
make_changeset(&[], &[]),
)
.unwrap(),
.insert_block::<io::Error>(
&H256::from_low_u64_be(0),
0,
&H256::from_low_u64_be(0),
make_changeset(&[], &[]),
)
.unwrap(),
);
let new_mode = PruningMode::Constrained(Constraints { max_blocks: Some(2), max_mem: None });
let state_db: Result<StateDb<H256, H256>, _> = StateDb::new(new_mode, false, &db);
substrate/client/state-db/src/noncanonical.rs
+178 -80
View File
@@ -22,11 +22,13 @@
//! All pending changes are kept in memory until next call to `apply_pending` or
//! `revert_pending`
use std::fmt;
use std::collections::{HashMap, VecDeque, hash_map::Entry};
use super::{Error, DBValue, ChangeSet, CommitSet, MetaDb, Hash, to_meta_key};
use codec::{Encode, Decode};
use super::{to_meta_key, ChangeSet, CommitSet, DBValue, Error, Hash, MetaDb};
use codec::{Decode, Encode};
use log::trace;
use std::{
collections::{hash_map::Entry, HashMap, VecDeque},
fmt,
};
const NON_CANONICAL_JOURNAL: &[u8] = b"noncanonical_journal";
const LAST_CANONICAL: &[u8] = b"last_canonical";
@@ -40,8 +42,8 @@ pub struct NonCanonicalOverlay<BlockHash: Hash, Key: Hash> {
parents: HashMap<BlockHash, BlockHash>,
pending_canonicalizations: Vec<BlockHash>,
pending_insertions: Vec<BlockHash>,
values: HashMap<Key, (u32, DBValue)>, //ref counted
//would be deleted but kept around because block is pinned, ref counted.
values: HashMap<Key, (u32, DBValue)>, // ref counted
// would be deleted but kept around because block is pinned, ref counted.
pinned: HashMap<BlockHash, u32>,
pinned_insertions: HashMap<BlockHash, (Vec<Key>, u32)>,
}
@@ -69,10 +71,7 @@ impl<BlockHash: Hash, Key: Hash> OverlayLevel<BlockHash, Key> {
}
fn new() -> OverlayLevel<BlockHash, Key> {
OverlayLevel {
blocks: Vec::new(),
used_indicies: 0,
}
OverlayLevel { blocks: Vec::new(), used_indicies: 0 }
}
}
@@ -98,7 +97,10 @@ struct BlockOverlay<BlockHash: Hash, Key: Hash> {
deleted: Vec<Key>,
}
fn insert_values<Key: Hash>(values: &mut HashMap<Key, (u32, DBValue)>, inserted: Vec<(Key, DBValue)>) {
fn insert_values<Key: Hash>(
values: &mut HashMap<Key, (u32, DBValue)>,
inserted: Vec<(Key, DBValue)>,
) {
for (k, v) in inserted {
debug_assert!(values.get(&k).map_or(true, |(_, value)| *value == v));
let (ref mut counter, _) = values.entry(k).or_insert_with(|| (0, v));
@@ -118,7 +120,7 @@ fn discard_values<Key: Hash>(values: &mut HashMap<Key, (u32, DBValue)>, inserted
},
Entry::Vacant(_) => {
debug_assert!(false, "Trying to discard missing value");
}
},
}
}
}
@@ -142,10 +144,12 @@ fn discard_descendants<BlockHash: Hash, Key: Hash>(
};
let mut pinned_children = 0;
if let Some(level) = first {
while let Some(i) = level.blocks.iter().position(|overlay| parents.get(&overlay.hash)
.expect("there is a parent entry for each entry in levels; qed")
== hash)
{
while let Some(i) = level.blocks.iter().position(|overlay| {
parents
.get(&overlay.hash)
.expect("there is a parent entry for each entry in levels; qed") ==
hash
}) {
let overlay = level.remove(i);
let mut num_pinned = discard_descendants(
&mut remainder,
@@ -153,7 +157,7 @@ fn discard_descendants<BlockHash: Hash, Key: Hash>(
parents,
pinned,
pinned_insertions,
&overlay.hash
&overlay.hash,
);
if pinned.contains_key(&overlay.hash) {
num_pinned += 1;
@@ -175,10 +179,11 @@ fn discard_descendants<BlockHash: Hash, Key: Hash>(
impl<BlockHash: Hash, Key: Hash> NonCanonicalOverlay<BlockHash, Key> {
/// Creates a new instance. Does not expect any metadata to be present in the DB.
pub fn new<D: MetaDb>(db: &D) -> Result<NonCanonicalOverlay<BlockHash, Key>, Error<D::Error>> {
let last_canonicalized = db.get_meta(&to_meta_key(LAST_CANONICAL, &()))
.map_err(|e| Error::Db(e))?;
let last_canonicalized =
db.get_meta(&to_meta_key(LAST_CANONICAL, &())).map_err(|e| Error::Db(e))?;
let last_canonicalized = last_canonicalized
.map(|buffer| <(BlockHash, u64)>::decode(&mut buffer.as_slice())).transpose()?;
.map(|buffer| <(BlockHash, u64)>::decode(&mut buffer.as_slice()))
.transpose()?;
let mut levels = VecDeque::new();
let mut parents = HashMap::new();
let mut values = HashMap::new();
@@ -189,16 +194,17 @@ impl<BlockHash: Hash, Key: Hash> NonCanonicalOverlay<BlockHash, Key> {
block += 1;
loop {
let mut level = OverlayLevel::new();
for index in 0 .. MAX_BLOCKS_PER_LEVEL {
for index in 0..MAX_BLOCKS_PER_LEVEL {
let journal_key = to_journal_key(block, index);
if let Some(record) = db.get_meta(&journal_key).map_err(|e| Error::Db(e))? {
let record: JournalRecord<BlockHash, Key> = Decode::decode(&mut record.as_slice())?;
let record: JournalRecord<BlockHash, Key> =
Decode::decode(&mut record.as_slice())?;
let inserted = record.inserted.iter().map(|(k, _)| k.clone()).collect();
let overlay = BlockOverlay {
hash: record.hash.clone(),
journal_index: index,
journal_key,
inserted: inserted,
inserted,
deleted: record.deleted,
};
insert_values(&mut values, record.inserted);
@@ -216,7 +222,7 @@ impl<BlockHash: Hash, Key: Hash> NonCanonicalOverlay<BlockHash, Key> {
}
}
if level.blocks.is_empty() {
break;
break
}
levels.push_back(level);
block += 1;
@@ -231,38 +237,55 @@ impl<BlockHash: Hash, Key: Hash> NonCanonicalOverlay<BlockHash, Key> {
pending_insertions: Default::default(),
pinned: Default::default(),
pinned_insertions: Default::default(),
values: values,
values,
})
}
/// Insert a new block into the overlay. If inserted on the second level or lover expects parent to be present in the window.
pub fn insert<E: fmt::Debug>(&mut self, hash: &BlockHash, number: u64, parent_hash: &BlockHash, changeset: ChangeSet<Key>) -> Result<CommitSet<Key>, Error<E>> {
pub fn insert<E: fmt::Debug>(
&mut self,
hash: &BlockHash,
number: u64,
parent_hash: &BlockHash,
changeset: ChangeSet<Key>,
) -> Result<CommitSet<Key>, Error<E>> {
let mut commit = CommitSet::default();
let front_block_number = self.front_block_number();
if self.levels.is_empty() && self.last_canonicalized.is_none() && number > 0 {
// assume that parent was canonicalized
let last_canonicalized = (parent_hash.clone(), number - 1);
commit.meta.inserted.push((to_meta_key(LAST_CANONICAL, &()), last_canonicalized.encode()));
commit
.meta
.inserted
.push((to_meta_key(LAST_CANONICAL, &()), last_canonicalized.encode()));
self.last_canonicalized = Some(last_canonicalized);
} else if self.last_canonicalized.is_some() {
if number < front_block_number || number >= front_block_number + self.levels.len() as u64 + 1 {
if number < front_block_number ||
number >= front_block_number + self.levels.len() as u64 + 1
{
trace!(target: "state-db", "Failed to insert block {}, current is {} .. {})",
number,
front_block_number,
front_block_number + self.levels.len() as u64,
);
return Err(Error::InvalidBlockNumber);
return Err(Error::InvalidBlockNumber)
}
// check for valid parent if inserting on second level or higher
if number == front_block_number {
if !self.last_canonicalized.as_ref().map_or(false, |&(ref h, n)| h == parent_hash && n == number - 1) {
return Err(Error::InvalidParent);
if !self
.last_canonicalized
.as_ref()
.map_or(false, |&(ref h, n)| h == parent_hash && n == number - 1)
{
return Err(Error::InvalidParent)
}
} else if !self.parents.contains_key(&parent_hash) {
return Err(Error::InvalidParent);
return Err(Error::InvalidParent)
}
}
let level = if self.levels.is_empty() || number == front_block_number + self.levels.len() as u64 {
let level = if self.levels.is_empty() ||
number == front_block_number + self.levels.len() as u64
{
self.levels.push_back(OverlayLevel::new());
self.levels.back_mut().expect("can't be empty after insertion; qed")
} else {
@@ -271,7 +294,7 @@ impl<BlockHash: Hash, Key: Hash> NonCanonicalOverlay<BlockHash, Key> {
};
if level.blocks.len() >= MAX_BLOCKS_PER_LEVEL as usize {
return Err(Error::TooManySiblingBlocks);
return Err(Error::TooManySiblingBlocks)
}
let index = level.available_index();
@@ -282,7 +305,7 @@ impl<BlockHash: Hash, Key: Hash> NonCanonicalOverlay<BlockHash, Key> {
hash: hash.clone(),
journal_index: index,
journal_key: journal_key.clone(),
inserted: inserted,
inserted,
deleted: changeset.deleted.clone(),
};
level.push(overlay);
@@ -305,15 +328,24 @@ impl<BlockHash: Hash, Key: Hash> NonCanonicalOverlay<BlockHash, Key> {
level_index: usize,
discarded_journals: &mut Vec<Vec<u8>>,
discarded_blocks: &mut Vec<BlockHash>,
hash: &BlockHash
hash: &BlockHash,
) {
if let Some(level) = self.levels.get(level_index) {
level.blocks.iter().for_each(|overlay| {
let parent = self.parents.get(&overlay.hash).expect("there is a parent entry for each entry in levels; qed").clone();
let parent = self
.parents
.get(&overlay.hash)
.expect("there is a parent entry for each entry in levels; qed")
.clone();
if parent == *hash {
discarded_journals.push(overlay.journal_key.clone());
discarded_blocks.push(overlay.hash.clone());
self.discard_journals(level_index + 1, discarded_journals, discarded_blocks, &overlay.hash);
self.discard_journals(
level_index + 1,
discarded_journals,
discarded_blocks,
&overlay.hash,
);
}
});
}
@@ -326,7 +358,8 @@ impl<BlockHash: Hash, Key: Hash> NonCanonicalOverlay<BlockHash, Key> {
pub fn last_canonicalized_block_number(&self) -> Option<u64> {
match self.last_canonicalized.as_ref().map(|&(_, n)| n) {
Some(n) => Some(n + self.pending_canonicalizations.len() as u64),
None if !self.pending_canonicalizations.is_empty() => Some(self.pending_canonicalizations.len() as u64),
None if !self.pending_canonicalizations.is_empty() =>
Some(self.pending_canonicalizations.len() as u64),
_ => None,
}
}
@@ -351,8 +384,12 @@ impl<BlockHash: Hash, Key: Hash> NonCanonicalOverlay<BlockHash, Key> {
commit: &mut CommitSet<Key>,
) -> Result<(), Error<E>> {
trace!(target: "state-db", "Canonicalizing {:?}", hash);
let level = self.levels.get(self.pending_canonicalizations.len()).ok_or_else(|| Error::InvalidBlock)?;
let index = level.blocks
let level = self
.levels
.get(self.pending_canonicalizations.len())
.ok_or_else(|| Error::InvalidBlock)?;
let index = level
.blocks
.iter()
.position(|overlay| overlay.hash == *hash)
.ok_or_else(|| Error::InvalidBlock)?;
@@ -365,7 +402,7 @@ impl<BlockHash: Hash, Key: Hash> NonCanonicalOverlay<BlockHash, Key> {
self.pending_canonicalizations.len() + 1,
&mut discarded_journals,
&mut discarded_blocks,
&overlay.hash
&overlay.hash,
);
}
discarded_journals.push(overlay.journal_key.clone());
@@ -374,13 +411,25 @@ impl<BlockHash: Hash, Key: Hash> NonCanonicalOverlay<BlockHash, Key> {
// get the one we need to canonicalize
let overlay = &level.blocks[index];
commit.data.inserted.extend(overlay.inserted.iter()
.map(|k| (k.clone(), self.values.get(k).expect("For each key in overlays there's a value in values").1.clone())));
commit.data.inserted.extend(overlay.inserted.iter().map(|k| {
(
k.clone(),
self.values
.get(k)
.expect("For each key in overlays there's a value in values")
.1
.clone(),
)
}));
commit.data.deleted.extend(overlay.deleted.clone());
commit.meta.deleted.append(&mut discarded_journals);
let canonicalized = (hash.clone(), self.front_block_number() + self.pending_canonicalizations.len() as u64);
commit.meta.inserted.push((to_meta_key(LAST_CANONICAL, &()), canonicalized.encode()));
let canonicalized =
(hash.clone(), self.front_block_number() + self.pending_canonicalizations.len() as u64);
commit
.meta
.inserted
.push((to_meta_key(LAST_CANONICAL, &()), canonicalized.encode()));
trace!(target: "state-db", "Discarding {} records", commit.meta.deleted.len());
self.pending_canonicalizations.push(hash.clone());
Ok(())
@@ -391,8 +440,10 @@ impl<BlockHash: Hash, Key: Hash> NonCanonicalOverlay<BlockHash, Key> {
let count = self.pending_canonicalizations.len() as u64;
for hash in self.pending_canonicalizations.drain(..) {
trace!(target: "state-db", "Post canonicalizing {:?}", hash);
let level = self.levels.pop_front().expect("Hash validity is checked in `canonicalize`");
let index = level.blocks
let level =
self.levels.pop_front().expect("Hash validity is checked in `canonicalize`");
let index = level
.blocks
.iter()
.position(|overlay| overlay.hash == hash)
.expect("Hash validity is checked in `canonicalize`");
@@ -415,7 +466,8 @@ impl<BlockHash: Hash, Key: Hash> NonCanonicalOverlay<BlockHash, Key> {
pinned_children += 1;
}
if pinned_children != 0 {
self.pinned_insertions.insert(overlay.hash.clone(), (overlay.inserted, pinned_children));
self.pinned_insertions
.insert(overlay.hash.clone(), (overlay.inserted, pinned_children));
} else {
self.parents.remove(&overlay.hash);
discard_values(&mut self.values, overlay.inserted);
@@ -423,7 +475,10 @@ impl<BlockHash: Hash, Key: Hash> NonCanonicalOverlay<BlockHash, Key> {
}
}
if let Some(hash) = last {
let last_canonicalized = (hash, self.last_canonicalized.as_ref().map(|(_, n)| n + count).unwrap_or(count - 1));
let last_canonicalized = (
hash,
self.last_canonicalized.as_ref().map(|(_, n)| n + count).unwrap_or(count - 1),
);
self.last_canonicalized = Some(last_canonicalized);
}
}
@@ -435,15 +490,15 @@ impl<BlockHash: Hash, Key: Hash> NonCanonicalOverlay<BlockHash, Key> {
Q: std::hash::Hash + Eq,
{
if let Some((_, value)) = self.values.get(&key) {
return Some(value.clone());
return Some(value.clone())
}
None
}
/// Check if the block is in the canonicalization queue.
pub fn have_block(&self, hash: &BlockHash) -> bool {
(self.parents.contains_key(hash) || self.pending_insertions.contains(hash))
&& !self.pending_canonicalizations.contains(hash)
(self.parents.contains_key(hash) || self.pending_insertions.contains(hash)) &&
!self.pending_canonicalizations.contains(hash)
}
/// Revert a single level. Returns commit set that deletes the journal or `None` if not possible.
@@ -471,13 +526,13 @@ impl<BlockHash: Hash, Key: Hash> NonCanonicalOverlay<BlockHash, Key> {
// Check that it does not have any children
if (level_index != level_count - 1) && self.parents.values().any(|h| h == hash) {
log::debug!(target: "state-db", "Trying to remove block {:?} with children", hash);
return None;
return None
}
let overlay = level.remove(index);
commit.meta.deleted.push(overlay.journal_key);
self.parents.remove(&overlay.hash);
discard_values(&mut self.values, overlay.inserted);
break;
break
}
if self.levels.back().map_or(false, |l| l.blocks.is_empty()) {
self.levels.pop_back();
@@ -494,9 +549,13 @@ impl<BlockHash: Hash, Key: Hash> NonCanonicalOverlay<BlockHash, Key> {
for hash in self.pending_insertions.drain(..) {
self.parents.remove(&hash);
// find a level. When iterating insertions backwards the hash is always last in the level.
let level_index =
self.levels.iter().position(|level|
level.blocks.last().expect("Hash is added in `insert` in reverse order").hash == hash)
let level_index = self
.levels
.iter()
.position(|level| {
level.blocks.last().expect("Hash is added in `insert` in reverse order").hash ==
hash
})
.expect("Hash is added in insert");
let overlay_index = self.levels[level_index].blocks.len() - 1;
@@ -526,7 +585,7 @@ impl<BlockHash: Hash, Key: Hash> NonCanonicalOverlay<BlockHash, Key> {
if self.pending_insertions.contains(hash) {
// Pinning pending state is not implemented. Pending states
// won't be pruned for quite some time anyway, so it's not a big deal.
return;
return
}
let refs = self.pinned.entry(hash.clone()).or_default();
if *refs == 0 {
@@ -576,14 +635,17 @@ impl<BlockHash: Hash, Key: Hash> NonCanonicalOverlay<BlockHash, Key> {
#[cfg(test)]
mod tests {
use std::io;
use super::{to_journal_key, NonCanonicalOverlay};
use crate::{
test::{make_changeset, make_db},
ChangeSet, CommitSet, MetaDb,
};
use sp_core::H256;
use super::{NonCanonicalOverlay, to_journal_key};
use crate::{ChangeSet, CommitSet, MetaDb};
use crate::test::{make_db, make_changeset};
use std::io;
fn contains(overlay: &NonCanonicalOverlay<H256, H256>, key: u64) -> bool {
overlay.get(&H256::from_low_u64_be(key)) == Some(H256::from_low_u64_be(key).as_bytes().to_vec())
overlay.get(&H256::from_low_u64_be(key)) ==
Some(H256::from_low_u64_be(key).as_bytes().to_vec())
}
#[test]
@@ -611,7 +673,9 @@ mod tests {
let h1 = H256::random();
let h2 = H256::random();
let mut overlay = NonCanonicalOverlay::<H256, H256>::new(&db).unwrap();
overlay.insert::<io::Error>(&h1, 2, &H256::default(), ChangeSet::default()).unwrap();
overlay
.insert::<io::Error>(&h1, 2, &H256::default(), ChangeSet::default())
.unwrap();
overlay.insert::<io::Error>(&h2, 1, &h1, ChangeSet::default()).unwrap();
}
@@ -622,7 +686,9 @@ mod tests {
let h2 = H256::random();
let db = make_db(&[]);
let mut overlay = NonCanonicalOverlay::<H256, H256>::new(&db).unwrap();
overlay.insert::<io::Error>(&h1, 1, &H256::default(), ChangeSet::default()).unwrap();
overlay
.insert::<io::Error>(&h1, 1, &H256::default(), ChangeSet::default())
.unwrap();
overlay.insert::<io::Error>(&h2, 3, &h1, ChangeSet::default()).unwrap();
}
@@ -633,8 +699,12 @@ mod tests {
let h1 = H256::random();
let h2 = H256::random();
let mut overlay = NonCanonicalOverlay::<H256, H256>::new(&db).unwrap();
overlay.insert::<io::Error>(&h1, 1, &H256::default(), ChangeSet::default()).unwrap();
overlay.insert::<io::Error>(&h2, 2, &H256::default(), ChangeSet::default()).unwrap();
overlay
.insert::<io::Error>(&h1, 1, &H256::default(), ChangeSet::default())
.unwrap();
overlay
.insert::<io::Error>(&h2, 2, &H256::default(), ChangeSet::default())
.unwrap();
}
#[test]
@@ -644,7 +714,9 @@ mod tests {
let h2 = H256::random();
let db = make_db(&[]);
let mut overlay = NonCanonicalOverlay::<H256, H256>::new(&db).unwrap();
overlay.insert::<io::Error>(&h1, 1, &H256::default(), ChangeSet::default()).unwrap();
overlay
.insert::<io::Error>(&h1, 1, &H256::default(), ChangeSet::default())
.unwrap();
let mut commit = CommitSet::default();
overlay.canonicalize::<io::Error>(&h2, &mut commit).unwrap();
}
@@ -655,7 +727,9 @@ mod tests {
let mut db = make_db(&[1, 2]);
let mut overlay = NonCanonicalOverlay::<H256, H256>::new(&db).unwrap();
let changeset = make_changeset(&[3, 4], &[2]);
let insertion = overlay.insert::<io::Error>(&h1, 1, &H256::default(), changeset.clone()).unwrap();
let insertion = overlay
.insert::<io::Error>(&h1, 1, &H256::default(), changeset.clone())
.unwrap();
assert_eq!(insertion.data.inserted.len(), 0);
assert_eq!(insertion.data.deleted.len(), 0);
assert_eq!(insertion.meta.inserted.len(), 2);
@@ -677,7 +751,11 @@ mod tests {
let h2 = H256::random();
let mut db = make_db(&[1, 2]);
let mut overlay = NonCanonicalOverlay::<H256, H256>::new(&db).unwrap();
db.commit(&overlay.insert::<io::Error>(&h1, 10, &H256::default(), make_changeset(&[3, 4], &[2])).unwrap());
db.commit(
&overlay
.insert::<io::Error>(&h1, 10, &H256::default(), make_changeset(&[3, 4], &[2]))
.unwrap(),
);
db.commit(&overlay.insert::<io::Error>(&h2, 11, &h1, make_changeset(&[5], &[3])).unwrap());
assert_eq!(db.meta.len(), 3);
@@ -693,7 +771,11 @@ mod tests {
let h2 = H256::random();
let mut db = make_db(&[1, 2]);
let mut overlay = NonCanonicalOverlay::<H256, H256>::new(&db).unwrap();
db.commit(&overlay.insert::<io::Error>(&h1, 10, &H256::default(), make_changeset(&[3, 4], &[2])).unwrap());
db.commit(
&overlay
.insert::<io::Error>(&h1, 10, &H256::default(), make_changeset(&[3, 4], &[2]))
.unwrap(),
);
db.commit(&overlay.insert::<io::Error>(&h2, 11, &h1, make_changeset(&[5], &[3])).unwrap());
let mut commit = CommitSet::default();
overlay.canonicalize::<io::Error>(&h1, &mut commit).unwrap();
@@ -768,7 +850,11 @@ mod tests {
let mut db = make_db(&[]);
let mut overlay = NonCanonicalOverlay::<H256, H256>::new(&db).unwrap();
let changeset = make_changeset(&[], &[]);
db.commit(&overlay.insert::<io::Error>(&h1, 1, &H256::default(), changeset.clone()).unwrap());
db.commit(
&overlay
.insert::<io::Error>(&h1, 1, &H256::default(), changeset.clone())
.unwrap(),
);
db.commit(&overlay.insert::<io::Error>(&h2, 2, &h1, changeset.clone()).unwrap());
overlay.apply_pending();
let mut commit = CommitSet::default();
@@ -1035,14 +1121,18 @@ mod tests {
let h21 = H256::random();
let mut db = make_db(&[]);
let mut overlay = NonCanonicalOverlay::<H256, H256>::new(&db).unwrap();
db.commit(&overlay.insert::<io::Error>(&root, 10, &H256::default(), make_changeset(&[], &[])).unwrap());
db.commit(
&overlay
.insert::<io::Error>(&root, 10, &H256::default(), make_changeset(&[], &[]))
.unwrap(),
);
db.commit(&overlay.insert::<io::Error>(&h1, 11, &root, make_changeset(&[1], &[])).unwrap());
db.commit(&overlay.insert::<io::Error>(&h2, 11, &root, make_changeset(&[2], &[])).unwrap());
db.commit(&overlay.insert::<io::Error>(&h11, 12, &h1, make_changeset(&[11], &[])).unwrap());
db.commit(&overlay.insert::<io::Error>(&h21, 12, &h2, make_changeset(&[21], &[])).unwrap());
let mut commit = CommitSet::default();
overlay.canonicalize::<io::Error>(&root, &mut commit).unwrap();
overlay.canonicalize::<io::Error>(&h2, &mut commit).unwrap(); // h11 should stay in the DB
overlay.canonicalize::<io::Error>(&h2, &mut commit).unwrap(); // h11 should stay in the DB
db.commit(&commit);
overlay.apply_pending();
assert_eq!(overlay.levels.len(), 1);
@@ -1056,7 +1146,7 @@ mod tests {
assert!(contains(&overlay, 21));
let mut commit = CommitSet::default();
overlay.canonicalize::<io::Error>(&h21, &mut commit).unwrap(); // h11 should stay in the DB
overlay.canonicalize::<io::Error>(&h21, &mut commit).unwrap(); // h11 should stay in the DB
db.commit(&commit);
overlay.apply_pending();
assert!(!contains(&overlay, 21));
@@ -1073,19 +1163,23 @@ mod tests {
let h21 = H256::random();
let mut db = make_db(&[]);
let mut overlay = NonCanonicalOverlay::<H256, H256>::new(&db).unwrap();
db.commit(&overlay.insert::<io::Error>(&root, 10, &H256::default(), make_changeset(&[], &[])).unwrap());
db.commit(
&overlay
.insert::<io::Error>(&root, 10, &H256::default(), make_changeset(&[], &[]))
.unwrap(),
);
db.commit(&overlay.insert::<io::Error>(&h1, 11, &root, make_changeset(&[1], &[])).unwrap());
db.commit(&overlay.insert::<io::Error>(&h2, 11, &root, make_changeset(&[2], &[])).unwrap());
db.commit(&overlay.insert::<io::Error>(&h11, 12, &h1, make_changeset(&[11], &[])).unwrap());
db.commit(&overlay.insert::<io::Error>(&h21, 12, &h2, make_changeset(&[21], &[])).unwrap());
let mut commit = CommitSet::default();
overlay.canonicalize::<io::Error>(&root, &mut commit).unwrap();
overlay.canonicalize::<io::Error>(&h2, &mut commit).unwrap(); // h11 should stay in the DB
overlay.canonicalize::<io::Error>(&h2, &mut commit).unwrap(); // h11 should stay in the DB
db.commit(&commit);
overlay.apply_pending();
// add another block at top level. It should reuse journal index 0 of previously discarded block
let h22 = H256::random();
let h22 = H256::random();
db.commit(&overlay.insert::<io::Error>(&h22, 12, &h2, make_changeset(&[22], &[])).unwrap());
assert_eq!(overlay.levels[0].blocks[0].journal_index, 1);
assert_eq!(overlay.levels[0].blocks[1].journal_index, 0);
@@ -1106,7 +1200,11 @@ mod tests {
let h21 = H256::random();
let mut db = make_db(&[]);
let mut overlay = NonCanonicalOverlay::<H256, H256>::new(&db).unwrap();
db.commit(&overlay.insert::<io::Error>(&root, 10, &H256::default(), make_changeset(&[], &[])).unwrap());
db.commit(
&overlay
.insert::<io::Error>(&root, 10, &H256::default(), make_changeset(&[], &[]))
.unwrap(),
);
db.commit(&overlay.insert::<io::Error>(&h1, 11, &root, make_changeset(&[1], &[])).unwrap());
db.commit(&overlay.insert::<io::Error>(&h2, 11, &root, make_changeset(&[2], &[])).unwrap());
db.commit(&overlay.insert::<io::Error>(&h11, 12, &h1, make_changeset(&[11], &[])).unwrap());
substrate/client/state-db/src/pruning.rs
+45 -28
View File
@@ -24,10 +24,10 @@
//! the death list.
//! The changes are journaled in the DB.
use std::collections::{HashMap, HashSet, VecDeque};
use codec::{Encode, Decode};
use crate::{CommitSet, Error, MetaDb, to_meta_key, Hash};
use crate::{to_meta_key, CommitSet, Error, Hash, MetaDb};
use codec::{Decode, Encode};
use log::{trace, warn};
use std::collections::{HashMap, HashSet, VecDeque};
const LAST_PRUNED: &[u8] = b"last_pruned";
const PRUNING_JOURNAL: &[u8] = b"pruning_journal";
@@ -72,9 +72,11 @@ fn to_journal_key(block: u64) -> Vec<u8> {
}
impl<BlockHash: Hash, Key: Hash> RefWindow<BlockHash, Key> {
pub fn new<D: MetaDb>(db: &D, count_insertions: bool) -> Result<RefWindow<BlockHash, Key>, Error<D::Error>> {
let last_pruned = db.get_meta(&to_meta_key(LAST_PRUNED, &()))
.map_err(|e| Error::Db(e))?;
pub fn new<D: MetaDb>(
db: &D,
count_insertions: bool,
) -> Result<RefWindow<BlockHash, Key>, Error<D::Error>> {
let last_pruned = db.get_meta(&to_meta_key(LAST_PRUNED, &())).map_err(|e| Error::Db(e))?;
let pending_number: u64 = match last_pruned {
Some(buffer) => u64::decode(&mut buffer.as_slice())? + 1,
None => 0,
@@ -83,7 +85,7 @@ impl<BlockHash: Hash, Key: Hash> RefWindow<BlockHash, Key> {
let mut pruning = RefWindow {
death_rows: Default::default(),
death_index: Default::default(),
pending_number: pending_number,
pending_number,
pending_canonicalizations: 0,
pending_prunings: 0,
count_insertions,
@@ -94,9 +96,15 @@ impl<BlockHash: Hash, Key: Hash> RefWindow<BlockHash, Key> {
let journal_key = to_journal_key(block);
match db.get_meta(&journal_key).map_err(|e| Error::Db(e))? {
Some(record) => {
let record: JournalRecord<BlockHash, Key> = Decode::decode(&mut record.as_slice())?;
let record: JournalRecord<BlockHash, Key> =
Decode::decode(&mut record.as_slice())?;
trace!(target: "state-db", "Pruning journal entry {} ({} inserted, {} deleted)", block, record.inserted.len(), record.deleted.len());
pruning.import(&record.hash, journal_key, record.inserted.into_iter(), record.deleted);
pruning.import(
&record.hash,
journal_key,
record.inserted.into_iter(),
record.deleted,
);
},
None => break,
}
@@ -105,7 +113,13 @@ impl<BlockHash: Hash, Key: Hash> RefWindow<BlockHash, Key> {
Ok(pruning)
}
fn import<I: IntoIterator<Item=Key>>(&mut self, hash: &BlockHash, journal_key: Vec<u8>, inserted: I, deleted: Vec<Key>) {
fn import<I: IntoIterator<Item = Key>>(
&mut self,
hash: &BlockHash,
journal_key: Vec<u8>,
inserted: I,
deleted: Vec<Key>,
) {
if self.count_insertions {
// remove all re-inserted keys from death rows
for k in inserted {
@@ -120,13 +134,11 @@ impl<BlockHash: Hash, Key: Hash> RefWindow<BlockHash, Key> {
self.death_index.insert(k.clone(), imported_block);
}
}
self.death_rows.push_back(
DeathRow {
hash: hash.clone(),
deleted: deleted.into_iter().collect(),
journal_key: journal_key,
}
);
self.death_rows.push_back(DeathRow {
hash: hash.clone(),
deleted: deleted.into_iter().collect(),
journal_key,
});
}
pub fn window_size(&self) -> u64 {
@@ -172,23 +184,27 @@ impl<BlockHash: Hash, Key: Hash> RefWindow<BlockHash, Key> {
Default::default()
};
let deleted = ::std::mem::take(&mut commit.data.deleted);
let journal_record = JournalRecord {
hash: hash.clone(),
inserted,
deleted,
};
let journal_record = JournalRecord { hash: hash.clone(), inserted, deleted };
let block = self.pending_number + self.death_rows.len() as u64;
let journal_key = to_journal_key(block);
commit.meta.inserted.push((journal_key.clone(), journal_record.encode()));
self.import(&journal_record.hash, journal_key, journal_record.inserted.into_iter(), journal_record.deleted);
self.import(
&journal_record.hash,
journal_key,
journal_record.inserted.into_iter(),
journal_record.deleted,
);
self.pending_canonicalizations += 1;
}
/// Apply all pending changes
pub fn apply_pending(&mut self) {
self.pending_canonicalizations = 0;
for _ in 0 .. self.pending_prunings {
let pruned = self.death_rows.pop_front().expect("pending_prunings is always < death_rows.len()");
for _ in 0..self.pending_prunings {
let pruned = self
.death_rows
.pop_front()
.expect("pending_prunings is always < death_rows.len()");
trace!(target: "state-db", "Applying pruning {:?} ({} deleted)", pruned.hash, pruned.deleted.len());
if self.count_insertions {
for k in pruned.deleted.iter() {
@@ -219,9 +235,11 @@ impl<BlockHash: Hash, Key: Hash> RefWindow<BlockHash, Key> {
#[cfg(test)]
mod tests {
use super::RefWindow;
use crate::{
test::{make_commit, make_db, TestDb},
CommitSet,
};
use sp_core::H256;
use crate::CommitSet;
use crate::test::{make_db, make_commit, TestDb};
fn check_journal(pruning: &RefWindow<H256, H256>, db: &TestDb) {
let restored: RefWindow<H256, H256> = RefWindow::new(db, pruning.count_insertions).unwrap();
@@ -419,5 +437,4 @@ mod tests {
assert!(db.data_eq(&make_db(&[1, 3])));
assert!(pruning.death_index.is_empty());
}
}
substrate/client/state-db/src/test.rs
+5 -13
View File
@@ -18,9 +18,9 @@
//! Test utils
use std::collections::HashMap;
use crate::{ChangeSet, CommitSet, DBValue, MetaDb, NodeDb};
use sp_core::H256;
use crate::{DBValue, ChangeSet, CommitSet, MetaDb, NodeDb};
use std::collections::HashMap;
#[derive(Default, Debug, Clone, PartialEq, Eq)]
pub struct TestDb {
@@ -67,30 +67,22 @@ pub fn make_changeset(inserted: &[u64], deleted: &[u64]) -> ChangeSet<H256> {
ChangeSet {
inserted: inserted
.iter()
.map(|v| {
(H256::from_low_u64_be(*v), H256::from_low_u64_be(*v).as_bytes().to_vec())
})
.map(|v| (H256::from_low_u64_be(*v), H256::from_low_u64_be(*v).as_bytes().to_vec()))
.collect(),
deleted: deleted.iter().map(|v| H256::from_low_u64_be(*v)).collect(),
}
}
pub fn make_commit(inserted: &[u64], deleted: &[u64]) -> CommitSet<H256> {
CommitSet {
data: make_changeset(inserted, deleted),
meta: ChangeSet::default(),
}
CommitSet { data: make_changeset(inserted, deleted), meta: ChangeSet::default() }
}
pub fn make_db(inserted: &[u64]) -> TestDb {
TestDb {
data: inserted
.iter()
.map(|v| {
(H256::from_low_u64_be(*v), H256::from_low_u64_be(*v).as_bytes().to_vec())
})
.map(|v| (H256::from_low_u64_be(*v), H256::from_low_u64_be(*v).as_bytes().to_vec()))
.collect(),
meta: Default::default(),
}
}