Remove BlockNumber <-> u64 conversions from light-client related code (#2666)

* Remove As usage from CHT

* Remove As usage from CHT (continue)

* Restrict BN <-> int conversions in CT

* more BN <-> u64 conversions removed

* upd spec_version

* Apply suggestions from code review

Co-Authored-By: Gavin Wood <github@gavwood.com>

* Apply suggestions from code review

Co-Authored-By: Gavin Wood <github@gavwood.com>

* more grumbles

* fix last grumbles + compilation

* too long lines

* too long lines
This commit is contained in:
Svyatoslav Nikolsky
2019-05-28 16:07:16 +03:00
committed by Gavin Wood
parent 25b88f1a1f
commit 549d9e1da1
41 changed files with 1087 additions and 654 deletions
@@ -19,12 +19,13 @@
use std::collections::{BTreeMap, BTreeSet};
use parity_codec::Decode;
use hash_db::Hasher;
use num_traits::One;
use crate::backend::Backend;
use crate::overlayed_changes::OverlayedChanges;
use crate::trie_backend_essence::{TrieBackendStorage, TrieBackendEssence};
use crate::trie_backend_essence::TrieBackendEssence;
use crate::changes_trie::build_iterator::digest_build_iterator;
use crate::changes_trie::input::{InputKey, InputPair, DigestIndex, ExtrinsicIndex};
use crate::changes_trie::{AnchorBlockId, Configuration, Storage};
use crate::changes_trie::{AnchorBlockId, Configuration, Storage, BlockNumber};
/// Prepare input pairs for building a changes trie of given block.
///
@@ -32,29 +33,25 @@ use crate::changes_trie::{AnchorBlockId, Configuration, Storage};
/// required data.
/// Returns Ok(None) data required to prepare input pairs is not collected
/// or storage is not provided.
pub fn prepare_input<'a, B, S, H>(
pub fn prepare_input<'a, B, S, H, Number>(
backend: &B,
storage: Option<&'a S>,
storage: &'a S,
config: &'a Configuration,
changes: &OverlayedChanges,
parent: &'a AnchorBlockId<H::Out>,
) -> Result<Option<Vec<InputPair>>, String>
parent: &'a AnchorBlockId<H::Out, Number>,
) -> Result<Option<Vec<InputPair<Number>>>, String>
where
B: Backend<H>,
S: Storage<H>,
&'a S: TrieBackendStorage<H>,
S: Storage<H, Number>,
H: Hasher,
Number: BlockNumber,
{
let (storage, config) = match (storage, changes.changes_trie_config.as_ref()) {
(Some(storage), Some(config)) => (storage, config),
_ => return Ok(None),
};
let mut input = Vec::new();
input.extend(prepare_extrinsics_input(
backend,
parent.number + 1,
parent.number.clone() + 1.into(),
changes)?);
input.extend(prepare_digest_input::<_, H>(
input.extend(prepare_digest_input::<_, H, Number>(
parent,
config,
storage)?);
@@ -63,14 +60,15 @@ pub fn prepare_input<'a, B, S, H>(
}
/// Prepare ExtrinsicIndex input pairs.
fn prepare_extrinsics_input<B, H>(
fn prepare_extrinsics_input<B, H, Number>(
backend: &B,
block: u64,
block: Number,
changes: &OverlayedChanges,
) -> Result<impl Iterator<Item=InputPair>, String>
) -> Result<impl Iterator<Item=InputPair<Number>>, String>
where
B: Backend<H>,
H: Hasher,
Number: BlockNumber,
{
let mut extrinsic_map = BTreeMap::<Vec<u8>, BTreeSet<u32>>::new();
for (key, val) in changes.prospective.top.iter().chain(changes.committed.top.iter()) {
@@ -93,47 +91,50 @@ fn prepare_extrinsics_input<B, H>(
Ok(extrinsic_map.into_iter()
.map(move |(key, extrinsics)| InputPair::ExtrinsicIndex(ExtrinsicIndex {
block,
block: block.clone(),
key,
}, extrinsics.iter().cloned().collect())))
}
/// Prepare DigestIndex input pairs.
fn prepare_digest_input<'a, S, H>(
parent: &'a AnchorBlockId<H::Out>,
fn prepare_digest_input<'a, S, H, Number>(
parent: &'a AnchorBlockId<H::Out, Number>,
config: &Configuration,
storage: &'a S
) -> Result<impl Iterator<Item=InputPair> + 'a, String>
) -> Result<impl Iterator<Item=InputPair<Number>> + 'a, String>
where
S: Storage<H>,
&'a S: TrieBackendStorage<H>,
S: Storage<H, Number>,
H: Hasher,
H::Out: 'a,
Number: BlockNumber,
{
let mut digest_map = BTreeMap::<Vec<u8>, BTreeSet<u64>>::new();
for digest_build_block in digest_build_iterator(config, parent.number + 1) {
let trie_root = storage.root(parent, digest_build_block)?;
let trie_root = trie_root.ok_or_else(|| format!("No changes trie root for block {}", digest_build_block))?;
let trie_storage = TrieBackendEssence::<_, H>::new(storage, trie_root);
let mut digest_map = BTreeMap::<Vec<u8>, BTreeSet<Number>>::new();
for digest_build_block in digest_build_iterator(config, parent.number.clone() + One::one()) {
let trie_root = storage.root(parent, digest_build_block.clone())?;
let trie_root = trie_root.ok_or_else(|| format!("No changes trie root for block {}", digest_build_block.clone()))?;
let trie_storage = TrieBackendEssence::<_, H>::new(
crate::changes_trie::TrieBackendStorageAdapter(storage),
trie_root,
);
let extrinsic_prefix = ExtrinsicIndex::key_neutral_prefix(digest_build_block);
let extrinsic_prefix = ExtrinsicIndex::key_neutral_prefix(digest_build_block.clone());
trie_storage.for_keys_with_prefix(&extrinsic_prefix, |key|
if let Some(InputKey::ExtrinsicIndex(trie_key)) = Decode::decode(&mut &key[..]) {
if let Some(InputKey::ExtrinsicIndex::<Number>(trie_key)) = Decode::decode(&mut &key[..]) {
digest_map.entry(trie_key.key).or_default()
.insert(digest_build_block);
.insert(digest_build_block.clone());
});
let digest_prefix = DigestIndex::key_neutral_prefix(digest_build_block);
let digest_prefix = DigestIndex::key_neutral_prefix(digest_build_block.clone());
trie_storage.for_keys_with_prefix(&digest_prefix, |key|
if let Some(InputKey::DigestIndex(trie_key)) = Decode::decode(&mut &key[..]) {
if let Some(InputKey::DigestIndex::<Number>(trie_key)) = Decode::decode(&mut &key[..]) {
digest_map.entry(trie_key.key).or_default()
.insert(digest_build_block);
.insert(digest_build_block.clone());
});
}
Ok(digest_map.into_iter()
.map(move |(key, set)| InputPair::DigestIndex(DigestIndex {
block: parent.number + 1,
block: parent.number.clone() + One::one(),
key
}, set.into_iter().collect())))
}
@@ -148,7 +149,7 @@ mod test {
use crate::overlayed_changes::OverlayedValue;
use super::*;
fn prepare_for_build() -> (InMemory<Blake2Hasher>, InMemoryStorage<Blake2Hasher>, OverlayedChanges) {
fn prepare_for_build() -> (InMemory<Blake2Hasher>, InMemoryStorage<Blake2Hasher, u64>, OverlayedChanges) {
let backend: InMemory<_> = vec![
(vec![100], vec![255]),
(vec![101], vec![255]),
@@ -225,7 +226,14 @@ mod test {
#[test]
fn build_changes_trie_nodes_on_non_digest_block() {
let (backend, storage, changes) = prepare_for_build();
let changes_trie_nodes = prepare_input(&backend, Some(&storage), &changes, &AnchorBlockId { hash: Default::default(), number: 4 }).unwrap();
let config = changes.changes_trie_config.as_ref().unwrap();
let changes_trie_nodes = prepare_input(
&backend,
&storage,
config,
&changes,
&AnchorBlockId { hash: Default::default(), number: 4 },
).unwrap();
assert_eq!(changes_trie_nodes, Some(vec![
InputPair::ExtrinsicIndex(ExtrinsicIndex { block: 5, key: vec![100] }, vec![0, 2, 3]),
InputPair::ExtrinsicIndex(ExtrinsicIndex { block: 5, key: vec![101] }, vec![1]),
@@ -236,7 +244,14 @@ mod test {
#[test]
fn build_changes_trie_nodes_on_digest_block_l1() {
let (backend, storage, changes) = prepare_for_build();
let changes_trie_nodes = prepare_input(&backend, Some(&storage), &changes, &AnchorBlockId { hash: Default::default(), number: 3 }).unwrap();
let config = changes.changes_trie_config.as_ref().unwrap();
let changes_trie_nodes = prepare_input(
&backend,
&storage,
config,
&changes,
&AnchorBlockId { hash: Default::default(), number: 3 },
).unwrap();
assert_eq!(changes_trie_nodes, Some(vec![
InputPair::ExtrinsicIndex(ExtrinsicIndex { block: 4, key: vec![100] }, vec![0, 2, 3]),
InputPair::ExtrinsicIndex(ExtrinsicIndex { block: 4, key: vec![101] }, vec![1]),
@@ -252,7 +267,14 @@ mod test {
#[test]
fn build_changes_trie_nodes_on_digest_block_l2() {
let (backend, storage, changes) = prepare_for_build();
let changes_trie_nodes = prepare_input(&backend, Some(&storage), &changes, &AnchorBlockId { hash: Default::default(), number: 15 }).unwrap();
let config = changes.changes_trie_config.as_ref().unwrap();
let changes_trie_nodes = prepare_input(
&backend,
&storage,
config,
&changes,
&AnchorBlockId { hash: Default::default(), number: 15 },
).unwrap();
assert_eq!(changes_trie_nodes, Some(vec![
InputPair::ExtrinsicIndex(ExtrinsicIndex { block: 16, key: vec![100] }, vec![0, 2, 3]),
InputPair::ExtrinsicIndex(ExtrinsicIndex { block: 16, key: vec![101] }, vec![1]),
@@ -276,7 +298,14 @@ mod test {
extrinsics: Some(vec![1].into_iter().collect())
});
let changes_trie_nodes = prepare_input(&backend, Some(&storage), &changes, &AnchorBlockId { hash: Default::default(), number: 3 }).unwrap();
let config = changes.changes_trie_config.as_ref().unwrap();
let changes_trie_nodes = prepare_input(
&backend,
&storage,
config,
&changes,
&AnchorBlockId { hash: Default::default(), number: 3 },
).unwrap();
assert_eq!(changes_trie_nodes, Some(vec![
InputPair::ExtrinsicIndex(ExtrinsicIndex { block: 4, key: vec![100] }, vec![0, 2, 3]),
InputPair::ExtrinsicIndex(ExtrinsicIndex { block: 4, key: vec![101] }, vec![1]),
@@ -17,13 +17,16 @@
//! Structures and functions to return blocks whose changes are to be included
//! in given block' changes trie.
use crate::changes_trie::Configuration;
use crate::changes_trie::{Configuration, BlockNumber};
/// Returns iterator of OTHER blocks that are required for inclusion into
/// changes trie of given block.
pub fn digest_build_iterator(config: &Configuration, block: u64) -> DigestBuildIterator {
pub fn digest_build_iterator<Number: BlockNumber>(
config: &Configuration,
block: Number,
) -> DigestBuildIterator<Number> {
// prepare digest build parameters
let (_, _, digest_step) = match config.digest_level_at_block(block) {
let (_, _, digest_step) = match config.digest_level_at_block(block.clone()) {
Some((current_level, digest_interval, digest_step)) =>
(current_level, digest_interval, digest_step),
None => return DigestBuildIterator::empty(),
@@ -35,24 +38,26 @@ pub fn digest_build_iterator(config: &Configuration, block: u64) -> DigestBuildI
/// Changes trie build iterator that returns numbers of OTHER blocks that are
/// required for inclusion into changes trie of given block.
#[derive(Debug)]
pub struct DigestBuildIterator {
pub struct DigestBuildIterator<Number: BlockNumber> {
/// Block we're building changes trie for.
block: u64,
block: Number,
/// Interval for creation digest blocks.
digest_interval: u64,
/// Step of current blocks range.
current_step: u64,
/// Current blocks range.
current_range: Option<::std::iter::StepBy<::std::ops::Range<u64>>>,
digest_interval: u32,
/// Max step of blocks range.
max_step: u64,
max_step: u32,
/// Step of current blocks range.
current_step: u32,
/// Current blocks range.
current_range: Option<BlocksRange<Number>>,
}
impl DigestBuildIterator {
impl<Number: BlockNumber> DigestBuildIterator<Number> {
/// Create new digest build iterator.
pub fn new(block: u64, digest_interval: u64, max_step: u64) -> Self {
pub fn new(block: Number, digest_interval: u32, max_step: u32) -> Self {
DigestBuildIterator {
block, digest_interval, max_step,
block,
digest_interval,
max_step,
current_step: 0,
current_range: None,
}
@@ -60,12 +65,12 @@ impl DigestBuildIterator {
/// Create empty digest build iterator.
pub fn empty() -> Self {
Self::new(0, 0, 0)
Self::new(0.into(), 0, 0)
}
}
impl Iterator for DigestBuildIterator {
type Item = u64;
impl<Number: BlockNumber> Iterator for DigestBuildIterator<Number> {
type Item = Number;
fn next(&mut self) -> Option<Self::Item> {
if let Some(next) = self.current_range.as_mut().and_then(|iter| iter.next()) {
@@ -82,10 +87,11 @@ impl Iterator for DigestBuildIterator {
}
self.current_step = next_step;
self.current_range = Some(
((self.block - self.current_step * self.digest_interval + self.current_step)..self.block)
.step_by(self.current_step as usize)
);
self.current_range = Some(BlocksRange::new(
self.block.clone() - (self.current_step * self.digest_interval - self.current_step).into(),
self.block.clone(),
self.current_step.into(),
));
Some(self.current_range.as_mut()
.expect("assigned one line above; qed")
@@ -94,20 +100,52 @@ impl Iterator for DigestBuildIterator {
}
}
/// Blocks range iterator with builtin step_by support.
#[derive(Debug)]
struct BlocksRange<Number: BlockNumber> {
current: Number,
end: Number,
step: Number,
}
impl<Number: BlockNumber> BlocksRange<Number> {
pub fn new(begin: Number, end: Number, step: Number) -> Self {
BlocksRange {
current: begin,
end,
step,
}
}
}
impl<Number: BlockNumber> Iterator for BlocksRange<Number> {
type Item = Number;
fn next(&mut self) -> Option<Self::Item> {
if self.current >= self.end {
return None;
}
let current = Some(self.current.clone());
self.current += self.step.clone();
current
}
}
#[cfg(test)]
mod tests {
use super::*;
fn digest_build_iterator(digest_interval: u64, digest_levels: u32, block: u64) -> DigestBuildIterator {
fn digest_build_iterator(digest_interval: u32, digest_levels: u32, block: u64) -> DigestBuildIterator<u64> {
super::digest_build_iterator(&Configuration { digest_interval, digest_levels }, block)
}
fn digest_build_iterator_basic(digest_interval: u64, digest_levels: u32, block: u64) -> (u64, u64, u64) {
fn digest_build_iterator_basic(digest_interval: u32, digest_levels: u32, block: u64) -> (u64, u32, u32) {
let iter = digest_build_iterator(digest_interval, digest_levels, block);
(iter.block, iter.digest_interval, iter.max_step)
}
fn digest_build_iterator_blocks(digest_interval: u64, digest_levels: u32, block: u64) -> Vec<u64> {
fn digest_build_iterator_blocks(digest_interval: u32, digest_levels: u32, block: u64) -> Vec<u64> {
digest_build_iterator(digest_interval, digest_levels, block).collect()
}
@@ -122,7 +160,11 @@ mod tests {
assert_eq!(digest_build_iterator_basic(4, 16, 2), empty, "digest is not required for this block");
assert_eq!(digest_build_iterator_basic(4, 16, 15), empty, "digest is not required for this block");
assert_eq!(digest_build_iterator_basic(4, 16, 17), empty, "digest is not required for this block");
assert_eq!(digest_build_iterator_basic(::std::u64::MAX / 2 + 1, 16, ::std::u64::MAX), empty, "digest_interval * 2 is greater than u64::MAX");
assert_eq!(digest_build_iterator_basic(
::std::u32::MAX / 2 + 1,
16,
::std::u64::MAX,
), empty, "digest_interval * 2 is greater than u64::MAX");
}
#[test]
@@ -21,8 +21,9 @@ use std::cell::RefCell;
use std::collections::VecDeque;
use parity_codec::{Decode, Encode};
use hash_db::{HashDB, Hasher};
use num_traits::One;
use trie::{Recorder, MemoryDB};
use crate::changes_trie::{AnchorBlockId, Configuration, RootsStorage, Storage};
use crate::changes_trie::{AnchorBlockId, Configuration, RootsStorage, Storage, BlockNumber};
use crate::changes_trie::input::{DigestIndex, ExtrinsicIndex, DigestIndexValue, ExtrinsicIndexValue};
use crate::changes_trie::storage::{TrieBackendAdapter, InMemoryStorage};
use crate::proving_backend::ProvingBackendEssence;
@@ -31,25 +32,25 @@ use crate::trie_backend_essence::{TrieBackendEssence};
/// Return changes of given key at given blocks range.
/// `max` is the number of best known block.
/// Changes are returned in descending order (i.e. last block comes first).
pub fn key_changes<'a, S: Storage<H>, H: Hasher>(
pub fn key_changes<'a, S: Storage<H, Number>, H: Hasher, Number: BlockNumber>(
config: &'a Configuration,
storage: &'a S,
begin: u64,
end: &'a AnchorBlockId<H::Out>,
max: u64,
begin: Number,
end: &'a AnchorBlockId<H::Out, Number>,
max: Number,
key: &'a [u8],
) -> Result<DrilldownIterator<'a, S, S, H>, String> {
) -> Result<DrilldownIterator<'a, S, S, H, Number>, String> {
// we can't query any roots before root
let max = ::std::cmp::min(max, end.number);
let max = ::std::cmp::min(max.clone(), end.number.clone());
Ok(DrilldownIterator {
essence: DrilldownIteratorEssence {
key,
roots_storage: storage,
storage,
begin,
begin: begin.clone(),
end,
surface: surface_iterator(config, max, begin, end.number)?,
surface: surface_iterator(config, max, begin, end.number.clone())?,
extrinsics: Default::default(),
blocks: Default::default(),
@@ -61,25 +62,25 @@ pub fn key_changes<'a, S: Storage<H>, H: Hasher>(
/// Returns proof of changes of given key at given blocks range.
/// `max` is the number of best known block.
pub fn key_changes_proof<S: Storage<H>, H: Hasher>(
pub fn key_changes_proof<S: Storage<H, Number>, H: Hasher, Number: BlockNumber>(
config: &Configuration,
storage: &S,
begin: u64,
end: &AnchorBlockId<H::Out>,
max: u64,
begin: Number,
end: &AnchorBlockId<H::Out, Number>,
max: Number,
key: &[u8],
) -> Result<Vec<Vec<u8>>, String> {
// we can't query any roots before root
let max = ::std::cmp::min(max, end.number);
let max = ::std::cmp::min(max.clone(), end.number.clone());
let mut iter = ProvingDrilldownIterator {
essence: DrilldownIteratorEssence {
key,
roots_storage: storage.clone(),
storage,
begin,
begin: begin.clone(),
end,
surface: surface_iterator(config, max, begin, end.number)?,
surface: surface_iterator(config, max, begin, end.number.clone())?,
extrinsics: Default::default(),
blocks: Default::default(),
@@ -100,17 +101,17 @@ pub fn key_changes_proof<S: Storage<H>, H: Hasher>(
/// Check key changes proog and return changes of the key at given blocks range.
/// `max` is the number of best known block.
/// Changes are returned in descending order (i.e. last block comes first).
pub fn key_changes_proof_check<S: RootsStorage<H>, H: Hasher>(
pub fn key_changes_proof_check<S: RootsStorage<H, Number>, H: Hasher, Number: BlockNumber>(
config: &Configuration,
roots_storage: &S,
proof: Vec<Vec<u8>>,
begin: u64,
end: &AnchorBlockId<H::Out>,
max: u64,
begin: Number,
end: &AnchorBlockId<H::Out, Number>,
max: Number,
key: &[u8]
) -> Result<Vec<(u64, u32)>, String> {
) -> Result<Vec<(Number, u32)>, String> {
// we can't query any roots before root
let max = ::std::cmp::min(max, end.number);
let max = ::std::cmp::min(max.clone(), end.number.clone());
let mut proof_db = MemoryDB::<H>::default();
for item in proof {
@@ -123,9 +124,9 @@ pub fn key_changes_proof_check<S: RootsStorage<H>, H: Hasher>(
key,
roots_storage,
storage: &proof_db,
begin,
begin: begin.clone(),
end,
surface: surface_iterator(config, max, begin, end.number)?,
surface: surface_iterator(config, max, begin, end.number.clone())?,
extrinsics: Default::default(),
blocks: Default::default(),
@@ -137,36 +138,36 @@ pub fn key_changes_proof_check<S: RootsStorage<H>, H: Hasher>(
/// Surface iterator - only traverses top-level digests from given range and tries to find
/// all digest changes for the key.
pub struct SurfaceIterator<'a> {
pub struct SurfaceIterator<'a, Number: BlockNumber> {
config: &'a Configuration,
begin: u64,
max: u64,
current: Option<u64>,
current_begin: u64,
digest_step: u64,
begin: Number,
max: Number,
current: Option<Number>,
current_begin: Number,
digest_step: u32,
digest_level: u32,
}
impl<'a> Iterator for SurfaceIterator<'a> {
type Item = Result<(u64, u32), String>;
impl<'a, Number: BlockNumber> Iterator for SurfaceIterator<'a, Number> {
type Item = Result<(Number, u32), String>;
fn next(&mut self) -> Option<Self::Item> {
let current = self.current?;
let current = self.current.clone()?;
let digest_level = self.digest_level;
if current < self.digest_step {
if current < self.digest_step.into() {
self.current = None;
}
else {
let next = current - self.digest_step;
if next == 0 || next < self.begin {
let next = current.clone() - self.digest_step.into();
if next.is_zero() || next < self.begin {
self.current = None;
}
else if next > self.current_begin {
self.current = Some(next);
} else {
let (current, current_begin, digest_step, digest_level) = match
lower_bound_max_digest(self.config, self.max, self.begin, next) {
lower_bound_max_digest(self.config, self.max.clone(), self.begin.clone(), next) {
Err(err) => return Some(Err(err)),
Ok(range) => range,
};
@@ -184,22 +185,36 @@ impl<'a> Iterator for SurfaceIterator<'a> {
/// Drilldown iterator - receives 'digest points' from surface iterator and explores
/// every point until extrinsic is found.
pub struct DrilldownIteratorEssence<'a, RS: 'a + RootsStorage<H>, S: 'a + Storage<H>, H: Hasher> where H::Out: 'a {
pub struct DrilldownIteratorEssence<'a, RS, S, H, Number>
where
RS: 'a + RootsStorage<H, Number>,
S: 'a + Storage<H, Number>,
H: Hasher,
Number: BlockNumber,
H::Out: 'a,
{
key: &'a [u8],
roots_storage: &'a RS,
storage: &'a S,
begin: u64,
end: &'a AnchorBlockId<H::Out>,
surface: SurfaceIterator<'a>,
begin: Number,
end: &'a AnchorBlockId<H::Out, Number>,
surface: SurfaceIterator<'a, Number>,
extrinsics: VecDeque<(u64, u32)>,
blocks: VecDeque<(u64, u32)>,
extrinsics: VecDeque<(Number, u32)>,
blocks: VecDeque<(Number, u32)>,
_hasher: ::std::marker::PhantomData<H>,
}
impl<'a, RS: 'a + RootsStorage<H>, S: Storage<H>, H: Hasher> DrilldownIteratorEssence<'a, RS, S, H> {
pub fn next<F>(&mut self, trie_reader: F) -> Option<Result<(u64, u32), String>>
impl<'a, RS, S, H, Number> DrilldownIteratorEssence<'a, RS, S, H, Number>
where
RS: 'a + RootsStorage<H, Number>,
S: 'a + Storage<H, Number>,
H: Hasher,
Number: BlockNumber,
H::Out: 'a,
{
pub fn next<F>(&mut self, trie_reader: F) -> Option<Result<(Number, u32), String>>
where
F: FnMut(&S, H::Out, &[u8]) -> Result<Option<Vec<u8>>, String>,
{
@@ -210,7 +225,7 @@ impl<'a, RS: 'a + RootsStorage<H>, S: Storage<H>, H: Hasher> DrilldownIteratorEs
}
}
fn do_next<F>(&mut self, mut trie_reader: F) -> Result<Option<(u64, u32)>, String>
fn do_next<F>(&mut self, mut trie_reader: F) -> Result<Option<(Number, u32)>, String>
where
F: FnMut(&S, H::Out, &[u8]) -> Result<Option<Vec<u8>>, String>,
{
@@ -223,33 +238,33 @@ impl<'a, RS: 'a + RootsStorage<H>, S: Storage<H>, H: Hasher> DrilldownIteratorEs
// not having a changes trie root is an error because:
// we never query roots for future blocks
// AND trie roots for old blocks are known (both on full + light node)
let trie_root = self.roots_storage.root(&self.end, block)?
.ok_or_else(|| format!("Changes trie root for block {} is not found", block))?;
let trie_root = self.roots_storage.root(&self.end, block.clone())?
.ok_or_else(|| format!("Changes trie root for block {} is not found", block.clone()))?;
// only return extrinsics for blocks before self.max
// most of blocks will be filtered out before pushing to `self.blocks`
// here we just throwing away changes at digest blocks we're processing
debug_assert!(block >= self.begin, "We shall not touch digests earlier than a range' begin");
if block <= self.end.number {
let extrinsics_key = ExtrinsicIndex { block, key: self.key.to_vec() }.encode();
let extrinsics_key = ExtrinsicIndex { block: block.clone(), key: self.key.to_vec() }.encode();
let extrinsics = trie_reader(&self.storage, trie_root, &extrinsics_key);
if let Some(extrinsics) = extrinsics? {
let extrinsics: Option<ExtrinsicIndexValue> = Decode::decode(&mut &extrinsics[..]);
if let Some(extrinsics) = extrinsics {
self.extrinsics.extend(extrinsics.into_iter().rev().map(|e| (block, e)));
self.extrinsics.extend(extrinsics.into_iter().rev().map(|e| (block.clone(), e)));
}
}
}
let blocks_key = DigestIndex { block, key: self.key.to_vec() }.encode();
let blocks_key = DigestIndex { block: block.clone(), key: self.key.to_vec() }.encode();
let blocks = trie_reader(&self.storage, trie_root, &blocks_key);
if let Some(blocks) = blocks? {
let blocks: Option<DigestIndexValue> = Decode::decode(&mut &blocks[..]);
let blocks: Option<DigestIndexValue<Number>> = Decode::decode(&mut &blocks[..]);
if let Some(blocks) = blocks {
// filter level0 blocks here because we tend to use digest blocks,
// AND digest block changes could also include changes for out-of-range blocks
let begin = self.begin;
let end = self.end.number;
let begin = self.begin.clone();
let end = self.end.number.clone();
self.blocks.extend(blocks.into_iter()
.rev()
.filter(|b| level > 1 || (*b >= begin && *b <= end))
@@ -271,14 +286,21 @@ impl<'a, RS: 'a + RootsStorage<H>, S: Storage<H>, H: Hasher> DrilldownIteratorEs
}
/// Exploring drilldown operator.
pub struct DrilldownIterator<'a, RS: 'a + RootsStorage<H>, S: 'a + Storage<H>, H: Hasher> where H::Out: 'a {
essence: DrilldownIteratorEssence<'a, RS, S, H>,
pub struct DrilldownIterator<'a, RS, S, H, Number>
where
Number: BlockNumber,
H: Hasher,
S: 'a + Storage<H, Number>,
RS: 'a + RootsStorage<H, Number>,
H::Out: 'a,
{
essence: DrilldownIteratorEssence<'a, RS, S, H, Number>,
}
impl<'a, RS: 'a + RootsStorage<H>, S: Storage<H>, H: Hasher> Iterator
for DrilldownIterator<'a, RS, S, H>
impl<'a, RS: 'a + RootsStorage<H, Number>, S: Storage<H, Number>, H: Hasher, Number: BlockNumber> Iterator
for DrilldownIterator<'a, RS, S, H, Number>
{
type Item = Result<(u64, u32), String>;
type Item = Result<(Number, u32), String>;
fn next(&mut self) -> Option<Self::Item> {
self.essence.next(|storage, root, key|
@@ -287,12 +309,26 @@ impl<'a, RS: 'a + RootsStorage<H>, S: Storage<H>, H: Hasher> Iterator
}
/// Proving drilldown iterator.
struct ProvingDrilldownIterator<'a, RS: 'a + RootsStorage<H>, S: 'a + Storage<H>, H: Hasher> where H::Out: 'a {
essence: DrilldownIteratorEssence<'a, RS, S, H>,
struct ProvingDrilldownIterator<'a, RS, S, H, Number>
where
Number: BlockNumber,
H: Hasher,
S: 'a + Storage<H, Number>,
RS: 'a + RootsStorage<H, Number>,
H::Out: 'a,
{
essence: DrilldownIteratorEssence<'a, RS, S, H, Number>,
proof_recorder: RefCell<Recorder<H::Out>>,
}
impl<'a, RS: 'a + RootsStorage<H>, S: Storage<H>, H: Hasher> ProvingDrilldownIterator<'a, RS, S, H> {
impl<'a, RS, S, H, Number> ProvingDrilldownIterator<'a, RS, S, H, Number>
where
Number: BlockNumber,
H: Hasher,
S: 'a + Storage<H, Number>,
RS: 'a + RootsStorage<H, Number>,
H::Out: 'a,
{
/// Consume the iterator, extracting the gathered proof in lexicographical order
/// by value.
pub fn extract_proof(self) -> Vec<Vec<u8>> {
@@ -303,8 +339,15 @@ impl<'a, RS: 'a + RootsStorage<H>, S: Storage<H>, H: Hasher> ProvingDrilldownIte
}
}
impl<'a, RS: 'a + RootsStorage<H>, S: Storage<H>, H: Hasher> Iterator for ProvingDrilldownIterator<'a, RS, S, H> {
type Item = Result<(u64, u32), String>;
impl<'a, RS, S, H, Number> Iterator for ProvingDrilldownIterator<'a, RS, S, H, Number>
where
Number: BlockNumber,
H: Hasher,
S: 'a + Storage<H, Number>,
RS: 'a + RootsStorage<H, Number>,
H::Out: 'a,
{
type Item = Result<(Number, u32), String>;
fn next(&mut self) -> Option<Self::Item> {
let proof_recorder = &mut *self.proof_recorder.try_borrow_mut()
@@ -318,8 +361,18 @@ impl<'a, RS: 'a + RootsStorage<H>, S: Storage<H>, H: Hasher> Iterator for Provin
}
/// Returns surface iterator for given range of blocks.
fn surface_iterator<'a>(config: &'a Configuration, max: u64, begin: u64, end: u64) -> Result<SurfaceIterator<'a>, String> {
let (current, current_begin, digest_step, digest_level) = lower_bound_max_digest(config, max, begin, end)?;
fn surface_iterator<'a, Number: BlockNumber>(
config: &'a Configuration,
max: Number,
begin: Number,
end: Number,
) -> Result<SurfaceIterator<'a, Number>, String> {
let (current, current_begin, digest_step, digest_level) = lower_bound_max_digest(
config,
max.clone(),
begin.clone(),
end,
)?;
Ok(SurfaceIterator {
config,
begin,
@@ -333,31 +386,32 @@ fn surface_iterator<'a>(config: &'a Configuration, max: u64, begin: u64, end: u6
/// Returns parameters of highest level digest block that includes the end of given range
/// and tends to include the whole range.
fn lower_bound_max_digest(
fn lower_bound_max_digest<Number: BlockNumber>(
config: &Configuration,
max: u64,
begin: u64,
end: u64,
) -> Result<(u64, u64, u64, u32), String> {
max: Number,
begin: Number,
end: Number,
) -> Result<(Number, Number, u32, u32), String> {
if end > max || begin > end {
return Err("invalid changes range".into());
}
let mut digest_level = 0u32;
let mut digest_step = 1u64;
let mut digest_interval = 0u64;
let mut current = end;
let mut current_begin = begin;
if begin != end {
let mut digest_step = 1u32;
let mut digest_interval = 0u32;
let mut current = end.clone();
let mut current_begin = begin.clone();
if current_begin != current {
while digest_level != config.digest_levels {
let new_digest_level = digest_level + 1;
let new_digest_step = digest_step * config.digest_interval;
let new_digest_interval = config.digest_interval * {
if digest_interval == 0 { 1 } else { digest_interval }
};
let new_digest_begin = ((current - 1) / new_digest_interval) * new_digest_interval;
let new_digest_end = new_digest_begin + new_digest_interval;
let new_current = new_digest_begin + new_digest_interval;
let new_digest_begin = ((current.clone() - One::one())
/ new_digest_interval.into()) * new_digest_interval.into();
let new_digest_end = new_digest_begin.clone() + new_digest_interval.into();
let new_current = new_digest_begin.clone() + new_digest_interval.into();
if new_digest_end > max {
if begin < new_digest_begin {
@@ -372,7 +426,7 @@ fn lower_bound_max_digest(
current = new_current;
current_begin = new_digest_begin;
if new_digest_begin <= begin && new_digest_end >= end {
if current_begin <= begin && new_digest_end >= end {
break;
}
}
@@ -394,7 +448,7 @@ mod tests {
use crate::changes_trie::storage::InMemoryStorage;
use super::*;
fn prepare_for_drilldown() -> (Configuration, InMemoryStorage<Blake2Hasher>) {
fn prepare_for_drilldown() -> (Configuration, InMemoryStorage<Blake2Hasher, u64>) {
let config = Configuration { digest_interval: 4, digest_levels: 2 };
let backend = InMemoryStorage::with_inputs(vec![
// digest: 1..4 => [(3, 0)]
@@ -436,27 +490,27 @@ mod tests {
#[test]
fn drilldown_iterator_works() {
let (config, storage) = prepare_for_drilldown();
let drilldown_result = key_changes::<InMemoryStorage<Blake2Hasher>, Blake2Hasher>(
let drilldown_result = key_changes::<InMemoryStorage<Blake2Hasher, u64>, Blake2Hasher, u64>(
&config, &storage, 0, &AnchorBlockId { hash: Default::default(), number: 16 }, 16, &[42])
.and_then(Result::from_iter);
assert_eq!(drilldown_result, Ok(vec![(8, 2), (8, 1), (6, 3), (3, 0)]));
let drilldown_result = key_changes::<InMemoryStorage<Blake2Hasher>, Blake2Hasher>(
let drilldown_result = key_changes::<InMemoryStorage<Blake2Hasher, u64>, Blake2Hasher, u64>(
&config, &storage, 0, &AnchorBlockId { hash: Default::default(), number: 2 }, 4, &[42])
.and_then(Result::from_iter);
assert_eq!(drilldown_result, Ok(vec![]));
let drilldown_result = key_changes::<InMemoryStorage<Blake2Hasher>, Blake2Hasher>(
let drilldown_result = key_changes::<InMemoryStorage<Blake2Hasher, u64>, Blake2Hasher, u64>(
&config, &storage, 0, &AnchorBlockId { hash: Default::default(), number: 3 }, 4, &[42])
.and_then(Result::from_iter);
assert_eq!(drilldown_result, Ok(vec![(3, 0)]));
let drilldown_result = key_changes::<InMemoryStorage<Blake2Hasher>, Blake2Hasher>(
let drilldown_result = key_changes::<InMemoryStorage<Blake2Hasher, u64>, Blake2Hasher, u64>(
&config, &storage, 7, &AnchorBlockId { hash: Default::default(), number: 8 }, 8, &[42])
.and_then(Result::from_iter);
assert_eq!(drilldown_result, Ok(vec![(8, 2), (8, 1)]));
let drilldown_result = key_changes::<InMemoryStorage<Blake2Hasher>, Blake2Hasher>(
let drilldown_result = key_changes::<InMemoryStorage<Blake2Hasher, u64>, Blake2Hasher, u64>(
&config, &storage, 5, &AnchorBlockId { hash: Default::default(), number: 7 }, 8, &[42])
.and_then(Result::from_iter);
assert_eq!(drilldown_result, Ok(vec![(6, 3)]));
@@ -467,7 +521,7 @@ mod tests {
let (config, storage) = prepare_for_drilldown();
storage.clear_storage();
assert!(key_changes::<InMemoryStorage<Blake2Hasher>, Blake2Hasher>(
assert!(key_changes::<InMemoryStorage<Blake2Hasher, u64>, Blake2Hasher, u64>(
&config, &storage, 0, &AnchorBlockId { hash: Default::default(), number: 100 }, 1000, &[42])
.and_then(|i| i.collect::<Result<Vec<_>, _>>()).is_err());
}
@@ -475,9 +529,9 @@ mod tests {
#[test]
fn drilldown_iterator_fails_when_range_is_invalid() {
let (config, storage) = prepare_for_drilldown();
assert!(key_changes::<InMemoryStorage<Blake2Hasher>, Blake2Hasher>(
assert!(key_changes::<InMemoryStorage<Blake2Hasher, u64>, Blake2Hasher, u64>(
&config, &storage, 0, &AnchorBlockId { hash: Default::default(), number: 100 }, 50, &[42]).is_err());
assert!(key_changes::<InMemoryStorage<Blake2Hasher>, Blake2Hasher>(
assert!(key_changes::<InMemoryStorage<Blake2Hasher, u64>, Blake2Hasher, u64>(
&config, &storage, 20, &AnchorBlockId { hash: Default::default(), number: 10 }, 100, &[42]).is_err());
}
@@ -488,7 +542,7 @@ mod tests {
// create drilldown iterator that records all trie nodes during drilldown
let (remote_config, remote_storage) = prepare_for_drilldown();
let remote_proof = key_changes_proof::<InMemoryStorage<Blake2Hasher>, Blake2Hasher>(
let remote_proof = key_changes_proof::<InMemoryStorage<Blake2Hasher, u64>, Blake2Hasher, u64>(
&remote_config, &remote_storage,
0, &AnchorBlockId { hash: Default::default(), number: 16 }, 16, &[42]).unwrap();
@@ -497,7 +551,7 @@ mod tests {
// create drilldown iterator that works the same, but only depends on trie
let (local_config, local_storage) = prepare_for_drilldown();
local_storage.clear_storage();
let local_result = key_changes_proof_check::<InMemoryStorage<Blake2Hasher>, Blake2Hasher>(
let local_result = key_changes_proof_check::<InMemoryStorage<Blake2Hasher, u64>, Blake2Hasher, u64>(
&local_config, &local_storage, remote_proof,
0, &AnchorBlockId { hash: Default::default(), number: 16 }, 16, &[42]);
@@ -17,12 +17,13 @@
//! Different types of changes trie input pairs.
use parity_codec::{Decode, Encode, Input, Output};
use crate::changes_trie::BlockNumber;
/// Key of { changed key => set of extrinsic indices } mapping.
#[derive(Clone, Debug, PartialEq, Eq)]
pub struct ExtrinsicIndex {
pub struct ExtrinsicIndex<Number: BlockNumber> {
/// Block at which this key has been inserted in the trie.
pub block: u64,
pub block: Number,
/// Storage key this node is responsible for.
pub key: Vec<u8>,
}
@@ -32,35 +33,35 @@ pub type ExtrinsicIndexValue = Vec<u32>;
/// Key of { changed key => block/digest block numbers } mapping.
#[derive(Clone, Debug, PartialEq, Eq)]
pub struct DigestIndex {
pub struct DigestIndex<Number: BlockNumber> {
/// Block at which this key has been inserted in the trie.
pub block: u64,
pub block: Number,
/// Storage key this node is responsible for.
pub key: Vec<u8>,
}
/// Value of { changed key => block/digest block numbers } mapping.
pub type DigestIndexValue = Vec<u64>;
pub type DigestIndexValue<Number> = Vec<Number>;
/// Single input pair of changes trie.
#[derive(Clone, Debug, PartialEq, Eq)]
pub enum InputPair {
pub enum InputPair<Number: BlockNumber> {
/// Element of { key => set of extrinsics where key has been changed } element mapping.
ExtrinsicIndex(ExtrinsicIndex, ExtrinsicIndexValue),
ExtrinsicIndex(ExtrinsicIndex<Number>, ExtrinsicIndexValue),
/// Element of { key => set of blocks/digest blocks where key has been changed } element mapping.
DigestIndex(DigestIndex, DigestIndexValue),
DigestIndex(DigestIndex<Number>, DigestIndexValue<Number>),
}
/// Single input key of changes trie.
#[derive(Clone, Debug, PartialEq, Eq)]
pub enum InputKey {
pub enum InputKey<Number: BlockNumber> {
/// Key of { key => set of extrinsics where key has been changed } element mapping.
ExtrinsicIndex(ExtrinsicIndex),
ExtrinsicIndex(ExtrinsicIndex<Number>),
/// Key of { key => set of blocks/digest blocks where key has been changed } element mapping.
DigestIndex(DigestIndex),
DigestIndex(DigestIndex<Number>),
}
impl Into<(Vec<u8>, Vec<u8>)> for InputPair {
impl<Number: BlockNumber> Into<(Vec<u8>, Vec<u8>)> for InputPair<Number> {
fn into(self) -> (Vec<u8>, Vec<u8>) {
match self {
InputPair::ExtrinsicIndex(key, value) => (key.encode(), value.encode()),
@@ -69,8 +70,8 @@ impl Into<(Vec<u8>, Vec<u8>)> for InputPair {
}
}
impl Into<InputKey> for InputPair {
fn into(self) -> InputKey {
impl<Number: BlockNumber> Into<InputKey<Number>> for InputPair<Number> {
fn into(self) -> InputKey<Number> {
match self {
InputPair::ExtrinsicIndex(key, _) => InputKey::ExtrinsicIndex(key),
InputPair::DigestIndex(key, _) => InputKey::DigestIndex(key),
@@ -78,15 +79,15 @@ impl Into<InputKey> for InputPair {
}
}
impl ExtrinsicIndex {
pub fn key_neutral_prefix(block: u64) -> Vec<u8> {
impl<Number: BlockNumber> ExtrinsicIndex<Number> {
pub fn key_neutral_prefix(block: Number) -> Vec<u8> {
let mut prefix = vec![1];
prefix.extend(block.encode());
prefix
}
}
impl Encode for ExtrinsicIndex {
impl<Number: BlockNumber> Encode for ExtrinsicIndex<Number> {
fn encode_to<W: Output>(&self, dest: &mut W) {
dest.push_byte(1);
self.block.encode_to(dest);
@@ -94,8 +95,8 @@ impl Encode for ExtrinsicIndex {
}
}
impl DigestIndex {
pub fn key_neutral_prefix(block: u64) -> Vec<u8> {
impl<Number: BlockNumber> DigestIndex<Number> {
pub fn key_neutral_prefix(block: Number) -> Vec<u8> {
let mut prefix = vec![2];
prefix.extend(block.encode());
prefix
@@ -103,7 +104,7 @@ impl DigestIndex {
}
impl Encode for DigestIndex {
impl<Number: BlockNumber> Encode for DigestIndex<Number> {
fn encode_to<W: Output>(&self, dest: &mut W) {
dest.push_byte(2);
self.block.encode_to(dest);
@@ -111,7 +112,7 @@ impl Encode for DigestIndex {
}
}
impl Decode for InputKey {
impl<Number: BlockNumber> Decode for InputKey<Number> {
fn decode<I: Input>(input: &mut I) -> Option<Self> {
match input.read_byte()? {
1 => Some(InputKey::ExtrinsicIndex(ExtrinsicIndex {
@@ -133,17 +134,17 @@ mod tests {
#[test]
fn extrinsic_index_serialized_and_deserialized() {
let original = ExtrinsicIndex { block: 777, key: vec![42] };
let original = ExtrinsicIndex { block: 777u64, key: vec![42] };
let serialized = original.encode();
let deserialized: InputKey = Decode::decode(&mut &serialized[..]).unwrap();
let deserialized: InputKey<u64> = Decode::decode(&mut &serialized[..]).unwrap();
assert_eq!(InputKey::ExtrinsicIndex(original), deserialized);
}
#[test]
fn digest_index_serialized_and_deserialized() {
let original = DigestIndex { block: 777, key: vec![42] };
let original = DigestIndex { block: 777u64, key: vec![42] };
let serialized = original.encode();
let deserialized: InputKey = Decode::decode(&mut &serialized[..]).unwrap();
let deserialized: InputKey<u64> = Decode::decode(&mut &serialized[..]).unwrap();
assert_eq!(InputKey::DigestIndex(original), deserialized);
}
}
@@ -48,58 +48,116 @@ pub use self::prune::{prune, oldest_non_pruned_trie};
use hash_db::Hasher;
use crate::backend::Backend;
use num_traits::{One, Zero};
use parity_codec::{Decode, Encode};
use primitives;
use crate::changes_trie::build::prepare_input;
use crate::overlayed_changes::OverlayedChanges;
use crate::trie_backend_essence::TrieBackendStorage;
use trie::{DBValue, trie_root};
/// Changes that are made outside of extrinsics are marked with this index;
pub const NO_EXTRINSIC_INDEX: u32 = 0xffffffff;
/// Requirements for block number that can be used with changes tries.
pub trait BlockNumber:
Send + Sync + 'static +
::std::fmt::Display +
Clone +
From<u32> + One + Zero +
PartialEq + Ord +
::std::ops::Add<Self, Output=Self> + ::std::ops::Sub<Self, Output=Self> +
::std::ops::Mul<Self, Output=Self> + ::std::ops::Div<Self, Output=Self> +
::std::ops::Rem<Self, Output=Self> +
::std::ops::AddAssign<Self> +
num_traits::CheckedMul + num_traits::CheckedSub +
Decode + Encode
{}
impl<T> BlockNumber for T where T:
Send + Sync + 'static +
::std::fmt::Display +
Clone +
From<u32> + One + Zero +
PartialEq + Ord +
::std::ops::Add<Self, Output=Self> + ::std::ops::Sub<Self, Output=Self> +
::std::ops::Mul<Self, Output=Self> + ::std::ops::Div<Self, Output=Self> +
::std::ops::Rem<Self, Output=Self> +
::std::ops::AddAssign<Self> +
num_traits::CheckedMul + num_traits::CheckedSub +
Decode + Encode,
{}
/// Block identifier that could be used to determine fork of this block.
#[derive(Debug)]
pub struct AnchorBlockId<Hash: ::std::fmt::Debug> {
pub struct AnchorBlockId<Hash: ::std::fmt::Debug, Number: BlockNumber> {
/// Hash of this block.
pub hash: Hash,
/// Number of this block.
pub number: u64,
pub number: Number,
}
/// Changes trie storage. Provides access to trie roots and trie nodes.
pub trait RootsStorage<H: Hasher>: Send + Sync {
pub trait RootsStorage<H: Hasher, Number: BlockNumber>: Send + Sync {
/// Resolve hash of the block into anchor.
fn build_anchor(&self, hash: H::Out) -> Result<AnchorBlockId<H::Out, Number>, String>;
/// Get changes trie root for the block with given number which is an ancestor (or the block
/// itself) of the anchor_block (i.e. anchor_block.number >= block).
fn root(&self, anchor: &AnchorBlockId<H::Out>, block: u64) -> Result<Option<H::Out>, String>;
fn root(&self, anchor: &AnchorBlockId<H::Out, Number>, block: Number) -> Result<Option<H::Out>, String>;
}
/// Changes trie storage. Provides access to trie roots and trie nodes.
pub trait Storage<H: Hasher>: RootsStorage<H> {
pub trait Storage<H: Hasher, Number: BlockNumber>: RootsStorage<H, Number> {
/// Get a trie node.
fn get(&self, key: &H::Out, prefix: &[u8]) -> Result<Option<DBValue>, String>;
}
/// Changes trie storage -> trie backend essence adapter.
pub struct TrieBackendStorageAdapter<'a, H: Hasher, Number: BlockNumber>(pub &'a Storage<H, Number>);
impl<'a, H: Hasher, N: BlockNumber> crate::TrieBackendStorage<H> for TrieBackendStorageAdapter<'a, H, N> {
type Overlay = trie::MemoryDB<H>;
fn get(&self, key: &H::Out, prefix: &[u8]) -> Result<Option<DBValue>, String> {
self.0.get(key, prefix)
}
}
/// Changes trie configuration.
pub type Configuration = primitives::ChangesTrieConfiguration;
/// Compute the changes trie root and transaction for given block.
/// Returns None if there's no data to perform computation.
pub fn compute_changes_trie_root<'a, B: Backend<H>, S: Storage<H>, H: Hasher>(
/// Returns Err(()) if unknown `parent_hash` has been passed.
/// Returns Ok(None) if there's no data to perform computation.
/// Panics if background storage returns an error.
pub fn compute_changes_trie_root<'a, B: Backend<H>, S: Storage<H, Number>, H: Hasher, Number: BlockNumber>(
backend: &B,
storage: Option<&'a S>,
changes: &OverlayedChanges,
parent: &'a AnchorBlockId<H::Out>,
) -> Option<(H::Out, Vec<(Vec<u8>, Vec<u8>)>)>
parent_hash: H::Out,
) -> Result<Option<(H::Out, Vec<(Vec<u8>, Vec<u8>)>)>, ()>
where
&'a S: TrieBackendStorage<H>,
H::Out: Ord,
H::Out: Ord + 'static,
{
let input_pairs = prepare_input::<B, S, H>(backend, storage, changes, parent)
.expect("storage is not allowed to fail within runtime")?;
let transaction = input_pairs.into_iter()
.map(Into::into)
.collect::<Vec<_>>();
let root = trie_root::<H, _, _, _>(transaction.iter().map(|(k, v)| (&*k, &*v)));
let (storage, config) = match (storage, changes.changes_trie_config.as_ref()) {
(Some(storage), Some(config)) => (storage, config),
_ => return Ok(None),
};
Some((root, transaction))
// build_anchor error should not be considered fatal
let parent = storage.build_anchor(parent_hash).map_err(|_| ())?;
// storage errors are considered fatal (similar to situations when runtime fetches values from storage)
let input_pairs = prepare_input::<B, S, H, Number>(backend, storage, config, changes, &parent)
.expect("storage is not allowed to fail within runtime");
match input_pairs {
Some(input_pairs) => {
let transaction = input_pairs.into_iter()
.map(Into::into)
.collect::<Vec<_>>();
let root = trie_root::<H, _, _, _>(transaction.iter().map(|(k, v)| (&*k, &*v)));
Ok(Some((root, transaction)))
},
None => Ok(None),
}
}
@@ -19,24 +19,26 @@
use hash_db::Hasher;
use trie::Recorder;
use log::warn;
use num_traits::One;
use crate::proving_backend::ProvingBackendEssence;
use crate::trie_backend_essence::TrieBackendEssence;
use crate::changes_trie::{AnchorBlockId, Configuration, Storage};
use crate::changes_trie::{AnchorBlockId, Configuration, Storage, BlockNumber};
use crate::changes_trie::storage::TrieBackendAdapter;
/// Get number of oldest block for which changes trie is not pruned
/// given changes trie configuration, pruning parameter and number of
/// best finalized block.
pub fn oldest_non_pruned_trie(
pub fn oldest_non_pruned_trie<Number: BlockNumber>(
config: &Configuration,
min_blocks_to_keep: u64,
best_finalized_block: u64,
) -> u64 {
min_blocks_to_keep: Number,
best_finalized_block: Number,
) -> Number {
let max_digest_interval = config.max_digest_interval();
let max_digest_block = best_finalized_block - best_finalized_block % max_digest_interval;
let best_finalized_block_rem = best_finalized_block.clone() % max_digest_interval.into();
let max_digest_block = best_finalized_block - best_finalized_block_rem;
match pruning_range(config, min_blocks_to_keep, max_digest_block) {
Some((_, last_pruned_block)) => last_pruned_block + 1,
None => 1,
Some((_, last_pruned_block)) => last_pruned_block + One::one(),
None => One::one(),
}
}
@@ -45,23 +47,32 @@ pub fn oldest_non_pruned_trie(
/// `min_blocks_to_keep` blocks. We only prune changes tries at `max_digest_interval`
/// ranges.
/// Returns MemoryDB that contains all deleted changes tries nodes.
pub fn prune<S: Storage<H>, H: Hasher, F: FnMut(H::Out)>(
pub fn prune<S: Storage<H, Number>, H: Hasher, Number: BlockNumber, F: FnMut(H::Out)>(
config: &Configuration,
storage: &S,
min_blocks_to_keep: u64,
current_block: &AnchorBlockId<H::Out>,
min_blocks_to_keep: Number,
current_block: &AnchorBlockId<H::Out, Number>,
mut remove_trie_node: F,
) {
// select range for pruning
let (first, last) = match pruning_range(config, min_blocks_to_keep, current_block.number) {
let (first, last) = match pruning_range(config, min_blocks_to_keep, current_block.number.clone()) {
Some((first, last)) => (first, last),
None => return,
};
// delete changes trie for every block in range
// FIXME: limit `max_digest_interval` so that this cycle won't involve huge ranges
for block in first..last+1 {
let root = match storage.root(current_block, block) {
let mut block = first;
loop {
if block >= last.clone() + One::one() {
break;
}
let prev_block = block.clone();
block += One::one();
let block = prev_block;
let root = match storage.root(current_block, block.clone()) {
Ok(Some(root)) => root,
Ok(None) => continue,
Err(error) => {
@@ -91,11 +102,15 @@ pub fn prune<S: Storage<H>, H: Hasher, F: FnMut(H::Out)>(
}
/// Select blocks range (inclusive from both ends) for pruning changes tries in.
fn pruning_range(config: &Configuration, min_blocks_to_keep: u64, block: u64) -> Option<(u64, u64)> {
fn pruning_range<Number: BlockNumber>(
config: &Configuration,
min_blocks_to_keep: Number,
block: Number,
) -> Option<(Number, Number)> {
// compute number of changes tries we actually want to keep
let (prune_interval, blocks_to_keep) = if config.is_digest_build_enabled() {
// we only CAN prune at block where max-level-digest is created
let max_digest_interval = match config.digest_level_at_block(block) {
let max_digest_interval = match config.digest_level_at_block(block.clone()) {
Some((digest_level, digest_interval, _)) if digest_level == config.digest_levels =>
digest_interval,
_ => return None,
@@ -107,7 +122,7 @@ fn pruning_range(config: &Configuration, min_blocks_to_keep: u64, block: u64) ->
// number of blocks BEFORE current block where changes tries are not pruned
(
max_digest_interval,
max_digest_intervals_to_keep.checked_mul(max_digest_interval)
max_digest_intervals_to_keep.checked_mul(&max_digest_interval.into())
)
} else {
(
@@ -117,11 +132,11 @@ fn pruning_range(config: &Configuration, min_blocks_to_keep: u64, block: u64) ->
};
// last block for which changes trie is pruned
let last_block_to_prune = blocks_to_keep.and_then(|b| block.checked_sub(b));
let first_block_to_prune = last_block_to_prune.clone().and_then(|b| b.checked_sub(prune_interval));
let last_block_to_prune = blocks_to_keep.and_then(|b| block.checked_sub(&b));
let first_block_to_prune = last_block_to_prune.clone().and_then(|b| b.checked_sub(&prune_interval.into()));
last_block_to_prune
.and_then(|last| first_block_to_prune.map(|first| (first + 1, last)))
.and_then(|last| first_block_to_prune.map(|first| (first + One::one(), last)))
}
/// Select pruning delay for the changes tries. To make sure we could build a changes
@@ -132,13 +147,16 @@ fn pruning_range(config: &Configuration, min_blocks_to_keep: u64, block: u64) ->
/// 0 or 1: means that only last changes trie is guaranteed to exists;
/// 2: the last chnages trie + previous changes trie
/// ...
fn max_digest_intervals_to_keep(min_blocks_to_keep: u64, max_digest_interval: u64) -> u64 {
fn max_digest_intervals_to_keep<Number: BlockNumber>(
min_blocks_to_keep: Number,
max_digest_interval: u32,
) -> Number {
// config.digest_level_at_block ensures that it is not zero
debug_assert!(max_digest_interval != 0);
let max_digest_intervals_to_keep = min_blocks_to_keep / max_digest_interval;
if max_digest_intervals_to_keep == 0 {
1
let max_digest_intervals_to_keep = min_blocks_to_keep / max_digest_interval.into();
if max_digest_intervals_to_keep.is_zero() {
One::one()
} else {
max_digest_intervals_to_keep
}
@@ -153,14 +171,14 @@ mod tests {
use crate::changes_trie::storage::InMemoryStorage;
use super::*;
fn config(interval: u64, levels: u32) -> Configuration {
fn config(interval: u32, levels: u32) -> Configuration {
Configuration {
digest_interval: interval,
digest_levels: levels,
}
}
fn prune_by_collect<S: Storage<H>, H: Hasher>(
fn prune_by_collect<S: Storage<H, u64>, H: Hasher>(
config: &Configuration,
storage: &S,
min_blocks_to_keep: u64,
@@ -174,7 +192,7 @@ mod tests {
#[test]
fn prune_works() {
fn prepare_storage() -> InMemoryStorage<Blake2Hasher> {
fn prepare_storage() -> InMemoryStorage<Blake2Hasher, u64> {
let mut mdb1 = MemoryDB::<Blake2Hasher>::default();
let root1 = insert_into_memory_db::<Blake2Hasher, _>(&mut mdb1, vec![(vec![10], vec![20])]).unwrap();
let mut mdb2 = MemoryDB::<Blake2Hasher>::default();
@@ -241,60 +259,60 @@ mod tests {
#[test]
fn pruning_range_works() {
// DIGESTS ARE NOT CREATED + NO TRIES ARE PRUNED
assert_eq!(pruning_range(&config(10, 0), 2, 2), None);
assert_eq!(pruning_range(&config(10, 0), 2u64, 2u64), None);
// DIGESTS ARE NOT CREATED + SOME TRIES ARE PRUNED
assert_eq!(pruning_range(&config(10, 0), 100, 110), Some((10, 10)));
assert_eq!(pruning_range(&config(10, 0), 100, 210), Some((110, 110)));
assert_eq!(pruning_range(&config(10, 0), 100u64, 110u64), Some((10, 10)));
assert_eq!(pruning_range(&config(10, 0), 100u64, 210u64), Some((110, 110)));
// DIGESTS ARE CREATED + NO TRIES ARE PRUNED
assert_eq!(pruning_range(&config(10, 2), 2, 0), None);
assert_eq!(pruning_range(&config(10, 2), 30, 100), None);
assert_eq!(pruning_range(&config(::std::u64::MAX, 2), 1, 1024), None);
assert_eq!(pruning_range(&config(::std::u64::MAX, 2), ::std::u64::MAX, 1024), None);
assert_eq!(pruning_range(&config(32, 2), 2048, 512), None);
assert_eq!(pruning_range(&config(32, 2), 2048, 1024), None);
assert_eq!(pruning_range(&config(10, 2), 2u64, 0u64), None);
assert_eq!(pruning_range(&config(10, 2), 30u64, 100u64), None);
assert_eq!(pruning_range(&config(::std::u32::MAX, 2), 1u64, 1024u64), None);
assert_eq!(pruning_range(&config(::std::u32::MAX, 2), ::std::u64::MAX, 1024u64), None);
assert_eq!(pruning_range(&config(32, 2), 2048u64, 512u64), None);
assert_eq!(pruning_range(&config(32, 2), 2048u64, 1024u64), None);
// DIGESTS ARE CREATED + SOME TRIES ARE PRUNED
// when we do not want to keep any highest-level-digests
// (system forces to keep at least one)
assert_eq!(pruning_range(&config(4, 2), 0, 32), Some((1, 16)));
assert_eq!(pruning_range(&config(4, 2), 0, 64), Some((33, 48)));
assert_eq!(pruning_range(&config(4, 2), 0u64, 32u64), Some((1, 16)));
assert_eq!(pruning_range(&config(4, 2), 0u64, 64u64), Some((33, 48)));
// when we want to keep 1 (last) highest-level-digest
assert_eq!(pruning_range(&config(4, 2), 16, 32), Some((1, 16)));
assert_eq!(pruning_range(&config(4, 2), 16, 64), Some((33, 48)));
assert_eq!(pruning_range(&config(4, 2), 16u64, 32u64), Some((1, 16)));
assert_eq!(pruning_range(&config(4, 2), 16u64, 64u64), Some((33, 48)));
// when we want to keep 1 (last) + 1 additional level digests
assert_eq!(pruning_range(&config(32, 2), 4096, 5120), Some((1, 1024)));
assert_eq!(pruning_range(&config(32, 2), 4096, 6144), Some((1025, 2048)));
assert_eq!(pruning_range(&config(32, 2), 4096u64, 5120u64), Some((1, 1024)));
assert_eq!(pruning_range(&config(32, 2), 4096u64, 6144u64), Some((1025, 2048)));
}
#[test]
fn max_digest_intervals_to_keep_works() {
assert_eq!(max_digest_intervals_to_keep(1024, 1025), 1);
assert_eq!(max_digest_intervals_to_keep(1024, 1023), 1);
assert_eq!(max_digest_intervals_to_keep(1024, 512), 2);
assert_eq!(max_digest_intervals_to_keep(1024, 511), 2);
assert_eq!(max_digest_intervals_to_keep(1024, 100), 10);
assert_eq!(max_digest_intervals_to_keep(1024u64, 1025), 1u64);
assert_eq!(max_digest_intervals_to_keep(1024u64, 1023), 1u64);
assert_eq!(max_digest_intervals_to_keep(1024u64, 512), 2u64);
assert_eq!(max_digest_intervals_to_keep(1024u64, 511), 2u64);
assert_eq!(max_digest_intervals_to_keep(1024u64, 100), 10u64);
}
#[test]
fn oldest_non_pruned_trie_works() {
// when digests are not created at all
assert_eq!(oldest_non_pruned_trie(&config(0, 0), 100, 10), 1);
assert_eq!(oldest_non_pruned_trie(&config(0, 0), 100, 110), 11);
assert_eq!(oldest_non_pruned_trie(&config(0, 0), 100u64, 10u64), 1);
assert_eq!(oldest_non_pruned_trie(&config(0, 0), 100u64, 110u64), 11);
// when only l1 digests are created
assert_eq!(oldest_non_pruned_trie(&config(100, 1), 100, 50), 1);
assert_eq!(oldest_non_pruned_trie(&config(100, 1), 100, 110), 1);
assert_eq!(oldest_non_pruned_trie(&config(100, 1), 100, 210), 101);
assert_eq!(oldest_non_pruned_trie(&config(100, 1), 100u64, 50u64), 1);
assert_eq!(oldest_non_pruned_trie(&config(100, 1), 100u64, 110u64), 1);
assert_eq!(oldest_non_pruned_trie(&config(100, 1), 100u64, 210u64), 101);
// when l2 digests are created
assert_eq!(oldest_non_pruned_trie(&config(100, 2), 100, 50), 1);
assert_eq!(oldest_non_pruned_trie(&config(100, 2), 100, 110), 1);
assert_eq!(oldest_non_pruned_trie(&config(100, 2), 100, 210), 1);
assert_eq!(oldest_non_pruned_trie(&config(100, 2), 100, 10110), 1);
assert_eq!(oldest_non_pruned_trie(&config(100, 2), 100, 20110), 10001);
assert_eq!(oldest_non_pruned_trie(&config(100, 2), 100u64, 50u64), 1);
assert_eq!(oldest_non_pruned_trie(&config(100, 2), 100u64, 110u64), 1);
assert_eq!(oldest_non_pruned_trie(&config(100, 2), 100u64, 210u64), 1);
assert_eq!(oldest_non_pruned_trie(&config(100, 2), 100u64, 10110u64), 1);
assert_eq!(oldest_non_pruned_trie(&config(100, 2), 100u64, 20110u64), 10001);
}
}
@@ -16,12 +16,12 @@
//! Changes trie storage utilities.
use std::collections::HashMap;
use std::collections::BTreeMap;
use hash_db::Hasher;
use trie::DBValue;
use trie::MemoryDB;
use parking_lot::RwLock;
use crate::changes_trie::{AnchorBlockId, RootsStorage, Storage};
use crate::changes_trie::{RootsStorage, Storage, AnchorBlockId, BlockNumber};
use crate::trie_backend_essence::TrieBackendStorage;
#[cfg(test)]
@@ -32,27 +32,27 @@ use crate::backend::insert_into_memory_db;
use crate::changes_trie::input::InputPair;
/// In-memory implementation of changes trie storage.
pub struct InMemoryStorage<H: Hasher> {
data: RwLock<InMemoryStorageData<H>>,
pub struct InMemoryStorage<H: Hasher, Number: BlockNumber> {
data: RwLock<InMemoryStorageData<H, Number>>,
}
/// Adapter for using changes trie storage as a TrieBackendEssence' storage.
pub struct TrieBackendAdapter<'a, H: Hasher, S: 'a + Storage<H>> {
pub struct TrieBackendAdapter<'a, H: Hasher, Number: BlockNumber, S: 'a + Storage<H, Number>> {
storage: &'a S,
_hasher: ::std::marker::PhantomData<H>,
_hasher: ::std::marker::PhantomData<(H, Number)>,
}
struct InMemoryStorageData<H: Hasher> {
roots: HashMap<u64, H::Out>,
struct InMemoryStorageData<H: Hasher, Number: BlockNumber> {
roots: BTreeMap<Number, H::Out>,
mdb: MemoryDB<H>,
}
impl<H: Hasher> InMemoryStorage<H> {
impl<H: Hasher, Number: BlockNumber> InMemoryStorage<H, Number> {
/// Create the storage from given in-memory database.
pub fn with_db(mdb: MemoryDB<H>) -> Self {
Self {
data: RwLock::new(InMemoryStorageData {
roots: HashMap::new(),
roots: BTreeMap::new(),
mdb,
}),
}
@@ -63,10 +63,20 @@ impl<H: Hasher> InMemoryStorage<H> {
Self::with_db(Default::default())
}
/// Create the storage with given blocks.
pub fn with_blocks(blocks: Vec<(Number, H::Out)>) -> Self {
Self {
data: RwLock::new(InMemoryStorageData {
roots: blocks.into_iter().collect(),
mdb: MemoryDB::default(),
}),
}
}
#[cfg(test)]
pub fn with_inputs(inputs: Vec<(u64, Vec<InputPair>)>) -> Self {
pub fn with_inputs(inputs: Vec<(Number, Vec<InputPair<Number>>)>) -> Self {
let mut mdb = MemoryDB::default();
let mut roots = HashMap::new();
let mut roots = BTreeMap::new();
for (block, pairs) in inputs {
let root = insert_into_memory_db::<H, _>(&mut mdb, pairs.into_iter().map(Into::into));
if let Some(root) = root {
@@ -101,32 +111,44 @@ impl<H: Hasher> InMemoryStorage<H> {
}
/// Insert changes trie for given block.
pub fn insert(&self, block: u64, changes_trie_root: H::Out, trie: MemoryDB<H>) {
pub fn insert(&self, block: Number, changes_trie_root: H::Out, trie: MemoryDB<H>) {
let mut data = self.data.write();
data.roots.insert(block, changes_trie_root);
data.mdb.consolidate(trie);
}
}
impl<H: Hasher> RootsStorage<H> for InMemoryStorage<H> {
fn root(&self, _anchor_block: &AnchorBlockId<H::Out>, block: u64) -> Result<Option<H::Out>, String> {
impl<H: Hasher, Number: BlockNumber> RootsStorage<H, Number> for InMemoryStorage<H, Number> {
fn build_anchor(&self, parent_hash: H::Out) -> Result<AnchorBlockId<H::Out, Number>, String> {
self.data.read().roots.iter()
.find(|(_, v)| **v == parent_hash)
.map(|(k, _)| AnchorBlockId { hash: parent_hash, number: k.clone() })
.ok_or_else(|| format!("Can't find associated number for block {:?}", parent_hash))
}
fn root(&self, _anchor_block: &AnchorBlockId<H::Out, Number>, block: Number) -> Result<Option<H::Out>, String> {
Ok(self.data.read().roots.get(&block).cloned())
}
}
impl<H: Hasher> Storage<H> for InMemoryStorage<H> {
impl<H: Hasher, Number: BlockNumber> Storage<H, Number> for InMemoryStorage<H, Number> {
fn get(&self, key: &H::Out, prefix: &[u8]) -> Result<Option<DBValue>, String> {
MemoryDB::<H>::get(&self.data.read().mdb, key, prefix)
}
}
impl<'a, H: Hasher, S: 'a + Storage<H>> TrieBackendAdapter<'a, H, S> {
impl<'a, H: Hasher, Number: BlockNumber, S: 'a + Storage<H, Number>> TrieBackendAdapter<'a, H, Number, S> {
pub fn new(storage: &'a S) -> Self {
Self { storage, _hasher: Default::default() }
}
}
impl<'a, H: Hasher, S: 'a + Storage<H>> TrieBackendStorage<H> for TrieBackendAdapter<'a, H, S> {
impl<'a, H, Number, S> TrieBackendStorage<H> for TrieBackendAdapter<'a, H, Number, S>
where
S: 'a + Storage<H, Number>,
Number: BlockNumber,
H: Hasher,
{
type Overlay = MemoryDB<H>;
fn get(&self, key: &H::Out, prefix: &[u8]) -> Result<Option<DBValue>, String> {