diff --git a/substrate/core/client/db/src/lib.rs b/substrate/core/client/db/src/lib.rs index 927359ecdf..a6301596ad 100644 --- a/substrate/core/client/db/src/lib.rs +++ b/substrate/core/client/db/src/lib.rs @@ -681,6 +681,10 @@ impl state_machine::ChangesTrieStorage> where Block: BlockT, { + fn as_roots_storage(&self) -> &dyn state_machine::ChangesTrieRootsStorage> { + self + } + fn get(&self, key: &H256, _prefix: Prefix) -> Result, String> { self.db.get(columns::CHANGES_TRIE, &key[..]) .map_err(|err| format!("{}", err)) diff --git a/substrate/core/client/src/client.rs b/substrate/core/client/src/client.rs index a57584575c..5b0cba3a40 100644 --- a/substrate/core/client/src/client.rs +++ b/substrate/core/client/src/client.rs @@ -43,7 +43,7 @@ use sr_primitives::{ use state_machine::{ DBValue, Backend as StateBackend, CodeExecutor, ChangesTrieAnchorBlockId, ExecutionStrategy, ExecutionManager, prove_read, prove_child_read, - ChangesTrieRootsStorage, ChangesTrieStorage, + ChangesTrieRootsStorage, ChangesTrieStorage, ChangesTrieConfigurationRange, key_changes, key_changes_proof, OverlayedChanges, NeverOffchainExt, }; use executor::{RuntimeVersion, RuntimeInfo}; @@ -554,8 +554,15 @@ impl Client where let last_number = self.backend.blockchain().expect_block_number_from_id(&last)?; let last_hash = self.backend.blockchain().expect_block_hash_from_id(&last)?; - key_changes::<_, Blake2Hasher, _>( - &config, + // FIXME: remove this in https://github.com/paritytech/substrate/pull/3201 + let config_range = ChangesTrieConfigurationRange { + config: &config, + zero: Zero::zero(), + end: None, + }; + + key_changes::( + config_range, &*storage, first, &ChangesTrieAnchorBlockId { @@ -632,6 +639,10 @@ impl Client where } impl<'a, Block: BlockT> ChangesTrieStorage> for AccessedRootsRecorder<'a, Block> { + fn as_roots_storage(&self) -> &dyn state_machine::ChangesTrieRootsStorage> { + self + } + fn get(&self, key: &H256, prefix: Prefix) -> Result, String> { self.storage.get(key, prefix) } @@ -651,13 +662,20 @@ impl Client where self.backend.blockchain().expect_block_number_from_id(&BlockId::Hash(max))?, ); + // FIXME: remove this in https://github.com/paritytech/substrate/pull/3201 + let config_range = ChangesTrieConfigurationRange { + config: &config, + zero: Zero::zero(), + end: None, + }; + // fetch key changes proof let first_number = self.backend.blockchain() .expect_block_number_from_id(&BlockId::Hash(first))?; let last_number = self.backend.blockchain() .expect_block_number_from_id(&BlockId::Hash(last))?; - let key_changes_proof = key_changes_proof::<_, Blake2Hasher, _>( - &config, + let key_changes_proof = key_changes_proof::( + config_range, &recording_storage, first_number, &ChangesTrieAnchorBlockId { diff --git a/substrate/core/client/src/in_mem.rs b/substrate/core/client/src/in_mem.rs index c43c4e3197..1a6881ea8d 100644 --- a/substrate/core/client/src/in_mem.rs +++ b/substrate/core/client/src/in_mem.rs @@ -758,6 +758,10 @@ impl state_machine::ChangesTrieStorage> for Change Block: BlockT, H: Hasher, { + fn as_roots_storage(&self) -> &dyn state_machine::ChangesTrieRootsStorage> { + self + } + fn get(&self, key: &H::Out, prefix: Prefix) -> Result, String> { self.0.get(key, prefix) } diff --git a/substrate/core/client/src/light/fetcher.rs b/substrate/core/client/src/light/fetcher.rs index 22d3e8fdb7..8502a19ba6 100644 --- a/substrate/core/client/src/light/fetcher.rs +++ b/substrate/core/client/src/light/fetcher.rs @@ -26,11 +26,14 @@ use codec::{Decode, Encode}; use primitives::{ChangesTrieConfiguration, convert_hash}; use sr_primitives::traits::{ Block as BlockT, Header as HeaderT, Hash, HashFor, NumberFor, - SimpleArithmetic, CheckedConversion, + SimpleArithmetic, CheckedConversion, Zero, }; -use state_machine::{CodeExecutor, ChangesTrieRootsStorage, ChangesTrieAnchorBlockId, +use state_machine::{ + CodeExecutor, ChangesTrieRootsStorage, + ChangesTrieAnchorBlockId, ChangesTrieConfigurationRange, TrieBackend, read_proof_check, key_changes_proof_check, - create_proof_check_backend_storage, read_child_proof_check}; + create_proof_check_backend_storage, read_child_proof_check, +}; use crate::cht; use crate::error::{Error as ClientError, Result as ClientResult}; @@ -283,9 +286,16 @@ impl, F> LightDataChecker( - &request.changes_trie_config, + key_changes_proof_check::( + changes_trie_config_range, &RootsStorage { roots: (request.tries_roots.0, &request.tries_roots.2), prev_roots: remote_roots, diff --git a/substrate/core/primitives/src/changes_trie.rs b/substrate/core/primitives/src/changes_trie.rs index fb7791f0a0..5e88485a03 100644 --- a/substrate/core/primitives/src/changes_trie.rs +++ b/substrate/core/primitives/src/changes_trie.rs @@ -39,19 +39,30 @@ pub struct ChangesTrieConfiguration { } impl ChangesTrieConfiguration { + /// Create new configuration given digest interval and levels. + pub fn new(digest_interval: u32, digest_levels: u32) -> Self { + Self { digest_interval, digest_levels } + } + /// Is digest build enabled? pub fn is_digest_build_enabled(&self) -> bool { self.digest_interval > 1 && self.digest_levels > 0 } /// Do we need to build digest at given block? - pub fn is_digest_build_required_at_block(&self, block: Number) -> bool + pub fn is_digest_build_required_at_block( + &self, + zero: Number, + block: Number, + ) -> bool where - Number: From + PartialEq + ::rstd::ops::Rem + Zero, + Number: From + PartialEq + + ::rstd::ops::Rem + ::rstd::ops::Sub + + ::rstd::cmp::PartialOrd + Zero, { - block != 0.into() + block > zero && self.is_digest_build_enabled() - && (block % self.digest_interval.into()).is_zero() + && ((block - zero) % self.digest_interval.into()).is_zero() } /// Returns max digest interval. One if digests are not created at all. @@ -71,6 +82,74 @@ impl ChangesTrieConfiguration { } } + /// Returns max level digest block number that has been created at block <= passed block number. + /// + /// Returns None if digests are not created at all. + pub fn prev_max_level_digest_block( + &self, + zero: Number, + block: Number, + ) -> Option + where + Number: Clone + From + PartialOrd + PartialEq + + ::rstd::ops::Add + ::rstd::ops::Sub + + ::rstd::ops::Div + ::rstd::ops::Mul + Zero, + { + if block <= zero { + return None; + } + + let (next_begin, next_end) = self.next_max_level_digest_range(zero.clone(), block.clone())?; + + // if 'next' digest includes our block, then it is a also a previous digest + if next_end == block { + return Some(block); + } + + // if previous digest ends at zero block, then there are no previous digest + let prev_end = next_begin - 1.into(); + if prev_end == zero { + None + } else { + Some(prev_end) + } + } + + /// Returns max level digest blocks range (inclusive) which includes passed block. + /// + /// Returns None if digests are not created at all. + /// It will return the first max-level digest if block is <= zero. + pub fn next_max_level_digest_range( + &self, + zero: Number, + mut block: Number, + ) -> Option<(Number, Number)> + where + Number: Clone + From + PartialOrd + PartialEq + + ::rstd::ops::Add + ::rstd::ops::Sub + + ::rstd::ops::Div + ::rstd::ops::Mul, + { + if !self.is_digest_build_enabled() { + return None; + } + + if block <= zero { + block = zero.clone() + 1.into(); + } + + let max_digest_interval: Number = self.max_digest_interval().into(); + let max_digests_since_zero = (block.clone() - zero.clone()) / max_digest_interval.clone(); + if max_digests_since_zero == 0.into() { + return Some((zero.clone() + 1.into(), zero + max_digest_interval)); + } + let last_max_digest_block = zero + max_digests_since_zero * max_digest_interval.clone(); + Some(if block == last_max_digest_block { + (block.clone() - max_digest_interval + 1.into(), block) + } else { + (last_max_digest_block.clone() + 1.into(), last_max_digest_block + max_digest_interval) + }) + } + /// Returns Some if digest must be built at given block number. /// The tuple is: /// ( @@ -78,20 +157,23 @@ impl ChangesTrieConfiguration { /// digest interval (in blocks) /// step between blocks we're interested in when digest is built /// ) - pub fn digest_level_at_block(&self, block: Number) -> Option<(u32, u32, u32)> + pub fn digest_level_at_block(&self, zero: Number, block: Number) -> Option<(u32, u32, u32)> where - Number: Clone + From + PartialEq + ::rstd::ops::Rem + Zero, + Number: Clone + From + PartialEq + + ::rstd::ops::Rem + ::rstd::ops::Sub + + ::rstd::cmp::PartialOrd + Zero, { - if !self.is_digest_build_required_at_block(block.clone()) { + if !self.is_digest_build_required_at_block(zero.clone(), block.clone()) { return None; } + let relative_block = block - zero; let mut digest_interval = self.digest_interval; let mut current_level = 1u32; let mut digest_step = 1u32; while current_level < self.digest_levels { let new_digest_interval = match digest_interval.checked_mul(self.digest_interval) { - Some(new_digest_interval) if (block.clone() % new_digest_interval.into()).is_zero() + Some(new_digest_interval) if (relative_block.clone() % new_digest_interval.into()).is_zero() => new_digest_interval, _ => break, }; @@ -131,31 +213,43 @@ mod tests { #[test] fn is_digest_build_required_at_block_works() { - assert!(!config(8, 4).is_digest_build_required_at_block(0u64)); - assert!(!config(8, 4).is_digest_build_required_at_block(1u64)); - assert!(!config(8, 4).is_digest_build_required_at_block(2u64)); - assert!(!config(8, 4).is_digest_build_required_at_block(4u64)); - assert!(config(8, 4).is_digest_build_required_at_block(8u64)); - assert!(!config(8, 4).is_digest_build_required_at_block(9u64)); - assert!(config(8, 4).is_digest_build_required_at_block(64u64)); - assert!(config(8, 4).is_digest_build_required_at_block(64u64)); - assert!(config(8, 4).is_digest_build_required_at_block(512u64)); - assert!(config(8, 4).is_digest_build_required_at_block(4096u64)); - assert!(!config(8, 4).is_digest_build_required_at_block(4103u64)); - assert!(config(8, 4).is_digest_build_required_at_block(4104u64)); - assert!(!config(8, 4).is_digest_build_required_at_block(4108u64)); + fn test_with_zero(zero: u64) { + assert!(!config(8, 4).is_digest_build_required_at_block(zero, zero + 0u64)); + assert!(!config(8, 4).is_digest_build_required_at_block(zero, zero + 1u64)); + assert!(!config(8, 4).is_digest_build_required_at_block(zero, zero + 2u64)); + assert!(!config(8, 4).is_digest_build_required_at_block(zero, zero + 4u64)); + assert!(config(8, 4).is_digest_build_required_at_block(zero, zero + 8u64)); + assert!(!config(8, 4).is_digest_build_required_at_block(zero, zero + 9u64)); + assert!(config(8, 4).is_digest_build_required_at_block(zero, zero + 64u64)); + assert!(config(8, 4).is_digest_build_required_at_block(zero, zero + 64u64)); + assert!(config(8, 4).is_digest_build_required_at_block(zero, zero + 512u64)); + assert!(config(8, 4).is_digest_build_required_at_block(zero, zero + 4096u64)); + assert!(!config(8, 4).is_digest_build_required_at_block(zero, zero + 4103u64)); + assert!(config(8, 4).is_digest_build_required_at_block(zero, zero + 4104u64)); + assert!(!config(8, 4).is_digest_build_required_at_block(zero, zero + 4108u64)); + } + + test_with_zero(0); + test_with_zero(8); + test_with_zero(17); } #[test] fn digest_level_at_block_works() { - assert_eq!(config(8, 4).digest_level_at_block(0u64), None); - assert_eq!(config(8, 4).digest_level_at_block(7u64), None); - assert_eq!(config(8, 4).digest_level_at_block(63u64), None); - assert_eq!(config(8, 4).digest_level_at_block(8u64), Some((1, 8, 1))); - assert_eq!(config(8, 4).digest_level_at_block(64u64), Some((2, 64, 8))); - assert_eq!(config(8, 4).digest_level_at_block(512u64), Some((3, 512, 64))); - assert_eq!(config(8, 4).digest_level_at_block(4096u64), Some((4, 4096, 512))); - assert_eq!(config(8, 4).digest_level_at_block(4112u64), Some((1, 8, 1))); + fn test_with_zero(zero: u64) { + assert_eq!(config(8, 4).digest_level_at_block(zero, zero + 0u64), None); + assert_eq!(config(8, 4).digest_level_at_block(zero, zero + 7u64), None); + assert_eq!(config(8, 4).digest_level_at_block(zero, zero + 63u64), None); + assert_eq!(config(8, 4).digest_level_at_block(zero, zero + 8u64), Some((1, 8, 1))); + assert_eq!(config(8, 4).digest_level_at_block(zero, zero + 64u64), Some((2, 64, 8))); + assert_eq!(config(8, 4).digest_level_at_block(zero, zero + 512u64), Some((3, 512, 64))); + assert_eq!(config(8, 4).digest_level_at_block(zero, zero + 4096u64), Some((4, 4096, 512))); + assert_eq!(config(8, 4).digest_level_at_block(zero, zero + 4112u64), Some((1, 8, 1))); + } + + test_with_zero(0); + test_with_zero(8); + test_with_zero(17); } #[test] @@ -165,4 +259,36 @@ mod tests { assert_eq!(config(8, 4).max_digest_interval(), 4096); assert_eq!(config(::std::u32::MAX, 1024).max_digest_interval(), ::std::u32::MAX); } + + #[test] + fn next_max_level_digest_range_works() { + assert_eq!(config(0, 0).next_max_level_digest_range(0u64, 16), None); + assert_eq!(config(1, 1).next_max_level_digest_range(0u64, 16), None); + assert_eq!(config(2, 1).next_max_level_digest_range(0u64, 16), Some((15, 16))); + assert_eq!(config(4, 1).next_max_level_digest_range(0u64, 16), Some((13, 16))); + assert_eq!(config(32, 1).next_max_level_digest_range(0u64, 16), Some((1, 32))); + assert_eq!(config(2, 3).next_max_level_digest_range(0u64, 10), Some((9, 16))); + assert_eq!(config(2, 3).next_max_level_digest_range(0u64, 8), Some((1, 8))); + assert_eq!(config(2, 1).next_max_level_digest_range(1u64, 1), Some((2, 3))); + assert_eq!(config(2, 2).next_max_level_digest_range(7u64, 9), Some((8, 11))); + + assert_eq!(config(2, 2).next_max_level_digest_range(7u64, 5), Some((8, 11))); + } + + #[test] + fn prev_max_level_digest_block_works() { + assert_eq!(config(0, 0).prev_max_level_digest_block(0u64, 16), None); + assert_eq!(config(1, 1).prev_max_level_digest_block(0u64, 16), None); + assert_eq!(config(2, 1).prev_max_level_digest_block(0u64, 16), Some(16)); + assert_eq!(config(4, 1).prev_max_level_digest_block(0u64, 16), Some(16)); + assert_eq!(config(4, 2).prev_max_level_digest_block(0u64, 16), Some(16)); + assert_eq!(config(4, 2).prev_max_level_digest_block(0u64, 17), Some(16)); + assert_eq!(config(4, 2).prev_max_level_digest_block(0u64, 33), Some(32)); + assert_eq!(config(32, 1).prev_max_level_digest_block(0u64, 16), None); + assert_eq!(config(2, 3).prev_max_level_digest_block(0u64, 10), Some(8)); + assert_eq!(config(2, 3).prev_max_level_digest_block(0u64, 8), Some(8)); + assert_eq!(config(2, 2).prev_max_level_digest_block(7u64, 8), None); + + assert_eq!(config(2, 2).prev_max_level_digest_block(7u64, 5), None); + } } diff --git a/substrate/core/state-machine/src/changes_trie/build.rs b/substrate/core/state-machine/src/changes_trie/build.rs index e1e3f6a808..cc2185ee79 100644 --- a/substrate/core/state-machine/src/changes_trie/build.rs +++ b/substrate/core/state-machine/src/changes_trie/build.rs @@ -26,22 +26,21 @@ use crate::overlayed_changes::OverlayedChanges; use crate::trie_backend_essence::TrieBackendEssence; use crate::changes_trie::build_iterator::digest_build_iterator; use crate::changes_trie::input::{InputKey, InputPair, DigestIndex, ExtrinsicIndex}; -use crate::changes_trie::{AnchorBlockId, Configuration, Storage, BlockNumber}; +use crate::changes_trie::{AnchorBlockId, ConfigurationRange, Storage, BlockNumber}; /// Prepare input pairs for building a changes trie of given block. /// /// Returns Err if storage error has occurred OR if storage haven't returned /// required data. -pub fn prepare_input<'a, B, S, H, Number>( +pub fn prepare_input<'a, B, H, Number>( backend: &'a B, - storage: &'a S, - config: &'a Configuration, + storage: &'a dyn Storage, + config: ConfigurationRange<'a, Number>, changes: &'a OverlayedChanges, parent: &'a AnchorBlockId, ) -> Result> + 'a, String> where B: Backend, - S: Storage, H: Hasher + 'a, Number: BlockNumber, { @@ -50,7 +49,7 @@ pub fn prepare_input<'a, B, S, H, Number>( backend, &number, changes)?; - let digest_input = prepare_digest_input::<_, H, Number>( + let digest_input = prepare_digest_input::( parent, config, number, @@ -111,19 +110,27 @@ fn prepare_extrinsics_input<'a, B, H, Number>( } /// Prepare DigestIndex input pairs. -fn prepare_digest_input<'a, S, H, Number>( +fn prepare_digest_input<'a, H, Number>( parent: &'a AnchorBlockId, - config: &Configuration, + config: ConfigurationRange<'a, Number>, block: Number, - storage: &'a S + storage: &'a dyn Storage, ) -> Result> + 'a, String> where - S: Storage, H: Hasher, H::Out: 'a, Number: BlockNumber, { - digest_build_iterator(config, block.clone()) + let build_skewed_digest = config.end.as_ref() == Some(&block); + let block_for_digest = if build_skewed_digest { + config.config.next_max_level_digest_range(config.zero.clone(), block.clone()) + .map(|(_, end)| end) + .unwrap_or_else(|| block.clone()) + } else { + block.clone() + }; + + digest_build_iterator(config, block_for_digest) .try_fold(BTreeMap::new(), move |mut map, digest_build_block| { let trie_root = storage.root(parent, digest_build_block.clone())?; let trie_root = trie_root.ok_or_else(|| format!("No changes trie root for block {}", digest_build_block.clone()))?; @@ -177,11 +184,18 @@ mod test { use primitives::Blake2Hasher; use primitives::storage::well_known_keys::EXTRINSIC_INDEX; use crate::backend::InMemory; + use crate::changes_trie::Configuration; use crate::changes_trie::storage::InMemoryStorage; use crate::overlayed_changes::OverlayedValue; use super::*; - fn prepare_for_build() -> (InMemory, InMemoryStorage, OverlayedChanges) { + fn prepare_for_build(zero: u64) -> ( + InMemory, + InMemoryStorage, + OverlayedChanges, + Configuration, + ) { + let config = Configuration { digest_interval: 4, digest_levels: 2 }; let backend: InMemory<_> = vec![ (vec![100], vec![255]), (vec![101], vec![255]), @@ -191,38 +205,38 @@ mod test { (vec![105], vec![255]), ].into_iter().collect::<::std::collections::HashMap<_, _>>().into(); let storage = InMemoryStorage::with_inputs(vec![ - (1, vec![ - InputPair::ExtrinsicIndex(ExtrinsicIndex { block: 1, key: vec![100] }, vec![1, 3]), - InputPair::ExtrinsicIndex(ExtrinsicIndex { block: 1, key: vec![101] }, vec![0, 2]), - InputPair::ExtrinsicIndex(ExtrinsicIndex { block: 1, key: vec![105] }, vec![0, 2, 4]), + (zero + 1, vec![ + InputPair::ExtrinsicIndex(ExtrinsicIndex { block: zero + 1, key: vec![100] }, vec![1, 3]), + InputPair::ExtrinsicIndex(ExtrinsicIndex { block: zero + 1, key: vec![101] }, vec![0, 2]), + InputPair::ExtrinsicIndex(ExtrinsicIndex { block: zero + 1, key: vec![105] }, vec![0, 2, 4]), ]), - (2, vec![ - InputPair::ExtrinsicIndex(ExtrinsicIndex { block: 2, key: vec![102] }, vec![0]), + (zero + 2, vec![ + InputPair::ExtrinsicIndex(ExtrinsicIndex { block: zero + 2, key: vec![102] }, vec![0]), ]), - (3, vec![ - InputPair::ExtrinsicIndex(ExtrinsicIndex { block: 3, key: vec![100] }, vec![0]), - InputPair::ExtrinsicIndex(ExtrinsicIndex { block: 3, key: vec![105] }, vec![1]), + (zero + 3, vec![ + InputPair::ExtrinsicIndex(ExtrinsicIndex { block: zero + 3, key: vec![100] }, vec![0]), + InputPair::ExtrinsicIndex(ExtrinsicIndex { block: zero + 3, key: vec![105] }, vec![1]), ]), - (4, vec![ - InputPair::ExtrinsicIndex(ExtrinsicIndex { block: 4, key: vec![100] }, vec![0, 2, 3]), - InputPair::ExtrinsicIndex(ExtrinsicIndex { block: 4, key: vec![101] }, vec![1]), - InputPair::ExtrinsicIndex(ExtrinsicIndex { block: 4, key: vec![103] }, vec![0, 1]), + (zero + 4, vec![ + InputPair::ExtrinsicIndex(ExtrinsicIndex { block: zero + 4, key: vec![100] }, vec![0, 2, 3]), + InputPair::ExtrinsicIndex(ExtrinsicIndex { block: zero + 4, key: vec![101] }, vec![1]), + InputPair::ExtrinsicIndex(ExtrinsicIndex { block: zero + 4, key: vec![103] }, vec![0, 1]), - InputPair::DigestIndex(DigestIndex { block: 4, key: vec![100] }, vec![1, 3]), - InputPair::DigestIndex(DigestIndex { block: 4, key: vec![101] }, vec![1]), - InputPair::DigestIndex(DigestIndex { block: 4, key: vec![102] }, vec![2]), - InputPair::DigestIndex(DigestIndex { block: 4, key: vec![105] }, vec![1, 3]), + InputPair::DigestIndex(DigestIndex { block: zero + 4, key: vec![100] }, vec![zero + 1, zero + 3]), + InputPair::DigestIndex(DigestIndex { block: zero + 4, key: vec![101] }, vec![zero + 1]), + InputPair::DigestIndex(DigestIndex { block: zero + 4, key: vec![102] }, vec![zero + 2]), + InputPair::DigestIndex(DigestIndex { block: zero + 4, key: vec![105] }, vec![zero + 1, zero + 3]), ]), - (5, Vec::new()), - (6, vec![ - InputPair::ExtrinsicIndex(ExtrinsicIndex { block: 6, key: vec![105] }, vec![2]), + (zero + 5, Vec::new()), + (zero + 6, vec![ + InputPair::ExtrinsicIndex(ExtrinsicIndex { block: zero + 6, key: vec![105] }, vec![2]), ]), - (7, Vec::new()), - (8, vec![ - InputPair::DigestIndex(DigestIndex { block: 8, key: vec![105] }, vec![6]), + (zero + 7, Vec::new()), + (zero + 8, vec![ + InputPair::DigestIndex(DigestIndex { block: zero + 8, key: vec![105] }, vec![zero + 6]), ]), - (9, Vec::new()), (10, Vec::new()), (11, Vec::new()), (12, Vec::new()), (13, Vec::new()), - (14, Vec::new()), (15, Vec::new()), + (zero + 9, Vec::new()), (zero + 10, Vec::new()), (zero + 11, Vec::new()), (zero + 12, Vec::new()), + (zero + 13, Vec::new()), (zero + 14, Vec::new()), (zero + 15, Vec::new()), ]); let changes = OverlayedChanges { prospective: vec![ @@ -249,108 +263,182 @@ mod test { extrinsics: Some(vec![1].into_iter().collect()) }), ].into_iter().collect(), - changes_trie_config: Some(Configuration { digest_interval: 4, digest_levels: 2 }), + changes_trie_config: Some(config.clone()), }; - (backend, storage, changes) + (backend, storage, changes, config) + } + + fn configuration_range<'a>(config: &'a Configuration, zero: u64) -> ConfigurationRange<'a, u64> { + ConfigurationRange { + config, + zero, + end: None, + } } #[test] fn build_changes_trie_nodes_on_non_digest_block() { - let (backend, storage, changes) = prepare_for_build(); - let config = changes.changes_trie_config.as_ref().unwrap(); - let parent = AnchorBlockId { hash: Default::default(), number: 4 }; - let changes_trie_nodes = prepare_input( - &backend, - &storage, - config, - &changes, - &parent, - ).unwrap(); - assert_eq!(changes_trie_nodes.collect::>>(), vec![ - InputPair::ExtrinsicIndex(ExtrinsicIndex { block: 5, key: vec![100] }, vec![0, 2, 3]), - InputPair::ExtrinsicIndex(ExtrinsicIndex { block: 5, key: vec![101] }, vec![1]), - InputPair::ExtrinsicIndex(ExtrinsicIndex { block: 5, key: vec![103] }, vec![0, 1]), - ]); + fn test_with_zero(zero: u64) { + let (backend, storage, changes, config) = prepare_for_build(zero); + let parent = AnchorBlockId { hash: Default::default(), number: zero + 4 }; + let changes_trie_nodes = prepare_input( + &backend, + &storage, + configuration_range(&config, zero), + &changes, + &parent, + ).unwrap(); + assert_eq!(changes_trie_nodes.collect::>>(), vec![ + InputPair::ExtrinsicIndex(ExtrinsicIndex { block: zero + 5, key: vec![100] }, vec![0, 2, 3]), + InputPair::ExtrinsicIndex(ExtrinsicIndex { block: zero + 5, key: vec![101] }, vec![1]), + InputPair::ExtrinsicIndex(ExtrinsicIndex { block: zero + 5, key: vec![103] }, vec![0, 1]), + ]); + } + + test_with_zero(0); + test_with_zero(16); + test_with_zero(17); } #[test] fn build_changes_trie_nodes_on_digest_block_l1() { - let (backend, storage, changes) = prepare_for_build(); - let config = changes.changes_trie_config.as_ref().unwrap(); - let parent = AnchorBlockId { hash: Default::default(), number: 3 }; - let changes_trie_nodes = prepare_input( - &backend, - &storage, - config, - &changes, - &parent, - ).unwrap(); - assert_eq!(changes_trie_nodes.collect::>>(), vec![ - InputPair::ExtrinsicIndex(ExtrinsicIndex { block: 4, key: vec![100] }, vec![0, 2, 3]), - InputPair::ExtrinsicIndex(ExtrinsicIndex { block: 4, key: vec![101] }, vec![1]), - InputPair::ExtrinsicIndex(ExtrinsicIndex { block: 4, key: vec![103] }, vec![0, 1]), + fn test_with_zero(zero: u64) { + let (backend, storage, changes, config) = prepare_for_build(zero); + let parent = AnchorBlockId { hash: Default::default(), number: zero + 3 }; + let changes_trie_nodes = prepare_input( + &backend, + &storage, + configuration_range(&config, zero), + &changes, + &parent, + ).unwrap(); + assert_eq!(changes_trie_nodes.collect::>>(), vec![ + InputPair::ExtrinsicIndex(ExtrinsicIndex { block: zero + 4, key: vec![100] }, vec![0, 2, 3]), + InputPair::ExtrinsicIndex(ExtrinsicIndex { block: zero + 4, key: vec![101] }, vec![1]), + InputPair::ExtrinsicIndex(ExtrinsicIndex { block: zero + 4, key: vec![103] }, vec![0, 1]), - InputPair::DigestIndex(DigestIndex { block: 4, key: vec![100] }, vec![1, 3]), - InputPair::DigestIndex(DigestIndex { block: 4, key: vec![101] }, vec![1]), - InputPair::DigestIndex(DigestIndex { block: 4, key: vec![102] }, vec![2]), - InputPair::DigestIndex(DigestIndex { block: 4, key: vec![105] }, vec![1, 3]), - ]); + InputPair::DigestIndex(DigestIndex { block: zero + 4, key: vec![100] }, vec![zero + 1, zero + 3]), + InputPair::DigestIndex(DigestIndex { block: zero + 4, key: vec![101] }, vec![zero + 1]), + InputPair::DigestIndex(DigestIndex { block: zero + 4, key: vec![102] }, vec![zero + 2]), + InputPair::DigestIndex(DigestIndex { block: zero + 4, key: vec![105] }, vec![zero + 1, zero + 3]), + ]); + } + + test_with_zero(0); + test_with_zero(16); + test_with_zero(17); } #[test] fn build_changes_trie_nodes_on_digest_block_l2() { - let (backend, storage, changes) = prepare_for_build(); - let config = changes.changes_trie_config.as_ref().unwrap(); - let parent = AnchorBlockId { hash: Default::default(), number: 15 }; - let changes_trie_nodes = prepare_input( - &backend, - &storage, - config, - &changes, - &parent, - ).unwrap(); - assert_eq!(changes_trie_nodes.collect::>>(), vec![ - InputPair::ExtrinsicIndex(ExtrinsicIndex { block: 16, key: vec![100] }, vec![0, 2, 3]), - InputPair::ExtrinsicIndex(ExtrinsicIndex { block: 16, key: vec![101] }, vec![1]), - InputPair::ExtrinsicIndex(ExtrinsicIndex { block: 16, key: vec![103] }, vec![0, 1]), + fn test_with_zero(zero: u64) { + let (backend, storage, changes, config) = prepare_for_build(zero); + let parent = AnchorBlockId { hash: Default::default(), number: zero + 15 }; + let changes_trie_nodes = prepare_input( + &backend, + &storage, + configuration_range(&config, zero), + &changes, + &parent, + ).unwrap(); + assert_eq!(changes_trie_nodes.collect::>>(), vec![ + InputPair::ExtrinsicIndex(ExtrinsicIndex { block: zero + 16, key: vec![100] }, vec![0, 2, 3]), + InputPair::ExtrinsicIndex(ExtrinsicIndex { block: zero + 16, key: vec![101] }, vec![1]), + InputPair::ExtrinsicIndex(ExtrinsicIndex { block: zero + 16, key: vec![103] }, vec![0, 1]), - InputPair::DigestIndex(DigestIndex { block: 16, key: vec![100] }, vec![4]), - InputPair::DigestIndex(DigestIndex { block: 16, key: vec![101] }, vec![4]), - InputPair::DigestIndex(DigestIndex { block: 16, key: vec![102] }, vec![4]), - InputPair::DigestIndex(DigestIndex { block: 16, key: vec![103] }, vec![4]), - InputPair::DigestIndex(DigestIndex { block: 16, key: vec![105] }, vec![4, 8]), - ]); + InputPair::DigestIndex(DigestIndex { block: zero + 16, key: vec![100] }, vec![zero + 4]), + InputPair::DigestIndex(DigestIndex { block: zero + 16, key: vec![101] }, vec![zero + 4]), + InputPair::DigestIndex(DigestIndex { block: zero + 16, key: vec![102] }, vec![zero + 4]), + InputPair::DigestIndex(DigestIndex { block: zero + 16, key: vec![103] }, vec![zero + 4]), + InputPair::DigestIndex(DigestIndex { block: zero + 16, key: vec![105] }, vec![zero + 4, zero + 8]), + ]); + } + + test_with_zero(0); + test_with_zero(16); + test_with_zero(17); + } + + #[test] + fn build_changes_trie_nodes_on_skewed_digest_block() { + fn test_with_zero(zero: u64) { + let (backend, storage, changes, config) = prepare_for_build(zero); + let parent = AnchorBlockId { hash: Default::default(), number: zero + 10 }; + + let mut configuration_range = configuration_range(&config, zero); + let changes_trie_nodes = prepare_input( + &backend, + &storage, + configuration_range.clone(), + &changes, + &parent, + ).unwrap(); + assert_eq!(changes_trie_nodes.collect::>>(), vec![ + InputPair::ExtrinsicIndex(ExtrinsicIndex { block: zero + 11, key: vec![100] }, vec![0, 2, 3]), + InputPair::ExtrinsicIndex(ExtrinsicIndex { block: zero + 11, key: vec![101] }, vec![1]), + InputPair::ExtrinsicIndex(ExtrinsicIndex { block: zero + 11, key: vec![103] }, vec![0, 1]), + ]); + + configuration_range.end = Some(zero + 11); + let changes_trie_nodes = prepare_input( + &backend, + &storage, + configuration_range, + &changes, + &parent, + ).unwrap(); + assert_eq!(changes_trie_nodes.collect::>>(), vec![ + InputPair::ExtrinsicIndex(ExtrinsicIndex { block: zero + 11, key: vec![100] }, vec![0, 2, 3]), + InputPair::ExtrinsicIndex(ExtrinsicIndex { block: zero + 11, key: vec![101] }, vec![1]), + InputPair::ExtrinsicIndex(ExtrinsicIndex { block: zero + 11, key: vec![103] }, vec![0, 1]), + + InputPair::DigestIndex(DigestIndex { block: zero + 11, key: vec![100] }, vec![zero + 4]), + InputPair::DigestIndex(DigestIndex { block: zero + 11, key: vec![101] }, vec![zero + 4]), + InputPair::DigestIndex(DigestIndex { block: zero + 11, key: vec![102] }, vec![zero + 4]), + InputPair::DigestIndex(DigestIndex { block: zero + 11, key: vec![103] }, vec![zero + 4]), + InputPair::DigestIndex(DigestIndex { block: zero + 11, key: vec![105] }, vec![zero + 4, zero + 8]), + ]); + } + + test_with_zero(0); + test_with_zero(16); + test_with_zero(17); } #[test] fn build_changes_trie_nodes_ignores_temporary_storage_values() { - let (backend, storage, mut changes) = prepare_for_build(); + fn test_with_zero(zero: u64) { + let (backend, storage, mut changes, config) = prepare_for_build(zero); - // 110: missing from backend, set to None in overlay - changes.prospective.top.insert(vec![110], OverlayedValue { - value: None, - extrinsics: Some(vec![1].into_iter().collect()) - }); + // 110: missing from backend, set to None in overlay + changes.prospective.top.insert(vec![110], OverlayedValue { + value: None, + extrinsics: Some(vec![1].into_iter().collect()) + }); - let config = changes.changes_trie_config.as_ref().unwrap(); - let parent = AnchorBlockId { hash: Default::default(), number: 3 }; - let changes_trie_nodes = prepare_input( - &backend, - &storage, - config, - &changes, - &parent, - ).unwrap(); - assert_eq!(changes_trie_nodes.collect::>>(), vec![ - InputPair::ExtrinsicIndex(ExtrinsicIndex { block: 4, key: vec![100] }, vec![0, 2, 3]), - InputPair::ExtrinsicIndex(ExtrinsicIndex { block: 4, key: vec![101] }, vec![1]), - InputPair::ExtrinsicIndex(ExtrinsicIndex { block: 4, key: vec![103] }, vec![0, 1]), + let parent = AnchorBlockId { hash: Default::default(), number: zero + 3 }; + let changes_trie_nodes = prepare_input( + &backend, + &storage, + configuration_range(&config, zero), + &changes, + &parent, + ).unwrap(); + assert_eq!(changes_trie_nodes.collect::>>(), vec![ + InputPair::ExtrinsicIndex(ExtrinsicIndex { block: zero + 4, key: vec![100] }, vec![0, 2, 3]), + InputPair::ExtrinsicIndex(ExtrinsicIndex { block: zero + 4, key: vec![101] }, vec![1]), + InputPair::ExtrinsicIndex(ExtrinsicIndex { block: zero + 4, key: vec![103] }, vec![0, 1]), - InputPair::DigestIndex(DigestIndex { block: 4, key: vec![100] }, vec![1, 3]), - InputPair::DigestIndex(DigestIndex { block: 4, key: vec![101] }, vec![1]), - InputPair::DigestIndex(DigestIndex { block: 4, key: vec![102] }, vec![2]), - InputPair::DigestIndex(DigestIndex { block: 4, key: vec![105] }, vec![1, 3]), - ]); + InputPair::DigestIndex(DigestIndex { block: zero + 4, key: vec![100] }, vec![zero + 1, zero + 3]), + InputPair::DigestIndex(DigestIndex { block: zero + 4, key: vec![101] }, vec![zero + 1]), + InputPair::DigestIndex(DigestIndex { block: zero + 4, key: vec![102] }, vec![zero + 2]), + InputPair::DigestIndex(DigestIndex { block: zero + 4, key: vec![105] }, vec![zero + 1, zero + 3]), + ]); + } + + test_with_zero(0); + test_with_zero(16); + test_with_zero(17); } } diff --git a/substrate/core/state-machine/src/changes_trie/build_iterator.rs b/substrate/core/state-machine/src/changes_trie/build_iterator.rs index f4fff28765..36b6dd1983 100644 --- a/substrate/core/state-machine/src/changes_trie/build_iterator.rs +++ b/substrate/core/state-machine/src/changes_trie/build_iterator.rs @@ -17,59 +17,73 @@ //! Structures and functions to return blocks whose changes are to be included //! in given block's changes trie. -use crate::changes_trie::{Configuration, BlockNumber}; +use num_traits::Zero; +use crate::changes_trie::{ConfigurationRange, BlockNumber}; /// Returns iterator of OTHER blocks that are required for inclusion into /// changes trie of given block. Blocks are guaranteed to be returned in /// ascending order. -pub fn digest_build_iterator( - config: &Configuration, +/// +/// Skewed digest is built IF block >= config.end. +pub fn digest_build_iterator<'a, Number: BlockNumber>( + config: ConfigurationRange<'a, Number>, block: Number, ) -> DigestBuildIterator { // prepare digest build parameters - let (_, _, digest_step) = match config.digest_level_at_block(block.clone()) { + let (_, _, digest_step) = match config.config.digest_level_at_block(config.zero, block.clone()) { Some((current_level, digest_interval, digest_step)) => (current_level, digest_interval, digest_step), None => return DigestBuildIterator::empty(), }; - DigestBuildIterator::new(block, config.digest_interval, digest_step) + DigestBuildIterator::new(block.clone(), config.end.unwrap_or(block), config.config.digest_interval, digest_step) } /// Changes trie build iterator that returns numbers of OTHER blocks that are /// required for inclusion into changes trie of given block. #[derive(Debug)] pub struct DigestBuildIterator { - /// Block we're building changes trie for. + /// Block we're building changes trie for. It could (logically) be a post-end block if we are creating + /// skewed digest. block: Number, - /// Interval for creation digest blocks. + /// Block that is a last block where current configuration is active. We have never yet created anything + /// after this block => digest that we're creating can't reference any blocks that are >= end. + end: Number, + /// Interval of L1 digest blocks. digest_interval: u32, - /// Max step of blocks range. + /// Max step that could be used when digest is created. max_step: u32, + + // Mutable data below: + /// Step of current blocks range. current_step: u32, /// Reverse step of current blocks range. current_step_reverse: u32, /// Current blocks range. current_range: Option>, + /// Last block that we have returned. + last_block: Option, } impl DigestBuildIterator { /// Create new digest build iterator. - pub fn new(block: Number, digest_interval: u32, max_step: u32) -> Self { + pub fn new(block: Number, end: Number, digest_interval: u32, max_step: u32) -> Self { DigestBuildIterator { block, + end, digest_interval, max_step, current_step: max_step, current_step_reverse: 0, current_range: None, + last_block: None, } } /// Create empty digest build iterator. pub fn empty() -> Self { - Self::new(0.into(), 0, 0) + Self::new(Zero::zero(), Zero::zero(), 0, 0) } } @@ -77,39 +91,44 @@ impl Iterator for DigestBuildIterator { type Item = Number; fn next(&mut self) -> Option { - if let Some(next) = self.current_range.as_mut().and_then(|iter| iter.next()) { - return Some(next); + // when we're building skewed digest, we might want to skip some blocks if + // they're not covered by current configuration + loop { + if let Some(next) = self.current_range.as_mut().and_then(|iter| iter.next()) { + if next < self.end { + self.last_block = Some(next.clone()); + return Some(next); + } + } + + // we are safe to use non-checking mul/sub versions here because: + // DigestBuildIterator is created only by internal function that is checking + // that all multiplications/subtractions are safe within max_step limit + + let next_step_reverse = if self.current_step_reverse == 0 { + 1 + } else { + self.current_step_reverse * self.digest_interval + }; + if next_step_reverse > self.max_step { + return None; + } + + self.current_step_reverse = next_step_reverse; + self.current_range = Some(BlocksRange::new( + match self.last_block.clone() { + Some(last_block) => last_block + self.current_step.into(), + None => self.block.clone() - (self.current_step * self.digest_interval - self.current_step).into(), + }, + self.block.clone(), + self.current_step.into(), + )); + + self.current_step = self.current_step / self.digest_interval; + if self.current_step == 0 { + self.current_step = 1; + } } - - // we are safe to use non-checking mul/sub versions here because: - // DigestBuildIterator is created only by internal function that is checking - // that all multiplications/subtractions are safe within max_step limit - - let next_step_reverse = if self.current_step_reverse == 0 { - 1 - } else { - self.current_step_reverse * self.digest_interval - }; - if next_step_reverse > self.max_step { - return None; - } - - self.current_step_reverse = next_step_reverse; - self.current_range = Some(BlocksRange::new( - self.block.clone() - (self.current_step * self.digest_interval - self.current_step).into(), - self.block.clone(), - self.current_step.into(), - )); - - self.current_step = self.current_step / self.digest_interval; - if self.current_step == 0 { - self.current_step = 1; - } - - Some(self.current_range.as_mut() - .expect("assigned one line above; qed") - .next() - .expect("X - I^(N+1) + I^N > X when X,I,N are > 1; qed")) } } @@ -147,102 +166,266 @@ impl Iterator for BlocksRange { #[cfg(test)] mod tests { + use crate::changes_trie::Configuration; use super::*; - fn digest_build_iterator(digest_interval: u32, digest_levels: u32, block: u64) -> DigestBuildIterator { - super::digest_build_iterator(&Configuration { digest_interval, digest_levels }, block) + fn digest_build_iterator( + digest_interval: u32, + digest_levels: u32, + zero: u64, + block: u64, + end: Option, + ) -> DigestBuildIterator { + super::digest_build_iterator( + ConfigurationRange { + config: &Configuration { + digest_interval, + digest_levels, + }, + zero, + end, + }, + block, + ) } - fn digest_build_iterator_basic(digest_interval: u32, digest_levels: u32, block: u64) -> (u64, u32, u32) { - let iter = digest_build_iterator(digest_interval, digest_levels, block); + fn digest_build_iterator_basic( + digest_interval: u32, + digest_levels: u32, + zero: u64, + block: u64, + ) -> (u64, u32, u32) { + let iter = digest_build_iterator(digest_interval, digest_levels, zero, block, None); (iter.block, iter.digest_interval, iter.max_step) } - fn digest_build_iterator_blocks(digest_interval: u32, digest_levels: u32, block: u64) -> Vec { - digest_build_iterator(digest_interval, digest_levels, block).collect() + fn digest_build_iterator_blocks( + digest_interval: u32, + digest_levels: u32, + zero: u64, + block: u64, + end: Option, + ) -> Vec { + digest_build_iterator(digest_interval, digest_levels, zero, block, end).collect() } #[test] fn suggest_digest_inclusion_returns_empty_iterator() { - let empty = (0, 0, 0); - assert_eq!(digest_build_iterator_basic(4, 16, 0), empty, "block is 0"); - assert_eq!(digest_build_iterator_basic(0, 16, 64), empty, "digest_interval is 0"); - assert_eq!(digest_build_iterator_basic(1, 16, 64), empty, "digest_interval is 1"); - assert_eq!(digest_build_iterator_basic(4, 0, 64), empty, "digest_levels is 0"); - assert_eq!(digest_build_iterator_basic(4, 16, 1), empty, "digest is not required for this block"); - assert_eq!(digest_build_iterator_basic(4, 16, 2), empty, "digest is not required for this block"); - assert_eq!(digest_build_iterator_basic(4, 16, 15), empty, "digest is not required for this block"); - assert_eq!(digest_build_iterator_basic(4, 16, 17), empty, "digest is not required for this block"); - assert_eq!(digest_build_iterator_basic( - ::std::u32::MAX / 2 + 1, - 16, - ::std::u64::MAX, - ), empty, "digest_interval * 2 is greater than u64::MAX"); + fn test_with_zero(zero: u64) { + let empty = (0, 0, 0); + assert_eq!(digest_build_iterator_basic(4, 16, zero, zero + 0), empty, "block is 0"); + assert_eq!(digest_build_iterator_basic(0, 16, zero, zero + 64), empty, "digest_interval is 0"); + assert_eq!(digest_build_iterator_basic(1, 16, zero, zero + 64), empty, "digest_interval is 1"); + assert_eq!(digest_build_iterator_basic(4, 0, zero, zero + 64), empty, "digest_levels is 0"); + assert_eq!( + digest_build_iterator_basic(4, 16, zero, zero + 1), + empty, + "digest is not required for this block", + ); + assert_eq!( + digest_build_iterator_basic(4, 16, zero, zero + 2), + empty, + "digest is not required for this block", + ); + assert_eq!( + digest_build_iterator_basic(4, 16, zero, zero + 15), + empty, + "digest is not required for this block", + ); + assert_eq!( + digest_build_iterator_basic(4, 16, zero, zero + 17), + empty, + "digest is not required for this block", + ); + assert_eq!(digest_build_iterator_basic( + ::std::u32::MAX / 2 + 1, + 16, + zero, + ::std::u64::MAX, + ), empty, "digest_interval * 2 is greater than u64::MAX"); + } + + test_with_zero(0); + test_with_zero(1); + test_with_zero(2); + test_with_zero(4); + test_with_zero(17); } #[test] fn suggest_digest_inclusion_returns_level1_iterator() { - assert_eq!(digest_build_iterator_basic(16, 1, 16), (16, 16, 1), "!(block % interval) && first digest level == block"); - assert_eq!(digest_build_iterator_basic(16, 1, 256), (256, 16, 1), "!(block % interval^2), but there's only 1 digest level"); - assert_eq!(digest_build_iterator_basic(16, 2, 32), (32, 16, 1), "second level digest is not required for this block"); - assert_eq!(digest_build_iterator_basic(16, 3, 4080), (4080, 16, 1), "second && third level digest are not required for this block"); + fn test_with_zero(zero: u64) { + assert_eq!( + digest_build_iterator_basic(16, 1, zero, zero + 16), + (zero + 16, 16, 1), + "!(block % interval) && first digest level == block", + ); + assert_eq!( + digest_build_iterator_basic(16, 1, zero, zero + 256), + (zero + 256, 16, 1), + "!(block % interval^2), but there's only 1 digest level", + ); + assert_eq!( + digest_build_iterator_basic(16, 2, zero, zero + 32), + (zero + 32, 16, 1), + "second level digest is not required for this block", + ); + assert_eq!( + digest_build_iterator_basic(16, 3, zero, zero + 4080), + (zero + 4080, 16, 1), + "second && third level digest are not required for this block", + ); + } + + test_with_zero(0); + test_with_zero(16); + test_with_zero(17); } #[test] fn suggest_digest_inclusion_returns_level2_iterator() { - assert_eq!(digest_build_iterator_basic(16, 2, 256), (256, 16, 16), "second level digest"); - assert_eq!(digest_build_iterator_basic(16, 2, 4096), (4096, 16, 16), "!(block % interval^3), but there's only 2 digest levels"); + fn test_with_zero(zero: u64) { + assert_eq!( + digest_build_iterator_basic(16, 2, zero, zero + 256), + (zero + 256, 16, 16), + "second level digest", + ); + assert_eq!( + digest_build_iterator_basic(16, 2, zero, zero + 4096), + (zero + 4096, 16, 16), + "!(block % interval^3), but there's only 2 digest levels", + ); + } + + test_with_zero(0); + test_with_zero(16); + test_with_zero(17); } #[test] fn suggest_digest_inclusion_returns_level3_iterator() { - assert_eq!(digest_build_iterator_basic(16, 3, 4096), (4096, 16, 256), "third level digest: beginning"); - assert_eq!(digest_build_iterator_basic(16, 3, 8192), (8192, 16, 256), "third level digest: next"); + fn test_with_zero(zero: u64) { + assert_eq!( + digest_build_iterator_basic(16, 3, zero, zero + 4096), + (zero + 4096, 16, 256), + "third level digest: beginning", + ); + assert_eq!( + digest_build_iterator_basic(16, 3, zero, zero + 8192), + (zero + 8192, 16, 256), + "third level digest: next", + ); + } + + test_with_zero(0); + test_with_zero(16); + test_with_zero(17); } #[test] fn digest_iterator_returns_level1_blocks() { - assert_eq!(digest_build_iterator_blocks(16, 1, 16), - vec![1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15]); - assert_eq!(digest_build_iterator_blocks(16, 1, 256), - vec![241, 242, 243, 244, 245, 246, 247, 248, 249, 250, 251, 252, 253, 254, 255]); - assert_eq!(digest_build_iterator_blocks(16, 2, 32), - vec![17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31]); - assert_eq!(digest_build_iterator_blocks(16, 3, 4080), - vec![4065, 4066, 4067, 4068, 4069, 4070, 4071, 4072, 4073, 4074, 4075, 4076, 4077, 4078, 4079]); + fn test_with_zero(zero: u64) { + assert_eq!(digest_build_iterator_blocks(16, 1, zero, zero + 16, None), + [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15] + .iter().map(|item| zero + item).collect::>()); + assert_eq!(digest_build_iterator_blocks(16, 1, zero, zero + 256, None), + [241, 242, 243, 244, 245, 246, 247, 248, 249, 250, 251, 252, 253, 254, 255] + .iter().map(|item| zero + item).collect::>()); + assert_eq!(digest_build_iterator_blocks(16, 2, zero, zero + 32, None), + [17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31] + .iter().map(|item| zero + item).collect::>()); + assert_eq!(digest_build_iterator_blocks(16, 3, zero, zero + 4080, None), + [4065, 4066, 4067, 4068, 4069, 4070, 4071, 4072, 4073, 4074, 4075, 4076, 4077, 4078, 4079] + .iter().map(|item| zero + item).collect::>()); + } + + test_with_zero(0); + test_with_zero(16); + test_with_zero(17); } #[test] fn digest_iterator_returns_level1_and_level2_blocks() { - assert_eq!(digest_build_iterator_blocks(16, 2, 256), - vec![ - // level2 points to previous 16-1 level1 digests: - 16, 32, 48, 64, 80, 96, 112, 128, 144, 160, 176, 192, 208, 224, 240, - // level2 is a level1 digest of 16-1 previous blocks: - 241, 242, 243, 244, 245, 246, 247, 248, 249, 250, 251, 252, 253, 254, 255, - ], - ); - assert_eq!(digest_build_iterator_blocks(16, 2, 4096), - vec![ - // level2 points to previous 16-1 level1 digests: - 3856, 3872, 3888, 3904, 3920, 3936, 3952, 3968, 3984, 4000, 4016, 4032, 4048, 4064, 4080, - // level2 is a level1 digest of 16-1 previous blocks: - 4081, 4082, 4083, 4084, 4085, 4086, 4087, 4088, 4089, 4090, 4091, 4092, 4093, 4094, 4095, - ], - ); + fn test_with_zero(zero: u64) { + assert_eq!(digest_build_iterator_blocks(16, 2, zero, zero + 256, None), + [ + // level2 points to previous 16-1 level1 digests: + 16, 32, 48, 64, 80, 96, 112, 128, 144, 160, 176, 192, 208, 224, 240, + // level2 is a level1 digest of 16-1 previous blocks: + 241, 242, 243, 244, 245, 246, 247, 248, 249, 250, 251, 252, 253, 254, 255, + ].iter().map(|item| zero + item).collect::>(), + ); + assert_eq!(digest_build_iterator_blocks(16, 2, zero, zero + 4096, None), + [ + // level2 points to previous 16-1 level1 digests: + 3856, 3872, 3888, 3904, 3920, 3936, 3952, 3968, 3984, 4000, 4016, 4032, 4048, 4064, 4080, + // level2 is a level1 digest of 16-1 previous blocks: + 4081, 4082, 4083, 4084, 4085, 4086, 4087, 4088, 4089, 4090, 4091, 4092, 4093, 4094, 4095, + ].iter().map(|item| zero + item).collect::>(), + ); + } + + test_with_zero(0); + test_with_zero(16); + test_with_zero(17); } #[test] fn digest_iterator_returns_level1_and_level2_and_level3_blocks() { - assert_eq!(digest_build_iterator_blocks(16, 3, 4096), - vec![ - // level3 points to previous 16-1 level2 digests: - 256, 512, 768, 1024, 1280, 1536, 1792, 2048, 2304, 2560, 2816, 3072, 3328, 3584, 3840, - // level3 points to previous 16-1 level1 digests: - 3856, 3872, 3888, 3904, 3920, 3936, 3952, 3968, 3984, 4000, 4016, 4032, 4048, 4064, 4080, - // level3 is a level1 digest of 16-1 previous blocks: - 4081, 4082, 4083, 4084, 4085, 4086, 4087, 4088, 4089, 4090, 4091, 4092, 4093, 4094, 4095, - ], - ); + fn test_with_zero(zero: u64) { + assert_eq!(digest_build_iterator_blocks(16, 3, zero, zero + 4096, None), + [ + // level3 points to previous 16-1 level2 digests: + 256, 512, 768, 1024, 1280, 1536, 1792, 2048, 2304, 2560, 2816, 3072, 3328, 3584, 3840, + // level3 points to previous 16-1 level1 digests: + 3856, 3872, 3888, 3904, 3920, 3936, 3952, 3968, 3984, 4000, 4016, 4032, 4048, 4064, 4080, + // level3 is a level1 digest of 16-1 previous blocks: + 4081, 4082, 4083, 4084, 4085, 4086, 4087, 4088, 4089, 4090, 4091, 4092, 4093, 4094, 4095, + ].iter().map(|item| zero + item).collect::>(), + ); + } + + test_with_zero(0); + test_with_zero(16); + test_with_zero(17); + } + + #[test] + fn digest_iterator_returns_skewed_digest_blocks() { + fn test_with_zero(zero: u64) { + assert_eq!(digest_build_iterator_blocks(16, 3, zero, zero + 4096, Some(zero + 1338)), + [ + // level3 MUST point to previous 16-1 level2 digests, BUT there are only 5: + 256, 512, 768, 1024, 1280, + // level3 MUST point to previous 16-1 level1 digests, BUT there are only 3: + 1296, 1312, 1328, + // level3 MUST be a level1 digest of 16-1 previous blocks, BUT there are only 9: + 1329, 1330, 1331, 1332, 1333, 1334, 1335, 1336, 1337, + ].iter().map(|item| zero + item).collect::>(), + ); + } + + test_with_zero(0); + test_with_zero(16); + test_with_zero(17); + } + + #[test] + fn digest_iterator_returns_skewed_digest_blocks_skipping_level() { + fn test_with_zero(zero: u64) { + assert_eq!(digest_build_iterator_blocks(16, 3, zero, zero + 4096, Some(zero + 1284)), + [ + // level3 MUST point to previous 16-1 level2 digests, BUT there are only 5: + 256, 512, 768, 1024, 1280, + // level3 MUST point to previous 16-1 level1 digests, BUT there are NO ANY L1-digests: + // level3 MUST be a level1 digest of 16-1 previous blocks, BUT there are only 3: + 1281, 1282, 1283, + ].iter().map(|item| zero + item).collect::>(), + ); + } + + test_with_zero(0); + test_with_zero(16); + test_with_zero(17); } } diff --git a/substrate/core/state-machine/src/changes_trie/changes_iterator.rs b/substrate/core/state-machine/src/changes_trie/changes_iterator.rs index f7342cc60f..397bc89857 100644 --- a/substrate/core/state-machine/src/changes_trie/changes_iterator.rs +++ b/substrate/core/state-machine/src/changes_trie/changes_iterator.rs @@ -20,37 +20,44 @@ use std::cell::RefCell; use std::collections::VecDeque; use codec::{Decode, Encode}; -use hash_db::{HashDB, Hasher, EMPTY_PREFIX}; -use num_traits::One; -use trie::{Recorder, MemoryDB}; -use crate::changes_trie::{AnchorBlockId, Configuration, RootsStorage, Storage, BlockNumber}; +use hash_db::Hasher; +use num_traits::Zero; +use trie::Recorder; +use crate::changes_trie::{AnchorBlockId, ConfigurationRange, RootsStorage, Storage, BlockNumber}; use crate::changes_trie::input::{DigestIndex, ExtrinsicIndex, DigestIndexValue, ExtrinsicIndexValue}; use crate::changes_trie::storage::{TrieBackendAdapter, InMemoryStorage}; +use crate::changes_trie::surface_iterator::{surface_iterator, SurfaceIterator}; use crate::proving_backend::ProvingBackendEssence; use crate::trie_backend_essence::{TrieBackendEssence}; /// Return changes of given key at given blocks range. /// `max` is the number of best known block. /// Changes are returned in descending order (i.e. last block comes first). -pub fn key_changes<'a, S: Storage, H: Hasher, Number: BlockNumber>( - config: &'a Configuration, - storage: &'a S, +pub fn key_changes<'a, H: Hasher, Number: BlockNumber>( + config: ConfigurationRange<'a, Number>, + storage: &'a dyn Storage, begin: Number, end: &'a AnchorBlockId, max: Number, key: &'a [u8], -) -> Result, String> { +) -> Result, String> { // we can't query any roots before root let max = ::std::cmp::min(max.clone(), end.number.clone()); Ok(DrilldownIterator { essence: DrilldownIteratorEssence { key, - roots_storage: storage, + roots_storage: storage.as_roots_storage(), storage, begin: begin.clone(), end, - surface: surface_iterator(config, max, begin, end.number.clone())?, + config: config.clone(), + surface: surface_iterator( + config, + max, + begin, + end.number.clone(), + )?, extrinsics: Default::default(), blocks: Default::default(), @@ -62,9 +69,9 @@ pub fn key_changes<'a, S: Storage, H: Hasher, Number: BlockNumber>( /// Returns proof of changes of given key at given blocks range. /// `max` is the number of best known block. -pub fn key_changes_proof, H: Hasher, Number: BlockNumber>( - config: &Configuration, - storage: &S, +pub fn key_changes_proof<'a, H: Hasher, Number: BlockNumber>( + config: ConfigurationRange<'a, Number>, + storage: &dyn Storage, begin: Number, end: &AnchorBlockId, max: Number, @@ -76,11 +83,17 @@ pub fn key_changes_proof, H: Hasher, Number: BlockNumber>( let mut iter = ProvingDrilldownIterator { essence: DrilldownIteratorEssence { key, - roots_storage: storage.clone(), + roots_storage: storage.as_roots_storage(), storage, begin: begin.clone(), end, - surface: surface_iterator(config, max, begin, end.number.clone())?, + config: config.clone(), + surface: surface_iterator( + config, + max, + begin, + end.number.clone(), + )?, extrinsics: Default::default(), blocks: Default::default(), @@ -101,32 +114,53 @@ pub fn key_changes_proof, H: Hasher, Number: BlockNumber>( /// Check key changes proof and return changes of the key at given blocks range. /// `max` is the number of best known block. /// Changes are returned in descending order (i.e. last block comes first). -pub fn key_changes_proof_check, H: Hasher, Number: BlockNumber>( - config: &Configuration, - roots_storage: &S, +pub fn key_changes_proof_check<'a, H: Hasher, Number: BlockNumber>( + config: ConfigurationRange<'a, Number>, + roots_storage: &dyn RootsStorage, proof: Vec>, begin: Number, end: &AnchorBlockId, max: Number, key: &[u8] +) -> Result, String> { + key_changes_proof_check_with_db( + config, + roots_storage, + &InMemoryStorage::with_proof(proof), + begin, + end, + max, + key, + ) +} + +/// Similar to the `key_changes_proof_check` function, but works with prepared proof storage. +pub fn key_changes_proof_check_with_db<'a, H: Hasher, Number: BlockNumber>( + config: ConfigurationRange<'a, Number>, + roots_storage: &dyn RootsStorage, + proof_db: &InMemoryStorage, + begin: Number, + end: &AnchorBlockId, + max: Number, + key: &[u8] ) -> Result, String> { // we can't query any roots before root let max = ::std::cmp::min(max.clone(), end.number.clone()); - let mut proof_db = MemoryDB::::default(); - for item in proof { - proof_db.insert(EMPTY_PREFIX, &item); - } - - let proof_db = InMemoryStorage::with_db(proof_db); DrilldownIterator { essence: DrilldownIteratorEssence { key, roots_storage, - storage: &proof_db, + storage: proof_db, begin: begin.clone(), end, - surface: surface_iterator(config, max, begin, end.number.clone())?, + config: config.clone(), + surface: surface_iterator( + config, + max, + begin, + end.number.clone(), + )?, extrinsics: Default::default(), blocks: Default::default(), @@ -136,87 +170,37 @@ pub fn key_changes_proof_check, H: Hasher, Number: Bl }.collect() } -/// Surface iterator - only traverses top-level digests from given range and tries to find -/// all digest changes for the key. -pub struct SurfaceIterator<'a, Number: BlockNumber> { - config: &'a Configuration, - begin: Number, - max: Number, - current: Option, - current_begin: Number, - digest_step: u32, - digest_level: u32, -} - -impl<'a, Number: BlockNumber> Iterator for SurfaceIterator<'a, Number> { - type Item = Result<(Number, u32), String>; - - fn next(&mut self) -> Option { - let current = self.current.clone()?; - let digest_level = self.digest_level; - - if current < self.digest_step.into() { - self.current = None; - } - else { - let next = current.clone() - self.digest_step.into(); - if next.is_zero() || next < self.begin { - self.current = None; - } - else if next > self.current_begin { - self.current = Some(next); - } else { - let (current, current_begin, digest_step, digest_level) = match - lower_bound_max_digest(self.config, self.max.clone(), self.begin.clone(), next) { - Err(err) => return Some(Err(err)), - Ok(range) => range, - }; - - self.current = Some(current); - self.current_begin = current_begin; - self.digest_step = digest_step; - self.digest_level = digest_level; - } - } - - Some(Ok((current, digest_level))) - } -} - /// Drilldown iterator - receives 'digest points' from surface iterator and explores /// every point until extrinsic is found. -pub struct DrilldownIteratorEssence<'a, RS, S, H, Number> +pub struct DrilldownIteratorEssence<'a, H, Number> where - RS: 'a + RootsStorage, - S: 'a + Storage, H: Hasher, Number: BlockNumber, H::Out: 'a, { key: &'a [u8], - roots_storage: &'a RS, - storage: &'a S, + roots_storage: &'a dyn RootsStorage, + storage: &'a dyn Storage, begin: Number, end: &'a AnchorBlockId, + config: ConfigurationRange<'a, Number>, surface: SurfaceIterator<'a, Number>, extrinsics: VecDeque<(Number, u32)>, - blocks: VecDeque<(Number, u32)>, + blocks: VecDeque<(Number, Option)>, _hasher: ::std::marker::PhantomData, } -impl<'a, RS, S, H, Number> DrilldownIteratorEssence<'a, RS, S, H, Number> +impl<'a, H, Number> DrilldownIteratorEssence<'a, H, Number> where - RS: 'a + RootsStorage, - S: 'a + Storage, H: Hasher, Number: BlockNumber, H::Out: 'a, { pub fn next(&mut self, trie_reader: F) -> Option> where - F: FnMut(&S, H::Out, &[u8]) -> Result>, String>, + F: FnMut(&dyn Storage, H::Out, &[u8]) -> Result>, String>, { match self.do_next(trie_reader) { Ok(Some(res)) => Some(Ok(res)), @@ -227,7 +211,7 @@ impl<'a, RS, S, H, Number> DrilldownIteratorEssence<'a, RS, S, H, Number> fn do_next(&mut self, mut trie_reader: F) -> Result, String> where - F: FnMut(&S, H::Out, &[u8]) -> Result>, String>, + F: FnMut(&dyn Storage, H::Out, &[u8]) -> Result>, String>, { loop { if let Some((block, extrinsic)) = self.extrinsics.pop_front() { @@ -247,7 +231,7 @@ impl<'a, RS, S, H, Number> DrilldownIteratorEssence<'a, RS, S, H, Number> debug_assert!(block >= self.begin, "We shall not touch digests earlier than a range' begin"); if block <= self.end.number { let extrinsics_key = ExtrinsicIndex { block: block.clone(), key: self.key.to_vec() }.encode(); - let extrinsics = trie_reader(&self.storage, trie_root, &extrinsics_key); + let extrinsics = trie_reader(self.storage, trie_root, &extrinsics_key); if let Some(extrinsics) = extrinsics? { if let Ok(extrinsics) = ExtrinsicIndexValue::decode(&mut &extrinsics[..]) { self.extrinsics.extend(extrinsics.into_iter().rev().map(|e| (block.clone(), e))); @@ -256,17 +240,26 @@ impl<'a, RS, S, H, Number> DrilldownIteratorEssence<'a, RS, S, H, Number> } let blocks_key = DigestIndex { block: block.clone(), key: self.key.to_vec() }.encode(); - let blocks = trie_reader(&self.storage, trie_root, &blocks_key); + let blocks = trie_reader(self.storage, trie_root, &blocks_key); if let Some(blocks) = blocks? { if let Ok(blocks) = >::decode(&mut &blocks[..]) { // filter level0 blocks here because we tend to use digest blocks, // AND digest block changes could also include changes for out-of-range blocks let begin = self.begin.clone(); let end = self.end.number.clone(); + let config = self.config.clone(); self.blocks.extend(blocks.into_iter() .rev() - .filter(|b| level > 1 || (*b >= begin && *b <= end)) - .map(|b| (b, level - 1)) + .filter(|b| level.map(|level| level > 1).unwrap_or(true) || (*b >= begin && *b <= end)) + .map(|b| { + let prev_level = level + .map(|level| Some(level - 1)) + .unwrap_or_else(|| + Some(config.config.digest_level_at_block(config.zero.clone(), b.clone()) + .map(|(level, _, _)| level) + .unwrap_or_else(|| Zero::zero()))); + (b, prev_level) + }) ); } } @@ -284,19 +277,17 @@ impl<'a, RS, S, H, Number> DrilldownIteratorEssence<'a, RS, S, H, Number> } /// Exploring drilldown operator. -pub struct DrilldownIterator<'a, RS, S, H, Number> +pub struct DrilldownIterator<'a, H, Number> where Number: BlockNumber, H: Hasher, - S: 'a + Storage, - RS: 'a + RootsStorage, H::Out: 'a, { - essence: DrilldownIteratorEssence<'a, RS, S, H, Number>, + essence: DrilldownIteratorEssence<'a, H, Number>, } -impl<'a, RS: 'a + RootsStorage, S: Storage, H: Hasher, Number: BlockNumber> Iterator - for DrilldownIterator<'a, RS, S, H, Number> +impl<'a, H: Hasher, Number: BlockNumber> Iterator + for DrilldownIterator<'a, H, Number> { type Item = Result<(Number, u32), String>; @@ -307,24 +298,20 @@ impl<'a, RS: 'a + RootsStorage, S: Storage, H: Hasher, Num } /// Proving drilldown iterator. -struct ProvingDrilldownIterator<'a, RS, S, H, Number> +struct ProvingDrilldownIterator<'a, H, Number> where Number: BlockNumber, H: Hasher, - S: 'a + Storage, - RS: 'a + RootsStorage, H::Out: 'a, { - essence: DrilldownIteratorEssence<'a, RS, S, H, Number>, + essence: DrilldownIteratorEssence<'a, H, Number>, proof_recorder: RefCell>, } -impl<'a, RS, S, H, Number> ProvingDrilldownIterator<'a, RS, S, H, Number> +impl<'a, H, Number> ProvingDrilldownIterator<'a, H, Number> where Number: BlockNumber, H: Hasher, - S: 'a + Storage, - RS: 'a + RootsStorage, H::Out: 'a, { /// Consume the iterator, extracting the gathered proof in lexicographical order @@ -337,12 +324,10 @@ impl<'a, RS, S, H, Number> ProvingDrilldownIterator<'a, RS, S, H, Number> } } -impl<'a, RS, S, H, Number> Iterator for ProvingDrilldownIterator<'a, RS, S, H, Number> +impl<'a, H, Number> Iterator for ProvingDrilldownIterator<'a, H, Number> where Number: BlockNumber, H: Hasher, - S: 'a + Storage, - RS: 'a + RootsStorage, H::Out: 'a, { type Item = Result<(Number, u32), String>; @@ -358,90 +343,11 @@ impl<'a, RS, S, H, Number> Iterator for ProvingDrilldownIterator<'a, RS, S, H, N } } -/// Returns surface iterator for given range of blocks. -fn surface_iterator<'a, Number: BlockNumber>( - config: &'a Configuration, - max: Number, - begin: Number, - end: Number, -) -> Result, String> { - let (current, current_begin, digest_step, digest_level) = lower_bound_max_digest( - config, - max.clone(), - begin.clone(), - end, - )?; - Ok(SurfaceIterator { - config, - begin, - max, - current: Some(current), - current_begin, - digest_step, - digest_level, - }) -} - -/// Returns parameters of highest level digest block that includes the end of given range -/// and tends to include the whole range. -fn lower_bound_max_digest( - config: &Configuration, - max: Number, - begin: Number, - end: Number, -) -> Result<(Number, Number, u32, u32), String> { - if end > max || begin > end { - return Err("invalid changes range".into()); - } - - let mut digest_level = 0u32; - let mut digest_step = 1u32; - let mut digest_interval = 0u32; - let mut current = end.clone(); - let mut current_begin = begin.clone(); - if current_begin != current { - while digest_level != config.digest_levels { - let new_digest_level = digest_level + 1; - let new_digest_step = digest_step * config.digest_interval; - let new_digest_interval = config.digest_interval * { - if digest_interval == 0 { 1 } else { digest_interval } - }; - let new_digest_begin = ((current.clone() - One::one()) - / new_digest_interval.into()) * new_digest_interval.into(); - let new_digest_end = new_digest_begin.clone() + new_digest_interval.into(); - let new_current = new_digest_begin.clone() + new_digest_interval.into(); - - if new_digest_end > max { - if begin < new_digest_begin { - current_begin = new_digest_begin; - } - break; - } - - digest_level = new_digest_level; - digest_step = new_digest_step; - digest_interval = new_digest_interval; - current = new_current; - current_begin = new_digest_begin; - - if current_begin <= begin && new_digest_end >= end { - break; - } - } - } - - Ok(( - current, - current_begin, - digest_step, - digest_level, - )) -} - #[cfg(test)] mod tests { use std::iter::FromIterator; use primitives::Blake2Hasher; + use crate::changes_trie::Configuration; use crate::changes_trie::input::InputPair; use crate::changes_trie::storage::InMemoryStorage; use super::*; @@ -485,32 +391,75 @@ mod tests { (config, backend) } + fn configuration_range<'a>(config: &'a Configuration, zero: u64) -> ConfigurationRange<'a, u64> { + ConfigurationRange { + config, + zero, + end: None, + } + } + #[test] fn drilldown_iterator_works() { let (config, storage) = prepare_for_drilldown(); - let drilldown_result = key_changes::, Blake2Hasher, u64>( - &config, &storage, 0, &AnchorBlockId { hash: Default::default(), number: 16 }, 16, &[42]) - .and_then(Result::from_iter); + let drilldown_result = key_changes::( + configuration_range(&config, 0), + &storage, + 1, + &AnchorBlockId { hash: Default::default(), number: 16 }, + 16, + &[42], + ).and_then(Result::from_iter); assert_eq!(drilldown_result, Ok(vec![(8, 2), (8, 1), (6, 3), (3, 0)])); - let drilldown_result = key_changes::, Blake2Hasher, u64>( - &config, &storage, 0, &AnchorBlockId { hash: Default::default(), number: 2 }, 4, &[42]) - .and_then(Result::from_iter); + let drilldown_result = key_changes::( + configuration_range(&config, 0), + &storage, + 1, + &AnchorBlockId { hash: Default::default(), number: 2 }, + 4, + &[42], + ).and_then(Result::from_iter); assert_eq!(drilldown_result, Ok(vec![])); - let drilldown_result = key_changes::, Blake2Hasher, u64>( - &config, &storage, 0, &AnchorBlockId { hash: Default::default(), number: 3 }, 4, &[42]) - .and_then(Result::from_iter); + let drilldown_result = key_changes::( + configuration_range(&config, 0), + &storage, + 1, + &AnchorBlockId { hash: Default::default(), number: 3 }, + 4, + &[42], + ).and_then(Result::from_iter); assert_eq!(drilldown_result, Ok(vec![(3, 0)])); - let drilldown_result = key_changes::, Blake2Hasher, u64>( - &config, &storage, 7, &AnchorBlockId { hash: Default::default(), number: 8 }, 8, &[42]) - .and_then(Result::from_iter); + let drilldown_result = key_changes::( + configuration_range(&config, 0), + &storage, + 1, + &AnchorBlockId { hash: Default::default(), number: 7 }, + 7, + &[42], + ).and_then(Result::from_iter); + assert_eq!(drilldown_result, Ok(vec![(6, 3), (3, 0)])); + + let drilldown_result = key_changes::( + configuration_range(&config, 0), + &storage, + 7, + &AnchorBlockId { hash: Default::default(), number: 8 }, + 8, + &[42], + ).and_then(Result::from_iter); assert_eq!(drilldown_result, Ok(vec![(8, 2), (8, 1)])); - let drilldown_result = key_changes::, Blake2Hasher, u64>( - &config, &storage, 5, &AnchorBlockId { hash: Default::default(), number: 7 }, 8, &[42]) - .and_then(Result::from_iter); + let drilldown_result = key_changes::( + configuration_range(&config, 0), + &storage, + 5, + &AnchorBlockId { hash: Default::default(), number: 7 }, + 8, + &[42], + ).and_then(Result::from_iter); assert_eq!(drilldown_result, Ok(vec![(6, 3)])); } @@ -519,18 +468,35 @@ mod tests { let (config, storage) = prepare_for_drilldown(); storage.clear_storage(); - assert!(key_changes::, Blake2Hasher, u64>( - &config, &storage, 0, &AnchorBlockId { hash: Default::default(), number: 100 }, 1000, &[42]) - .and_then(|i| i.collect::, _>>()).is_err()); + assert!(key_changes::( + configuration_range(&config, 0), + &storage, + 1, + &AnchorBlockId { hash: Default::default(), number: 100 }, + 1000, + &[42], + ).and_then(|i| i.collect::, _>>()).is_err()); } #[test] fn drilldown_iterator_fails_when_range_is_invalid() { let (config, storage) = prepare_for_drilldown(); - assert!(key_changes::, Blake2Hasher, u64>( - &config, &storage, 0, &AnchorBlockId { hash: Default::default(), number: 100 }, 50, &[42]).is_err()); - assert!(key_changes::, Blake2Hasher, u64>( - &config, &storage, 20, &AnchorBlockId { hash: Default::default(), number: 10 }, 100, &[42]).is_err()); + assert!(key_changes::( + configuration_range(&config, 0), + &storage, + 1, + &AnchorBlockId { hash: Default::default(), number: 100 }, + 50, + &[42], + ).is_err()); + assert!(key_changes::( + configuration_range(&config, 0), + &storage, + 20, + &AnchorBlockId { hash: Default::default(), number: 10 }, + 100, + &[42], + ).is_err()); } @@ -540,20 +506,48 @@ mod tests { // create drilldown iterator that records all trie nodes during drilldown let (remote_config, remote_storage) = prepare_for_drilldown(); - let remote_proof = key_changes_proof::, Blake2Hasher, u64>( - &remote_config, &remote_storage, - 0, &AnchorBlockId { hash: Default::default(), number: 16 }, 16, &[42]).unwrap(); + let remote_proof = key_changes_proof::( + configuration_range(&remote_config, 0), &remote_storage, + 1, &AnchorBlockId { hash: Default::default(), number: 16 }, 16, &[42]).unwrap(); // happens on local light node: // create drilldown iterator that works the same, but only depends on trie let (local_config, local_storage) = prepare_for_drilldown(); local_storage.clear_storage(); - let local_result = key_changes_proof_check::, Blake2Hasher, u64>( - &local_config, &local_storage, remote_proof, - 0, &AnchorBlockId { hash: Default::default(), number: 16 }, 16, &[42]); + let local_result = key_changes_proof_check::( + configuration_range(&local_config, 0), &local_storage, remote_proof, + 1, &AnchorBlockId { hash: Default::default(), number: 16 }, 16, &[42]); // check that drilldown result is the same as if it was happening at the full node assert_eq!(local_result, Ok(vec![(8, 2), (8, 1), (6, 3), (3, 0)])); } + + #[test] + fn drilldown_iterator_works_with_skewed_digest() { + let config = Configuration { digest_interval: 4, digest_levels: 3 }; + let mut config_range = configuration_range(&config, 0); + config_range.end = Some(91); + + // when 4^3 deactivates at block 91: + // last L3 digest has been created at block#64 + // skewed digest covers: + // L2 digests at blocks: 80 + // L1 digests at blocks: 84, 88 + // regular blocks: 89, 90, 91 + let mut input = (1u64..92u64).map(|b| (b, vec![])).collect::>(); + // changed at block#63 and covered by L3 digest at block#64 + input[63 - 1].1.push(InputPair::ExtrinsicIndex(ExtrinsicIndex { block: 63, key: vec![42] }, vec![0])); + input[64 - 1].1.push(InputPair::DigestIndex(DigestIndex { block: 64, key: vec![42] }, vec![63])); + // changed at block#79 and covered by L2 digest at block#80 + skewed digest at block#91 + input[79 - 1].1.push(InputPair::ExtrinsicIndex(ExtrinsicIndex { block: 79, key: vec![42] }, vec![1])); + input[80 - 1].1.push(InputPair::DigestIndex(DigestIndex { block: 80, key: vec![42] }, vec![79])); + input[91 - 1].1.push(InputPair::DigestIndex(DigestIndex { block: 91, key: vec![42] }, vec![80])); + let storage = InMemoryStorage::with_inputs(input); + + let drilldown_result = key_changes::( + config_range, &storage, 1, &AnchorBlockId { hash: Default::default(), number: 91 }, 100_000u64, &[42]) + .and_then(Result::from_iter); + assert_eq!(drilldown_result, Ok(vec![(79, 1), (63, 0)])); + } } diff --git a/substrate/core/state-machine/src/changes_trie/mod.rs b/substrate/core/state-machine/src/changes_trie/mod.rs index b29a515d74..5a607cee01 100644 --- a/substrate/core/state-machine/src/changes_trie/mod.rs +++ b/substrate/core/state-machine/src/changes_trie/mod.rs @@ -32,6 +32,19 @@ //! the last N*digest_level-1 blocks (except for genesis block), mapping these keys //! to the set of lower-level digest blocks. //! +//! Changes trie configuration could change within a time. The range of blocks, where +//! configuration has been active, is given by two blocks: zero and end. Zero block is +//! the block where configuration has been set. But the first changes trie that uses +//! this configuration will be built at the block zero+1. If configuration deactivates +//! at some block, this will be the end block of the configuration. It is also the +//! zero block of the next configuration. +//! +//! If configuration has the end block, it also means that 'skewed digest' has/should +//! been built at that block. If this is the block where max-level digest should have +//! been created, than it is simply max-level digest of this configuration. Otherwise, +//! it is the digest that covers all blocks since last max-level digest block was +//! created. +//! //! Changes trie only contains the top level storage changes. Sub-level changes //! are propagated through its storage root on the top level storage. @@ -41,11 +54,16 @@ mod changes_iterator; mod input; mod prune; mod storage; +mod surface_iterator; pub use self::storage::InMemoryStorage; -pub use self::changes_iterator::{key_changes, key_changes_proof, key_changes_proof_check}; +pub use self::changes_iterator::{ + key_changes, key_changes_proof, + key_changes_proof_check, key_changes_proof_check_with_db, +}; pub use self::prune::{prune, oldest_non_pruned_trie}; +use std::convert::TryInto; use hash_db::{Hasher, Prefix}; use crate::backend::Backend; use num_traits::{One, Zero}; @@ -64,7 +82,7 @@ pub trait BlockNumber: Send + Sync + 'static + ::std::fmt::Display + Clone + - From + One + Zero + + From + TryInto + One + Zero + PartialEq + Ord + ::std::ops::Add + ::std::ops::Sub + ::std::ops::Mul + ::std::ops::Div + @@ -78,7 +96,7 @@ impl BlockNumber for T where T: Send + Sync + 'static + ::std::fmt::Display + Clone + - From + One + Zero + + From + TryInto + One + Zero + PartialEq + Ord + ::std::ops::Add + ::std::ops::Sub + ::std::ops::Mul + ::std::ops::Div + @@ -108,6 +126,8 @@ pub trait RootsStorage: Send + Sync { /// Changes trie storage. Provides access to trie roots and trie nodes. pub trait Storage: RootsStorage { + /// Casts from self reference to RootsStorage reference. + fn as_roots_storage(&self) -> &dyn RootsStorage; /// Get a trie node. fn get(&self, key: &H::Out, prefix: Prefix) -> Result, String>; } @@ -126,6 +146,17 @@ impl<'a, H: Hasher, N: BlockNumber> crate::TrieBackendStorage for TrieBackend /// Changes trie configuration. pub type Configuration = primitives::ChangesTrieConfiguration; +/// Blocks range where configuration has been constant. +#[derive(Clone)] +pub struct ConfigurationRange<'a, N> { + /// Active configuration. + pub config: &'a Configuration, + /// Zero block of this configuration. The configuration is active starting from the next block. + pub zero: N, + /// End block of this configuration. It is the last block where configuration has been active. + pub end: Option, +} + /// Compute the changes trie root and transaction for given block. /// Returns Err(()) if unknown `parent_hash` has been passed. /// Returns Ok(None) if there's no data to perform computation. @@ -144,11 +175,18 @@ pub fn build_changes_trie<'a, B: Backend, S: Storage, H: Hasher, N _ => return Ok(None), }; + // FIXME: remove this in https://github.com/paritytech/substrate/pull/3201 + let config = ConfigurationRange { + config, + zero: Zero::zero(), + end: None, + }; + // build_anchor error should not be considered fatal let parent = storage.build_anchor(parent_hash).map_err(|_| ())?; // storage errors are considered fatal (similar to situations when runtime fetches values from storage) - let input_pairs = prepare_input::(backend, storage, config, changes, &parent) + let input_pairs = prepare_input::(backend, storage, config, changes, &parent) .expect("changes trie: storage access is not allowed to fail within runtime"); let mut root = Default::default(); let mut mdb = MemoryDB::default(); diff --git a/substrate/core/state-machine/src/changes_trie/prune.rs b/substrate/core/state-machine/src/changes_trie/prune.rs index 08e7c02b8c..01d52b23ce 100644 --- a/substrate/core/state-machine/src/changes_trie/prune.rs +++ b/substrate/core/state-machine/src/changes_trie/prune.rs @@ -19,7 +19,7 @@ use hash_db::Hasher; use trie::Recorder; use log::warn; -use num_traits::One; +use num_traits::{One, Zero}; use crate::proving_backend::ProvingBackendEssence; use crate::trie_backend_essence::TrieBackendEssence; use crate::changes_trie::{AnchorBlockId, Configuration, Storage, BlockNumber}; @@ -110,7 +110,7 @@ fn pruning_range( // compute number of changes tries we actually want to keep let (prune_interval, blocks_to_keep) = if config.is_digest_build_enabled() { // we only CAN prune at block where max-level-digest is created - let max_digest_interval = match config.digest_level_at_block(block.clone()) { + let max_digest_interval = match config.digest_level_at_block(Zero::zero(), block.clone()) { Some((digest_level, digest_interval, _)) if digest_level == config.digest_levels => digest_interval, _ => return None, diff --git a/substrate/core/state-machine/src/changes_trie/storage.rs b/substrate/core/state-machine/src/changes_trie/storage.rs index bb2256235a..4cf7741763 100644 --- a/substrate/core/state-machine/src/changes_trie/storage.rs +++ b/substrate/core/state-machine/src/changes_trie/storage.rs @@ -17,7 +17,7 @@ //! Changes trie storage utilities. use std::collections::BTreeMap; -use hash_db::{Hasher, Prefix}; +use hash_db::{Hasher, Prefix, EMPTY_PREFIX}; use trie::DBValue; use trie::MemoryDB; use parking_lot::RwLock; @@ -37,8 +37,8 @@ pub struct InMemoryStorage { } /// Adapter for using changes trie storage as a TrieBackendEssence' storage. -pub struct TrieBackendAdapter<'a, H: Hasher, Number: BlockNumber, S: 'a + Storage> { - storage: &'a S, +pub struct TrieBackendAdapter<'a, H: Hasher, Number: BlockNumber> { + storage: &'a dyn Storage, _hasher: ::std::marker::PhantomData<(H, Number)>, } @@ -48,7 +48,7 @@ struct InMemoryStorageData { } impl InMemoryStorage { - /// Create the storage from given in-memory database. + /// Creates storage from given in-memory database. pub fn with_db(mdb: MemoryDB) -> Self { Self { data: RwLock::new(InMemoryStorageData { @@ -58,11 +58,22 @@ impl InMemoryStorage { } } - /// Create the storage with empty database. + /// Creates storage with empty database. pub fn new() -> Self { Self::with_db(Default::default()) } + /// Creates storage with given proof. + pub fn with_proof(proof: Vec>) -> Self { + use hash_db::HashDB; + + let mut proof_db = MemoryDB::::default(); + for item in proof { + proof_db.insert(EMPTY_PREFIX, &item); + } + Self::with_db(proof_db) + } + /// Create the storage with given blocks. pub fn with_blocks(blocks: Vec<(Number, H::Out)>) -> Self { Self { @@ -132,20 +143,23 @@ impl RootsStorage for InMemoryStorage } impl Storage for InMemoryStorage { + fn as_roots_storage(&self) -> &dyn RootsStorage { + self + } + fn get(&self, key: &H::Out, prefix: Prefix) -> Result, String> { MemoryDB::::get(&self.data.read().mdb, key, prefix) } } -impl<'a, H: Hasher, Number: BlockNumber, S: 'a + Storage> TrieBackendAdapter<'a, H, Number, S> { - pub fn new(storage: &'a S) -> Self { +impl<'a, H: Hasher, Number: BlockNumber> TrieBackendAdapter<'a, H, Number> { + pub fn new(storage: &'a dyn Storage) -> Self { Self { storage, _hasher: Default::default() } } } -impl<'a, H, Number, S> TrieBackendStorage for TrieBackendAdapter<'a, H, Number, S> +impl<'a, H, Number> TrieBackendStorage for TrieBackendAdapter<'a, H, Number> where - S: 'a + Storage, Number: BlockNumber, H: Hasher, { diff --git a/substrate/core/state-machine/src/changes_trie/surface_iterator.rs b/substrate/core/state-machine/src/changes_trie/surface_iterator.rs new file mode 100644 index 0000000000..f3583e2f57 --- /dev/null +++ b/substrate/core/state-machine/src/changes_trie/surface_iterator.rs @@ -0,0 +1,285 @@ +// Copyright 2017-2019 Parity Technologies (UK) Ltd. +// This file is part of Substrate. + +// Substrate is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Substrate is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Substrate. If not, see . + +//! The best way to understand how this iterator works is to imagine some 2D terrain that have some mountains +//! (digest changes tries) and valleys (changes tries for regular blocks). There are gems (blocks) beneath the +//! terrain. Given the request to find all gems in the range [X1; X2] this iterator will return **minimal set** +//! of points at the terrain (mountains and valleys) inside this range that have to be drilled down to +//! search for gems. + +use num_traits::One; +use crate::changes_trie::{ConfigurationRange, BlockNumber}; + +/// Returns surface iterator for given range of blocks. +/// +/// `max` is the number of best block, known to caller. We can't access any changes tries +/// that are built after this block, even though we may have them built already. +pub fn surface_iterator<'a, Number: BlockNumber>( + config: ConfigurationRange<'a, Number>, + max: Number, + begin: Number, + end: Number, +) -> Result, String> { + let (current, current_begin, digest_step, digest_level) = lower_bound_max_digest( + config.clone(), + max.clone(), + begin.clone(), + end, + )?; + Ok(SurfaceIterator { + config, + begin, + max, + current: Some(current), + current_begin, + digest_step, + digest_level, + }) +} + +/// Surface iterator - only traverses top-level digests from given range and tries to find +/// all valid digest changes. +/// +/// Iterator item is the tuple of (last block of the current point + digest level of the current point). +/// Digest level is Some(0) when it is regular block, is Some(non-zero) when it is digest block and None +/// if it is skewed digest block. +pub struct SurfaceIterator<'a, Number: BlockNumber> { + config: ConfigurationRange<'a, Number>, + begin: Number, + max: Number, + current: Option, + current_begin: Number, + digest_step: u32, + digest_level: Option, +} + +impl<'a, Number: BlockNumber> Iterator for SurfaceIterator<'a, Number> { + type Item = Result<(Number, Option), String>; + + fn next(&mut self) -> Option { + let current = self.current.clone()?; + let digest_level = self.digest_level; + + if current < self.digest_step.into() { + self.current = None; + } else { + let next = current.clone() - self.digest_step.into(); + if next.is_zero() || next < self.begin { + self.current = None; + } else if next > self.current_begin { + self.current = Some(next); + } else { + let max_digest_interval = lower_bound_max_digest( + self.config.clone(), + self.max.clone(), + self.begin.clone(), + next, + ); + let (current, current_begin, digest_step, digest_level) = match max_digest_interval { + Err(err) => return Some(Err(err)), + Ok(range) => range, + }; + + self.current = Some(current); + self.current_begin = current_begin; + self.digest_step = digest_step; + self.digest_level = digest_level; + } + } + + Some(Ok((current, digest_level))) + } +} + +/// Returns parameters of highest level digest block that includes the end of given range +/// and tends to include the whole range. +fn lower_bound_max_digest<'a, Number: BlockNumber>( + config: ConfigurationRange<'a, Number>, + max: Number, + begin: Number, + end: Number, +) -> Result<(Number, Number, u32, Option), String> { + if end > max || begin > end { + return Err(format!("invalid changes range: {}..{}/{}", begin, end, max)); + } + if begin <= config.zero || config.end.as_ref().map(|config_end| end > *config_end).unwrap_or(false) { + return Err(format!("changes trie range is not covered by configuration: {}..{}/{}..{}", + begin, end, config.zero, match config.end.as_ref() { + Some(config_end) => format!("{}", config_end), + None => "None".into(), + })); + } + + let mut digest_level = 0u32; + let mut digest_step = 1u32; + let mut digest_interval = 0u32; + let mut current = end.clone(); + let mut current_begin = begin.clone(); + if current_begin != current { + while digest_level != config.config.digest_levels { + // try to use next level digest + let new_digest_level = digest_level + 1; + let new_digest_step = digest_step * config.config.digest_interval; + let new_digest_interval = config.config.digest_interval * { + if digest_interval == 0 { 1 } else { digest_interval } + }; + let new_digest_begin = config.zero.clone() + ((current.clone() - One::one() - config.zero.clone()) + / new_digest_interval.into()) * new_digest_interval.into(); + let new_digest_end = new_digest_begin.clone() + new_digest_interval.into(); + let new_current = new_digest_begin.clone() + new_digest_interval.into(); + + // check if we met skewed digest + if let Some(skewed_digest_end) = config.end.as_ref() { + if new_digest_end > *skewed_digest_end { + let skewed_digest_start = config.config.prev_max_level_digest_block( + config.zero.clone(), + skewed_digest_end.clone(), + ); + if let Some(skewed_digest_start) = skewed_digest_start { + let skewed_digest_range = (skewed_digest_end.clone() - skewed_digest_start.clone()) + .try_into().ok() + .expect("skewed digest range is always <= max level digest range;\ + max level digest range always fits u32; qed"); + return Ok(( + skewed_digest_end.clone(), + skewed_digest_start, + skewed_digest_range, + None, + )); + } + } + } + + // we can't use next level digest if it touches any unknown (> max) blocks + if new_digest_end > max { + if begin < new_digest_begin { + current_begin = new_digest_begin; + } + break; + } + + // we can (and will) use this digest + digest_level = new_digest_level; + digest_step = new_digest_step; + digest_interval = new_digest_interval; + current = new_current; + current_begin = new_digest_begin; + + // if current digest covers the whole range => no need to use next level digest + if current_begin <= begin && new_digest_end >= end { + break; + } + } + } + + Ok(( + current, + current_begin, + digest_step, + Some(digest_level), + )) +} + +#[cfg(test)] +mod tests { + use crate::changes_trie::{Configuration}; + use super::*; + + fn configuration_range<'a>(config: &'a Configuration, zero: u64) -> ConfigurationRange<'a, u64> { + ConfigurationRange { + config, + zero, + end: None, + } + } + + #[test] + fn lower_bound_max_digest_works() { + let config = Configuration { digest_interval: 4, digest_levels: 2 }; + + // when config activates at 0 + assert_eq!( + lower_bound_max_digest(configuration_range(&config, 0u64), 100_000u64, 20u64, 180u64).unwrap(), + (192, 176, 16, Some(2)), + ); + + // when config activates at 30 + assert_eq!( + lower_bound_max_digest(configuration_range(&config, 30u64), 100_000u64, 50u64, 210u64).unwrap(), + (222, 206, 16, Some(2)), + ); + } + + #[test] + fn surface_iterator_works() { + let config = Configuration { digest_interval: 4, digest_levels: 2 }; + + // when config activates at 0 + assert_eq!( + surface_iterator( + configuration_range(&config, 0u64), + 100_000u64, + 40u64, + 180u64, + ).unwrap().collect::>(), + vec![ + Ok((192, Some(2))), Ok((176, Some(2))), Ok((160, Some(2))), Ok((144, Some(2))), + Ok((128, Some(2))), Ok((112, Some(2))), Ok((96, Some(2))), Ok((80, Some(2))), + Ok((64, Some(2))), Ok((48, Some(2))), + ], + ); + + // when config activates at 30 + assert_eq!( + surface_iterator( + configuration_range(&config, 30u64), + 100_000u64, + 40u64, + 180u64, + ).unwrap().collect::>(), + vec![ + Ok((190, Some(2))), Ok((174, Some(2))), Ok((158, Some(2))), Ok((142, Some(2))), Ok((126, Some(2))), + Ok((110, Some(2))), Ok((94, Some(2))), Ok((78, Some(2))), Ok((62, Some(2))), Ok((46, Some(2))), + ], + ); + + // when config activates at 0 AND max block is before next digest + assert_eq!( + surface_iterator(configuration_range(&config, 0u64), 183u64, 40u64, 183u64).unwrap().collect::>(), + vec![ + Ok((183, Some(0))), Ok((182, Some(0))), Ok((181, Some(0))), Ok((180, Some(1))), + Ok((176, Some(2))), Ok((160, Some(2))), Ok((144, Some(2))), Ok((128, Some(2))), Ok((112, Some(2))), + Ok((96, Some(2))), Ok((80, Some(2))), Ok((64, Some(2))), Ok((48, Some(2))), + ], + ); + } + + #[test] + fn surface_iterator_works_with_skewed_digest() { + let config = Configuration { digest_interval: 4, digest_levels: 2 }; + let mut config_range = configuration_range(&config, 0u64); + + // when config activates at 0 AND ends at 170 + config_range.end = Some(170); + assert_eq!( + surface_iterator(config_range, 100_000u64, 40u64, 170u64).unwrap().collect::>(), + vec![ + Ok((170, None)), Ok((160, Some(2))), Ok((144, Some(2))), Ok((128, Some(2))), Ok((112, Some(2))), + Ok((96, Some(2))), Ok((80, Some(2))), Ok((64, Some(2))), Ok((48, Some(2))), + ], + ); + } +} diff --git a/substrate/core/state-machine/src/lib.rs b/substrate/core/state-machine/src/lib.rs index 8c2046e591..acc7c366b8 100644 --- a/substrate/core/state-machine/src/lib.rs +++ b/substrate/core/state-machine/src/lib.rs @@ -50,6 +50,7 @@ pub use changes_trie::{ Storage as ChangesTrieStorage, RootsStorage as ChangesTrieRootsStorage, InMemoryStorage as InMemoryChangesTrieStorage, + ConfigurationRange as ChangesTrieConfigurationRange, key_changes, key_changes_proof, key_changes_proof_check, prune as prune_changes_tries, oldest_non_pruned_trie as oldest_non_pruned_changes_trie