diff --git a/substrate/Cargo.lock b/substrate/Cargo.lock index 5d2a6c5b29..a40f588d29 100644 --- a/substrate/Cargo.lock +++ b/substrate/Cargo.lock @@ -4268,6 +4268,7 @@ dependencies = [ "hex 0.3.2 (registry+https://github.com/rust-lang/crates.io-index)", "hex-literal 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)", "impl-serde 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)", + "num-traits 0.2.6 (registry+https://github.com/rust-lang/crates.io-index)", "parity-codec 3.5.1 (registry+https://github.com/rust-lang/crates.io-index)", "pretty_assertions 0.6.1 (registry+https://github.com/rust-lang/crates.io-index)", "primitive-types 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)", @@ -4410,6 +4411,7 @@ dependencies = [ "hash-db 0.12.2 (registry+https://github.com/rust-lang/crates.io-index)", "hex-literal 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)", "log 0.4.6 (registry+https://github.com/rust-lang/crates.io-index)", + "num-traits 0.2.6 (registry+https://github.com/rust-lang/crates.io-index)", "parity-codec 3.5.1 (registry+https://github.com/rust-lang/crates.io-index)", "parking_lot 0.7.1 (registry+https://github.com/rust-lang/crates.io-index)", "substrate-panic-handler 2.0.0", diff --git a/substrate/core/client/db/src/cache/mod.rs b/substrate/core/client/db/src/cache/mod.rs index d8ffaea723..d79b01ab6b 100644 --- a/substrate/core/client/db/src/cache/mod.rs +++ b/substrate/core/client/db/src/cache/mod.rs @@ -35,7 +35,7 @@ mod list_cache; mod list_entry; mod list_storage; -/// Minimal post-finalization age age of finalized blocks before they'll pruned. +/// Minimal post-finalization age of finalized blocks before they'll pruned. const PRUNE_DEPTH: u32 = 1024; /// The type of entry that is inserted to the cache. diff --git a/substrate/core/client/db/src/lib.rs b/substrate/core/client/db/src/lib.rs index 8f54099a78..e9e9a61a76 100644 --- a/substrate/core/client/db/src/lib.rs +++ b/substrate/core/client/db/src/lib.rs @@ -48,7 +48,7 @@ use primitives::storage::well_known_keys; use runtime_primitives::{generic::BlockId, Justification, StorageOverlay, ChildrenStorageOverlay}; use runtime_primitives::traits::{ Block as BlockT, Header as HeaderT, NumberFor, Zero, One, Digest, DigestItem, - SaturatedConversion, + SaturatedConversion }; use runtime_primitives::BuildStorage; use state_machine::backend::Backend as StateBackend; @@ -67,7 +67,7 @@ pub use state_db::PruningMode; use client::in_mem::Backend as InMemoryBackend; const CANONICALIZATION_DELAY: u64 = 4096; -const MIN_BLOCKS_TO_KEEP_CHANGES_TRIES_FOR: u64 = 32768; +const MIN_BLOCKS_TO_KEEP_CHANGES_TRIES_FOR: u32 = 32768; /// DB-backed patricia trie state, transaction type is an overlay of changes to commit. pub type DbState = state_machine::TrieBackend>, Blake2Hasher>; @@ -423,11 +423,11 @@ impl state_machine::Storage for DbGenesisStorage { pub struct DbChangesTrieStorage { db: Arc, meta: Arc, Block::Hash>>>, - min_blocks_to_keep: Option, + min_blocks_to_keep: Option, _phantom: ::std::marker::PhantomData, } -impl DbChangesTrieStorage { +impl> DbChangesTrieStorage { /// Commit new changes trie. pub fn commit(&self, tx: &mut DBTransaction, mut changes_trie: MemoryDB) { for (key, (val, _)) in changes_trie.drain() { @@ -446,53 +446,79 @@ impl DbChangesTrieStorage { state_machine::prune_changes_tries( config, &*self, - min_blocks_to_keep, + min_blocks_to_keep.into(), &state_machine::ChangesTrieAnchorBlockId { hash: convert_hash(&block_hash), - number: block_num.saturated_into::(), + number: block_num, }, |node| tx.delete(columns::CHANGES_TRIE, node.as_ref())); } } -impl client::backend::PrunableStateChangesTrieStorage for DbChangesTrieStorage { +impl client::backend::PrunableStateChangesTrieStorage + for DbChangesTrieStorage +where + Block: BlockT, +{ fn oldest_changes_trie_block( &self, config: &ChangesTrieConfiguration, - best_finalized_block: u64 - ) -> u64 { + best_finalized_block: NumberFor, + ) -> NumberFor { match self.min_blocks_to_keep { Some(min_blocks_to_keep) => state_machine::oldest_non_pruned_changes_trie( config, - min_blocks_to_keep, + min_blocks_to_keep.into(), best_finalized_block, ), - None => 1, + None => One::one(), } } } -impl state_machine::ChangesTrieRootsStorage for DbChangesTrieStorage { - fn root(&self, anchor: &state_machine::ChangesTrieAnchorBlockId, block: u64) -> Result, String> { +impl state_machine::ChangesTrieRootsStorage> + for DbChangesTrieStorage +where + Block: BlockT, +{ + fn build_anchor( + &self, + hash: H256, + ) -> Result>, String> { + utils::read_header::(&*self.db, columns::KEY_LOOKUP, columns::HEADER, BlockId::Hash(hash)) + .map_err(|e| e.to_string()) + .and_then(|maybe_header| maybe_header.map(|header| + state_machine::ChangesTrieAnchorBlockId { + hash, + number: *header.number(), + } + ).ok_or_else(|| format!("Unknown header: {}", hash))) + } + + fn root( + &self, + anchor: &state_machine::ChangesTrieAnchorBlockId>, + block: NumberFor, + ) -> Result, String> { // check API requirement: we can't get NEXT block(s) based on anchor if block > anchor.number { return Err(format!("Can't get changes trie root at {} using anchor at {}", block, anchor.number)); } // we need to get hash of the block to resolve changes trie root - let block_id = if block <= self.meta.read().finalized_number.saturated_into::() { + let block_id = if block <= self.meta.read().finalized_number { // if block is finalized, we could just read canonical hash - BlockId::Number(block.saturated_into()) + BlockId::Number(block) } else { // the block is not finalized let mut current_num = anchor.number; let mut current_hash: Block::Hash = convert_hash(&anchor.hash); let maybe_anchor_header: Block::Header = utils::require_header::( - &*self.db, columns::KEY_LOOKUP, columns::HEADER, BlockId::Number(current_num.saturated_into()) + &*self.db, columns::KEY_LOOKUP, columns::HEADER, BlockId::Number(current_num) ).map_err(|e| e.to_string())?; if maybe_anchor_header.hash() == current_hash { // if anchor is canonicalized, then the block is also canonicalized - BlockId::Number(block.saturated_into()) + BlockId::Number(block) } else { // else (block is not finalized + anchor is not canonicalized): // => we should find the required block hash by traversing @@ -503,7 +529,7 @@ impl state_machine::ChangesTrieRootsStorage for DbC ).map_err(|e| e.to_string())?; current_hash = *current_header.parent_hash(); - current_num = current_num - 1; + current_num = current_num - One::one(); } BlockId::Hash(current_hash) @@ -517,7 +543,11 @@ impl state_machine::ChangesTrieRootsStorage for DbC } } -impl state_machine::ChangesTrieStorage for DbChangesTrieStorage { +impl state_machine::ChangesTrieStorage> + for DbChangesTrieStorage +where + Block: BlockT, +{ fn get(&self, key: &H256, _prefix: &[u8]) -> Result, String> { self.db.get(columns::CHANGES_TRIE, &key[..]) .map_err(|err| format!("{}", err)) diff --git a/substrate/core/client/db/src/light.rs b/substrate/core/client/db/src/light.rs index 02525455ba..cbac1b730e 100644 --- a/substrate/core/client/db/src/light.rs +++ b/substrate/core/client/db/src/light.rs @@ -32,8 +32,9 @@ use client::light::blockchain::Storage as LightBlockchainStorage; use parity_codec::{Decode, Encode}; use primitives::Blake2Hasher; use runtime_primitives::generic::BlockId; -use runtime_primitives::traits::{Block as BlockT, Header as HeaderT, - Zero, One, SaturatedConversion, NumberFor, Digest, DigestItem +use runtime_primitives::traits::{ + Block as BlockT, Header as HeaderT, + Zero, One, NumberFor, Digest, DigestItem, }; use consensus_common::well_known_cache_keys; use crate::cache::{DbCacheSync, DbCache, ComplexBlockId, EntryType as CacheEntryType}; @@ -269,12 +270,18 @@ impl LightStorage { transaction.put(columns::META, meta_keys::FINALIZED_BLOCK, &lookup_key); // build new CHT(s) if required - if let Some(new_cht_number) = cht::is_build_required(cht::SIZE, *header.number()) { - let new_cht_start: NumberFor = cht::start_number(cht::SIZE, new_cht_number); + if let Some(new_cht_number) = cht::is_build_required(cht::size(), *header.number()) { + let new_cht_start: NumberFor = cht::start_number(cht::size(), new_cht_number); + + let mut current_num = new_cht_start; + let cht_range = ::std::iter::from_fn(|| { + let old_current_num = current_num; + current_num = current_num + One::one(); + Some(old_current_num) + }); let new_header_cht_root = cht::compute_root::( - cht::SIZE, new_cht_number, (new_cht_start.saturated_into::()..) - .map(|num| self.hash(num.saturated_into())) + cht::size(), new_cht_number, cht_range.map(|num| self.hash(num)) )?; transaction.put( columns::CHT, @@ -284,9 +291,15 @@ impl LightStorage { // if the header includes changes trie root, let's build a changes tries roots CHT if header.digest().log(DigestItem::as_changes_trie_root).is_some() { + let mut current_num = new_cht_start; + let cht_range = ::std::iter::from_fn(|| { + let old_current_num = current_num; + current_num = current_num + One::one(); + Some(old_current_num) + }); let new_changes_trie_cht_root = cht::compute_root::( - cht::SIZE, new_cht_number, (new_cht_start.saturated_into::()..) - .map(|num| self.changes_trie_root(BlockId::Number(num.saturated_into()))) + cht::size(), new_cht_number, cht_range + .map(|num| self.changes_trie_root(BlockId::Number(num))) )?; transaction.put( columns::CHT, @@ -297,7 +310,7 @@ impl LightStorage { // prune headers that are replaced with CHT let mut prune_block = new_cht_start; - let new_cht_end = cht::end_number(cht::SIZE, new_cht_number); + let new_cht_end = cht::end_number(cht::size(), new_cht_number); trace!(target: "db", "Replacing blocks [{}..{}] with CHT#{}", new_cht_start, new_cht_end, new_cht_number); @@ -330,7 +343,7 @@ impl LightStorage { fn read_cht_root( &self, cht_type: u8, - cht_size: u64, + cht_size: NumberFor, block: NumberFor ) -> ClientResult { let no_cht_for_block = || ClientError::Backend(format!("CHT for block {} not exists", block)); @@ -482,11 +495,19 @@ impl LightBlockchainStorage for LightStorage } } - fn header_cht_root(&self, cht_size: u64, block: NumberFor) -> ClientResult { + fn header_cht_root( + &self, + cht_size: NumberFor, + block: NumberFor, + ) -> ClientResult { self.read_cht_root(HEADER_CHT_PREFIX, cht_size, block) } - fn changes_trie_cht_root(&self, cht_size: u64, block: NumberFor) -> ClientResult { + fn changes_trie_cht_root( + &self, + cht_size: NumberFor, + block: NumberFor, + ) -> ClientResult { self.read_cht_root(CHANGES_TRIE_CHT_PREFIX, cht_size, block) } @@ -670,33 +691,44 @@ pub(crate) mod tests { fn finalized_ancient_headers_are_replaced_with_cht() { fn insert_headers Header>(header_producer: F) -> LightStorage { let db = LightStorage::new_test(); + let cht_size: u64 = cht::size(); + let ucht_size: usize = cht_size as _; // insert genesis block header (never pruned) let mut prev_hash = insert_final_block(&db, HashMap::new(), || header_producer(&Default::default(), 0)); // insert SIZE blocks && ensure that nothing is pruned - for number in 0..cht::SIZE { + + for number in 0..cht::size() { prev_hash = insert_block(&db, HashMap::new(), || header_producer(&prev_hash, 1 + number)); } - assert_eq!(db.db.iter(columns::HEADER).count(), (1 + cht::SIZE) as usize); + assert_eq!(db.db.iter(columns::HEADER).count(), 1 + ucht_size); assert_eq!(db.db.iter(columns::CHT).count(), 0); // insert next SIZE blocks && ensure that nothing is pruned - for number in 0..cht::SIZE { - prev_hash = insert_block(&db, HashMap::new(), || header_producer(&prev_hash, 1 + cht::SIZE + number)); + for number in 0..(cht_size as _) { + prev_hash = insert_block( + &db, + HashMap::new(), + || header_producer(&prev_hash, 1 + cht_size + number), + ); } - assert_eq!(db.db.iter(columns::HEADER).count(), (1 + cht::SIZE + cht::SIZE) as usize); + assert_eq!(db.db.iter(columns::HEADER).count(), 1 + ucht_size + ucht_size); assert_eq!(db.db.iter(columns::CHT).count(), 0); - // insert block #{2 * cht::SIZE + 1} && check that new CHT is created + headers of this CHT are pruned + // insert block #{2 * cht::size() + 1} && check that new CHT is created + headers of this CHT are pruned // nothing is yet finalized, so nothing is pruned. - prev_hash = insert_block(&db, HashMap::new(), || header_producer(&prev_hash, 1 + cht::SIZE + cht::SIZE)); - assert_eq!(db.db.iter(columns::HEADER).count(), (2 + cht::SIZE + cht::SIZE) as usize); + prev_hash = insert_block( + &db, + HashMap::new(), + || header_producer(&prev_hash, 1 + cht_size + cht_size), + ); + assert_eq!(db.db.iter(columns::HEADER).count(), 2 + ucht_size + ucht_size); assert_eq!(db.db.iter(columns::CHT).count(), 0); // now finalize the block. - for i in (0..(cht::SIZE + cht::SIZE)).map(|i| i + 1) { - db.finalize_header(BlockId::Number(i)).unwrap(); + for i in (0..(ucht_size + ucht_size)).map(|i| i + 1) { + db.finalize_header(BlockId::Number(i as _)).unwrap(); } db.finalize_header(BlockId::Hash(prev_hash)).unwrap(); db @@ -704,34 +736,36 @@ pub(crate) mod tests { // when headers are created without changes tries roots let db = insert_headers(default_header); - assert_eq!(db.db.iter(columns::HEADER).count(), (1 + cht::SIZE + 1) as usize); - assert_eq!(db.db.iter(columns::KEY_LOOKUP).count(), (2 * (1 + cht::SIZE + 1)) as usize); + let cht_size: u64 = cht::size(); + assert_eq!(db.db.iter(columns::HEADER).count(), (1 + cht_size + 1) as usize); + assert_eq!(db.db.iter(columns::KEY_LOOKUP).count(), (2 * (1 + cht_size + 1)) as usize); assert_eq!(db.db.iter(columns::CHT).count(), 1); - assert!((0..cht::SIZE).all(|i| db.header(BlockId::Number(1 + i)).unwrap().is_none())); - assert!(db.header_cht_root(cht::SIZE, cht::SIZE / 2).is_ok()); - assert!(db.header_cht_root(cht::SIZE, cht::SIZE + cht::SIZE / 2).is_err()); - assert!(db.changes_trie_cht_root(cht::SIZE, cht::SIZE / 2).is_err()); - assert!(db.changes_trie_cht_root(cht::SIZE, cht::SIZE + cht::SIZE / 2).is_err()); + assert!((0..cht_size as _).all(|i| db.header(BlockId::Number(1 + i)).unwrap().is_none())); + assert!(db.header_cht_root(cht_size, cht_size / 2).is_ok()); + assert!(db.header_cht_root(cht_size, cht_size + cht_size / 2).is_err()); + assert!(db.changes_trie_cht_root(cht_size, cht_size / 2).is_err()); + assert!(db.changes_trie_cht_root(cht_size, cht_size + cht_size / 2).is_err()); // when headers are created with changes tries roots let db = insert_headers(header_with_changes_trie); - assert_eq!(db.db.iter(columns::HEADER).count(), (1 + cht::SIZE + 1) as usize); + assert_eq!(db.db.iter(columns::HEADER).count(), (1 + cht_size + 1) as usize); assert_eq!(db.db.iter(columns::CHT).count(), 2); - assert!((0..cht::SIZE).all(|i| db.header(BlockId::Number(1 + i)).unwrap().is_none())); - assert!(db.header_cht_root(cht::SIZE, cht::SIZE / 2).is_ok()); - assert!(db.header_cht_root(cht::SIZE, cht::SIZE + cht::SIZE / 2).is_err()); - assert!(db.changes_trie_cht_root(cht::SIZE, cht::SIZE / 2).is_ok()); - assert!(db.changes_trie_cht_root(cht::SIZE, cht::SIZE + cht::SIZE / 2).is_err()); + assert!((0..cht_size as _).all(|i| db.header(BlockId::Number(1 + i)).unwrap().is_none())); + assert!(db.header_cht_root(cht_size, cht_size / 2).is_ok()); + assert!(db.header_cht_root(cht_size, cht_size + cht_size / 2).is_err()); + assert!(db.changes_trie_cht_root(cht_size, cht_size / 2).is_ok()); + assert!(db.changes_trie_cht_root(cht_size, cht_size + cht_size / 2).is_err()); } #[test] fn get_cht_fails_for_genesis_block() { - assert!(LightStorage::::new_test().header_cht_root(cht::SIZE, 0).is_err()); + assert!(LightStorage::::new_test().header_cht_root(cht::size(), 0).is_err()); } #[test] fn get_cht_fails_for_non_existant_cht() { - assert!(LightStorage::::new_test().header_cht_root(cht::SIZE, (cht::SIZE / 2) as u64).is_err()); + let cht_size: u64 = cht::size(); + assert!(LightStorage::::new_test().header_cht_root(cht_size, cht_size / 2).is_err()); } #[test] @@ -740,20 +774,22 @@ pub(crate) mod tests { // insert 1 + SIZE + SIZE + 1 blocks so that CHT#0 is created let mut prev_hash = insert_final_block(&db, HashMap::new(), || header_with_changes_trie(&Default::default(), 0)); - for i in 1..1 + cht::SIZE + cht::SIZE + 1 { + let cht_size: u64 = cht::size(); + let ucht_size: usize = cht_size as _; + for i in 1..1 + ucht_size + ucht_size + 1 { prev_hash = insert_block(&db, HashMap::new(), || header_with_changes_trie(&prev_hash, i as u64)); db.finalize_header(BlockId::Hash(prev_hash)).unwrap(); } - let cht_root_1 = db.header_cht_root(cht::SIZE, cht::start_number(cht::SIZE, 0)).unwrap(); - let cht_root_2 = db.header_cht_root(cht::SIZE, (cht::start_number(cht::SIZE, 0) + cht::SIZE / 2) as u64).unwrap(); - let cht_root_3 = db.header_cht_root(cht::SIZE, cht::end_number(cht::SIZE, 0)).unwrap(); + let cht_root_1 = db.header_cht_root(cht_size, cht::start_number(cht_size, 0)).unwrap(); + let cht_root_2 = db.header_cht_root(cht_size, cht::start_number(cht_size, 0) + cht_size / 2).unwrap(); + let cht_root_3 = db.header_cht_root(cht_size, cht::end_number(cht_size, 0)).unwrap(); assert_eq!(cht_root_1, cht_root_2); assert_eq!(cht_root_2, cht_root_3); - let cht_root_1 = db.changes_trie_cht_root(cht::SIZE, cht::start_number(cht::SIZE, 0)).unwrap(); - let cht_root_2 = db.changes_trie_cht_root(cht::SIZE, (cht::start_number(cht::SIZE, 0) + cht::SIZE / 2) as u64).unwrap(); - let cht_root_3 = db.changes_trie_cht_root(cht::SIZE, cht::end_number(cht::SIZE, 0)).unwrap(); + let cht_root_1 = db.changes_trie_cht_root(cht_size, cht::start_number(cht_size, 0)).unwrap(); + let cht_root_2 = db.changes_trie_cht_root(cht_size, cht::start_number(cht_size, 0) + cht_size / 2).unwrap(); + let cht_root_3 = db.changes_trie_cht_root(cht_size, cht::end_number(cht_size, 0)).unwrap(); assert_eq!(cht_root_1, cht_root_2); assert_eq!(cht_root_2, cht_root_3); } diff --git a/substrate/core/client/src/backend.rs b/substrate/core/client/src/backend.rs index 09faab1a12..d2cc086ecd 100644 --- a/substrate/core/client/src/backend.rs +++ b/substrate/core/client/src/backend.rs @@ -127,7 +127,7 @@ pub trait Backend: AuxStore + Send + Sync where /// Associated state backend type. type State: StateBackend; /// Changes trie storage. - type ChangesTrieStorage: PrunableStateChangesTrieStorage; + type ChangesTrieStorage: PrunableStateChangesTrieStorage; /// Begin a new block insertion transaction with given parent block id. /// When constructing the genesis, this is called with all-zero hash. @@ -177,9 +177,15 @@ pub trait Backend: AuxStore + Send + Sync where } /// Changes trie storage that supports pruning. -pub trait PrunableStateChangesTrieStorage: StateChangesTrieStorage { +pub trait PrunableStateChangesTrieStorage: + StateChangesTrieStorage> +{ /// Get number block of oldest, non-pruned changes trie. - fn oldest_changes_trie_block(&self, config: &ChangesTrieConfiguration, best_finalized: u64) -> u64; + fn oldest_changes_trie_block( + &self, + config: &ChangesTrieConfiguration, + best_finalized: NumberFor, + ) -> NumberFor; } /// Mark for all Backend implementations, that are making use of state data, stored locally. diff --git a/substrate/core/client/src/cht.rs b/substrate/core/client/src/cht.rs index 42b4654d34..996a6b37f1 100644 --- a/substrate/core/client/src/cht.rs +++ b/substrate/core/client/src/cht.rs @@ -26,15 +26,11 @@ use std::collections::HashSet; use hash_db; +use parity_codec::Encode; use trie; use primitives::{H256, convert_hash}; -// We're using saturatedconversion in order to go back and forth to `u64`. this is stupid. -// instead we should just make the CHT generic over the block number. -use runtime_primitives::traits::{ - Header as HeaderT, SimpleArithmetic, One, SaturatedConversion, - UniqueSaturatedInto -}; +use runtime_primitives::traits::{Header as HeaderT, SimpleArithmetic, Zero, One}; use state_machine::backend::InMemory as InMemoryState; use state_machine::{MemoryDB, TrieBackend, Backend as StateBackend, prove_read_on_trie_backend, read_proof_check, read_proof_check_on_proving_backend}; @@ -43,14 +39,19 @@ use crate::error::{Error as ClientError, Result as ClientResult}; /// The size of each CHT. This value is passed to every CHT-related function from /// production code. Other values are passed from tests. -pub const SIZE: u64 = 2048; +const SIZE: u32 = 2048; + +/// Gets default CHT size. +pub fn size>() -> N { + SIZE.into() +} /// Returns Some(cht_number) if CHT is need to be built when the block with given number is canonized. -pub fn is_build_required(cht_size: u64, block_num: N) -> Option +pub fn is_build_required(cht_size: N, block_num: N) -> Option where N: Clone + SimpleArithmetic, { - let block_cht_num = block_to_cht_number(cht_size, block_num.clone())?; + let block_cht_num = block_to_cht_number(cht_size.clone(), block_num.clone())?; let two = N::one() + N::one(); if block_cht_num < two { return None; @@ -67,7 +68,7 @@ pub fn is_build_required(cht_size: u64, block_num: N) -> Option /// SIZE items. The items are assumed to proceed sequentially from `start_number(cht_num)`. /// Discards the trie's nodes. pub fn compute_root( - cht_size: u64, + cht_size: Header::Number, cht_num: Header::Number, hashes: I, ) -> ClientResult @@ -84,7 +85,7 @@ pub fn compute_root( /// Build CHT-based header proof. pub fn build_proof( - cht_size: u64, + cht_size: Header::Number, cht_num: Header::Number, blocks: BlocksI, hashes: HashesI @@ -175,7 +176,7 @@ fn do_check_proof( /// Group ordered blocks by CHT number and call functor with blocks of each group. pub fn for_each_cht_group( - cht_size: u64, + cht_size: Header::Number, blocks: I, mut functor: F, mut functor_param: P, @@ -188,7 +189,7 @@ pub fn for_each_cht_group( let mut current_cht_num = None; let mut current_cht_blocks = Vec::new(); for block in blocks { - let new_cht_num = match block_to_cht_number(cht_size, block.saturated_into()) { + let new_cht_num = match block_to_cht_number(cht_size, block) { Some(new_cht_num) => new_cht_num, None => return Err(ClientError::Backend(format!( "Cannot compute CHT root for the block #{}", block)).into() @@ -203,7 +204,7 @@ pub fn for_each_cht_group( functor_param = functor( functor_param, - current_cht_num.saturated_into(), + current_cht_num, ::std::mem::replace(&mut current_cht_blocks, Vec::new()), )?; } @@ -215,7 +216,7 @@ pub fn for_each_cht_group( if let Some(current_cht_num) = current_cht_num { functor( functor_param, - current_cht_num.saturated_into(), + current_cht_num, ::std::mem::replace(&mut current_cht_blocks, Vec::new()), )?; } @@ -225,7 +226,7 @@ pub fn for_each_cht_group( /// Build pairs for computing CHT. fn build_pairs( - cht_size: u64, + cht_size: Header::Number, cht_num: Header::Number, hashes: I ) -> ClientResult, Vec)>> @@ -235,28 +236,25 @@ fn build_pairs( { let start_num = start_number(cht_size, cht_num); let mut pairs = Vec::new(); - let mut hash_number = start_num; - for hash in hashes.into_iter().take(cht_size as usize) { + let mut hash_index = Header::Number::zero(); + for hash in hashes.into_iter() { let hash = hash?.ok_or_else(|| ClientError::from( - ClientError::MissingHashRequiredForCHT( - cht_num.saturated_into::(), - hash_number.saturated_into::() - ) + ClientError::MissingHashRequiredForCHT ))?; pairs.push(( - encode_cht_key(hash_number).to_vec(), + encode_cht_key(start_num + hash_index).to_vec(), encode_cht_value(hash) )); - hash_number += Header::Number::one(); + hash_index += Header::Number::one(); + if hash_index == cht_size { + break; + } } - if pairs.len() as u64 == cht_size { + if hash_index == cht_size { Ok(pairs) } else { - Err(ClientError::MissingHashRequiredForCHT( - cht_num.saturated_into::(), - hash_number.saturated_into::() - )) + Err(ClientError::MissingHashRequiredForCHT) } } @@ -266,39 +264,28 @@ fn build_pairs( /// More generally: CHT N includes block (1 + N*SIZE)...((N+1)*SIZE). /// This is because the genesis hash is assumed to be known /// and including it would be redundant. -pub fn start_number(cht_size: u64, cht_num: N) -> N { - (cht_num * cht_size.saturated_into()) + N::one() +pub fn start_number(cht_size: N, cht_num: N) -> N { + (cht_num * cht_size) + N::one() } /// Get the ending block of a given CHT. -pub fn end_number(cht_size: u64, cht_num: N) -> N { - (cht_num + N::one()) * cht_size.saturated_into() +pub fn end_number(cht_size: N, cht_num: N) -> N { + (cht_num + N::one()) * cht_size } /// Convert a block number to a CHT number. /// Returns `None` for `block_num` == 0, `Some` otherwise. -pub fn block_to_cht_number(cht_size: u64, block_num: N) -> Option { +pub fn block_to_cht_number(cht_size: N, block_num: N) -> Option { if block_num == N::zero() { None } else { - Some((block_num - N::one()) / cht_size.saturated_into()) + Some((block_num - N::one()) / cht_size) } } /// Convert header number into CHT key. -pub fn encode_cht_key>(number: N) -> Vec { - // why not just use Encode? - let number: u64 = number.saturated_into(); - vec![ - (number >> 56) as u8, - ((number >> 48) & 0xff) as u8, - ((number >> 40) & 0xff) as u8, - ((number >> 32) & 0xff) as u8, - ((number >> 24) & 0xff) as u8, - ((number >> 16) & 0xff) as u8, - ((number >> 8) & 0xff) as u8, - (number & 0xff) as u8 - ] +pub fn encode_cht_key(number: N) -> Vec { + number.encode() } /// Convert header hash into CHT value. @@ -323,8 +310,8 @@ mod tests { #[test] fn is_build_required_works() { - assert_eq!(is_build_required(SIZE, 0u64), None); - assert_eq!(is_build_required(SIZE, 1u64), None); + assert_eq!(is_build_required(SIZE, 0u32.into()), None); + assert_eq!(is_build_required(SIZE, 1u32.into()), None); assert_eq!(is_build_required(SIZE, SIZE), None); assert_eq!(is_build_required(SIZE, SIZE + 1), None); assert_eq!(is_build_required(SIZE, 2 * SIZE), None); @@ -335,73 +322,101 @@ mod tests { #[test] fn start_number_works() { - assert_eq!(start_number(SIZE, 0u64), 1u64); - assert_eq!(start_number(SIZE, 1u64), SIZE + 1); - assert_eq!(start_number(SIZE, 2u64), SIZE + SIZE + 1); + assert_eq!(start_number(SIZE, 0u32), 1u32); + assert_eq!(start_number(SIZE, 1u32), SIZE + 1); + assert_eq!(start_number(SIZE, 2u32), SIZE + SIZE + 1); } #[test] fn end_number_works() { - assert_eq!(end_number(SIZE, 0u64), SIZE); - assert_eq!(end_number(SIZE, 1u64), SIZE + SIZE); - assert_eq!(end_number(SIZE, 2u64), SIZE + SIZE + SIZE); + assert_eq!(end_number(SIZE, 0u32), SIZE); + assert_eq!(end_number(SIZE, 1u32), SIZE + SIZE); + assert_eq!(end_number(SIZE, 2u32), SIZE + SIZE + SIZE); } #[test] fn build_pairs_fails_when_no_enough_blocks() { - assert!(build_pairs::(SIZE, 0, + assert!(build_pairs::(SIZE as _, 0, ::std::iter::repeat_with(|| Ok(Some(H256::from_low_u64_be(1)))).take(SIZE as usize / 2)).is_err()); } #[test] fn build_pairs_fails_when_missing_block() { - assert!(build_pairs::(SIZE, 0, ::std::iter::repeat_with(|| Ok(Some(H256::from_low_u64_be(1)))).take(SIZE as usize / 2) - .chain(::std::iter::once(Ok(None))) - .chain(::std::iter::repeat_with(|| Ok(Some(H256::from_low_u64_be(2)))).take(SIZE as usize / 2 - 1))).is_err()); + assert!(build_pairs::( + SIZE as _, + 0, + ::std::iter::repeat_with(|| Ok(Some(H256::from_low_u64_be(1)))) + .take(SIZE as usize / 2) + .chain(::std::iter::once(Ok(None))) + .chain(::std::iter::repeat_with(|| Ok(Some(H256::from_low_u64_be(2)))) + .take(SIZE as usize / 2 - 1)) + ).is_err()); } #[test] fn compute_root_works() { - assert!(compute_root::(SIZE, 42, - ::std::iter::repeat_with(|| Ok(Some(H256::from_low_u64_be(1)))).take(SIZE as usize)).is_ok()); + assert!(compute_root::( + SIZE as _, + 42, + ::std::iter::repeat_with(|| Ok(Some(H256::from_low_u64_be(1)))) + .take(SIZE as usize) + ).is_ok()); } #[test] #[should_panic] fn build_proof_panics_when_querying_wrong_block() { assert!(build_proof::( - SIZE, 0, vec![(SIZE * 1000) as u64], - ::std::iter::repeat_with(|| Ok(Some(H256::from_low_u64_be(1)))).take(SIZE as usize)).is_err()); + SIZE as _, + 0, + vec![(SIZE * 1000) as u64], + ::std::iter::repeat_with(|| Ok(Some(H256::from_low_u64_be(1)))) + .take(SIZE as usize) + ).is_err()); } #[test] fn build_proof_works() { assert!(build_proof::( - SIZE, 0, vec![(SIZE / 2) as u64], - ::std::iter::repeat_with(|| Ok(Some(H256::from_low_u64_be(1)))).take(SIZE as usize)).is_ok()); + SIZE as _, + 0, + vec![(SIZE / 2) as u64], + ::std::iter::repeat_with(|| Ok(Some(H256::from_low_u64_be(1)))) + .take(SIZE as usize) + ).is_ok()); } #[test] #[should_panic] fn for_each_cht_group_panics() { - let _ = for_each_cht_group::(SIZE, vec![SIZE * 5, SIZE * 2], |_, _, _| Ok(()), ()); + let cht_size = SIZE as u64; + let _ = for_each_cht_group::( + cht_size, + vec![cht_size * 5, cht_size * 2], + |_, _, _| Ok(()), + (), + ); } #[test] fn for_each_cht_group_works() { - let _ = for_each_cht_group::(SIZE, vec![ - SIZE * 2 + 1, SIZE * 2 + 2, SIZE * 2 + 5, - SIZE * 4 + 1, SIZE * 4 + 7, - SIZE * 6 + 1 - ], |_, cht_num, blocks| { - match cht_num { - 2 => assert_eq!(blocks, vec![SIZE * 2 + 1, SIZE * 2 + 2, SIZE * 2 + 5]), - 4 => assert_eq!(blocks, vec![SIZE * 4 + 1, SIZE * 4 + 7]), - 6 => assert_eq!(blocks, vec![SIZE * 6 + 1]), - _ => unreachable!(), - } + let cht_size = SIZE as u64; + let _ = for_each_cht_group::( + cht_size, + vec![ + cht_size * 2 + 1, cht_size * 2 + 2, cht_size * 2 + 5, + cht_size * 4 + 1, cht_size * 4 + 7, + cht_size * 6 + 1 + ], |_, cht_num, blocks| { + match cht_num { + 2 => assert_eq!(blocks, vec![cht_size * 2 + 1, cht_size * 2 + 2, cht_size * 2 + 5]), + 4 => assert_eq!(blocks, vec![cht_size * 4 + 1, cht_size * 4 + 7]), + 6 => assert_eq!(blocks, vec![cht_size * 6 + 1]), + _ => unreachable!(), + } - Ok(()) - }, ()); + Ok(()) + }, () + ); } } diff --git a/substrate/core/client/src/client.rs b/substrate/core/client/src/client.rs index a7e1e18847..f36992a14e 100644 --- a/substrate/core/client/src/client.rs +++ b/substrate/core/client/src/client.rs @@ -448,7 +448,7 @@ impl Client where /// Reads given header and generates CHT-based header proof. pub fn header_proof(&self, id: &BlockId) -> error::Result<(Block::Header, Vec>)> { - self.header_proof_with_cht_size(id, cht::SIZE) + self.header_proof_with_cht_size(id, cht::size()) } /// Get block hash by number. @@ -459,16 +459,23 @@ impl Client where } /// Reads given header and generates CHT-based header proof for CHT of given size. - pub fn header_proof_with_cht_size(&self, + pub fn header_proof_with_cht_size( + &self, id: &BlockId, - cht_size: u64 + cht_size: NumberFor, ) -> error::Result<(Block::Header, Vec>)> { let proof_error = || error::Error::Backend(format!("Failed to generate header proof for {:?}", id)); let header = self.backend.blockchain().expect_header(*id)?; let block_num = *header.number(); let cht_num = cht::block_to_cht_number(cht_size, block_num).ok_or_else(proof_error)?; let cht_start = cht::start_number(cht_size, cht_num); - let headers = (cht_start.saturated_into()..).map(|num| self.block_hash(num.saturated_into())); + let mut current_num = cht_start; + let cht_range = ::std::iter::from_fn(|| { + let old_current_num = current_num; + current_num = current_num + One::one(); + Some(old_current_num) + }); + let headers = cht_range.map(|num| self.block_hash(num)); let proof = cht::build_proof::(cht_size, cht_num, ::std::iter::once(block_num), headers)?; Ok((header, proof)) } @@ -486,14 +493,13 @@ impl Client where Some((config, storage)) => (config, storage), None => return Ok(None), }; - let first = first.saturated_into::(); - let last_num = self.backend.blockchain().expect_block_number_from_id(&last)?.saturated_into::(); + let last_num = self.backend.blockchain().expect_block_number_from_id(&last)?; if first > last_num { return Err(error::Error::ChangesTrieAccessFailed("Invalid changes trie range".into())); } let finalized_number = self.backend.blockchain().info()?.finalized_number; - let oldest = storage.oldest_changes_trie_block(&config, finalized_number.saturated_into::()); - let first = ::std::cmp::max(first, oldest).saturated_into::>(); + let oldest = storage.oldest_changes_trie_block(&config, finalized_number); + let first = ::std::cmp::max(first, oldest); Ok(Some((first, last))) } @@ -506,20 +512,20 @@ impl Client where key: &StorageKey ) -> error::Result, u32)>> { let (config, storage) = self.require_changes_trie()?; - let last_number = self.backend.blockchain().expect_block_number_from_id(&last)?.saturated_into::(); + let last_number = self.backend.blockchain().expect_block_number_from_id(&last)?; let last_hash = self.backend.blockchain().expect_block_hash_from_id(&last)?; - key_changes::<_, Blake2Hasher>( + key_changes::<_, Blake2Hasher, _>( &config, &*storage, - first.saturated_into::(), + first, &ChangesTrieAnchorBlockId { hash: convert_hash(&last_hash), number: last_number, }, - self.backend.blockchain().info()?.best_number.saturated_into::(), + self.backend.blockchain().info()?.best_number, &key.0) - .and_then(|r| r.map(|r| r.map(|(block, tx)| (block.saturated_into(), tx))).collect::>()) + .and_then(|r| r.map(|r| r.map(|(block, tx)| (block, tx))).collect::>()) .map_err(|err| error::Error::ChangesTrieAccessFailed(err)) } @@ -543,7 +549,7 @@ impl Client where min, max, key, - cht::SIZE, + cht::size(), ) } @@ -555,21 +561,29 @@ impl Client where min: Block::Hash, max: Block::Hash, key: &StorageKey, - cht_size: u64, + cht_size: NumberFor, ) -> error::Result> { struct AccessedRootsRecorder<'a, Block: BlockT> { - storage: &'a ChangesTrieStorage, - min: u64, + storage: &'a ChangesTrieStorage>, + min: NumberFor, required_roots_proofs: Mutex, H256>>, }; - impl<'a, Block: BlockT> ChangesTrieRootsStorage for AccessedRootsRecorder<'a, Block> { - fn root(&self, anchor: &ChangesTrieAnchorBlockId, block: u64) -> Result, String> { + impl<'a, Block: BlockT> ChangesTrieRootsStorage> for AccessedRootsRecorder<'a, Block> { + fn build_anchor(&self, hash: H256) -> Result>, String> { + self.storage.build_anchor(hash) + } + + fn root( + &self, + anchor: &ChangesTrieAnchorBlockId>, + block: NumberFor, + ) -> Result, String> { let root = self.storage.root(anchor, block)?; if block < self.min { if let Some(ref root) = root { self.required_roots_proofs.lock().insert( - block.saturated_into(), + block, root.clone() ); } @@ -578,7 +592,7 @@ impl Client where } } - impl<'a, Block: BlockT> ChangesTrieStorage for AccessedRootsRecorder<'a, Block> { + impl<'a, Block: BlockT> ChangesTrieStorage> for AccessedRootsRecorder<'a, Block> { fn get(&self, key: &H256, prefix: &[u8]) -> Result, String> { self.storage.get(key, prefix) } @@ -589,7 +603,7 @@ impl Client where let recording_storage = AccessedRootsRecorder:: { storage, - min: min_number.saturated_into::(), + min: min_number, required_roots_proofs: Mutex::new(BTreeMap::new()), }; @@ -600,12 +614,10 @@ impl Client where // fetch key changes proof let first_number = self.backend.blockchain() - .expect_block_number_from_id(&BlockId::Hash(first))? - .saturated_into::(); + .expect_block_number_from_id(&BlockId::Hash(first))?; let last_number = self.backend.blockchain() - .expect_block_number_from_id(&BlockId::Hash(last))? - .saturated_into::(); - let key_changes_proof = key_changes_proof::<_, Blake2Hasher>( + .expect_block_number_from_id(&BlockId::Hash(last))?; + let key_changes_proof = key_changes_proof::<_, Blake2Hasher, _>( &config, &recording_storage, first_number, @@ -613,7 +625,7 @@ impl Client where hash: convert_hash(&last), number: last_number, }, - max_number.saturated_into::(), + max_number, &key.0 ) .map_err(|err| error::Error::from(error::Error::ChangesTrieAccessFailed(err)))?; @@ -634,7 +646,7 @@ impl Client where /// Generate CHT-based proof for roots of changes tries at given blocks. fn changes_trie_roots_proof>>( &self, - cht_size: u64, + cht_size: NumberFor, blocks: I ) -> error::Result>> { // most probably we have touched several changes tries that are parts of the single CHT @@ -653,12 +665,19 @@ impl Client where /// Generates CHT-based proof for roots of changes tries at given blocks (that are part of single CHT). fn changes_trie_roots_proof_at_cht( &self, - cht_size: u64, + cht_size: NumberFor, cht_num: NumberFor, blocks: Vec> ) -> error::Result>> { let cht_start = cht::start_number(cht_size, cht_num); - let roots = (cht_start.saturated_into()..).map(|num| self.header(&BlockId::Number(num.saturated_into())) + let mut current_num = cht_start; + let cht_range = ::std::iter::from_fn(|| { + let old_current_num = current_num; + current_num = current_num + One::one(); + Some(old_current_num) + }); + let roots = cht_range + .map(|num| self.header(&BlockId::Number(num)) .map(|block| block.and_then(|block| block.digest().log(DigestItem::as_changes_trie_root).cloned()))); let proof = cht::build_proof::(cht_size, cht_num, blocks, roots)?; Ok(proof) diff --git a/substrate/core/client/src/error.rs b/substrate/core/client/src/error.rs index b201c092e0..bad279f2e4 100644 --- a/substrate/core/client/src/error.rs +++ b/substrate/core/client/src/error.rs @@ -92,8 +92,8 @@ pub enum Error { #[display(fmt = "Potential long-range attack: block not in finalized chain.")] NotInFinalizedChain, /// Hash that is required for building CHT is missing. - #[display(fmt = "Failed to get hash of block#{} for building CHT#{}", _0, _1)] - MissingHashRequiredForCHT(u64, u64), + #[display(fmt = "Failed to get hash of block for building CHT")] + MissingHashRequiredForCHT, /// A convenience variant for String #[display(fmt = "{}", _0)] Msg(String), diff --git a/substrate/core/client/src/genesis.rs b/substrate/core/client/src/genesis.rs index 74bc74360a..73bd1e0368 100644 --- a/substrate/core/client/src/genesis.rs +++ b/substrate/core/client/src/genesis.rs @@ -84,7 +84,7 @@ mod tests { state_machine::new( backend, - Some(&InMemoryChangesTrieStorage::new()), + Some(&InMemoryChangesTrieStorage::<_, u64>::new()), state_machine::NeverOffchainExt::new(), &mut overlay, &executor(), @@ -97,7 +97,7 @@ mod tests { for tx in transactions.iter() { state_machine::new( backend, - Some(&InMemoryChangesTrieStorage::new()), + Some(&InMemoryChangesTrieStorage::<_, u64>::new()), state_machine::NeverOffchainExt::new(), &mut overlay, &executor(), @@ -110,7 +110,7 @@ mod tests { let (ret_data, _, _) = state_machine::new( backend, - Some(&InMemoryChangesTrieStorage::new()), + Some(&InMemoryChangesTrieStorage::<_, u64>::new()), state_machine::NeverOffchainExt::new(), &mut overlay, &executor(), @@ -157,7 +157,7 @@ mod tests { let mut overlay = OverlayedChanges::default(); let _ = state_machine::new( &backend, - Some(&InMemoryChangesTrieStorage::new()), + Some(&InMemoryChangesTrieStorage::<_, u64>::new()), state_machine::NeverOffchainExt::new(), &mut overlay, &executor(), @@ -186,7 +186,7 @@ mod tests { let mut overlay = OverlayedChanges::default(); let _ = state_machine::new( &backend, - Some(&InMemoryChangesTrieStorage::new()), + Some(&InMemoryChangesTrieStorage::<_, u64>::new()), state_machine::NeverOffchainExt::new(), &mut overlay, &executor(), @@ -215,7 +215,7 @@ mod tests { let mut overlay = OverlayedChanges::default(); let r = state_machine::new( &backend, - Some(&InMemoryChangesTrieStorage::new()), + Some(&InMemoryChangesTrieStorage::<_, u64>::new()), state_machine::NeverOffchainExt::new(), &mut overlay, &Executor::new(None), diff --git a/substrate/core/client/src/in_mem.rs b/substrate/core/client/src/in_mem.rs index 558643e887..1172939b34 100644 --- a/substrate/core/client/src/in_mem.rs +++ b/substrate/core/client/src/in_mem.rs @@ -23,7 +23,7 @@ use primitives::{ChangesTrieConfiguration, storage::well_known_keys}; use runtime_primitives::generic::BlockId; use runtime_primitives::traits::{ Block as BlockT, Header as HeaderT, Zero, - NumberFor, SaturatedConversion, Digest, DigestItem + NumberFor, Digest, DigestItem }; use runtime_primitives::{Justification, StorageOverlay, ChildrenStorageOverlay}; use state_machine::backend::{Backend as StateBackend, InMemory}; @@ -415,12 +415,20 @@ impl light::blockchain::Storage for Blockchain Blockchain::finalize_header(self, id, None) } - fn header_cht_root(&self, _cht_size: u64, block: NumberFor) -> error::Result { + fn header_cht_root( + &self, + _cht_size: NumberFor, + block: NumberFor, + ) -> error::Result { self.storage.read().header_cht_roots.get(&block).cloned() .ok_or_else(|| error::Error::Backend(format!("Header CHT for block {} not exists", block))) } - fn changes_trie_cht_root(&self, _cht_size: u64, block: NumberFor) -> error::Result { + fn changes_trie_cht_root( + &self, + _cht_size: NumberFor, + block: NumberFor, + ) -> error::Result { self.storage.read().changes_trie_cht_roots.get(&block).cloned() .ok_or_else(|| error::Error::Backend(format!("Changes trie CHT for block {} not exists", block))) } @@ -531,7 +539,7 @@ where H::Out: Ord, { states: RwLock>>, - changes_trie_storage: ChangesTrieStorage, + changes_trie_storage: ChangesTrieStorage, blockchain: Blockchain, } @@ -581,7 +589,7 @@ where type BlockImportOperation = BlockImportOperation; type Blockchain = Blockchain; type State = InMemory; - type ChangesTrieStorage = ChangesTrieStorage; + type ChangesTrieStorage = ChangesTrieStorage; fn begin_operation(&self) -> error::Result { let old_state = self.state_at(BlockId::Hash(Default::default()))?; @@ -622,7 +630,7 @@ where if let Some(changes_trie_update) = operation.changes_trie_update { let changes_trie_root: H::Out = changes_trie_root.into(); self.changes_trie_storage.0.insert( - (*header.number()).saturated_into::(), + *header.number(), changes_trie_root, changes_trie_update ); @@ -699,22 +707,45 @@ where } /// Prunable in-memory changes trie storage. -pub struct ChangesTrieStorage(InMemoryChangesTrieStorage); -impl backend::PrunableStateChangesTrieStorage for ChangesTrieStorage { - fn oldest_changes_trie_block(&self, _config: &ChangesTrieConfiguration, _best_finalized: u64) -> u64 { - 0 +pub struct ChangesTrieStorage(InMemoryChangesTrieStorage>); +impl backend::PrunableStateChangesTrieStorage for ChangesTrieStorage { + fn oldest_changes_trie_block( + &self, + _config: &ChangesTrieConfiguration, + _best_finalized: NumberFor, + ) -> NumberFor { + Zero::zero() } } -impl state_machine::ChangesTrieRootsStorage for ChangesTrieStorage { - fn root(&self, anchor: &ChangesTrieAnchorBlockId, block: u64) -> Result, String> { - self.0.root(anchor, block) +impl state_machine::ChangesTrieRootsStorage> for ChangesTrieStorage + where + Block: BlockT, + H: Hasher, +{ + fn build_anchor( + &self, + _hash: H::Out, + ) -> Result>, String> { + Err("Dummy implementation".into()) + } + + fn root( + &self, + _anchor: &ChangesTrieAnchorBlockId>, + _block: NumberFor, + ) -> Result, String> { + Err("Dummy implementation".into()) } } -impl state_machine::ChangesTrieStorage for ChangesTrieStorage { - fn get(&self, key: &H::Out, prefix: &[u8]) -> Result, String> { - self.0.get(key, prefix) +impl state_machine::ChangesTrieStorage> for ChangesTrieStorage + where + Block: BlockT, + H: Hasher, +{ + fn get(&self, _key: &H::Out, _prefix: &[u8]) -> Result, String> { + Err("Dummy implementation".into()) } } diff --git a/substrate/core/client/src/light/backend.rs b/substrate/core/client/src/light/backend.rs index 0579c8c8d4..bf5eb11f26 100644 --- a/substrate/core/client/src/light/backend.rs +++ b/substrate/core/client/src/light/backend.rs @@ -112,7 +112,7 @@ impl ClientBackend for Backend where type BlockImportOperation = ImportOperation; type Blockchain = Blockchain; type State = OnDemandOrGenesisState; - type ChangesTrieStorage = in_mem::ChangesTrieStorage; + type ChangesTrieStorage = in_mem::ChangesTrieStorage; fn begin_operation(&self) -> ClientResult { Ok(ImportOperation { diff --git a/substrate/core/client/src/light/blockchain.rs b/substrate/core/client/src/light/blockchain.rs index d7fcd44232..b231f7e8aa 100644 --- a/substrate/core/client/src/light/blockchain.rs +++ b/substrate/core/client/src/light/blockchain.rs @@ -56,10 +56,18 @@ pub trait Storage: AuxStore + BlockchainHeaderBackend { fn last_finalized(&self) -> ClientResult; /// Get headers CHT root for given block. Fails if the block is not pruned (not a part of any CHT). - fn header_cht_root(&self, cht_size: u64, block: NumberFor) -> ClientResult; + fn header_cht_root( + &self, + cht_size: NumberFor, + block: NumberFor, + ) -> ClientResult; /// Get changes trie CHT root for given block. Fails if the block is not pruned (not a part of any CHT). - fn changes_trie_cht_root(&self, cht_size: u64, block: NumberFor) -> ClientResult; + fn changes_trie_cht_root( + &self, + cht_size: NumberFor, + block: NumberFor, + ) -> ClientResult; /// Get storage cache. fn cache(&self) -> Option>>; @@ -116,7 +124,7 @@ impl BlockchainHeaderBackend for Blockchain where Bloc self.fetcher().upgrade().ok_or(ClientError::NotAvailableOnLightClient)? .remote_header(RemoteHeaderRequest { - cht_root: self.storage.header_cht_root(cht::SIZE, number)?, + cht_root: self.storage.header_cht_root(cht::size(), number)?, block: number, retry_count: None, }) diff --git a/substrate/core/client/src/light/call_executor.rs b/substrate/core/client/src/light/call_executor.rs index a45a166674..982f106f2a 100644 --- a/substrate/core/client/src/light/call_executor.rs +++ b/substrate/core/client/src/light/call_executor.rs @@ -435,7 +435,7 @@ pub fn check_execution_proof( Header: HeaderT, E: CodeExecutor, H: Hasher, - H::Out: Ord, + H::Out: Ord + 'static, { let local_state_root = request.header.state_root(); let root: H::Out = convert_hash(&local_state_root); diff --git a/substrate/core/client/src/light/fetcher.rs b/substrate/core/client/src/light/fetcher.rs index 29fd6f4c39..39563aa83c 100644 --- a/substrate/core/client/src/light/fetcher.rs +++ b/substrate/core/client/src/light/fetcher.rs @@ -22,11 +22,11 @@ use std::marker::PhantomData; use futures::IntoFuture; use hash_db::{HashDB, Hasher}; -use parity_codec::Encode; +use parity_codec::{Decode, Encode}; use primitives::{ChangesTrieConfiguration, convert_hash}; use runtime_primitives::traits::{ Block as BlockT, Header as HeaderT, Hash, HashFor, NumberFor, - UniqueSaturatedInto, UniqueSaturatedFrom, SaturatedConversion + SimpleArithmetic, CheckedConversion, }; use state_machine::{CodeExecutor, ChangesTrieRootsStorage, ChangesTrieAnchorBlockId, TrieBackend, read_proof_check, key_changes_proof_check, @@ -236,7 +236,7 @@ impl, F> LightDataChecker, remote_proof: ChangesProof, - cht_size: u64, + cht_size: NumberFor, ) -> ClientResult, u32)>> where H: Hasher, @@ -284,30 +284,27 @@ impl, F> LightDataChecker( + key_changes_proof_check::<_, H, _>( &request.changes_trie_config, &RootsStorage { roots: (request.tries_roots.0, &request.tries_roots.2), prev_roots: remote_roots, }, remote_proof, - request.first_block.0.saturated_into::(), + request.first_block.0, &ChangesTrieAnchorBlockId { hash: convert_hash(&request.last_block.1), - number: request.last_block.0.saturated_into::(), + number: request.last_block.0, }, - remote_max_block.saturated_into::(), + remote_max_block, &request.key) - .map(|pairs| pairs.into_iter().map(|(b, x)| - (b.saturated_into::>(), x) - ).collect()) .map_err(|err| ClientError::ChangesTrieAccessFailed(err)) } /// Check CHT-based proof for changes tries roots. fn check_changes_tries_proof( &self, - cht_size: u64, + cht_size: NumberFor, remote_roots: &BTreeMap, B::Hash>, remote_roots_proof: Vec>, ) -> ClientResult<()> @@ -363,7 +360,7 @@ impl FetchChecker for LightDataChecker, H: Hasher, - H::Out: Ord, + H::Out: Ord + 'static, S: BlockchainStorage, F: Send + Sync, { @@ -419,7 +416,7 @@ impl FetchChecker for LightDataChecker, remote_proof: ChangesProof ) -> ClientResult, u32)>> { - self.check_changes_proof_with_cht_size(request, remote_proof, cht::SIZE) + self.check_changes_proof_with_cht_size(request, remote_proof, cht::size()) } fn check_body_proof( @@ -443,26 +440,38 @@ impl FetchChecker for LightDataChecker as a changes trie roots storage. -struct RootsStorage<'a, Number: UniqueSaturatedInto + UniqueSaturatedFrom, Hash: 'a> { +struct RootsStorage<'a, Number: SimpleArithmetic, Hash: 'a> { roots: (Number, &'a [Hash]), prev_roots: BTreeMap, } -impl<'a, H, Number, Hash> ChangesTrieRootsStorage for RootsStorage<'a, Number, Hash> +impl<'a, H, Number, Hash> ChangesTrieRootsStorage for RootsStorage<'a, Number, Hash> where H: Hasher, - Number: Send + Sync + Eq + ::std::cmp::Ord + Copy + UniqueSaturatedInto - + UniqueSaturatedFrom, + Number: ::std::fmt::Display + Clone + SimpleArithmetic + Encode + Decode + Send + Sync + 'static, Hash: 'a + Send + Sync + Clone + AsRef<[u8]>, { - fn root(&self, _anchor: &ChangesTrieAnchorBlockId, block: u64) -> Result, String> { + fn build_anchor( + &self, + _hash: H::Out, + ) -> Result, String> { + Err("build_anchor is only called when building block".into()) + } + + fn root( + &self, + _anchor: &ChangesTrieAnchorBlockId, + block: Number, + ) -> Result, String> { // we can't ask for roots from parallel forks here => ignore anchor - let root = if block < self.roots.0.saturated_into::() { + let root = if block < self.roots.0 { self.prev_roots.get(&Number::unique_saturated_from(block)).cloned() } else { - block.checked_sub(self.roots.0.saturated_into::()) - .and_then(|index| self.roots.1.get(index as usize)) - .cloned() + let index: Option = block.checked_sub(&self.roots.0).and_then(|index| index.checked_into()); + match index { + Some(index) => self.roots.1.get(index as usize).cloned(), + None => None, + } }; Ok(root.map(|root| { diff --git a/substrate/core/executor/src/sandbox.rs b/substrate/core/executor/src/sandbox.rs index 24c8ad66ea..377294f7b1 100644 --- a/substrate/core/executor/src/sandbox.rs +++ b/substrate/core/executor/src/sandbox.rs @@ -569,9 +569,11 @@ mod tests { use crate::allocator; use crate::sandbox::trap; use crate::wasm_executor::WasmExecutor; - use state_machine::TestExternalities; + use state_machine::TestExternalities as CoreTestExternalities; use wabt; + type TestExternalities = CoreTestExternalities; + #[test] fn sandbox_should_work() { let mut ext = TestExternalities::::default(); diff --git a/substrate/core/executor/src/wasm_executor.rs b/substrate/core/executor/src/wasm_executor.rs index 988b6ddcf3..bed60666f7 100644 --- a/substrate/core/executor/src/wasm_executor.rs +++ b/substrate/core/executor/src/wasm_executor.rs @@ -414,7 +414,7 @@ impl_function_executor!(this: FunctionExecutor<'e, E>, .map_err(|_| UserError("Invalid attempt to write written_out in ext_child_storage_root"))?; Ok(offset) }, - ext_storage_changes_root(parent_hash_data: *const u8, parent_hash_len: u32, parent_number: u64, result: *mut u8) -> u32 => { + ext_storage_changes_root(parent_hash_data: *const u8, parent_hash_len: u32, result: *mut u8) -> u32 => { let mut parent_hash = H256::default(); if parent_hash_len != parent_hash.as_ref().len() as u32 { return Err(UserError("Invalid parent_hash_len in ext_storage_changes_root").into()); @@ -422,7 +422,8 @@ impl_function_executor!(this: FunctionExecutor<'e, E>, let raw_parent_hash = this.memory.get(parent_hash_data, parent_hash_len as usize) .map_err(|_| UserError("Invalid attempt to get parent_hash in ext_storage_changes_root"))?; parent_hash.as_mut().copy_from_slice(&raw_parent_hash[..]); - let r = this.ext.storage_changes_root(parent_hash, parent_number); + let r = this.ext.storage_changes_root(parent_hash) + .map_err(|_| UserError("Invaid parent_hash passed to ext_storage_changes_root"))?; if let Some(r) = r { this.memory.set(result, &r[..]).map_err(|_| UserError("Invalid attempt to set memory in ext_storage_changes_root"))?; Ok(1) @@ -896,10 +897,12 @@ mod tests { use parity_codec::Encode; - use state_machine::TestExternalities; + use state_machine::TestExternalities as CoreTestExternalities; use hex_literal::hex; use primitives::map; + type TestExternalities = CoreTestExternalities; + #[test] fn returning_should_work() { let mut ext = TestExternalities::default(); diff --git a/substrate/core/executor/wasm/Cargo.lock b/substrate/core/executor/wasm/Cargo.lock index b65d8e0869..8c53b790d8 100644 --- a/substrate/core/executor/wasm/Cargo.lock +++ b/substrate/core/executor/wasm/Cargo.lock @@ -8,6 +8,11 @@ dependencies = [ "nodrop 0.1.12 (registry+https://github.com/rust-lang/crates.io-index)", ] +[[package]] +name = "autocfg" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" + [[package]] name = "byteorder" version = "1.3.1" @@ -52,6 +57,14 @@ name = "nodrop" version = "0.1.12" source = "registry+https://github.com/rust-lang/crates.io-index" +[[package]] +name = "num-traits" +version = "0.2.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "autocfg 0.1.4 (registry+https://github.com/rust-lang/crates.io-index)", +] + [[package]] name = "parity-codec" version = "3.5.1" @@ -186,6 +199,7 @@ dependencies = [ "byteorder 1.3.1 (registry+https://github.com/rust-lang/crates.io-index)", "hash-db 0.12.0 (registry+https://github.com/rust-lang/crates.io-index)", "hash256-std-hasher 0.12.0 (registry+https://github.com/rust-lang/crates.io-index)", + "num-traits 0.2.8 (registry+https://github.com/rust-lang/crates.io-index)", "parity-codec 3.5.1 (registry+https://github.com/rust-lang/crates.io-index)", "primitive-types 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)", "rustc-hex 2.0.1 (registry+https://github.com/rust-lang/crates.io-index)", @@ -227,6 +241,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" [metadata] "checksum arrayvec 0.4.7 (registry+https://github.com/rust-lang/crates.io-index)" = "a1e964f9e24d588183fcb43503abda40d288c8657dfc27311516ce2f05675aef" +"checksum autocfg 0.1.4 (registry+https://github.com/rust-lang/crates.io-index)" = "0e49efa51329a5fd37e7c79db4621af617cd4e3e5bc224939808d076077077bf" "checksum byteorder 1.3.1 (registry+https://github.com/rust-lang/crates.io-index)" = "a019b10a2a7cdeb292db131fc8113e57ea2a908f6e7894b0c3c671893b65dbeb" "checksum crunchy 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)" = "c240f247c278fa08a6d4820a6a222bfc6e0d999e51ba67be94f44c905b2161f2" "checksum fixed-hash 0.3.0 (registry+https://github.com/rust-lang/crates.io-index)" = "a557e80084b05c32b455963ff565a9de6f2866da023d6671705c6aff6f65e01c" @@ -234,6 +249,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" "checksum hash256-std-hasher 0.12.0 (registry+https://github.com/rust-lang/crates.io-index)" = "1224388a21c88a80ae7087a2a245ca6d80acc97a9186b75789fb3eeefd0609af" "checksum impl-codec 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)" = "d2050d823639fbeae26b2b5ba09aca8907793117324858070ade0673c49f793b" "checksum nodrop 0.1.12 (registry+https://github.com/rust-lang/crates.io-index)" = "9a2228dca57108069a5262f2ed8bd2e82496d2e074a06d1ccc7ce1687b6ae0a2" +"checksum num-traits 0.2.8 (registry+https://github.com/rust-lang/crates.io-index)" = "6ba9a427cfca2be13aa6f6403b0b7e7368fe982bfa16fccc450ce74c46cd9b32" "checksum parity-codec 3.5.1 (registry+https://github.com/rust-lang/crates.io-index)" = "dcb43c05fb71c03b4ea7327bf15694da1e0f23f19d5b1e95bab6c6d74097e336" "checksum parity-codec-derive 3.3.0 (registry+https://github.com/rust-lang/crates.io-index)" = "00a486fd383382ddcb2de928364b1f82571c1e48274fc43b7667a4738ee4056c" "checksum primitive-types 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)" = "edb92f1ebfc177432c03287b15d48c202e6e2c95993a7af3ba039abb43b1492e" diff --git a/substrate/core/primitives/Cargo.toml b/substrate/core/primitives/Cargo.toml index 8eb62ddf84..0875de55b7 100644 --- a/substrate/core/primitives/Cargo.toml +++ b/substrate/core/primitives/Cargo.toml @@ -25,7 +25,8 @@ sha2 = { version = "0.8", optional = true } substrate-bip39 = { git = "https://github.com/paritytech/substrate-bip39", optional = true } tiny-bip39 = { version = "0.6.1", optional = true } hex = { version = "0.3", optional = true } -regex = {version = "1.1", optional = true } +regex = { version = "1.1", optional = true } +num-traits = { version = "0.2", default-features = false } [dev-dependencies] substrate-serializer = { path = "../serializer" } @@ -71,4 +72,5 @@ std = [ "sha2", "schnorrkel", "regex", + "num-traits/std", ] diff --git a/substrate/core/primitives/src/changes_trie.rs b/substrate/core/primitives/src/changes_trie.rs index 2fa11f5641..eb6a75454f 100644 --- a/substrate/core/primitives/src/changes_trie.rs +++ b/substrate/core/primitives/src/changes_trie.rs @@ -19,6 +19,7 @@ #[cfg(any(feature = "std", test))] use serde::{Serialize, Deserialize}; use parity_codec::{Encode, Decode}; +use num_traits::Zero; /// Substrate changes trie configuration. #[cfg_attr(any(feature = "std", test), derive(Serialize, Deserialize))] @@ -26,10 +27,14 @@ use parity_codec::{Encode, Decode}; pub struct ChangesTrieConfiguration { /// Interval (in blocks) at which level1-digests are created. Digests are not /// created when this is less or equal to 1. - pub digest_interval: u64, + pub digest_interval: u32, /// Maximal number of digest levels in hierarchy. 0 means that digests are not /// created at all (even level1 digests). 1 means only level1-digests are created. /// 2 means that every digest_interval^2 there will be a level2-digest, and so on. + /// Please ensure that maximum digest interval (i.e. digest_interval^digest_levels) + /// is within `u32` limits. Otherwise you'll never see digests covering such intervals + /// && maximal digests interval will be truncated to the last interval that fits + /// `u32` limits. pub digest_levels: u32, } @@ -40,20 +45,30 @@ impl ChangesTrieConfiguration { } /// Do we need to build digest at given block? - pub fn is_digest_build_required_at_block(&self, block: u64) -> bool { - block != 0 + pub fn is_digest_build_required_at_block(&self, block: Number) -> bool + where + Number: From + PartialEq + ::rstd::ops::Rem + Zero, + { + block != 0.into() && self.is_digest_build_enabled() - && block % self.digest_interval == 0 + && (block % self.digest_interval.into()).is_zero() } /// Returns max digest interval. One if digests are not created at all. - /// Returns ::std::u64::MAX instead of panic in the case of overflow. - pub fn max_digest_interval(&self) -> u64 { + pub fn max_digest_interval(&self) -> u32 { if !self.is_digest_build_enabled() { return 1; } - self.digest_interval.saturating_pow(self.digest_levels) + // we'll get >1 loop iteration only when bad configuration parameters are selected + let mut current_level = self.digest_levels; + loop { + if let Some(max_digest_interval) = self.digest_interval.checked_pow(current_level) { + return max_digest_interval; + } + + current_level = current_level - 1; + } } /// Returns Some if digest must be built at given block number. @@ -63,17 +78,21 @@ impl ChangesTrieConfiguration { /// digest interval (in blocks) /// step between blocks we're interested in when digest is built /// ) - pub fn digest_level_at_block(&self, block: u64) -> Option<(u32, u64, u64)> { - if !self.is_digest_build_required_at_block(block) { + pub fn digest_level_at_block(&self, block: Number) -> Option<(u32, u32, u32)> + where + Number: Clone + From + PartialEq + ::rstd::ops::Rem + Zero, + { + if !self.is_digest_build_required_at_block(block.clone()) { return None; } let mut digest_interval = self.digest_interval; let mut current_level = 1u32; - let mut digest_step = 1u64; + let mut digest_step = 1u32; while current_level < self.digest_levels { let new_digest_interval = match digest_interval.checked_mul(self.digest_interval) { - Some(new_digest_interval) if block % new_digest_interval == 0 => new_digest_interval, + Some(new_digest_interval) if (block.clone() % new_digest_interval.into()).is_zero() + => new_digest_interval, _ => break, }; @@ -94,7 +113,7 @@ impl ChangesTrieConfiguration { mod tests { use super::ChangesTrieConfiguration; - fn config(interval: u64, levels: u32) -> ChangesTrieConfiguration { + fn config(interval: u32, levels: u32) -> ChangesTrieConfiguration { ChangesTrieConfiguration { digest_interval: interval, digest_levels: levels, @@ -112,31 +131,31 @@ mod tests { #[test] fn is_digest_build_required_at_block_works() { - assert!(!config(8, 4).is_digest_build_required_at_block(0)); - assert!(!config(8, 4).is_digest_build_required_at_block(1)); - assert!(!config(8, 4).is_digest_build_required_at_block(2)); - assert!(!config(8, 4).is_digest_build_required_at_block(4)); - assert!(config(8, 4).is_digest_build_required_at_block(8)); - assert!(!config(8, 4).is_digest_build_required_at_block(9)); - assert!(config(8, 4).is_digest_build_required_at_block(64)); - assert!(config(8, 4).is_digest_build_required_at_block(64)); - assert!(config(8, 4).is_digest_build_required_at_block(512)); - assert!(config(8, 4).is_digest_build_required_at_block(4096)); - assert!(!config(8, 4).is_digest_build_required_at_block(4103)); - assert!(config(8, 4).is_digest_build_required_at_block(4104)); - assert!(!config(8, 4).is_digest_build_required_at_block(4108)); + assert!(!config(8, 4).is_digest_build_required_at_block(0u64)); + assert!(!config(8, 4).is_digest_build_required_at_block(1u64)); + assert!(!config(8, 4).is_digest_build_required_at_block(2u64)); + assert!(!config(8, 4).is_digest_build_required_at_block(4u64)); + assert!(config(8, 4).is_digest_build_required_at_block(8u64)); + assert!(!config(8, 4).is_digest_build_required_at_block(9u64)); + assert!(config(8, 4).is_digest_build_required_at_block(64u64)); + assert!(config(8, 4).is_digest_build_required_at_block(64u64)); + assert!(config(8, 4).is_digest_build_required_at_block(512u64)); + assert!(config(8, 4).is_digest_build_required_at_block(4096u64)); + assert!(!config(8, 4).is_digest_build_required_at_block(4103u64)); + assert!(config(8, 4).is_digest_build_required_at_block(4104u64)); + assert!(!config(8, 4).is_digest_build_required_at_block(4108u64)); } #[test] fn digest_level_at_block_works() { - assert_eq!(config(8, 4).digest_level_at_block(0), None); - assert_eq!(config(8, 4).digest_level_at_block(7), None); - assert_eq!(config(8, 4).digest_level_at_block(63), None); - assert_eq!(config(8, 4).digest_level_at_block(8), Some((1, 8, 1))); - assert_eq!(config(8, 4).digest_level_at_block(64), Some((2, 64, 8))); - assert_eq!(config(8, 4).digest_level_at_block(512), Some((3, 512, 64))); - assert_eq!(config(8, 4).digest_level_at_block(4096), Some((4, 4096, 512))); - assert_eq!(config(8, 4).digest_level_at_block(4112), Some((1, 8, 1))); + assert_eq!(config(8, 4).digest_level_at_block(0u64), None); + assert_eq!(config(8, 4).digest_level_at_block(7u64), None); + assert_eq!(config(8, 4).digest_level_at_block(63u64), None); + assert_eq!(config(8, 4).digest_level_at_block(8u64), Some((1, 8, 1))); + assert_eq!(config(8, 4).digest_level_at_block(64u64), Some((2, 64, 8))); + assert_eq!(config(8, 4).digest_level_at_block(512u64), Some((3, 512, 64))); + assert_eq!(config(8, 4).digest_level_at_block(4096u64), Some((4, 4096, 512))); + assert_eq!(config(8, 4).digest_level_at_block(4112u64), Some((1, 8, 1))); } #[test] @@ -144,6 +163,6 @@ mod tests { assert_eq!(config(0, 0).max_digest_interval(), 1); assert_eq!(config(2, 2).max_digest_interval(), 4); assert_eq!(config(8, 4).max_digest_interval(), 4096); - assert_eq!(config(::std::u64::MAX, 1024).max_digest_interval(), ::std::u64::MAX); + assert_eq!(config(::std::u32::MAX, 1024).max_digest_interval(), ::std::u32::MAX); } } diff --git a/substrate/core/sr-io/src/lib.rs b/substrate/core/sr-io/src/lib.rs index 47af4de98b..314aa3b61e 100644 --- a/substrate/core/sr-io/src/lib.rs +++ b/substrate/core/sr-io/src/lib.rs @@ -138,7 +138,7 @@ export_api! { fn child_storage_root(storage_key: &[u8]) -> Vec; /// "Commit" all existing operations and get the resultant storage change root. - fn storage_changes_root(parent_hash: [u8; 32], parent_num: u64) -> Option<[u8; 32]>; + fn storage_changes_root(parent_hash: [u8; 32]) -> Option<[u8; 32]>; /// A trie root formed from the enumerated items. /// TODO [#2382] remove (just use `ordered_trie_root` (NOTE currently not implemented for without_std)) @@ -251,6 +251,10 @@ mod imp { } #[cfg(feature = "std")] -pub use self::imp::{StorageOverlay, ChildrenStorageOverlay, with_storage, with_externalities, TestExternalities}; +pub use self::imp::{StorageOverlay, ChildrenStorageOverlay, with_storage, with_externalities}; #[cfg(not(feature = "std"))] pub use self::imp::ext::*; + +/// Type alias for Externalities implementation used in tests. +#[cfg(feature = "std")] +pub type TestExternalities = self::imp::TestExternalities; diff --git a/substrate/core/sr-io/with_std.rs b/substrate/core/sr-io/with_std.rs index d73ccf4b66..1bc0141c1d 100644 --- a/substrate/core/sr-io/with_std.rs +++ b/substrate/core/sr-io/with_std.rs @@ -158,10 +158,10 @@ impl StorageApi for () { }).expect("child_storage_root cannot be called outside of an Externalities-provided environment.") } - fn storage_changes_root(parent_hash: [u8; 32], parent_num: u64) -> Option<[u8; 32]> { + fn storage_changes_root(parent_hash: [u8; 32]) -> Option<[u8; 32]> { ext::with(|ext| - ext.storage_changes_root(parent_hash.into(), parent_num).map(Into::into) - ).unwrap_or(None) + ext.storage_changes_root(parent_hash.into()).map(|h| h.map(|h| h.into())) + ).unwrap_or(Ok(None)).expect("Invalid parent hash passed to storage_changes_root") } fn enumerated_trie_root(input: &[&[u8]]) -> H::Out diff --git a/substrate/core/sr-io/without_std.rs b/substrate/core/sr-io/without_std.rs index 2f2b482f5e..de87aeddbd 100644 --- a/substrate/core/sr-io/without_std.rs +++ b/substrate/core/sr-io/without_std.rs @@ -242,7 +242,7 @@ pub mod ext { /// /// - `1` if the change trie root was found. /// - `0` if the change trie root was not found. - fn ext_storage_changes_root(parent_hash_data: *const u8, parent_hash_len: u32, parent_num: u64, result: *mut u8) -> u32; + fn ext_storage_changes_root(parent_hash_data: *const u8, parent_hash_len: u32, result: *mut u8) -> u32; /// A child storage function. /// @@ -498,10 +498,10 @@ impl StorageApi for () { } } - fn storage_changes_root(parent_hash: [u8; 32], parent_num: u64) -> Option<[u8; 32]> { + fn storage_changes_root(parent_hash: [u8; 32]) -> Option<[u8; 32]> { let mut result: [u8; 32] = Default::default(); let is_set = unsafe { - ext_storage_changes_root.get()(parent_hash.as_ptr(), parent_hash.len() as u32, parent_num, result.as_mut_ptr()) + ext_storage_changes_root.get()(parent_hash.as_ptr(), parent_hash.len() as u32, result.as_mut_ptr()) }; if is_set != 0 { diff --git a/substrate/core/sr-primitives/src/traits.rs b/substrate/core/sr-primitives/src/traits.rs index 8906bbf074..dcfe8ffd24 100644 --- a/substrate/core/sr-primitives/src/traits.rs +++ b/substrate/core/sr-primitives/src/traits.rs @@ -435,7 +435,7 @@ pub trait Hash: 'static + MaybeSerializeDebug + Clone + Eq + PartialEq { // Stup fn storage_root() -> Self::Output; /// Acquire the global storage changes root. - fn storage_changes_root(parent_hash: Self::Output, parent_number: u64) -> Option; + fn storage_changes_root(parent_hash: Self::Output) -> Option; } /// Blake2-256 Hash implementation. @@ -468,8 +468,8 @@ impl Hash for BlakeTwo256 { fn storage_root() -> Self::Output { runtime_io::storage_root().into() } - fn storage_changes_root(parent_hash: Self::Output, parent_number: u64) -> Option { - runtime_io::storage_changes_root(parent_hash.into(), parent_number).map(Into::into) + fn storage_changes_root(parent_hash: Self::Output) -> Option { + runtime_io::storage_changes_root(parent_hash.into()).map(Into::into) } } diff --git a/substrate/core/state-machine/Cargo.toml b/substrate/core/state-machine/Cargo.toml index 405e62bacc..222c543f54 100644 --- a/substrate/core/state-machine/Cargo.toml +++ b/substrate/core/state-machine/Cargo.toml @@ -15,6 +15,7 @@ trie = { package = "substrate-trie", path = "../trie" } primitives = { package = "substrate-primitives", path = "../primitives" } panic-handler = { package = "substrate-panic-handler", path = "../panic-handler" } parity-codec = "3.3" +num-traits = "0.2" [dev-dependencies] hex-literal = "0.2.0" diff --git a/substrate/core/state-machine/src/basic.rs b/substrate/core/state-machine/src/basic.rs index 3021ddfd28..6cb3efd267 100644 --- a/substrate/core/state-machine/src/basic.rs +++ b/substrate/core/state-machine/src/basic.rs @@ -151,8 +151,8 @@ impl Externalities for BasicExternalities where H::Out: Ord { vec![42] } - fn storage_changes_root(&mut self, _parent: H::Out, _parent_num: u64) -> Option { - None + fn storage_changes_root(&mut self, _parent: H::Out) -> Result, ()> { + Ok(None) } fn submit_extrinsic(&mut self, _extrinsic: Vec) -> Result<(), ()> { diff --git a/substrate/core/state-machine/src/changes_trie/build.rs b/substrate/core/state-machine/src/changes_trie/build.rs index 9af058515a..487fde2e35 100644 --- a/substrate/core/state-machine/src/changes_trie/build.rs +++ b/substrate/core/state-machine/src/changes_trie/build.rs @@ -19,12 +19,13 @@ use std::collections::{BTreeMap, BTreeSet}; use parity_codec::Decode; use hash_db::Hasher; +use num_traits::One; use crate::backend::Backend; use crate::overlayed_changes::OverlayedChanges; -use crate::trie_backend_essence::{TrieBackendStorage, TrieBackendEssence}; +use crate::trie_backend_essence::TrieBackendEssence; use crate::changes_trie::build_iterator::digest_build_iterator; use crate::changes_trie::input::{InputKey, InputPair, DigestIndex, ExtrinsicIndex}; -use crate::changes_trie::{AnchorBlockId, Configuration, Storage}; +use crate::changes_trie::{AnchorBlockId, Configuration, Storage, BlockNumber}; /// Prepare input pairs for building a changes trie of given block. /// @@ -32,29 +33,25 @@ use crate::changes_trie::{AnchorBlockId, Configuration, Storage}; /// required data. /// Returns Ok(None) data required to prepare input pairs is not collected /// or storage is not provided. -pub fn prepare_input<'a, B, S, H>( +pub fn prepare_input<'a, B, S, H, Number>( backend: &B, - storage: Option<&'a S>, + storage: &'a S, + config: &'a Configuration, changes: &OverlayedChanges, - parent: &'a AnchorBlockId, -) -> Result>, String> + parent: &'a AnchorBlockId, +) -> Result>>, String> where B: Backend, - S: Storage, - &'a S: TrieBackendStorage, + S: Storage, H: Hasher, + Number: BlockNumber, { - let (storage, config) = match (storage, changes.changes_trie_config.as_ref()) { - (Some(storage), Some(config)) => (storage, config), - _ => return Ok(None), - }; - let mut input = Vec::new(); input.extend(prepare_extrinsics_input( backend, - parent.number + 1, + parent.number.clone() + 1.into(), changes)?); - input.extend(prepare_digest_input::<_, H>( + input.extend(prepare_digest_input::<_, H, Number>( parent, config, storage)?); @@ -63,14 +60,15 @@ pub fn prepare_input<'a, B, S, H>( } /// Prepare ExtrinsicIndex input pairs. -fn prepare_extrinsics_input( +fn prepare_extrinsics_input( backend: &B, - block: u64, + block: Number, changes: &OverlayedChanges, -) -> Result, String> +) -> Result>, String> where B: Backend, H: Hasher, + Number: BlockNumber, { let mut extrinsic_map = BTreeMap::, BTreeSet>::new(); for (key, val) in changes.prospective.top.iter().chain(changes.committed.top.iter()) { @@ -93,47 +91,50 @@ fn prepare_extrinsics_input( Ok(extrinsic_map.into_iter() .map(move |(key, extrinsics)| InputPair::ExtrinsicIndex(ExtrinsicIndex { - block, + block: block.clone(), key, }, extrinsics.iter().cloned().collect()))) } /// Prepare DigestIndex input pairs. -fn prepare_digest_input<'a, S, H>( - parent: &'a AnchorBlockId, +fn prepare_digest_input<'a, S, H, Number>( + parent: &'a AnchorBlockId, config: &Configuration, storage: &'a S -) -> Result + 'a, String> +) -> Result> + 'a, String> where - S: Storage, - &'a S: TrieBackendStorage, + S: Storage, H: Hasher, H::Out: 'a, + Number: BlockNumber, { - let mut digest_map = BTreeMap::, BTreeSet>::new(); - for digest_build_block in digest_build_iterator(config, parent.number + 1) { - let trie_root = storage.root(parent, digest_build_block)?; - let trie_root = trie_root.ok_or_else(|| format!("No changes trie root for block {}", digest_build_block))?; - let trie_storage = TrieBackendEssence::<_, H>::new(storage, trie_root); + let mut digest_map = BTreeMap::, BTreeSet>::new(); + for digest_build_block in digest_build_iterator(config, parent.number.clone() + One::one()) { + let trie_root = storage.root(parent, digest_build_block.clone())?; + let trie_root = trie_root.ok_or_else(|| format!("No changes trie root for block {}", digest_build_block.clone()))?; + let trie_storage = TrieBackendEssence::<_, H>::new( + crate::changes_trie::TrieBackendStorageAdapter(storage), + trie_root, + ); - let extrinsic_prefix = ExtrinsicIndex::key_neutral_prefix(digest_build_block); + let extrinsic_prefix = ExtrinsicIndex::key_neutral_prefix(digest_build_block.clone()); trie_storage.for_keys_with_prefix(&extrinsic_prefix, |key| - if let Some(InputKey::ExtrinsicIndex(trie_key)) = Decode::decode(&mut &key[..]) { + if let Some(InputKey::ExtrinsicIndex::(trie_key)) = Decode::decode(&mut &key[..]) { digest_map.entry(trie_key.key).or_default() - .insert(digest_build_block); + .insert(digest_build_block.clone()); }); - let digest_prefix = DigestIndex::key_neutral_prefix(digest_build_block); + let digest_prefix = DigestIndex::key_neutral_prefix(digest_build_block.clone()); trie_storage.for_keys_with_prefix(&digest_prefix, |key| - if let Some(InputKey::DigestIndex(trie_key)) = Decode::decode(&mut &key[..]) { + if let Some(InputKey::DigestIndex::(trie_key)) = Decode::decode(&mut &key[..]) { digest_map.entry(trie_key.key).or_default() - .insert(digest_build_block); + .insert(digest_build_block.clone()); }); } Ok(digest_map.into_iter() .map(move |(key, set)| InputPair::DigestIndex(DigestIndex { - block: parent.number + 1, + block: parent.number.clone() + One::one(), key }, set.into_iter().collect()))) } @@ -148,7 +149,7 @@ mod test { use crate::overlayed_changes::OverlayedValue; use super::*; - fn prepare_for_build() -> (InMemory, InMemoryStorage, OverlayedChanges) { + fn prepare_for_build() -> (InMemory, InMemoryStorage, OverlayedChanges) { let backend: InMemory<_> = vec![ (vec![100], vec![255]), (vec![101], vec![255]), @@ -225,7 +226,14 @@ mod test { #[test] fn build_changes_trie_nodes_on_non_digest_block() { let (backend, storage, changes) = prepare_for_build(); - let changes_trie_nodes = prepare_input(&backend, Some(&storage), &changes, &AnchorBlockId { hash: Default::default(), number: 4 }).unwrap(); + let config = changes.changes_trie_config.as_ref().unwrap(); + let changes_trie_nodes = prepare_input( + &backend, + &storage, + config, + &changes, + &AnchorBlockId { hash: Default::default(), number: 4 }, + ).unwrap(); assert_eq!(changes_trie_nodes, Some(vec![ InputPair::ExtrinsicIndex(ExtrinsicIndex { block: 5, key: vec![100] }, vec![0, 2, 3]), InputPair::ExtrinsicIndex(ExtrinsicIndex { block: 5, key: vec![101] }, vec![1]), @@ -236,7 +244,14 @@ mod test { #[test] fn build_changes_trie_nodes_on_digest_block_l1() { let (backend, storage, changes) = prepare_for_build(); - let changes_trie_nodes = prepare_input(&backend, Some(&storage), &changes, &AnchorBlockId { hash: Default::default(), number: 3 }).unwrap(); + let config = changes.changes_trie_config.as_ref().unwrap(); + let changes_trie_nodes = prepare_input( + &backend, + &storage, + config, + &changes, + &AnchorBlockId { hash: Default::default(), number: 3 }, + ).unwrap(); assert_eq!(changes_trie_nodes, Some(vec![ InputPair::ExtrinsicIndex(ExtrinsicIndex { block: 4, key: vec![100] }, vec![0, 2, 3]), InputPair::ExtrinsicIndex(ExtrinsicIndex { block: 4, key: vec![101] }, vec![1]), @@ -252,7 +267,14 @@ mod test { #[test] fn build_changes_trie_nodes_on_digest_block_l2() { let (backend, storage, changes) = prepare_for_build(); - let changes_trie_nodes = prepare_input(&backend, Some(&storage), &changes, &AnchorBlockId { hash: Default::default(), number: 15 }).unwrap(); + let config = changes.changes_trie_config.as_ref().unwrap(); + let changes_trie_nodes = prepare_input( + &backend, + &storage, + config, + &changes, + &AnchorBlockId { hash: Default::default(), number: 15 }, + ).unwrap(); assert_eq!(changes_trie_nodes, Some(vec![ InputPair::ExtrinsicIndex(ExtrinsicIndex { block: 16, key: vec![100] }, vec![0, 2, 3]), InputPair::ExtrinsicIndex(ExtrinsicIndex { block: 16, key: vec![101] }, vec![1]), @@ -276,7 +298,14 @@ mod test { extrinsics: Some(vec![1].into_iter().collect()) }); - let changes_trie_nodes = prepare_input(&backend, Some(&storage), &changes, &AnchorBlockId { hash: Default::default(), number: 3 }).unwrap(); + let config = changes.changes_trie_config.as_ref().unwrap(); + let changes_trie_nodes = prepare_input( + &backend, + &storage, + config, + &changes, + &AnchorBlockId { hash: Default::default(), number: 3 }, + ).unwrap(); assert_eq!(changes_trie_nodes, Some(vec![ InputPair::ExtrinsicIndex(ExtrinsicIndex { block: 4, key: vec![100] }, vec![0, 2, 3]), InputPair::ExtrinsicIndex(ExtrinsicIndex { block: 4, key: vec![101] }, vec![1]), diff --git a/substrate/core/state-machine/src/changes_trie/build_iterator.rs b/substrate/core/state-machine/src/changes_trie/build_iterator.rs index f9c6ba6e7b..5d8a8318ab 100644 --- a/substrate/core/state-machine/src/changes_trie/build_iterator.rs +++ b/substrate/core/state-machine/src/changes_trie/build_iterator.rs @@ -17,13 +17,16 @@ //! Structures and functions to return blocks whose changes are to be included //! in given block' changes trie. -use crate::changes_trie::Configuration; +use crate::changes_trie::{Configuration, BlockNumber}; /// Returns iterator of OTHER blocks that are required for inclusion into /// changes trie of given block. -pub fn digest_build_iterator(config: &Configuration, block: u64) -> DigestBuildIterator { +pub fn digest_build_iterator( + config: &Configuration, + block: Number, +) -> DigestBuildIterator { // prepare digest build parameters - let (_, _, digest_step) = match config.digest_level_at_block(block) { + let (_, _, digest_step) = match config.digest_level_at_block(block.clone()) { Some((current_level, digest_interval, digest_step)) => (current_level, digest_interval, digest_step), None => return DigestBuildIterator::empty(), @@ -35,24 +38,26 @@ pub fn digest_build_iterator(config: &Configuration, block: u64) -> DigestBuildI /// Changes trie build iterator that returns numbers of OTHER blocks that are /// required for inclusion into changes trie of given block. #[derive(Debug)] -pub struct DigestBuildIterator { +pub struct DigestBuildIterator { /// Block we're building changes trie for. - block: u64, + block: Number, /// Interval for creation digest blocks. - digest_interval: u64, - /// Step of current blocks range. - current_step: u64, - /// Current blocks range. - current_range: Option<::std::iter::StepBy<::std::ops::Range>>, + digest_interval: u32, /// Max step of blocks range. - max_step: u64, + max_step: u32, + /// Step of current blocks range. + current_step: u32, + /// Current blocks range. + current_range: Option>, } -impl DigestBuildIterator { +impl DigestBuildIterator { /// Create new digest build iterator. - pub fn new(block: u64, digest_interval: u64, max_step: u64) -> Self { + pub fn new(block: Number, digest_interval: u32, max_step: u32) -> Self { DigestBuildIterator { - block, digest_interval, max_step, + block, + digest_interval, + max_step, current_step: 0, current_range: None, } @@ -60,12 +65,12 @@ impl DigestBuildIterator { /// Create empty digest build iterator. pub fn empty() -> Self { - Self::new(0, 0, 0) + Self::new(0.into(), 0, 0) } } -impl Iterator for DigestBuildIterator { - type Item = u64; +impl Iterator for DigestBuildIterator { + type Item = Number; fn next(&mut self) -> Option { if let Some(next) = self.current_range.as_mut().and_then(|iter| iter.next()) { @@ -82,10 +87,11 @@ impl Iterator for DigestBuildIterator { } self.current_step = next_step; - self.current_range = Some( - ((self.block - self.current_step * self.digest_interval + self.current_step)..self.block) - .step_by(self.current_step as usize) - ); + self.current_range = Some(BlocksRange::new( + self.block.clone() - (self.current_step * self.digest_interval - self.current_step).into(), + self.block.clone(), + self.current_step.into(), + )); Some(self.current_range.as_mut() .expect("assigned one line above; qed") @@ -94,20 +100,52 @@ impl Iterator for DigestBuildIterator { } } +/// Blocks range iterator with builtin step_by support. +#[derive(Debug)] +struct BlocksRange { + current: Number, + end: Number, + step: Number, +} + +impl BlocksRange { + pub fn new(begin: Number, end: Number, step: Number) -> Self { + BlocksRange { + current: begin, + end, + step, + } + } +} + +impl Iterator for BlocksRange { + type Item = Number; + + fn next(&mut self) -> Option { + if self.current >= self.end { + return None; + } + + let current = Some(self.current.clone()); + self.current += self.step.clone(); + current + } +} + #[cfg(test)] mod tests { use super::*; - fn digest_build_iterator(digest_interval: u64, digest_levels: u32, block: u64) -> DigestBuildIterator { + fn digest_build_iterator(digest_interval: u32, digest_levels: u32, block: u64) -> DigestBuildIterator { super::digest_build_iterator(&Configuration { digest_interval, digest_levels }, block) } - fn digest_build_iterator_basic(digest_interval: u64, digest_levels: u32, block: u64) -> (u64, u64, u64) { + fn digest_build_iterator_basic(digest_interval: u32, digest_levels: u32, block: u64) -> (u64, u32, u32) { let iter = digest_build_iterator(digest_interval, digest_levels, block); (iter.block, iter.digest_interval, iter.max_step) } - fn digest_build_iterator_blocks(digest_interval: u64, digest_levels: u32, block: u64) -> Vec { + fn digest_build_iterator_blocks(digest_interval: u32, digest_levels: u32, block: u64) -> Vec { digest_build_iterator(digest_interval, digest_levels, block).collect() } @@ -122,7 +160,11 @@ mod tests { assert_eq!(digest_build_iterator_basic(4, 16, 2), empty, "digest is not required for this block"); assert_eq!(digest_build_iterator_basic(4, 16, 15), empty, "digest is not required for this block"); assert_eq!(digest_build_iterator_basic(4, 16, 17), empty, "digest is not required for this block"); - assert_eq!(digest_build_iterator_basic(::std::u64::MAX / 2 + 1, 16, ::std::u64::MAX), empty, "digest_interval * 2 is greater than u64::MAX"); + assert_eq!(digest_build_iterator_basic( + ::std::u32::MAX / 2 + 1, + 16, + ::std::u64::MAX, + ), empty, "digest_interval * 2 is greater than u64::MAX"); } #[test] diff --git a/substrate/core/state-machine/src/changes_trie/changes_iterator.rs b/substrate/core/state-machine/src/changes_trie/changes_iterator.rs index c8d6216926..0e4716ccab 100644 --- a/substrate/core/state-machine/src/changes_trie/changes_iterator.rs +++ b/substrate/core/state-machine/src/changes_trie/changes_iterator.rs @@ -21,8 +21,9 @@ use std::cell::RefCell; use std::collections::VecDeque; use parity_codec::{Decode, Encode}; use hash_db::{HashDB, Hasher}; +use num_traits::One; use trie::{Recorder, MemoryDB}; -use crate::changes_trie::{AnchorBlockId, Configuration, RootsStorage, Storage}; +use crate::changes_trie::{AnchorBlockId, Configuration, RootsStorage, Storage, BlockNumber}; use crate::changes_trie::input::{DigestIndex, ExtrinsicIndex, DigestIndexValue, ExtrinsicIndexValue}; use crate::changes_trie::storage::{TrieBackendAdapter, InMemoryStorage}; use crate::proving_backend::ProvingBackendEssence; @@ -31,25 +32,25 @@ use crate::trie_backend_essence::{TrieBackendEssence}; /// Return changes of given key at given blocks range. /// `max` is the number of best known block. /// Changes are returned in descending order (i.e. last block comes first). -pub fn key_changes<'a, S: Storage, H: Hasher>( +pub fn key_changes<'a, S: Storage, H: Hasher, Number: BlockNumber>( config: &'a Configuration, storage: &'a S, - begin: u64, - end: &'a AnchorBlockId, - max: u64, + begin: Number, + end: &'a AnchorBlockId, + max: Number, key: &'a [u8], -) -> Result, String> { +) -> Result, String> { // we can't query any roots before root - let max = ::std::cmp::min(max, end.number); + let max = ::std::cmp::min(max.clone(), end.number.clone()); Ok(DrilldownIterator { essence: DrilldownIteratorEssence { key, roots_storage: storage, storage, - begin, + begin: begin.clone(), end, - surface: surface_iterator(config, max, begin, end.number)?, + surface: surface_iterator(config, max, begin, end.number.clone())?, extrinsics: Default::default(), blocks: Default::default(), @@ -61,25 +62,25 @@ pub fn key_changes<'a, S: Storage, H: Hasher>( /// Returns proof of changes of given key at given blocks range. /// `max` is the number of best known block. -pub fn key_changes_proof, H: Hasher>( +pub fn key_changes_proof, H: Hasher, Number: BlockNumber>( config: &Configuration, storage: &S, - begin: u64, - end: &AnchorBlockId, - max: u64, + begin: Number, + end: &AnchorBlockId, + max: Number, key: &[u8], ) -> Result>, String> { // we can't query any roots before root - let max = ::std::cmp::min(max, end.number); + let max = ::std::cmp::min(max.clone(), end.number.clone()); let mut iter = ProvingDrilldownIterator { essence: DrilldownIteratorEssence { key, roots_storage: storage.clone(), storage, - begin, + begin: begin.clone(), end, - surface: surface_iterator(config, max, begin, end.number)?, + surface: surface_iterator(config, max, begin, end.number.clone())?, extrinsics: Default::default(), blocks: Default::default(), @@ -100,17 +101,17 @@ pub fn key_changes_proof, H: Hasher>( /// Check key changes proog and return changes of the key at given blocks range. /// `max` is the number of best known block. /// Changes are returned in descending order (i.e. last block comes first). -pub fn key_changes_proof_check, H: Hasher>( +pub fn key_changes_proof_check, H: Hasher, Number: BlockNumber>( config: &Configuration, roots_storage: &S, proof: Vec>, - begin: u64, - end: &AnchorBlockId, - max: u64, + begin: Number, + end: &AnchorBlockId, + max: Number, key: &[u8] -) -> Result, String> { +) -> Result, String> { // we can't query any roots before root - let max = ::std::cmp::min(max, end.number); + let max = ::std::cmp::min(max.clone(), end.number.clone()); let mut proof_db = MemoryDB::::default(); for item in proof { @@ -123,9 +124,9 @@ pub fn key_changes_proof_check, H: Hasher>( key, roots_storage, storage: &proof_db, - begin, + begin: begin.clone(), end, - surface: surface_iterator(config, max, begin, end.number)?, + surface: surface_iterator(config, max, begin, end.number.clone())?, extrinsics: Default::default(), blocks: Default::default(), @@ -137,36 +138,36 @@ pub fn key_changes_proof_check, H: Hasher>( /// Surface iterator - only traverses top-level digests from given range and tries to find /// all digest changes for the key. -pub struct SurfaceIterator<'a> { +pub struct SurfaceIterator<'a, Number: BlockNumber> { config: &'a Configuration, - begin: u64, - max: u64, - current: Option, - current_begin: u64, - digest_step: u64, + begin: Number, + max: Number, + current: Option, + current_begin: Number, + digest_step: u32, digest_level: u32, } -impl<'a> Iterator for SurfaceIterator<'a> { - type Item = Result<(u64, u32), String>; +impl<'a, Number: BlockNumber> Iterator for SurfaceIterator<'a, Number> { + type Item = Result<(Number, u32), String>; fn next(&mut self) -> Option { - let current = self.current?; + let current = self.current.clone()?; let digest_level = self.digest_level; - if current < self.digest_step { + if current < self.digest_step.into() { self.current = None; } else { - let next = current - self.digest_step; - if next == 0 || next < self.begin { + let next = current.clone() - self.digest_step.into(); + if next.is_zero() || next < self.begin { self.current = None; } else if next > self.current_begin { self.current = Some(next); } else { let (current, current_begin, digest_step, digest_level) = match - lower_bound_max_digest(self.config, self.max, self.begin, next) { + lower_bound_max_digest(self.config, self.max.clone(), self.begin.clone(), next) { Err(err) => return Some(Err(err)), Ok(range) => range, }; @@ -184,22 +185,36 @@ impl<'a> Iterator for SurfaceIterator<'a> { /// Drilldown iterator - receives 'digest points' from surface iterator and explores /// every point until extrinsic is found. -pub struct DrilldownIteratorEssence<'a, RS: 'a + RootsStorage, S: 'a + Storage, H: Hasher> where H::Out: 'a { +pub struct DrilldownIteratorEssence<'a, RS, S, H, Number> + where + RS: 'a + RootsStorage, + S: 'a + Storage, + H: Hasher, + Number: BlockNumber, + H::Out: 'a, +{ key: &'a [u8], roots_storage: &'a RS, storage: &'a S, - begin: u64, - end: &'a AnchorBlockId, - surface: SurfaceIterator<'a>, + begin: Number, + end: &'a AnchorBlockId, + surface: SurfaceIterator<'a, Number>, - extrinsics: VecDeque<(u64, u32)>, - blocks: VecDeque<(u64, u32)>, + extrinsics: VecDeque<(Number, u32)>, + blocks: VecDeque<(Number, u32)>, _hasher: ::std::marker::PhantomData, } -impl<'a, RS: 'a + RootsStorage, S: Storage, H: Hasher> DrilldownIteratorEssence<'a, RS, S, H> { - pub fn next(&mut self, trie_reader: F) -> Option> +impl<'a, RS, S, H, Number> DrilldownIteratorEssence<'a, RS, S, H, Number> + where + RS: 'a + RootsStorage, + S: 'a + Storage, + H: Hasher, + Number: BlockNumber, + H::Out: 'a, +{ + pub fn next(&mut self, trie_reader: F) -> Option> where F: FnMut(&S, H::Out, &[u8]) -> Result>, String>, { @@ -210,7 +225,7 @@ impl<'a, RS: 'a + RootsStorage, S: Storage, H: Hasher> DrilldownIteratorEs } } - fn do_next(&mut self, mut trie_reader: F) -> Result, String> + fn do_next(&mut self, mut trie_reader: F) -> Result, String> where F: FnMut(&S, H::Out, &[u8]) -> Result>, String>, { @@ -223,33 +238,33 @@ impl<'a, RS: 'a + RootsStorage, S: Storage, H: Hasher> DrilldownIteratorEs // not having a changes trie root is an error because: // we never query roots for future blocks // AND trie roots for old blocks are known (both on full + light node) - let trie_root = self.roots_storage.root(&self.end, block)? - .ok_or_else(|| format!("Changes trie root for block {} is not found", block))?; + let trie_root = self.roots_storage.root(&self.end, block.clone())? + .ok_or_else(|| format!("Changes trie root for block {} is not found", block.clone()))?; // only return extrinsics for blocks before self.max // most of blocks will be filtered out before pushing to `self.blocks` // here we just throwing away changes at digest blocks we're processing debug_assert!(block >= self.begin, "We shall not touch digests earlier than a range' begin"); if block <= self.end.number { - let extrinsics_key = ExtrinsicIndex { block, key: self.key.to_vec() }.encode(); + let extrinsics_key = ExtrinsicIndex { block: block.clone(), key: self.key.to_vec() }.encode(); let extrinsics = trie_reader(&self.storage, trie_root, &extrinsics_key); if let Some(extrinsics) = extrinsics? { let extrinsics: Option = Decode::decode(&mut &extrinsics[..]); if let Some(extrinsics) = extrinsics { - self.extrinsics.extend(extrinsics.into_iter().rev().map(|e| (block, e))); + self.extrinsics.extend(extrinsics.into_iter().rev().map(|e| (block.clone(), e))); } } } - let blocks_key = DigestIndex { block, key: self.key.to_vec() }.encode(); + let blocks_key = DigestIndex { block: block.clone(), key: self.key.to_vec() }.encode(); let blocks = trie_reader(&self.storage, trie_root, &blocks_key); if let Some(blocks) = blocks? { - let blocks: Option = Decode::decode(&mut &blocks[..]); + let blocks: Option> = Decode::decode(&mut &blocks[..]); if let Some(blocks) = blocks { // filter level0 blocks here because we tend to use digest blocks, // AND digest block changes could also include changes for out-of-range blocks - let begin = self.begin; - let end = self.end.number; + let begin = self.begin.clone(); + let end = self.end.number.clone(); self.blocks.extend(blocks.into_iter() .rev() .filter(|b| level > 1 || (*b >= begin && *b <= end)) @@ -271,14 +286,21 @@ impl<'a, RS: 'a + RootsStorage, S: Storage, H: Hasher> DrilldownIteratorEs } /// Exploring drilldown operator. -pub struct DrilldownIterator<'a, RS: 'a + RootsStorage, S: 'a + Storage, H: Hasher> where H::Out: 'a { - essence: DrilldownIteratorEssence<'a, RS, S, H>, +pub struct DrilldownIterator<'a, RS, S, H, Number> + where + Number: BlockNumber, + H: Hasher, + S: 'a + Storage, + RS: 'a + RootsStorage, + H::Out: 'a, +{ + essence: DrilldownIteratorEssence<'a, RS, S, H, Number>, } -impl<'a, RS: 'a + RootsStorage, S: Storage, H: Hasher> Iterator - for DrilldownIterator<'a, RS, S, H> +impl<'a, RS: 'a + RootsStorage, S: Storage, H: Hasher, Number: BlockNumber> Iterator + for DrilldownIterator<'a, RS, S, H, Number> { - type Item = Result<(u64, u32), String>; + type Item = Result<(Number, u32), String>; fn next(&mut self) -> Option { self.essence.next(|storage, root, key| @@ -287,12 +309,26 @@ impl<'a, RS: 'a + RootsStorage, S: Storage, H: Hasher> Iterator } /// Proving drilldown iterator. -struct ProvingDrilldownIterator<'a, RS: 'a + RootsStorage, S: 'a + Storage, H: Hasher> where H::Out: 'a { - essence: DrilldownIteratorEssence<'a, RS, S, H>, +struct ProvingDrilldownIterator<'a, RS, S, H, Number> + where + Number: BlockNumber, + H: Hasher, + S: 'a + Storage, + RS: 'a + RootsStorage, + H::Out: 'a, +{ + essence: DrilldownIteratorEssence<'a, RS, S, H, Number>, proof_recorder: RefCell>, } -impl<'a, RS: 'a + RootsStorage, S: Storage, H: Hasher> ProvingDrilldownIterator<'a, RS, S, H> { +impl<'a, RS, S, H, Number> ProvingDrilldownIterator<'a, RS, S, H, Number> + where + Number: BlockNumber, + H: Hasher, + S: 'a + Storage, + RS: 'a + RootsStorage, + H::Out: 'a, +{ /// Consume the iterator, extracting the gathered proof in lexicographical order /// by value. pub fn extract_proof(self) -> Vec> { @@ -303,8 +339,15 @@ impl<'a, RS: 'a + RootsStorage, S: Storage, H: Hasher> ProvingDrilldownIte } } -impl<'a, RS: 'a + RootsStorage, S: Storage, H: Hasher> Iterator for ProvingDrilldownIterator<'a, RS, S, H> { - type Item = Result<(u64, u32), String>; +impl<'a, RS, S, H, Number> Iterator for ProvingDrilldownIterator<'a, RS, S, H, Number> + where + Number: BlockNumber, + H: Hasher, + S: 'a + Storage, + RS: 'a + RootsStorage, + H::Out: 'a, +{ + type Item = Result<(Number, u32), String>; fn next(&mut self) -> Option { let proof_recorder = &mut *self.proof_recorder.try_borrow_mut() @@ -318,8 +361,18 @@ impl<'a, RS: 'a + RootsStorage, S: Storage, H: Hasher> Iterator for Provin } /// Returns surface iterator for given range of blocks. -fn surface_iterator<'a>(config: &'a Configuration, max: u64, begin: u64, end: u64) -> Result, String> { - let (current, current_begin, digest_step, digest_level) = lower_bound_max_digest(config, max, begin, end)?; +fn surface_iterator<'a, Number: BlockNumber>( + config: &'a Configuration, + max: Number, + begin: Number, + end: Number, +) -> Result, String> { + let (current, current_begin, digest_step, digest_level) = lower_bound_max_digest( + config, + max.clone(), + begin.clone(), + end, + )?; Ok(SurfaceIterator { config, begin, @@ -333,31 +386,32 @@ fn surface_iterator<'a>(config: &'a Configuration, max: u64, begin: u64, end: u6 /// Returns parameters of highest level digest block that includes the end of given range /// and tends to include the whole range. -fn lower_bound_max_digest( +fn lower_bound_max_digest( config: &Configuration, - max: u64, - begin: u64, - end: u64, -) -> Result<(u64, u64, u64, u32), String> { + max: Number, + begin: Number, + end: Number, +) -> Result<(Number, Number, u32, u32), String> { if end > max || begin > end { return Err("invalid changes range".into()); } let mut digest_level = 0u32; - let mut digest_step = 1u64; - let mut digest_interval = 0u64; - let mut current = end; - let mut current_begin = begin; - if begin != end { + let mut digest_step = 1u32; + let mut digest_interval = 0u32; + let mut current = end.clone(); + let mut current_begin = begin.clone(); + if current_begin != current { while digest_level != config.digest_levels { let new_digest_level = digest_level + 1; let new_digest_step = digest_step * config.digest_interval; let new_digest_interval = config.digest_interval * { if digest_interval == 0 { 1 } else { digest_interval } }; - let new_digest_begin = ((current - 1) / new_digest_interval) * new_digest_interval; - let new_digest_end = new_digest_begin + new_digest_interval; - let new_current = new_digest_begin + new_digest_interval; + let new_digest_begin = ((current.clone() - One::one()) + / new_digest_interval.into()) * new_digest_interval.into(); + let new_digest_end = new_digest_begin.clone() + new_digest_interval.into(); + let new_current = new_digest_begin.clone() + new_digest_interval.into(); if new_digest_end > max { if begin < new_digest_begin { @@ -372,7 +426,7 @@ fn lower_bound_max_digest( current = new_current; current_begin = new_digest_begin; - if new_digest_begin <= begin && new_digest_end >= end { + if current_begin <= begin && new_digest_end >= end { break; } } @@ -394,7 +448,7 @@ mod tests { use crate::changes_trie::storage::InMemoryStorage; use super::*; - fn prepare_for_drilldown() -> (Configuration, InMemoryStorage) { + fn prepare_for_drilldown() -> (Configuration, InMemoryStorage) { let config = Configuration { digest_interval: 4, digest_levels: 2 }; let backend = InMemoryStorage::with_inputs(vec![ // digest: 1..4 => [(3, 0)] @@ -436,27 +490,27 @@ mod tests { #[test] fn drilldown_iterator_works() { let (config, storage) = prepare_for_drilldown(); - let drilldown_result = key_changes::, Blake2Hasher>( + let drilldown_result = key_changes::, Blake2Hasher, u64>( &config, &storage, 0, &AnchorBlockId { hash: Default::default(), number: 16 }, 16, &[42]) .and_then(Result::from_iter); assert_eq!(drilldown_result, Ok(vec![(8, 2), (8, 1), (6, 3), (3, 0)])); - let drilldown_result = key_changes::, Blake2Hasher>( + let drilldown_result = key_changes::, Blake2Hasher, u64>( &config, &storage, 0, &AnchorBlockId { hash: Default::default(), number: 2 }, 4, &[42]) .and_then(Result::from_iter); assert_eq!(drilldown_result, Ok(vec![])); - let drilldown_result = key_changes::, Blake2Hasher>( + let drilldown_result = key_changes::, Blake2Hasher, u64>( &config, &storage, 0, &AnchorBlockId { hash: Default::default(), number: 3 }, 4, &[42]) .and_then(Result::from_iter); assert_eq!(drilldown_result, Ok(vec![(3, 0)])); - let drilldown_result = key_changes::, Blake2Hasher>( + let drilldown_result = key_changes::, Blake2Hasher, u64>( &config, &storage, 7, &AnchorBlockId { hash: Default::default(), number: 8 }, 8, &[42]) .and_then(Result::from_iter); assert_eq!(drilldown_result, Ok(vec![(8, 2), (8, 1)])); - let drilldown_result = key_changes::, Blake2Hasher>( + let drilldown_result = key_changes::, Blake2Hasher, u64>( &config, &storage, 5, &AnchorBlockId { hash: Default::default(), number: 7 }, 8, &[42]) .and_then(Result::from_iter); assert_eq!(drilldown_result, Ok(vec![(6, 3)])); @@ -467,7 +521,7 @@ mod tests { let (config, storage) = prepare_for_drilldown(); storage.clear_storage(); - assert!(key_changes::, Blake2Hasher>( + assert!(key_changes::, Blake2Hasher, u64>( &config, &storage, 0, &AnchorBlockId { hash: Default::default(), number: 100 }, 1000, &[42]) .and_then(|i| i.collect::, _>>()).is_err()); } @@ -475,9 +529,9 @@ mod tests { #[test] fn drilldown_iterator_fails_when_range_is_invalid() { let (config, storage) = prepare_for_drilldown(); - assert!(key_changes::, Blake2Hasher>( + assert!(key_changes::, Blake2Hasher, u64>( &config, &storage, 0, &AnchorBlockId { hash: Default::default(), number: 100 }, 50, &[42]).is_err()); - assert!(key_changes::, Blake2Hasher>( + assert!(key_changes::, Blake2Hasher, u64>( &config, &storage, 20, &AnchorBlockId { hash: Default::default(), number: 10 }, 100, &[42]).is_err()); } @@ -488,7 +542,7 @@ mod tests { // create drilldown iterator that records all trie nodes during drilldown let (remote_config, remote_storage) = prepare_for_drilldown(); - let remote_proof = key_changes_proof::, Blake2Hasher>( + let remote_proof = key_changes_proof::, Blake2Hasher, u64>( &remote_config, &remote_storage, 0, &AnchorBlockId { hash: Default::default(), number: 16 }, 16, &[42]).unwrap(); @@ -497,7 +551,7 @@ mod tests { // create drilldown iterator that works the same, but only depends on trie let (local_config, local_storage) = prepare_for_drilldown(); local_storage.clear_storage(); - let local_result = key_changes_proof_check::, Blake2Hasher>( + let local_result = key_changes_proof_check::, Blake2Hasher, u64>( &local_config, &local_storage, remote_proof, 0, &AnchorBlockId { hash: Default::default(), number: 16 }, 16, &[42]); diff --git a/substrate/core/state-machine/src/changes_trie/input.rs b/substrate/core/state-machine/src/changes_trie/input.rs index 3154aff715..ae939c028b 100644 --- a/substrate/core/state-machine/src/changes_trie/input.rs +++ b/substrate/core/state-machine/src/changes_trie/input.rs @@ -17,12 +17,13 @@ //! Different types of changes trie input pairs. use parity_codec::{Decode, Encode, Input, Output}; +use crate::changes_trie::BlockNumber; /// Key of { changed key => set of extrinsic indices } mapping. #[derive(Clone, Debug, PartialEq, Eq)] -pub struct ExtrinsicIndex { +pub struct ExtrinsicIndex { /// Block at which this key has been inserted in the trie. - pub block: u64, + pub block: Number, /// Storage key this node is responsible for. pub key: Vec, } @@ -32,35 +33,35 @@ pub type ExtrinsicIndexValue = Vec; /// Key of { changed key => block/digest block numbers } mapping. #[derive(Clone, Debug, PartialEq, Eq)] -pub struct DigestIndex { +pub struct DigestIndex { /// Block at which this key has been inserted in the trie. - pub block: u64, + pub block: Number, /// Storage key this node is responsible for. pub key: Vec, } /// Value of { changed key => block/digest block numbers } mapping. -pub type DigestIndexValue = Vec; +pub type DigestIndexValue = Vec; /// Single input pair of changes trie. #[derive(Clone, Debug, PartialEq, Eq)] -pub enum InputPair { +pub enum InputPair { /// Element of { key => set of extrinsics where key has been changed } element mapping. - ExtrinsicIndex(ExtrinsicIndex, ExtrinsicIndexValue), + ExtrinsicIndex(ExtrinsicIndex, ExtrinsicIndexValue), /// Element of { key => set of blocks/digest blocks where key has been changed } element mapping. - DigestIndex(DigestIndex, DigestIndexValue), + DigestIndex(DigestIndex, DigestIndexValue), } /// Single input key of changes trie. #[derive(Clone, Debug, PartialEq, Eq)] -pub enum InputKey { +pub enum InputKey { /// Key of { key => set of extrinsics where key has been changed } element mapping. - ExtrinsicIndex(ExtrinsicIndex), + ExtrinsicIndex(ExtrinsicIndex), /// Key of { key => set of blocks/digest blocks where key has been changed } element mapping. - DigestIndex(DigestIndex), + DigestIndex(DigestIndex), } -impl Into<(Vec, Vec)> for InputPair { +impl Into<(Vec, Vec)> for InputPair { fn into(self) -> (Vec, Vec) { match self { InputPair::ExtrinsicIndex(key, value) => (key.encode(), value.encode()), @@ -69,8 +70,8 @@ impl Into<(Vec, Vec)> for InputPair { } } -impl Into for InputPair { - fn into(self) -> InputKey { +impl Into> for InputPair { + fn into(self) -> InputKey { match self { InputPair::ExtrinsicIndex(key, _) => InputKey::ExtrinsicIndex(key), InputPair::DigestIndex(key, _) => InputKey::DigestIndex(key), @@ -78,15 +79,15 @@ impl Into for InputPair { } } -impl ExtrinsicIndex { - pub fn key_neutral_prefix(block: u64) -> Vec { +impl ExtrinsicIndex { + pub fn key_neutral_prefix(block: Number) -> Vec { let mut prefix = vec![1]; prefix.extend(block.encode()); prefix } } -impl Encode for ExtrinsicIndex { +impl Encode for ExtrinsicIndex { fn encode_to(&self, dest: &mut W) { dest.push_byte(1); self.block.encode_to(dest); @@ -94,8 +95,8 @@ impl Encode for ExtrinsicIndex { } } -impl DigestIndex { - pub fn key_neutral_prefix(block: u64) -> Vec { +impl DigestIndex { + pub fn key_neutral_prefix(block: Number) -> Vec { let mut prefix = vec![2]; prefix.extend(block.encode()); prefix @@ -103,7 +104,7 @@ impl DigestIndex { } -impl Encode for DigestIndex { +impl Encode for DigestIndex { fn encode_to(&self, dest: &mut W) { dest.push_byte(2); self.block.encode_to(dest); @@ -111,7 +112,7 @@ impl Encode for DigestIndex { } } -impl Decode for InputKey { +impl Decode for InputKey { fn decode(input: &mut I) -> Option { match input.read_byte()? { 1 => Some(InputKey::ExtrinsicIndex(ExtrinsicIndex { @@ -133,17 +134,17 @@ mod tests { #[test] fn extrinsic_index_serialized_and_deserialized() { - let original = ExtrinsicIndex { block: 777, key: vec![42] }; + let original = ExtrinsicIndex { block: 777u64, key: vec![42] }; let serialized = original.encode(); - let deserialized: InputKey = Decode::decode(&mut &serialized[..]).unwrap(); + let deserialized: InputKey = Decode::decode(&mut &serialized[..]).unwrap(); assert_eq!(InputKey::ExtrinsicIndex(original), deserialized); } #[test] fn digest_index_serialized_and_deserialized() { - let original = DigestIndex { block: 777, key: vec![42] }; + let original = DigestIndex { block: 777u64, key: vec![42] }; let serialized = original.encode(); - let deserialized: InputKey = Decode::decode(&mut &serialized[..]).unwrap(); + let deserialized: InputKey = Decode::decode(&mut &serialized[..]).unwrap(); assert_eq!(InputKey::DigestIndex(original), deserialized); } } diff --git a/substrate/core/state-machine/src/changes_trie/mod.rs b/substrate/core/state-machine/src/changes_trie/mod.rs index f82d8b33df..15ea1b474f 100644 --- a/substrate/core/state-machine/src/changes_trie/mod.rs +++ b/substrate/core/state-machine/src/changes_trie/mod.rs @@ -48,58 +48,116 @@ pub use self::prune::{prune, oldest_non_pruned_trie}; use hash_db::Hasher; use crate::backend::Backend; +use num_traits::{One, Zero}; +use parity_codec::{Decode, Encode}; use primitives; use crate::changes_trie::build::prepare_input; use crate::overlayed_changes::OverlayedChanges; -use crate::trie_backend_essence::TrieBackendStorage; use trie::{DBValue, trie_root}; /// Changes that are made outside of extrinsics are marked with this index; pub const NO_EXTRINSIC_INDEX: u32 = 0xffffffff; +/// Requirements for block number that can be used with changes tries. +pub trait BlockNumber: + Send + Sync + 'static + + ::std::fmt::Display + + Clone + + From + One + Zero + + PartialEq + Ord + + ::std::ops::Add + ::std::ops::Sub + + ::std::ops::Mul + ::std::ops::Div + + ::std::ops::Rem + + ::std::ops::AddAssign + + num_traits::CheckedMul + num_traits::CheckedSub + + Decode + Encode +{} + +impl BlockNumber for T where T: + Send + Sync + 'static + + ::std::fmt::Display + + Clone + + From + One + Zero + + PartialEq + Ord + + ::std::ops::Add + ::std::ops::Sub + + ::std::ops::Mul + ::std::ops::Div + + ::std::ops::Rem + + ::std::ops::AddAssign + + num_traits::CheckedMul + num_traits::CheckedSub + + Decode + Encode, +{} + /// Block identifier that could be used to determine fork of this block. #[derive(Debug)] -pub struct AnchorBlockId { +pub struct AnchorBlockId { /// Hash of this block. pub hash: Hash, /// Number of this block. - pub number: u64, + pub number: Number, } /// Changes trie storage. Provides access to trie roots and trie nodes. -pub trait RootsStorage: Send + Sync { +pub trait RootsStorage: Send + Sync { + /// Resolve hash of the block into anchor. + fn build_anchor(&self, hash: H::Out) -> Result, String>; /// Get changes trie root for the block with given number which is an ancestor (or the block /// itself) of the anchor_block (i.e. anchor_block.number >= block). - fn root(&self, anchor: &AnchorBlockId, block: u64) -> Result, String>; + fn root(&self, anchor: &AnchorBlockId, block: Number) -> Result, String>; } /// Changes trie storage. Provides access to trie roots and trie nodes. -pub trait Storage: RootsStorage { +pub trait Storage: RootsStorage { /// Get a trie node. fn get(&self, key: &H::Out, prefix: &[u8]) -> Result, String>; } +/// Changes trie storage -> trie backend essence adapter. +pub struct TrieBackendStorageAdapter<'a, H: Hasher, Number: BlockNumber>(pub &'a Storage); + +impl<'a, H: Hasher, N: BlockNumber> crate::TrieBackendStorage for TrieBackendStorageAdapter<'a, H, N> { + type Overlay = trie::MemoryDB; + + fn get(&self, key: &H::Out, prefix: &[u8]) -> Result, String> { + self.0.get(key, prefix) + } +} + /// Changes trie configuration. pub type Configuration = primitives::ChangesTrieConfiguration; /// Compute the changes trie root and transaction for given block. -/// Returns None if there's no data to perform computation. -pub fn compute_changes_trie_root<'a, B: Backend, S: Storage, H: Hasher>( +/// Returns Err(()) if unknown `parent_hash` has been passed. +/// Returns Ok(None) if there's no data to perform computation. +/// Panics if background storage returns an error. +pub fn compute_changes_trie_root<'a, B: Backend, S: Storage, H: Hasher, Number: BlockNumber>( backend: &B, storage: Option<&'a S>, changes: &OverlayedChanges, - parent: &'a AnchorBlockId, -) -> Option<(H::Out, Vec<(Vec, Vec)>)> + parent_hash: H::Out, +) -> Result, Vec)>)>, ()> where - &'a S: TrieBackendStorage, - H::Out: Ord, + H::Out: Ord + 'static, { - let input_pairs = prepare_input::(backend, storage, changes, parent) - .expect("storage is not allowed to fail within runtime")?; - let transaction = input_pairs.into_iter() - .map(Into::into) - .collect::>(); - let root = trie_root::(transaction.iter().map(|(k, v)| (&*k, &*v))); + let (storage, config) = match (storage, changes.changes_trie_config.as_ref()) { + (Some(storage), Some(config)) => (storage, config), + _ => return Ok(None), + }; - Some((root, transaction)) + // build_anchor error should not be considered fatal + let parent = storage.build_anchor(parent_hash).map_err(|_| ())?; + + // storage errors are considered fatal (similar to situations when runtime fetches values from storage) + let input_pairs = prepare_input::(backend, storage, config, changes, &parent) + .expect("storage is not allowed to fail within runtime"); + match input_pairs { + Some(input_pairs) => { + let transaction = input_pairs.into_iter() + .map(Into::into) + .collect::>(); + let root = trie_root::(transaction.iter().map(|(k, v)| (&*k, &*v))); + + Ok(Some((root, transaction))) + }, + None => Ok(None), + } } diff --git a/substrate/core/state-machine/src/changes_trie/prune.rs b/substrate/core/state-machine/src/changes_trie/prune.rs index 09e53315a5..3aedf66f75 100644 --- a/substrate/core/state-machine/src/changes_trie/prune.rs +++ b/substrate/core/state-machine/src/changes_trie/prune.rs @@ -19,24 +19,26 @@ use hash_db::Hasher; use trie::Recorder; use log::warn; +use num_traits::One; use crate::proving_backend::ProvingBackendEssence; use crate::trie_backend_essence::TrieBackendEssence; -use crate::changes_trie::{AnchorBlockId, Configuration, Storage}; +use crate::changes_trie::{AnchorBlockId, Configuration, Storage, BlockNumber}; use crate::changes_trie::storage::TrieBackendAdapter; /// Get number of oldest block for which changes trie is not pruned /// given changes trie configuration, pruning parameter and number of /// best finalized block. -pub fn oldest_non_pruned_trie( +pub fn oldest_non_pruned_trie( config: &Configuration, - min_blocks_to_keep: u64, - best_finalized_block: u64, -) -> u64 { + min_blocks_to_keep: Number, + best_finalized_block: Number, +) -> Number { let max_digest_interval = config.max_digest_interval(); - let max_digest_block = best_finalized_block - best_finalized_block % max_digest_interval; + let best_finalized_block_rem = best_finalized_block.clone() % max_digest_interval.into(); + let max_digest_block = best_finalized_block - best_finalized_block_rem; match pruning_range(config, min_blocks_to_keep, max_digest_block) { - Some((_, last_pruned_block)) => last_pruned_block + 1, - None => 1, + Some((_, last_pruned_block)) => last_pruned_block + One::one(), + None => One::one(), } } @@ -45,23 +47,32 @@ pub fn oldest_non_pruned_trie( /// `min_blocks_to_keep` blocks. We only prune changes tries at `max_digest_interval` /// ranges. /// Returns MemoryDB that contains all deleted changes tries nodes. -pub fn prune, H: Hasher, F: FnMut(H::Out)>( +pub fn prune, H: Hasher, Number: BlockNumber, F: FnMut(H::Out)>( config: &Configuration, storage: &S, - min_blocks_to_keep: u64, - current_block: &AnchorBlockId, + min_blocks_to_keep: Number, + current_block: &AnchorBlockId, mut remove_trie_node: F, ) { // select range for pruning - let (first, last) = match pruning_range(config, min_blocks_to_keep, current_block.number) { + let (first, last) = match pruning_range(config, min_blocks_to_keep, current_block.number.clone()) { Some((first, last)) => (first, last), None => return, }; // delete changes trie for every block in range // FIXME: limit `max_digest_interval` so that this cycle won't involve huge ranges - for block in first..last+1 { - let root = match storage.root(current_block, block) { + let mut block = first; + loop { + if block >= last.clone() + One::one() { + break; + } + + let prev_block = block.clone(); + block += One::one(); + + let block = prev_block; + let root = match storage.root(current_block, block.clone()) { Ok(Some(root)) => root, Ok(None) => continue, Err(error) => { @@ -91,11 +102,15 @@ pub fn prune, H: Hasher, F: FnMut(H::Out)>( } /// Select blocks range (inclusive from both ends) for pruning changes tries in. -fn pruning_range(config: &Configuration, min_blocks_to_keep: u64, block: u64) -> Option<(u64, u64)> { +fn pruning_range( + config: &Configuration, + min_blocks_to_keep: Number, + block: Number, +) -> Option<(Number, Number)> { // compute number of changes tries we actually want to keep let (prune_interval, blocks_to_keep) = if config.is_digest_build_enabled() { // we only CAN prune at block where max-level-digest is created - let max_digest_interval = match config.digest_level_at_block(block) { + let max_digest_interval = match config.digest_level_at_block(block.clone()) { Some((digest_level, digest_interval, _)) if digest_level == config.digest_levels => digest_interval, _ => return None, @@ -107,7 +122,7 @@ fn pruning_range(config: &Configuration, min_blocks_to_keep: u64, block: u64) -> // number of blocks BEFORE current block where changes tries are not pruned ( max_digest_interval, - max_digest_intervals_to_keep.checked_mul(max_digest_interval) + max_digest_intervals_to_keep.checked_mul(&max_digest_interval.into()) ) } else { ( @@ -117,11 +132,11 @@ fn pruning_range(config: &Configuration, min_blocks_to_keep: u64, block: u64) -> }; // last block for which changes trie is pruned - let last_block_to_prune = blocks_to_keep.and_then(|b| block.checked_sub(b)); - let first_block_to_prune = last_block_to_prune.clone().and_then(|b| b.checked_sub(prune_interval)); + let last_block_to_prune = blocks_to_keep.and_then(|b| block.checked_sub(&b)); + let first_block_to_prune = last_block_to_prune.clone().and_then(|b| b.checked_sub(&prune_interval.into())); last_block_to_prune - .and_then(|last| first_block_to_prune.map(|first| (first + 1, last))) + .and_then(|last| first_block_to_prune.map(|first| (first + One::one(), last))) } /// Select pruning delay for the changes tries. To make sure we could build a changes @@ -132,13 +147,16 @@ fn pruning_range(config: &Configuration, min_blocks_to_keep: u64, block: u64) -> /// 0 or 1: means that only last changes trie is guaranteed to exists; /// 2: the last chnages trie + previous changes trie /// ... -fn max_digest_intervals_to_keep(min_blocks_to_keep: u64, max_digest_interval: u64) -> u64 { +fn max_digest_intervals_to_keep( + min_blocks_to_keep: Number, + max_digest_interval: u32, +) -> Number { // config.digest_level_at_block ensures that it is not zero debug_assert!(max_digest_interval != 0); - let max_digest_intervals_to_keep = min_blocks_to_keep / max_digest_interval; - if max_digest_intervals_to_keep == 0 { - 1 + let max_digest_intervals_to_keep = min_blocks_to_keep / max_digest_interval.into(); + if max_digest_intervals_to_keep.is_zero() { + One::one() } else { max_digest_intervals_to_keep } @@ -153,14 +171,14 @@ mod tests { use crate::changes_trie::storage::InMemoryStorage; use super::*; - fn config(interval: u64, levels: u32) -> Configuration { + fn config(interval: u32, levels: u32) -> Configuration { Configuration { digest_interval: interval, digest_levels: levels, } } - fn prune_by_collect, H: Hasher>( + fn prune_by_collect, H: Hasher>( config: &Configuration, storage: &S, min_blocks_to_keep: u64, @@ -174,7 +192,7 @@ mod tests { #[test] fn prune_works() { - fn prepare_storage() -> InMemoryStorage { + fn prepare_storage() -> InMemoryStorage { let mut mdb1 = MemoryDB::::default(); let root1 = insert_into_memory_db::(&mut mdb1, vec![(vec![10], vec![20])]).unwrap(); let mut mdb2 = MemoryDB::::default(); @@ -241,60 +259,60 @@ mod tests { #[test] fn pruning_range_works() { // DIGESTS ARE NOT CREATED + NO TRIES ARE PRUNED - assert_eq!(pruning_range(&config(10, 0), 2, 2), None); + assert_eq!(pruning_range(&config(10, 0), 2u64, 2u64), None); // DIGESTS ARE NOT CREATED + SOME TRIES ARE PRUNED - assert_eq!(pruning_range(&config(10, 0), 100, 110), Some((10, 10))); - assert_eq!(pruning_range(&config(10, 0), 100, 210), Some((110, 110))); + assert_eq!(pruning_range(&config(10, 0), 100u64, 110u64), Some((10, 10))); + assert_eq!(pruning_range(&config(10, 0), 100u64, 210u64), Some((110, 110))); // DIGESTS ARE CREATED + NO TRIES ARE PRUNED - assert_eq!(pruning_range(&config(10, 2), 2, 0), None); - assert_eq!(pruning_range(&config(10, 2), 30, 100), None); - assert_eq!(pruning_range(&config(::std::u64::MAX, 2), 1, 1024), None); - assert_eq!(pruning_range(&config(::std::u64::MAX, 2), ::std::u64::MAX, 1024), None); - assert_eq!(pruning_range(&config(32, 2), 2048, 512), None); - assert_eq!(pruning_range(&config(32, 2), 2048, 1024), None); + assert_eq!(pruning_range(&config(10, 2), 2u64, 0u64), None); + assert_eq!(pruning_range(&config(10, 2), 30u64, 100u64), None); + assert_eq!(pruning_range(&config(::std::u32::MAX, 2), 1u64, 1024u64), None); + assert_eq!(pruning_range(&config(::std::u32::MAX, 2), ::std::u64::MAX, 1024u64), None); + assert_eq!(pruning_range(&config(32, 2), 2048u64, 512u64), None); + assert_eq!(pruning_range(&config(32, 2), 2048u64, 1024u64), None); // DIGESTS ARE CREATED + SOME TRIES ARE PRUNED // when we do not want to keep any highest-level-digests // (system forces to keep at least one) - assert_eq!(pruning_range(&config(4, 2), 0, 32), Some((1, 16))); - assert_eq!(pruning_range(&config(4, 2), 0, 64), Some((33, 48))); + assert_eq!(pruning_range(&config(4, 2), 0u64, 32u64), Some((1, 16))); + assert_eq!(pruning_range(&config(4, 2), 0u64, 64u64), Some((33, 48))); // when we want to keep 1 (last) highest-level-digest - assert_eq!(pruning_range(&config(4, 2), 16, 32), Some((1, 16))); - assert_eq!(pruning_range(&config(4, 2), 16, 64), Some((33, 48))); + assert_eq!(pruning_range(&config(4, 2), 16u64, 32u64), Some((1, 16))); + assert_eq!(pruning_range(&config(4, 2), 16u64, 64u64), Some((33, 48))); // when we want to keep 1 (last) + 1 additional level digests - assert_eq!(pruning_range(&config(32, 2), 4096, 5120), Some((1, 1024))); - assert_eq!(pruning_range(&config(32, 2), 4096, 6144), Some((1025, 2048))); + assert_eq!(pruning_range(&config(32, 2), 4096u64, 5120u64), Some((1, 1024))); + assert_eq!(pruning_range(&config(32, 2), 4096u64, 6144u64), Some((1025, 2048))); } #[test] fn max_digest_intervals_to_keep_works() { - assert_eq!(max_digest_intervals_to_keep(1024, 1025), 1); - assert_eq!(max_digest_intervals_to_keep(1024, 1023), 1); - assert_eq!(max_digest_intervals_to_keep(1024, 512), 2); - assert_eq!(max_digest_intervals_to_keep(1024, 511), 2); - assert_eq!(max_digest_intervals_to_keep(1024, 100), 10); + assert_eq!(max_digest_intervals_to_keep(1024u64, 1025), 1u64); + assert_eq!(max_digest_intervals_to_keep(1024u64, 1023), 1u64); + assert_eq!(max_digest_intervals_to_keep(1024u64, 512), 2u64); + assert_eq!(max_digest_intervals_to_keep(1024u64, 511), 2u64); + assert_eq!(max_digest_intervals_to_keep(1024u64, 100), 10u64); } #[test] fn oldest_non_pruned_trie_works() { // when digests are not created at all - assert_eq!(oldest_non_pruned_trie(&config(0, 0), 100, 10), 1); - assert_eq!(oldest_non_pruned_trie(&config(0, 0), 100, 110), 11); + assert_eq!(oldest_non_pruned_trie(&config(0, 0), 100u64, 10u64), 1); + assert_eq!(oldest_non_pruned_trie(&config(0, 0), 100u64, 110u64), 11); // when only l1 digests are created - assert_eq!(oldest_non_pruned_trie(&config(100, 1), 100, 50), 1); - assert_eq!(oldest_non_pruned_trie(&config(100, 1), 100, 110), 1); - assert_eq!(oldest_non_pruned_trie(&config(100, 1), 100, 210), 101); + assert_eq!(oldest_non_pruned_trie(&config(100, 1), 100u64, 50u64), 1); + assert_eq!(oldest_non_pruned_trie(&config(100, 1), 100u64, 110u64), 1); + assert_eq!(oldest_non_pruned_trie(&config(100, 1), 100u64, 210u64), 101); // when l2 digests are created - assert_eq!(oldest_non_pruned_trie(&config(100, 2), 100, 50), 1); - assert_eq!(oldest_non_pruned_trie(&config(100, 2), 100, 110), 1); - assert_eq!(oldest_non_pruned_trie(&config(100, 2), 100, 210), 1); - assert_eq!(oldest_non_pruned_trie(&config(100, 2), 100, 10110), 1); - assert_eq!(oldest_non_pruned_trie(&config(100, 2), 100, 20110), 10001); + assert_eq!(oldest_non_pruned_trie(&config(100, 2), 100u64, 50u64), 1); + assert_eq!(oldest_non_pruned_trie(&config(100, 2), 100u64, 110u64), 1); + assert_eq!(oldest_non_pruned_trie(&config(100, 2), 100u64, 210u64), 1); + assert_eq!(oldest_non_pruned_trie(&config(100, 2), 100u64, 10110u64), 1); + assert_eq!(oldest_non_pruned_trie(&config(100, 2), 100u64, 20110u64), 10001); } } diff --git a/substrate/core/state-machine/src/changes_trie/storage.rs b/substrate/core/state-machine/src/changes_trie/storage.rs index 8363ae4221..8da2052515 100644 --- a/substrate/core/state-machine/src/changes_trie/storage.rs +++ b/substrate/core/state-machine/src/changes_trie/storage.rs @@ -16,12 +16,12 @@ //! Changes trie storage utilities. -use std::collections::HashMap; +use std::collections::BTreeMap; use hash_db::Hasher; use trie::DBValue; use trie::MemoryDB; use parking_lot::RwLock; -use crate::changes_trie::{AnchorBlockId, RootsStorage, Storage}; +use crate::changes_trie::{RootsStorage, Storage, AnchorBlockId, BlockNumber}; use crate::trie_backend_essence::TrieBackendStorage; #[cfg(test)] @@ -32,27 +32,27 @@ use crate::backend::insert_into_memory_db; use crate::changes_trie::input::InputPair; /// In-memory implementation of changes trie storage. -pub struct InMemoryStorage { - data: RwLock>, +pub struct InMemoryStorage { + data: RwLock>, } /// Adapter for using changes trie storage as a TrieBackendEssence' storage. -pub struct TrieBackendAdapter<'a, H: Hasher, S: 'a + Storage> { +pub struct TrieBackendAdapter<'a, H: Hasher, Number: BlockNumber, S: 'a + Storage> { storage: &'a S, - _hasher: ::std::marker::PhantomData, + _hasher: ::std::marker::PhantomData<(H, Number)>, } -struct InMemoryStorageData { - roots: HashMap, +struct InMemoryStorageData { + roots: BTreeMap, mdb: MemoryDB, } -impl InMemoryStorage { +impl InMemoryStorage { /// Create the storage from given in-memory database. pub fn with_db(mdb: MemoryDB) -> Self { Self { data: RwLock::new(InMemoryStorageData { - roots: HashMap::new(), + roots: BTreeMap::new(), mdb, }), } @@ -63,10 +63,20 @@ impl InMemoryStorage { Self::with_db(Default::default()) } + /// Create the storage with given blocks. + pub fn with_blocks(blocks: Vec<(Number, H::Out)>) -> Self { + Self { + data: RwLock::new(InMemoryStorageData { + roots: blocks.into_iter().collect(), + mdb: MemoryDB::default(), + }), + } + } + #[cfg(test)] - pub fn with_inputs(inputs: Vec<(u64, Vec)>) -> Self { + pub fn with_inputs(inputs: Vec<(Number, Vec>)>) -> Self { let mut mdb = MemoryDB::default(); - let mut roots = HashMap::new(); + let mut roots = BTreeMap::new(); for (block, pairs) in inputs { let root = insert_into_memory_db::(&mut mdb, pairs.into_iter().map(Into::into)); if let Some(root) = root { @@ -101,32 +111,44 @@ impl InMemoryStorage { } /// Insert changes trie for given block. - pub fn insert(&self, block: u64, changes_trie_root: H::Out, trie: MemoryDB) { + pub fn insert(&self, block: Number, changes_trie_root: H::Out, trie: MemoryDB) { let mut data = self.data.write(); data.roots.insert(block, changes_trie_root); data.mdb.consolidate(trie); } } -impl RootsStorage for InMemoryStorage { - fn root(&self, _anchor_block: &AnchorBlockId, block: u64) -> Result, String> { +impl RootsStorage for InMemoryStorage { + fn build_anchor(&self, parent_hash: H::Out) -> Result, String> { + self.data.read().roots.iter() + .find(|(_, v)| **v == parent_hash) + .map(|(k, _)| AnchorBlockId { hash: parent_hash, number: k.clone() }) + .ok_or_else(|| format!("Can't find associated number for block {:?}", parent_hash)) + } + + fn root(&self, _anchor_block: &AnchorBlockId, block: Number) -> Result, String> { Ok(self.data.read().roots.get(&block).cloned()) } } -impl Storage for InMemoryStorage { +impl Storage for InMemoryStorage { fn get(&self, key: &H::Out, prefix: &[u8]) -> Result, String> { MemoryDB::::get(&self.data.read().mdb, key, prefix) } } -impl<'a, H: Hasher, S: 'a + Storage> TrieBackendAdapter<'a, H, S> { +impl<'a, H: Hasher, Number: BlockNumber, S: 'a + Storage> TrieBackendAdapter<'a, H, Number, S> { pub fn new(storage: &'a S) -> Self { Self { storage, _hasher: Default::default() } } } -impl<'a, H: Hasher, S: 'a + Storage> TrieBackendStorage for TrieBackendAdapter<'a, H, S> { +impl<'a, H, Number, S> TrieBackendStorage for TrieBackendAdapter<'a, H, Number, S> + where + S: 'a + Storage, + Number: BlockNumber, + H: Hasher, +{ type Overlay = MemoryDB; fn get(&self, key: &H::Out, prefix: &[u8]) -> Result, String> { diff --git a/substrate/core/state-machine/src/ext.rs b/substrate/core/state-machine/src/ext.rs index b9c035a778..f0638edf65 100644 --- a/substrate/core/state-machine/src/ext.rs +++ b/substrate/core/state-machine/src/ext.rs @@ -19,7 +19,7 @@ use std::{error, fmt, cmp::Ord}; use log::warn; use crate::backend::Backend; -use crate::changes_trie::{AnchorBlockId, Storage as ChangesTrieStorage, compute_changes_trie_root}; +use crate::changes_trie::{Storage as ChangesTrieStorage, compute_changes_trie_root}; use crate::{Externalities, OverlayedChanges, OffchainExt, ChildStorageKey}; use hash_db::Hasher; use primitives::storage::well_known_keys::is_child_storage_key; @@ -57,10 +57,9 @@ impl error::Error for Error { } /// Wraps a read-only backend, call executor, and current overlayed changes. -pub struct Ext<'a, H, B, T, O> +pub struct Ext<'a, H, N, B, T, O> where H: Hasher, - B: 'a + Backend, { /// The overlayed changes to write to. @@ -78,20 +77,23 @@ where /// This differs from `storage_transaction` behavior, because the moment when /// `storage_changes_root` is called matters + we need to remember additional /// data at this moment (block number). - changes_trie_transaction: Option<(u64, MemoryDB, H::Out)>, + changes_trie_transaction: Option<(MemoryDB, H::Out)>, /// Additional externalities for offchain workers. /// /// If None, some methods from the trait might not supported. offchain_externalities: Option<&'a mut O>, + /// Dummy usage of N arg. + _phantom: ::std::marker::PhantomData, } -impl<'a, H, B, T, O> Ext<'a, H, B, T, O> +impl<'a, H, N, B, T, O> Ext<'a, H, N, B, T, O> where H: Hasher, B: 'a + Backend, - T: 'a + ChangesTrieStorage, + T: 'a + ChangesTrieStorage, O: 'a + OffchainExt, - H::Out: Ord, + H::Out: Ord + 'static, + N: crate::changes_trie::BlockNumber, { /// Create a new `Ext` from overlayed changes and read-only backend pub fn new( @@ -107,6 +109,7 @@ where changes_trie_storage, changes_trie_transaction: None, offchain_externalities, + _phantom: Default::default(), } } @@ -118,7 +121,7 @@ where self.storage_transaction .expect("storage_transaction always set after calling storage root; qed"), self.changes_trie_transaction - .map(|(_, tx, _)| tx), + .map(|(tx, _)| tx), ); ( @@ -137,13 +140,13 @@ where } #[cfg(test)] -impl<'a, H, B, T, O> Ext<'a, H, B, T, O> +impl<'a, H, N, B, T, O> Ext<'a, H, N, B, T, O> where H: Hasher, - B: 'a + Backend, - T: 'a + ChangesTrieStorage, + T: 'a + ChangesTrieStorage, O: 'a + OffchainExt, + N: crate::changes_trie::BlockNumber, { pub fn storage_pairs(&self) -> Vec<(Vec, Vec)> { use std::collections::HashMap; @@ -159,13 +162,14 @@ where } } -impl<'a, B, T, H, O> Externalities for Ext<'a, H, B, T, O> +impl<'a, B, T, H, N, O> Externalities for Ext<'a, H, N, B, T, O> where H: Hasher, B: 'a + Backend, - T: 'a + ChangesTrieStorage, + T: 'a + ChangesTrieStorage, O: 'a + OffchainExt, - H::Out: Ord, + H::Out: Ord + 'static, + N: crate::changes_trie::BlockNumber, { fn storage(&self, key: &[u8]) -> Option> { let _guard = panic_handler::AbortGuard::new(true); @@ -313,14 +317,14 @@ where } } - fn storage_changes_root(&mut self, parent: H::Out, parent_num: u64) -> Option { + fn storage_changes_root(&mut self, parent_hash: H::Out) -> Result, ()> { let _guard = panic_handler::AbortGuard::new(true); - let root_and_tx = compute_changes_trie_root::<_, T, H>( + let root_and_tx = compute_changes_trie_root::<_, T, H, N>( self.backend, self.changes_trie_storage.clone(), self.overlay, - &AnchorBlockId { hash: parent, number: parent_num }, - ); + parent_hash, + )?; let root_and_tx = root_and_tx.map(|(root, changes)| { let mut calculated_root = Default::default(); let mut mdb = MemoryDB::default(); @@ -331,11 +335,11 @@ where } } - (parent_num + 1, mdb, root) + (mdb, root) }); - let root = root_and_tx.as_ref().map(|(_, _, root)| root.clone()); + let root = root_and_tx.as_ref().map(|(_, root)| root.clone()); self.changes_trie_transaction = root_and_tx; - root + Ok(root) } fn submit_extrinsic(&mut self, extrinsic: Vec) -> Result<(), ()> { @@ -363,8 +367,8 @@ mod tests { use super::*; type TestBackend = InMemory; - type TestChangesTrieStorage = InMemoryChangesTrieStorage; - type TestExt<'a> = Ext<'a, Blake2Hasher, TestBackend, TestChangesTrieStorage, crate::NeverOffchainExt>; + type TestChangesTrieStorage = InMemoryChangesTrieStorage; + type TestExt<'a> = Ext<'a, Blake2Hasher, u64, TestBackend, TestChangesTrieStorage, crate::NeverOffchainExt>; fn prepare_overlay_with_changes() -> OverlayedChanges { OverlayedChanges { @@ -391,26 +395,26 @@ mod tests { let mut overlay = prepare_overlay_with_changes(); let backend = TestBackend::default(); let mut ext = TestExt::new(&mut overlay, &backend, None, None); - assert_eq!(ext.storage_changes_root(Default::default(), 100), None); + assert_eq!(ext.storage_changes_root(Default::default()).unwrap(), None); } #[test] fn storage_changes_root_is_none_when_extrinsic_changes_are_none() { let mut overlay = prepare_overlay_with_changes(); overlay.changes_trie_config = None; - let storage = TestChangesTrieStorage::new(); + let storage = TestChangesTrieStorage::with_blocks(vec![(100, Default::default())]); let backend = TestBackend::default(); let mut ext = TestExt::new(&mut overlay, &backend, Some(&storage), None); - assert_eq!(ext.storage_changes_root(Default::default(), 100), None); + assert_eq!(ext.storage_changes_root(Default::default()).unwrap(), None); } #[test] fn storage_changes_root_is_some_when_extrinsic_changes_are_non_empty() { let mut overlay = prepare_overlay_with_changes(); - let storage = TestChangesTrieStorage::new(); + let storage = TestChangesTrieStorage::with_blocks(vec![(99, Default::default())]); let backend = TestBackend::default(); let mut ext = TestExt::new(&mut overlay, &backend, Some(&storage), None); - assert_eq!(ext.storage_changes_root(Default::default(), 99), + assert_eq!(ext.storage_changes_root(Default::default()).unwrap(), Some(hex!("5b829920b9c8d554a19ee2a1ba593c4f2ee6fc32822d083e04236d693e8358d5").into())); } @@ -418,10 +422,10 @@ mod tests { fn storage_changes_root_is_some_when_extrinsic_changes_are_empty() { let mut overlay = prepare_overlay_with_changes(); overlay.prospective.top.get_mut(&vec![1]).unwrap().value = None; - let storage = TestChangesTrieStorage::new(); + let storage = TestChangesTrieStorage::with_blocks(vec![(99, Default::default())]); let backend = TestBackend::default(); let mut ext = TestExt::new(&mut overlay, &backend, Some(&storage), None); - assert_eq!(ext.storage_changes_root(Default::default(), 99), + assert_eq!(ext.storage_changes_root(Default::default()).unwrap(), Some(hex!("bcf494e41e29a15c9ae5caa053fe3cb8b446ee3e02a254efbdec7a19235b76e4").into())); } } diff --git a/substrate/core/state-machine/src/lib.rs b/substrate/core/state-machine/src/lib.rs index daf8f915a1..4f51677fff 100644 --- a/substrate/core/state-machine/src/lib.rs +++ b/substrate/core/state-machine/src/lib.rs @@ -219,7 +219,7 @@ pub trait Externalities { fn child_storage_root(&mut self, storage_key: ChildStorageKey) -> Vec; /// Get the change trie root of the current storage overlay at a block with given parent. - fn storage_changes_root(&mut self, parent: H::Out, parent_num: u64) -> Option where H::Out: Ord; + fn storage_changes_root(&mut self, parent: H::Out) -> Result, ()> where H::Out: Ord; /// Submit extrinsic. /// @@ -338,7 +338,7 @@ pub fn always_wasm() -> ExecutionManager> { } /// Creates new substrate state machine. -pub fn new<'a, H, B, T, O, Exec>( +pub fn new<'a, H, N, B, T, O, Exec>( backend: &'a B, changes_trie_storage: Option<&'a T>, offchain_ext: Option<&'a mut O>, @@ -346,7 +346,7 @@ pub fn new<'a, H, B, T, O, Exec>( exec: &'a Exec, method: &'a str, call_data: &'a [u8], -) -> StateMachine<'a, H, B, T, O, Exec> { +) -> StateMachine<'a, H, N, B, T, O, Exec> { StateMachine { backend, changes_trie_storage, @@ -360,7 +360,7 @@ pub fn new<'a, H, B, T, O, Exec>( } /// The substrate state machine. -pub struct StateMachine<'a, H, B, T, O, Exec> { +pub struct StateMachine<'a, H, N, B, T, O, Exec> { backend: &'a B, changes_trie_storage: Option<&'a T>, offchain_ext: Option<&'a mut O>, @@ -368,16 +368,17 @@ pub struct StateMachine<'a, H, B, T, O, Exec> { exec: &'a Exec, method: &'a str, call_data: &'a [u8], - _hasher: PhantomData, + _hasher: PhantomData<(H, N)>, } -impl<'a, H, B, T, O, Exec> StateMachine<'a, H, B, T, O, Exec> where +impl<'a, H, N, B, T, O, Exec> StateMachine<'a, H, N, B, T, O, Exec> where H: Hasher, Exec: CodeExecutor, B: Backend, - T: ChangesTrieStorage, + T: ChangesTrieStorage, O: OffchainExt, - H::Out: Ord, + H::Out: Ord + 'static, + N: crate::changes_trie::BlockNumber, { /// Execute a call using the given state backend, overlayed changes, and call executor. /// Produces a state-backend-specific "transaction" which can be used to apply the changes @@ -568,7 +569,7 @@ where B: Backend, H: Hasher, Exec: CodeExecutor, - H::Out: Ord, + H::Out: Ord + 'static, { let trie_backend = backend.try_into_trie_backend() .ok_or_else(|| Box::new(ExecutionError::UnableToGenerateProof) as Box)?; @@ -595,12 +596,12 @@ where S: trie_backend_essence::TrieBackendStorage, H: Hasher, Exec: CodeExecutor, - H::Out: Ord, + H::Out: Ord + 'static, { let proving_backend = proving_backend::ProvingBackend::new(trie_backend); let mut sm = StateMachine { backend: &proving_backend, - changes_trie_storage: None as Option<&changes_trie::InMemoryStorage>, + changes_trie_storage: None as Option<&changes_trie::InMemoryStorage>, offchain_ext: NeverOffchainExt::new(), overlay, exec, @@ -629,7 +630,7 @@ pub fn execution_proof_check( where H: Hasher, Exec: CodeExecutor, - H::Out: Ord, + H::Out: Ord + 'static, { let trie_backend = create_proof_check_backend::(root.into(), proof)?; execution_proof_check_on_trie_backend(&trie_backend, overlay, exec, method, call_data) @@ -646,11 +647,11 @@ pub fn execution_proof_check_on_trie_backend( where H: Hasher, Exec: CodeExecutor, - H::Out: Ord, + H::Out: Ord + 'static, { let mut sm = StateMachine { backend: trie_backend, - changes_trie_storage: None as Option<&changes_trie::InMemoryStorage>, + changes_trie_storage: None as Option<&changes_trie::InMemoryStorage>, offchain_ext: NeverOffchainExt::new(), overlay, exec, @@ -892,7 +893,7 @@ mod tests { fn execute_works() { assert_eq!(new( &trie_backend::tests::test_trie(), - Some(&InMemoryChangesTrieStorage::new()), + Some(&InMemoryChangesTrieStorage::::new()), NeverOffchainExt::new(), &mut Default::default(), &DummyCodeExecutor { @@ -913,7 +914,7 @@ mod tests { fn execute_works_with_native_else_wasm() { assert_eq!(new( &trie_backend::tests::test_trie(), - Some(&InMemoryChangesTrieStorage::new()), + Some(&InMemoryChangesTrieStorage::::new()), NeverOffchainExt::new(), &mut Default::default(), &DummyCodeExecutor { @@ -934,7 +935,7 @@ mod tests { let mut consensus_failed = false; assert!(new( &trie_backend::tests::test_trie(), - Some(&InMemoryChangesTrieStorage::new()), + Some(&InMemoryChangesTrieStorage::::new()), NeverOffchainExt::new(), &mut Default::default(), &DummyCodeExecutor { @@ -1003,7 +1004,7 @@ mod tests { }; { - let changes_trie_storage = InMemoryChangesTrieStorage::new(); + let changes_trie_storage = InMemoryChangesTrieStorage::::new(); let mut ext = Ext::new(&mut overlay, &backend, Some(&changes_trie_storage), NeverOffchainExt::new()); ext.clear_prefix(b"ab"); } @@ -1026,7 +1027,7 @@ mod tests { #[test] fn set_child_storage_works() { let backend = InMemory::::default().try_into_trie_backend().unwrap(); - let changes_trie_storage = InMemoryChangesTrieStorage::new(); + let changes_trie_storage = InMemoryChangesTrieStorage::::new(); let mut overlay = OverlayedChanges::default(); let mut ext = Ext::new( &mut overlay, @@ -1106,7 +1107,7 @@ mod tests { fn cannot_change_changes_trie_config() { assert!(new( &trie_backend::tests::test_trie(), - Some(&InMemoryChangesTrieStorage::new()), + Some(&InMemoryChangesTrieStorage::::new()), NeverOffchainExt::new(), &mut Default::default(), &DummyCodeExecutor { @@ -1126,7 +1127,7 @@ mod tests { fn cannot_change_changes_trie_config_with_native_else_wasm() { assert!(new( &trie_backend::tests::test_trie(), - Some(&InMemoryChangesTrieStorage::new()), + Some(&InMemoryChangesTrieStorage::::new()), NeverOffchainExt::new(), &mut Default::default(), &DummyCodeExecutor { diff --git a/substrate/core/state-machine/src/overlayed_changes.rs b/substrate/core/state-machine/src/overlayed_changes.rs index e595cff0e4..663e4ff72e 100644 --- a/substrate/core/state-machine/src/overlayed_changes.rs +++ b/substrate/core/state-machine/src/overlayed_changes.rs @@ -361,7 +361,7 @@ mod tests { ..Default::default() }; - let changes_trie_storage = InMemoryChangesTrieStorage::new(); + let changes_trie_storage = InMemoryChangesTrieStorage::::new(); let mut ext = Ext::new( &mut overlay, &backend, diff --git a/substrate/core/state-machine/src/testing.rs b/substrate/core/state-machine/src/testing.rs index 03909f384c..52d208142d 100644 --- a/substrate/core/state-machine/src/testing.rs +++ b/substrate/core/state-machine/src/testing.rs @@ -18,12 +18,12 @@ use std::collections::{HashMap, BTreeMap}; use std::iter::FromIterator; -use std::marker::PhantomData; use hash_db::Hasher; use crate::backend::{InMemory, Backend}; use primitives::storage::well_known_keys::is_child_storage_key; use crate::changes_trie::{ - compute_changes_trie_root, InMemoryStorage as ChangesTrieInMemoryStorage, AnchorBlockId + compute_changes_trie_root, InMemoryStorage as ChangesTrieInMemoryStorage, + BlockNumber as ChangesTrieBlockNumber, }; use primitives::storage::well_known_keys::{CHANGES_TRIE_CONFIG, CODE, HEAP_PAGES}; use parity_codec::Encode; @@ -32,14 +32,13 @@ use super::{ChildStorageKey, Externalities, OverlayedChanges}; const EXT_NOT_ALLOWED_TO_FAIL: &str = "Externalities not allowed to fail within runtime"; /// Simple HashMap-based Externalities impl. -pub struct TestExternalities { +pub struct TestExternalities { overlay: OverlayedChanges, backend: InMemory, - changes_trie_storage: ChangesTrieInMemoryStorage, - _hasher: PhantomData, + changes_trie_storage: ChangesTrieInMemoryStorage, } -impl TestExternalities { +impl TestExternalities { /// Create a new instance of `TestExternalities` pub fn new(inner: HashMap, Vec>) -> Self { Self::new_with_code(&[], inner) @@ -62,7 +61,6 @@ impl TestExternalities { overlay, changes_trie_storage: ChangesTrieInMemoryStorage::new(), backend: inner.into(), - _hasher: Default::default(), } } @@ -81,23 +79,28 @@ impl TestExternalities { .into_iter() .filter_map(|(k, maybe_val)| maybe_val.map(|val| (k, val))) } + + /// Get mutable reference to changes trie storage. + pub fn changes_trie_storage(&mut self) -> &mut ChangesTrieInMemoryStorage { + &mut self.changes_trie_storage + } } -impl ::std::fmt::Debug for TestExternalities { +impl ::std::fmt::Debug for TestExternalities { fn fmt(&self, f: &mut ::std::fmt::Formatter) -> ::std::fmt::Result { write!(f, "overlay: {:?}\nbackend: {:?}", self.overlay, self.backend.pairs()) } } -impl PartialEq for TestExternalities { +impl PartialEq for TestExternalities { /// This doesn't test if they are in the same state, only if they contains the /// same data at this state - fn eq(&self, other: &TestExternalities) -> bool { + fn eq(&self, other: &TestExternalities) -> bool { self.iter_pairs_in_order().eq(other.iter_pairs_in_order()) } } -impl FromIterator<(Vec, Vec)> for TestExternalities { +impl FromIterator<(Vec, Vec)> for TestExternalities { fn from_iter, Vec)>>(iter: I) -> Self { let mut t = Self::new(Default::default()); t.backend = t.backend.update(iter.into_iter().map(|(k, v)| (None, k, Some(v))).collect()); @@ -105,23 +108,28 @@ impl FromIterator<(Vec, Vec)> for TestExternalities { } } -impl Default for TestExternalities { +impl Default for TestExternalities { fn default() -> Self { Self::new(Default::default()) } } -impl From> for HashMap, Vec> { - fn from(tex: TestExternalities) -> Self { +impl From> for HashMap, Vec> { + fn from(tex: TestExternalities) -> Self { tex.iter_pairs_in_order().collect() } } -impl From< HashMap, Vec> > for TestExternalities { +impl From< HashMap, Vec> > for TestExternalities { fn from(hashmap: HashMap, Vec>) -> Self { Self::from_iter(hashmap) } } -impl Externalities for TestExternalities where H::Out: Ord { +impl Externalities for TestExternalities + where + H: Hasher, + N: ChangesTrieBlockNumber, + H::Out: Ord + 'static +{ fn storage(&self, key: &[u8]) -> Option> { self.overlay.storage(key).map(|x| x.map(|x| x.to_vec())).unwrap_or_else(|| self.backend.storage(key).expect(EXT_NOT_ALLOWED_TO_FAIL)) @@ -209,13 +217,13 @@ impl Externalities for TestExternalities where H::Out: Ord { root } - fn storage_changes_root(&mut self, parent: H::Out, parent_num: u64) -> Option { - compute_changes_trie_root::<_, ChangesTrieInMemoryStorage, H>( + fn storage_changes_root(&mut self, parent: H::Out) -> Result, ()> { + Ok(compute_changes_trie_root::<_, _, H, N>( &self.backend, Some(&self.changes_trie_storage), &self.overlay, - &AnchorBlockId { hash: parent, number: parent_num }, - ).map(|(root, _)| root.clone()) + parent, + )?.map(|(root, _)| root.clone())) } fn submit_extrinsic(&mut self, _extrinsic: Vec) -> Result<(), ()> { @@ -231,7 +239,7 @@ mod tests { #[test] fn commit_should_work() { - let mut ext = TestExternalities::::default(); + let mut ext = TestExternalities::::default(); ext.set_storage(b"doe".to_vec(), b"reindeer".to_vec()); ext.set_storage(b"dog".to_vec(), b"puppy".to_vec()); ext.set_storage(b"dogglesworth".to_vec(), b"cat".to_vec()); @@ -241,7 +249,7 @@ mod tests { #[test] fn set_and_retrieve_code() { - let mut ext = TestExternalities::::default(); + let mut ext = TestExternalities::::default(); let code = vec![1, 2, 3]; ext.set_storage(CODE.to_vec(), code.clone()); diff --git a/substrate/core/state-machine/src/trie_backend_essence.rs b/substrate/core/state-machine/src/trie_backend_essence.rs index dfb6cae08c..89a5b42a84 100644 --- a/substrate/core/state-machine/src/trie_backend_essence.rs +++ b/substrate/core/state-machine/src/trie_backend_essence.rs @@ -22,7 +22,6 @@ use std::sync::Arc; use log::{debug, warn}; use hash_db::{self, Hasher}; use trie::{TrieDB, Trie, MemoryDB, PrefixedMemoryDB, DBValue, TrieError, default_child_trie_root, read_trie_value, read_child_trie_value, for_keys_in_child_trie}; -use crate::changes_trie::Storage as ChangesTrieStorage; use crate::backend::Consolidate; /// Patricia trie-based storage trait. @@ -300,12 +299,3 @@ impl TrieBackendStorage for MemoryDB { Ok(hash_db::HashDB::get(self, key, prefix)) } } - -// This implementation is used by changes trie clients. -impl<'a, S, H: Hasher> TrieBackendStorage for &'a S where S: ChangesTrieStorage { - type Overlay = MemoryDB; - - fn get(&self, key: &H::Out, prefix: &[u8]) -> Result, String> { - ChangesTrieStorage::::get(*self, key, prefix) - } -} diff --git a/substrate/core/test-runtime/src/system.rs b/substrate/core/test-runtime/src/system.rs index 5345dc0cdb..804d5411d6 100644 --- a/substrate/core/test-runtime/src/system.rs +++ b/substrate/core/test-runtime/src/system.rs @@ -100,7 +100,7 @@ pub fn polish_block(block: &mut Block) { // check digest let mut digest = Digest::default(); - if let Some(storage_changes_root) = storage_changes_root(header.parent_hash.into(), header.number - 1) { + if let Some(storage_changes_root) = storage_changes_root(header.parent_hash.into()) { digest.push(generic::DigestItem::ChangesTrieRoot(storage_changes_root.into())); } if let Some(new_authorities) = ::take() { @@ -133,7 +133,7 @@ pub fn execute_block(block: Block) { // check digest let mut digest = Digest::default(); - if let Some(storage_changes_root) = storage_changes_root(header.parent_hash.into(), header.number - 1) { + if let Some(storage_changes_root) = storage_changes_root(header.parent_hash.into()) { digest.push(generic::DigestItem::ChangesTrieRoot(storage_changes_root.into())); } if let Some(new_authorities) = ::take() { @@ -213,7 +213,7 @@ pub fn finalize_block() -> Header { let number = ::take().expect("Number is set by `initialize_block`"); let parent_hash = ::take(); let storage_root = BlakeTwo256::storage_root(); - let storage_changes_root = BlakeTwo256::storage_changes_root(parent_hash, number - 1); + let storage_changes_root = BlakeTwo256::storage_changes_root(parent_hash); let mut digest = Digest::default(); if let Some(storage_changes_root) = storage_changes_root { diff --git a/substrate/node/executor/src/lib.rs b/substrate/node/executor/src/lib.rs index c6903529c4..8813850bbd 100644 --- a/substrate/node/executor/src/lib.rs +++ b/substrate/node/executor/src/lib.rs @@ -33,7 +33,7 @@ mod tests { use parity_codec::{Encode, Decode, Joiner}; use keyring::{AuthorityKeyring, AccountKeyring}; use runtime_support::{Hashable, StorageValue, StorageMap, traits::Currency}; - use state_machine::{CodeExecutor, Externalities, TestExternalities}; + use state_machine::{CodeExecutor, Externalities, TestExternalities as CoreTestExternalities}; use primitives::{twox_128, blake2_256, Blake2Hasher, ChangesTrieConfiguration, NeverNativeValue, NativeOrEncoded}; use node_primitives::{Hash, BlockNumber, AccountId}; @@ -52,6 +52,8 @@ mod tests { const COMPACT_CODE: &[u8] = include_bytes!("../../runtime/wasm/target/wasm32-unknown-unknown/release/node_runtime.compact.wasm"); const GENESIS_HASH: [u8; 32] = [69u8; 32]; + type TestExternalities = CoreTestExternalities; + fn alice() -> AccountId { AccountKeyring::Alice.into() } @@ -258,7 +260,7 @@ mod tests { fn new_test_ext(code: &[u8], support_changes_trie: bool) -> TestExternalities { let three = AccountId::from_raw([3u8; 32]); - TestExternalities::new_with_code(code, GenesisConfig { + let mut ext = TestExternalities::new_with_code(code, GenesisConfig { consensus: Some(Default::default()), system: Some(SystemConfig { changes_trie_config: if support_changes_trie { Some(ChangesTrieConfiguration { @@ -322,7 +324,9 @@ mod tests { grandpa: Some(GrandpaConfig { authorities: vec![], }), - }.build_storage().unwrap().0) + }.build_storage().unwrap().0); + ext.changes_trie_storage().insert(0, GENESIS_HASH.into(), Default::default()); + ext } fn construct_block( @@ -879,7 +883,7 @@ mod tests { None, ).0.unwrap(); - assert!(t.storage_changes_root(Default::default(), 0).is_some()); + assert!(t.storage_changes_root(GENESIS_HASH.into()).unwrap().is_some()); } #[test] @@ -889,7 +893,7 @@ mod tests { let mut t = new_test_ext(COMPACT_CODE, true); WasmExecutor::new().call(&mut t, 8, COMPACT_CODE, "Core_execute_block", &block1.0).unwrap(); - assert!(t.storage_changes_root(Default::default(), 0).is_some()); + assert!(t.storage_changes_root(GENESIS_HASH.into()).unwrap().is_some()); } #[cfg(feature = "benchmarks")] diff --git a/substrate/node/runtime/src/lib.rs b/substrate/node/runtime/src/lib.rs index 0ec44093b4..bb066b1b98 100644 --- a/substrate/node/runtime/src/lib.rs +++ b/substrate/node/runtime/src/lib.rs @@ -58,8 +58,8 @@ pub const VERSION: RuntimeVersion = RuntimeVersion { spec_name: create_runtime_str!("node"), impl_name: create_runtime_str!("substrate-node"), authoring_version: 10, - spec_version: 81, - impl_version: 83, + spec_version: 82, + impl_version: 82, apis: RUNTIME_API_VERSIONS, }; diff --git a/substrate/srml/system/src/lib.rs b/substrate/srml/system/src/lib.rs index 9018b109e8..cf5f473bdc 100644 --- a/substrate/srml/system/src/lib.rs +++ b/substrate/srml/system/src/lib.rs @@ -78,7 +78,7 @@ use rstd::prelude::*; use rstd::map; use primitives::traits::{self, CheckEqual, SimpleArithmetic, SimpleBitOps, One, Bounded, Lookup, Hash, Member, MaybeDisplay, EnsureOrigin, Digest as DigestT, CurrentHeight, BlockNumberToHash, - MaybeSerializeDebugButNotDeserialize, MaybeSerializeDebug, StaticLookup, SaturatedConversion + MaybeSerializeDebugButNotDeserialize, MaybeSerializeDebug, StaticLookup, }; #[cfg(any(feature = "std", test))] use primitives::traits::Zero; @@ -493,8 +493,7 @@ impl Module { let mut digest = >::take(); let extrinsics_root = >::take(); let storage_root = T::Hashing::storage_root(); - let number_u64 = number.saturated_into::(); - let storage_changes_root = T::Hashing::storage_changes_root(parent_hash, number_u64 - 1); + let storage_changes_root = T::Hashing::storage_changes_root(parent_hash); // we can't compute changes trie root earlier && put it to the Digest // because it will include all currently existing temporaries.