pezkuwichain/pezkuwi-subxt
1
0
Fork 0
mirror of https://github.com/pezkuwichain/pezkuwi-subxt.git synced 2026-06-11 23:31:07 +00:00
Code Issues Packages Projects Releases Wiki Activity

Kill the light client, CHTs and change tries. (#10080)

Browse Source 
* Remove light client, change tries and CHTs

* Update tests

* fmt

* Restore changes_root

* Fixed benches

* Cargo fmt

* fmt

* fmt
This commit is contained in:
Arkadiy Paronyan
2021-11-12 14:15:01 +01:00
committed by GitHub
parent 112b7dac47
commit 4cbbf0cf43
141 changed files with 532 additions and 17807 deletions
Download Patch File Download Diff File Expand all files Collapse all files
substrate/client/db/src/cache/list_cache.rs
Vendored
-2351
View File
File diff suppressed because it is too large Load Diff
substrate/client/db/src/cache/list_entry.rs
Vendored
-187
View File
@@ -1,187 +0,0 @@
// This file is part of Substrate.
// Copyright (C) 2017-2021 Parity Technologies (UK) Ltd.
// SPDX-License-Identifier: GPL-3.0-or-later WITH Classpath-exception-2.0
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
// You should have received a copy of the GNU General Public License
// along with this program. If not, see <https://www.gnu.org/licenses/>.
//! List-cache storage entries.
use codec::{Decode, Encode};
use sp_blockchain::Result as ClientResult;
use sp_runtime::traits::{Block as BlockT, NumberFor};
use crate::cache::{list_storage::Storage, CacheItemT, ComplexBlockId};
/// Single list-based cache entry.
#[derive(Debug)]
#[cfg_attr(test, derive(PartialEq))]
pub struct Entry<Block: BlockT, T> {
/// first block, when this value became actual.
pub valid_from: ComplexBlockId<Block>,
/// Value stored at this entry.
pub value: T,
}
/// Internal representation of the single list-based cache entry. The entry points to the
/// previous entry in the cache, allowing us to traverse back in time in list-style.
#[derive(Debug, Encode, Decode)]
#[cfg_attr(test, derive(Clone, PartialEq))]
pub struct StorageEntry<Block: BlockT, T: CacheItemT> {
/// None if valid from the beginning.
pub prev_valid_from: Option<ComplexBlockId<Block>>,
/// Value stored at this entry.
pub value: T,
}
impl<Block: BlockT, T: CacheItemT> Entry<Block, T> {
/// Returns Some if the entry should be updated with the new value.
pub fn try_update(&self, value: Option<T>) -> Option<StorageEntry<Block, T>> {
match value {
Some(value) => match self.value == value {
true => None,
false =>
Some(StorageEntry { prev_valid_from: Some(self.valid_from.clone()), value }),
},
None => None,
}
}
/// Wrapper that calls search_before to get range where the given block fits.
pub fn search_best_range_before<S: Storage<Block, T>>(
&self,
storage: &S,
block: NumberFor<Block>,
) -> ClientResult<Option<(ComplexBlockId<Block>, Option<ComplexBlockId<Block>>)>> {
Ok(self
.search_best_before(storage, block)?
.map(|(entry, next)| (entry.valid_from, next)))
}
/// Searches the list, ending with THIS entry for the best entry preceding (or at)
/// given block number.
/// If the entry is found, result is the entry and the block id of next entry (if exists).
/// NOTE that this function does not check that the passed block is actually linked to
/// the blocks it found.
pub fn search_best_before<S: Storage<Block, T>>(
&self,
storage: &S,
block: NumberFor<Block>,
) -> ClientResult<Option<(Entry<Block, T>, Option<ComplexBlockId<Block>>)>> {
// we're looking for the best value
let mut next = None;
let mut current = self.valid_from.clone();
if block >= self.valid_from.number {
let value = self.value.clone();
return Ok(Some((Entry { valid_from: current, value }, next)))
}
// else - travel back in time
loop {
let entry = storage.require_entry(&current)?;
if block >= current.number {
return Ok(Some((Entry { valid_from: current, value: entry.value }, next)))
}
next = Some(current);
current = match entry.prev_valid_from {
Some(prev_valid_from) => prev_valid_from,
None => return Ok(None),
};
}
}
}
impl<Block: BlockT, T: CacheItemT> StorageEntry<Block, T> {
/// Converts storage entry into an entry, valid from given block.
pub fn into_entry(self, valid_from: ComplexBlockId<Block>) -> Entry<Block, T> {
Entry { valid_from, value: self.value }
}
}
#[cfg(test)]
mod tests {
use super::*;
use crate::cache::list_storage::tests::{DummyStorage, FaultyStorage};
use substrate_test_runtime_client::runtime::{Block, H256};
fn test_id(number: u64) -> ComplexBlockId<Block> {
ComplexBlockId::new(H256::from_low_u64_be(number), number)
}
#[test]
fn entry_try_update_works() {
// when trying to update with None value
assert_eq!(Entry::<_, u64> { valid_from: test_id(1), value: 42 }.try_update(None), None);
// when trying to update with the same Some value
assert_eq!(Entry { valid_from: test_id(1), value: 1 }.try_update(Some(1)), None);
// when trying to update with different Some value
assert_eq!(
Entry { valid_from: test_id(1), value: 1 }.try_update(Some(2)),
Some(StorageEntry { prev_valid_from: Some(test_id(1)), value: 2 })
);
}
#[test]
fn entry_search_best_before_fails() {
// when storage returns error
assert!(Entry::<_, u64> { valid_from: test_id(100), value: 42 }
.search_best_before(&FaultyStorage, 50)
.is_err());
}
#[test]
fn entry_search_best_before_works() {
// when block is better than our best block
assert_eq!(
Entry::<_, u64> { valid_from: test_id(100), value: 100 }
.search_best_before(&DummyStorage::new(), 150)
.unwrap(),
Some((Entry::<_, u64> { valid_from: test_id(100), value: 100 }, None))
);
// when block is found between two entries
assert_eq!(
Entry::<_, u64> { valid_from: test_id(100), value: 100 }
.search_best_before(
&DummyStorage::new()
.with_entry(
test_id(100),
StorageEntry { prev_valid_from: Some(test_id(50)), value: 100 }
)
.with_entry(
test_id(50),
StorageEntry { prev_valid_from: Some(test_id(30)), value: 50 }
),
75
)
.unwrap(),
Some((Entry::<_, u64> { valid_from: test_id(50), value: 50 }, Some(test_id(100))))
);
// when block is not found
assert_eq!(
Entry::<_, u64> { valid_from: test_id(100), value: 100 }
.search_best_before(
&DummyStorage::new()
.with_entry(
test_id(100),
StorageEntry { prev_valid_from: Some(test_id(50)), value: 100 }
)
.with_entry(test_id(50), StorageEntry { prev_valid_from: None, value: 50 }),
30
)
.unwrap(),
None
);
}
}
substrate/client/db/src/cache/list_storage.rs
Vendored
-441
View File
@@ -1,441 +0,0 @@
// This file is part of Substrate.
// Copyright (C) 2017-2021 Parity Technologies (UK) Ltd.
// SPDX-License-Identifier: GPL-3.0-or-later WITH Classpath-exception-2.0
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
// You should have received a copy of the GNU General Public License
// along with this program. If not, see <https://www.gnu.org/licenses/>.
//! List-cache storage definition and implementation.
use std::sync::Arc;
use crate::utils::{self, meta_keys};
use codec::{Decode, Encode};
use sp_blockchain::{Error as ClientError, Result as ClientResult};
use sp_database::{Database, Transaction};
use sp_runtime::{
generic::BlockId,
traits::{Block as BlockT, Header as HeaderT, NumberFor},
};
use crate::{
cache::{
list_cache::{CommitOperation, Fork},
list_entry::{Entry, StorageEntry},
CacheItemT, ComplexBlockId,
},
DbHash,
};
/// Single list-cache metadata.
#[derive(Debug)]
#[cfg_attr(test, derive(Clone, PartialEq))]
pub struct Metadata<Block: BlockT> {
/// Block at which best finalized entry is stored.
pub finalized: Option<ComplexBlockId<Block>>,
/// A set of blocks at which best unfinalized entries are stored.
pub unfinalized: Vec<ComplexBlockId<Block>>,
}
/// Readonly list-cache storage trait.
pub trait Storage<Block: BlockT, T: CacheItemT> {
/// Reads hash of the block at given number.
fn read_id(&self, at: NumberFor<Block>) -> ClientResult<Option<Block::Hash>>;
/// Reads header of the block with given hash.
fn read_header(&self, at: &Block::Hash) -> ClientResult<Option<Block::Header>>;
/// Reads cache metadata: best finalized entry (if some) and the list.
fn read_meta(&self) -> ClientResult<Metadata<Block>>;
/// Reads cache entry from the storage.
fn read_entry(
&self,
at: &ComplexBlockId<Block>,
) -> ClientResult<Option<StorageEntry<Block, T>>>;
/// Reads referenced (and thus existing) cache entry from the storage.
fn require_entry(&self, at: &ComplexBlockId<Block>) -> ClientResult<StorageEntry<Block, T>> {
self.read_entry(at).and_then(|entry| {
entry.ok_or_else(|| {
ClientError::from(ClientError::Backend(format!(
"Referenced cache entry at {:?} is not found",
at
)))
})
})
}
}
/// List-cache storage transaction.
pub trait StorageTransaction<Block: BlockT, T: CacheItemT> {
/// Insert storage entry at given block.
fn insert_storage_entry(&mut self, at: &ComplexBlockId<Block>, entry: &StorageEntry<Block, T>);
/// Delete storage entry at given block.
fn remove_storage_entry(&mut self, at: &ComplexBlockId<Block>);
/// Update metadata of the cache.
fn update_meta(
&mut self,
best_finalized_entry: Option<&Entry<Block, T>>,
unfinalized: &[Fork<Block, T>],
operation: &CommitOperation<Block, T>,
);
}
/// A set of columns used by the DbStorage.
#[derive(Debug)]
pub struct DbColumns {
/// Column holding cache meta.
pub meta: u32,
/// Column holding the mapping of { block number => block hash } for blocks of the best chain.
pub key_lookup: u32,
/// Column holding the mapping of { block hash => block header }.
pub header: u32,
/// Column holding cache entries.
pub cache: u32,
}
/// Database-backed list cache storage.
pub struct DbStorage {
name: Vec<u8>,
meta_key: Vec<u8>,
db: Arc<dyn Database<DbHash>>,
columns: DbColumns,
}
impl DbStorage {
/// Create new database-backed list cache storage.
pub fn new(name: Vec<u8>, db: Arc<dyn Database<DbHash>>, columns: DbColumns) -> Self {
let meta_key = meta::key(&name);
DbStorage { name, meta_key, db, columns }
}
/// Get reference to the database.
pub fn db(&self) -> &Arc<dyn Database<DbHash>> {
&self.db
}
/// Get reference to the database columns.
pub fn columns(&self) -> &DbColumns {
&self.columns
}
/// Encode block id for storing as a key in cache column.
/// We append prefix to the actual encoding to allow several caches
/// store entries in the same column.
pub fn encode_block_id<Block: BlockT>(&self, block: &ComplexBlockId<Block>) -> Vec<u8> {
let mut encoded = self.name.clone();
encoded.extend(block.hash.as_ref());
encoded
}
}
impl<Block: BlockT, T: CacheItemT> Storage<Block, T> for DbStorage {
fn read_id(&self, at: NumberFor<Block>) -> ClientResult<Option<Block::Hash>> {
utils::read_header::<Block>(
&*self.db,
self.columns.key_lookup,
self.columns.header,
BlockId::Number(at),
)
.map(|maybe_header| maybe_header.map(|header| header.hash()))
}
fn read_header(&self, at: &Block::Hash) -> ClientResult<Option<Block::Header>> {
utils::read_header::<Block>(
&*self.db,
self.columns.key_lookup,
self.columns.header,
BlockId::Hash(*at),
)
}
fn read_meta(&self) -> ClientResult<Metadata<Block>> {
match self.db.get(self.columns.meta, &self.meta_key) {
Some(meta) => meta::decode(&*meta),
None => Ok(Metadata { finalized: None, unfinalized: Vec::new() }),
}
}
fn read_entry(
&self,
at: &ComplexBlockId<Block>,
) -> ClientResult<Option<StorageEntry<Block, T>>> {
match self.db.get(self.columns.cache, &self.encode_block_id(at)) {
Some(entry) => StorageEntry::<Block, T>::decode(&mut &entry[..])
.map_err(|_| ClientError::Backend("Failed to decode cache entry".into()))
.map(Some),
None => Ok(None),
}
}
}
/// Database-backed list cache storage transaction.
pub struct DbStorageTransaction<'a> {
storage: &'a DbStorage,
tx: &'a mut Transaction<DbHash>,
}
impl<'a> DbStorageTransaction<'a> {
/// Create new database transaction.
pub fn new(storage: &'a DbStorage, tx: &'a mut Transaction<DbHash>) -> Self {
DbStorageTransaction { storage, tx }
}
}
impl<'a, Block: BlockT, T: CacheItemT> StorageTransaction<Block, T> for DbStorageTransaction<'a> {
fn insert_storage_entry(&mut self, at: &ComplexBlockId<Block>, entry: &StorageEntry<Block, T>) {
self.tx.set_from_vec(
self.storage.columns.cache,
&self.storage.encode_block_id(at),
entry.encode(),
);
}
fn remove_storage_entry(&mut self, at: &ComplexBlockId<Block>) {
self.tx.remove(self.storage.columns.cache, &self.storage.encode_block_id(at));
}
fn update_meta(
&mut self,
best_finalized_entry: Option<&Entry<Block, T>>,
unfinalized: &[Fork<Block, T>],
operation: &CommitOperation<Block, T>,
) {
self.tx.set_from_vec(
self.storage.columns.meta,
&self.storage.meta_key,
meta::encode(best_finalized_entry, unfinalized, operation),
);
}
}
/// Metadata related functions.
mod meta {
use super::*;
/// Convert cache name into cache metadata key.
pub fn key(name: &[u8]) -> Vec<u8> {
let mut key_name = meta_keys::CACHE_META_PREFIX.to_vec();
key_name.extend_from_slice(name);
key_name
}
/// Encode cache metadata 'applying' commit operation before encoding.
pub fn encode<Block: BlockT, T: CacheItemT>(
best_finalized_entry: Option<&Entry<Block, T>>,
unfinalized: &[Fork<Block, T>],
op: &CommitOperation<Block, T>,
) -> Vec<u8> {
let mut finalized = best_finalized_entry.as_ref().map(|entry| &entry.valid_from);
let mut unfinalized =
unfinalized.iter().map(|fork| &fork.head().valid_from).collect::<Vec<_>>();
match op {
CommitOperation::AppendNewBlock(_, _) => (),
CommitOperation::AppendNewEntry(index, ref entry) => {
unfinalized[*index] = &entry.valid_from;
},
CommitOperation::AddNewFork(ref entry) => {
unfinalized.push(&entry.valid_from);
},
CommitOperation::BlockFinalized(_, ref finalizing_entry, ref forks) => {
if let Some(finalizing_entry) = finalizing_entry.as_ref() {
finalized = Some(&finalizing_entry.valid_from);
}
for fork_index in forks.iter().rev() {
unfinalized.remove(*fork_index);
}
},
CommitOperation::BlockReverted(ref forks) => {
for (fork_index, updated_fork) in forks.iter().rev() {
match updated_fork {
Some(updated_fork) =>
unfinalized[*fork_index] = &updated_fork.head().valid_from,
None => {
unfinalized.remove(*fork_index);
},
}
}
},
}
(finalized, unfinalized).encode()
}
/// Decode meta information.
pub fn decode<Block: BlockT>(encoded: &[u8]) -> ClientResult<Metadata<Block>> {
let input = &mut &*encoded;
let finalized: Option<ComplexBlockId<Block>> = Decode::decode(input).map_err(|_| {
ClientError::from(ClientError::Backend("Error decoding cache meta".into()))
})?;
let unfinalized: Vec<ComplexBlockId<Block>> = Decode::decode(input).map_err(|_| {
ClientError::from(ClientError::Backend("Error decoding cache meta".into()))
})?;
Ok(Metadata { finalized, unfinalized })
}
}
#[cfg(test)]
pub mod tests {
use super::*;
use std::collections::{HashMap, HashSet};
pub struct FaultyStorage;
impl<Block: BlockT, T: CacheItemT> Storage<Block, T> for FaultyStorage {
fn read_id(&self, _at: NumberFor<Block>) -> ClientResult<Option<Block::Hash>> {
Err(ClientError::Backend("TestError".into()))
}
fn read_header(&self, _at: &Block::Hash) -> ClientResult<Option<Block::Header>> {
Err(ClientError::Backend("TestError".into()))
}
fn read_meta(&self) -> ClientResult<Metadata<Block>> {
Err(ClientError::Backend("TestError".into()))
}
fn read_entry(
&self,
_at: &ComplexBlockId<Block>,
) -> ClientResult<Option<StorageEntry<Block, T>>> {
Err(ClientError::Backend("TestError".into()))
}
}
pub struct DummyStorage<Block: BlockT, T: CacheItemT> {
meta: Metadata<Block>,
ids: HashMap<NumberFor<Block>, Block::Hash>,
headers: HashMap<Block::Hash, Block::Header>,
entries: HashMap<Block::Hash, StorageEntry<Block, T>>,
}
impl<Block: BlockT, T: CacheItemT> DummyStorage<Block, T> {
pub fn new() -> Self {
DummyStorage {
meta: Metadata { finalized: None, unfinalized: Vec::new() },
ids: HashMap::new(),
headers: HashMap::new(),
entries: HashMap::new(),
}
}
pub fn with_meta(
mut self,
finalized: Option<ComplexBlockId<Block>>,
unfinalized: Vec<ComplexBlockId<Block>>,
) -> Self {
self.meta.finalized = finalized;
self.meta.unfinalized = unfinalized;
self
}
pub fn with_id(mut self, at: NumberFor<Block>, id: Block::Hash) -> Self {
self.ids.insert(at, id);
self
}
pub fn with_header(mut self, header: Block::Header) -> Self {
self.headers.insert(header.hash(), header);
self
}
pub fn with_entry(
mut self,
at: ComplexBlockId<Block>,
entry: StorageEntry<Block, T>,
) -> Self {
self.entries.insert(at.hash, entry);
self
}
}
impl<Block: BlockT, T: CacheItemT> Storage<Block, T> for DummyStorage<Block, T> {
fn read_id(&self, at: NumberFor<Block>) -> ClientResult<Option<Block::Hash>> {
Ok(self.ids.get(&at).cloned())
}
fn read_header(&self, at: &Block::Hash) -> ClientResult<Option<Block::Header>> {
Ok(self.headers.get(&at).cloned())
}
fn read_meta(&self) -> ClientResult<Metadata<Block>> {
Ok(self.meta.clone())
}
fn read_entry(
&self,
at: &ComplexBlockId<Block>,
) -> ClientResult<Option<StorageEntry<Block, T>>> {
Ok(self.entries.get(&at.hash).cloned())
}
}
pub struct DummyTransaction<Block: BlockT> {
updated_meta: Option<Metadata<Block>>,
inserted_entries: HashSet<Block::Hash>,
removed_entries: HashSet<Block::Hash>,
}
impl<Block: BlockT> DummyTransaction<Block> {
pub fn new() -> Self {
DummyTransaction {
updated_meta: None,
inserted_entries: HashSet::new(),
removed_entries: HashSet::new(),
}
}
pub fn inserted_entries(&self) -> &HashSet<Block::Hash> {
&self.inserted_entries
}
pub fn removed_entries(&self) -> &HashSet<Block::Hash> {
&self.removed_entries
}
pub fn updated_meta(&self) -> &Option<Metadata<Block>> {
&self.updated_meta
}
}
impl<Block: BlockT, T: CacheItemT> StorageTransaction<Block, T> for DummyTransaction<Block> {
fn insert_storage_entry(
&mut self,
at: &ComplexBlockId<Block>,
_entry: &StorageEntry<Block, T>,
) {
self.inserted_entries.insert(at.hash);
}
fn remove_storage_entry(&mut self, at: &ComplexBlockId<Block>) {
self.removed_entries.insert(at.hash);
}
fn update_meta(
&mut self,
best_finalized_entry: Option<&Entry<Block, T>>,
unfinalized: &[Fork<Block, T>],
operation: &CommitOperation<Block, T>,
) {
self.updated_meta = Some(
meta::decode(&meta::encode(best_finalized_entry, unfinalized, operation)).unwrap(),
);
}
}
}
substrate/client/db/src/cache/mod.rs
Vendored
-413
View File
@@ -1,413 +0,0 @@
// This file is part of Substrate.
// Copyright (C) 2017-2021 Parity Technologies (UK) Ltd.
// SPDX-License-Identifier: GPL-3.0-or-later WITH Classpath-exception-2.0
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
// You should have received a copy of the GNU General Public License
// along with this program. If not, see <https://www.gnu.org/licenses/>.
//! DB-backed cache of blockchain data.
use parking_lot::RwLock;
use std::{
collections::{hash_map::Entry, HashMap},
sync::Arc,
};
use crate::{
utils::{self, COLUMN_META},
DbHash,
};
use codec::{Decode, Encode};
use sc_client_api::blockchain::{
well_known_cache_keys::{self, Id as CacheKeyId},
Cache as BlockchainCache,
};
use sp_blockchain::{HeaderMetadataCache, Result as ClientResult};
use sp_database::{Database, Transaction};
use sp_runtime::{
generic::BlockId,
traits::{Block as BlockT, Header as HeaderT, NumberFor, Zero},
};
use self::list_cache::{ListCache, PruningStrategy};
mod list_cache;
mod list_entry;
mod list_storage;
/// Minimal post-finalization age of finalized blocks before they'll pruned.
const PRUNE_DEPTH: u32 = 1024;
/// The type of entry that is inserted to the cache.
#[derive(Clone, Copy, Debug, PartialEq)]
pub enum EntryType {
/// Non-final entry.
NonFinal,
/// Final entry.
Final,
/// Genesis entry (inserted during cache initialization).
Genesis,
}
/// Block identifier that holds both hash and number.
#[derive(Clone, Debug, Encode, Decode, PartialEq)]
pub struct ComplexBlockId<Block: BlockT> {
/// Hash of the block.
pub(crate) hash: Block::Hash,
/// Number of the block.
pub(crate) number: NumberFor<Block>,
}
impl<Block: BlockT> ComplexBlockId<Block> {
/// Create new complex block id.
pub fn new(hash: Block::Hash, number: NumberFor<Block>) -> Self {
ComplexBlockId { hash, number }
}
}
impl<Block: BlockT> ::std::cmp::PartialOrd for ComplexBlockId<Block> {
fn partial_cmp(&self, other: &ComplexBlockId<Block>) -> Option<::std::cmp::Ordering> {
self.number.partial_cmp(&other.number)
}
}
/// All cache items must implement this trait.
pub trait CacheItemT: Clone + Decode + Encode + PartialEq {}
impl<T> CacheItemT for T where T: Clone + Decode + Encode + PartialEq {}
/// Database-backed blockchain data cache.
pub struct DbCache<Block: BlockT> {
cache_at: HashMap<CacheKeyId, ListCache<Block, Vec<u8>, self::list_storage::DbStorage>>,
header_metadata_cache: Arc<HeaderMetadataCache<Block>>,
db: Arc<dyn Database<DbHash>>,
key_lookup_column: u32,
header_column: u32,
cache_column: u32,
genesis_hash: Block::Hash,
best_finalized_block: ComplexBlockId<Block>,
}
impl<Block: BlockT> DbCache<Block> {
/// Create new cache.
pub fn new(
db: Arc<dyn Database<DbHash>>,
header_metadata_cache: Arc<HeaderMetadataCache<Block>>,
key_lookup_column: u32,
header_column: u32,
cache_column: u32,
genesis_hash: Block::Hash,
best_finalized_block: ComplexBlockId<Block>,
) -> Self {
Self {
cache_at: HashMap::new(),
db,
header_metadata_cache,
key_lookup_column,
header_column,
cache_column,
genesis_hash,
best_finalized_block,
}
}
/// Set genesis block hash.
pub fn set_genesis_hash(&mut self, genesis_hash: Block::Hash) {
self.genesis_hash = genesis_hash;
}
/// Begin cache transaction.
pub fn transaction<'a>(
&'a mut self,
tx: &'a mut Transaction<DbHash>,
) -> DbCacheTransaction<'a, Block> {
DbCacheTransaction {
cache: self,
tx,
cache_at_ops: HashMap::new(),
best_finalized_block: None,
}
}
/// Begin cache transaction with given ops.
pub fn transaction_with_ops<'a>(
&'a mut self,
tx: &'a mut Transaction<DbHash>,
ops: DbCacheTransactionOps<Block>,
) -> DbCacheTransaction<'a, Block> {
DbCacheTransaction {
cache: self,
tx,
cache_at_ops: ops.cache_at_ops,
best_finalized_block: ops.best_finalized_block,
}
}
/// Run post-commit cache operations.
pub fn commit(&mut self, ops: DbCacheTransactionOps<Block>) -> ClientResult<()> {
for (name, ops) in ops.cache_at_ops.into_iter() {
self.get_cache(name)?.on_transaction_commit(ops);
}
if let Some(best_finalized_block) = ops.best_finalized_block {
self.best_finalized_block = best_finalized_block;
}
Ok(())
}
/// Creates `ListCache` with the given name or returns a reference to the existing.
pub(crate) fn get_cache(
&mut self,
name: CacheKeyId,
) -> ClientResult<&mut ListCache<Block, Vec<u8>, self::list_storage::DbStorage>> {
get_cache_helper(
&mut self.cache_at,
name,
&self.db,
self.key_lookup_column,
self.header_column,
self.cache_column,
&self.best_finalized_block,
)
}
}
// This helper is needed because otherwise the borrow checker will require to
// clone all parameters outside of the closure.
fn get_cache_helper<'a, Block: BlockT>(
cache_at: &'a mut HashMap<CacheKeyId, ListCache<Block, Vec<u8>, self::list_storage::DbStorage>>,
name: CacheKeyId,
db: &Arc<dyn Database<DbHash>>,
key_lookup: u32,
header: u32,
cache: u32,
best_finalized_block: &ComplexBlockId<Block>,
) -> ClientResult<&'a mut ListCache<Block, Vec<u8>, self::list_storage::DbStorage>> {
match cache_at.entry(name) {
Entry::Occupied(entry) => Ok(entry.into_mut()),
Entry::Vacant(entry) => {
let cache = ListCache::new(
self::list_storage::DbStorage::new(
name.to_vec(),
db.clone(),
self::list_storage::DbColumns { meta: COLUMN_META, key_lookup, header, cache },
),
cache_pruning_strategy(name),
best_finalized_block.clone(),
)?;
Ok(entry.insert(cache))
},
}
}
/// Cache operations that are to be committed after database transaction is committed.
#[derive(Default)]
pub struct DbCacheTransactionOps<Block: BlockT> {
cache_at_ops: HashMap<CacheKeyId, self::list_cache::CommitOperations<Block, Vec<u8>>>,
best_finalized_block: Option<ComplexBlockId<Block>>,
}
impl<Block: BlockT> DbCacheTransactionOps<Block> {
/// Empty transaction ops.
pub fn empty() -> DbCacheTransactionOps<Block> {
DbCacheTransactionOps { cache_at_ops: HashMap::new(), best_finalized_block: None }
}
}
/// Database-backed blockchain data cache transaction valid for single block import.
pub struct DbCacheTransaction<'a, Block: BlockT> {
cache: &'a mut DbCache<Block>,
tx: &'a mut Transaction<DbHash>,
cache_at_ops: HashMap<CacheKeyId, self::list_cache::CommitOperations<Block, Vec<u8>>>,
best_finalized_block: Option<ComplexBlockId<Block>>,
}
impl<'a, Block: BlockT> DbCacheTransaction<'a, Block> {
/// Convert transaction into post-commit operations set.
pub fn into_ops(self) -> DbCacheTransactionOps<Block> {
DbCacheTransactionOps {
cache_at_ops: self.cache_at_ops,
best_finalized_block: self.best_finalized_block,
}
}
/// When new block is inserted into database.
pub fn on_block_insert(
mut self,
parent: ComplexBlockId<Block>,
block: ComplexBlockId<Block>,
data_at: HashMap<CacheKeyId, Vec<u8>>,
entry_type: EntryType,
) -> ClientResult<Self> {
// prepare list of caches that are not update
// (we might still need to do some cache maintenance in this case)
let missed_caches = self
.cache
.cache_at
.keys()
.filter(|cache| !data_at.contains_key(*cache))
.cloned()
.collect::<Vec<_>>();
let mut insert_op = |name: CacheKeyId,
value: Option<Vec<u8>>|
-> Result<(), sp_blockchain::Error> {
let cache = self.cache.get_cache(name)?;
let cache_ops = self.cache_at_ops.entry(name).or_default();
cache.on_block_insert(
&mut self::list_storage::DbStorageTransaction::new(cache.storage(), &mut self.tx),
parent.clone(),
block.clone(),
value,
entry_type,
cache_ops,
)?;
Ok(())
};
data_at.into_iter().try_for_each(|(name, data)| insert_op(name, Some(data)))?;
missed_caches.into_iter().try_for_each(|name| insert_op(name, None))?;
match entry_type {
EntryType::Final | EntryType::Genesis => self.best_finalized_block = Some(block),
EntryType::NonFinal => (),
}
Ok(self)
}
/// When previously inserted block is finalized.
pub fn on_block_finalize(
mut self,
parent: ComplexBlockId<Block>,
block: ComplexBlockId<Block>,
) -> ClientResult<Self> {
for (name, cache) in self.cache.cache_at.iter() {
let cache_ops = self.cache_at_ops.entry(*name).or_default();
cache.on_block_finalize(
&mut self::list_storage::DbStorageTransaction::new(cache.storage(), &mut self.tx),
parent.clone(),
block.clone(),
cache_ops,
)?;
}
self.best_finalized_block = Some(block);
Ok(self)
}
/// When block is reverted.
pub fn on_block_revert(mut self, reverted_block: &ComplexBlockId<Block>) -> ClientResult<Self> {
for (name, cache) in self.cache.cache_at.iter() {
let cache_ops = self.cache_at_ops.entry(*name).or_default();
cache.on_block_revert(
&mut self::list_storage::DbStorageTransaction::new(cache.storage(), &mut self.tx),
reverted_block,
cache_ops,
)?;
}
Ok(self)
}
}
/// Synchronous implementation of database-backed blockchain data cache.
pub struct DbCacheSync<Block: BlockT>(pub RwLock<DbCache<Block>>);
impl<Block: BlockT> BlockchainCache<Block> for DbCacheSync<Block> {
fn initialize(&self, key: &CacheKeyId, data: Vec<u8>) -> ClientResult<()> {
let mut cache = self.0.write();
let genesis_hash = cache.genesis_hash;
let cache_contents = vec![(*key, data)].into_iter().collect();
let db = cache.db.clone();
let mut dbtx = Transaction::new();
let tx = cache.transaction(&mut dbtx);
let tx = tx.on_block_insert(
ComplexBlockId::new(Default::default(), Zero::zero()),
ComplexBlockId::new(genesis_hash, Zero::zero()),
cache_contents,
EntryType::Genesis,
)?;
let tx_ops = tx.into_ops();
db.commit(dbtx)?;
cache.commit(tx_ops)?;
Ok(())
}
fn get_at(
&self,
key: &CacheKeyId,
at: &BlockId<Block>,
) -> ClientResult<
Option<((NumberFor<Block>, Block::Hash), Option<(NumberFor<Block>, Block::Hash)>, Vec<u8>)>,
> {
let mut cache = self.0.write();
let header_metadata_cache = cache.header_metadata_cache.clone();
let cache = cache.get_cache(*key)?;
let storage = cache.storage();
let db = storage.db();
let columns = storage.columns();
let at = match *at {
BlockId::Hash(hash) => match header_metadata_cache.header_metadata(hash) {
Some(metadata) => ComplexBlockId::new(hash, metadata.number),
None => {
let header = utils::require_header::<Block>(
&**db,
columns.key_lookup,
columns.header,
BlockId::Hash(hash.clone()),
)?;
ComplexBlockId::new(hash, *header.number())
},
},
BlockId::Number(number) => {
let hash = utils::require_header::<Block>(
&**db,
columns.key_lookup,
columns.header,
BlockId::Number(number.clone()),
)?
.hash();
ComplexBlockId::new(hash, number)
},
};
cache.value_at_block(&at).map(|block_and_value| {
block_and_value.map(|(begin_block, end_block, value)| {
(
(begin_block.number, begin_block.hash),
end_block.map(|end_block| (end_block.number, end_block.hash)),
value,
)
})
})
}
}
/// Get pruning strategy for given cache.
fn cache_pruning_strategy<N: From<u32>>(cache: CacheKeyId) -> PruningStrategy<N> {
// the cache is mostly used to store data from consensus engines
// this kind of data is only required for non-finalized blocks
// => by default we prune finalized cached entries
match cache {
// we need to keep changes tries configurations forever (or at least until changes tries,
// that were built using this configuration, are pruned) to make it possible to refer
// to old changes tries
well_known_cache_keys::CHANGES_TRIE_CONFIG => PruningStrategy::NeverPrune,
_ => PruningStrategy::ByDepth(PRUNE_DEPTH.into()),
}
}
substrate/client/db/src/changes_tries_storage.rs
-1168
View File
File diff suppressed because it is too large Load Diff
substrate/client/db/src/lib.rs
+8 -265
View File
@@ -28,14 +28,11 @@
#![warn(missing_docs)]
pub mod light;
pub mod offchain;
#[cfg(any(feature = "with-kvdb-rocksdb", test))]
pub mod bench;
mod cache;
mod changes_tries_storage;
mod children;
#[cfg(feature = "with-parity-db")]
mod parity_db;
@@ -56,7 +53,6 @@ use std::{
};
use crate::{
changes_tries_storage::{DbChangesTrieStorage, DbChangesTrieStorageTransaction},
stats::StateUsageStats,
storage_cache::{new_shared_cache, CachingState, SharedCache, SyncingCachingState},
utils::{meta_keys, read_db, read_meta, DatabaseType, Meta},
@@ -64,8 +60,7 @@ use crate::{
use codec::{Decode, Encode};
use hash_db::Prefix;
use sc_client_api::{
backend::{NewBlockState, ProvideChtRoots, PrunableStateChangesTrieStorage},
cht,
backend::NewBlockState,
leaves::{FinalizationDisplaced, LeafSet},
utils::is_descendent_of,
IoInfo, MemoryInfo, MemorySize, UsageInfo,
@@ -79,11 +74,10 @@ use sp_blockchain::{
use sp_core::{
offchain::OffchainOverlayedChange,
storage::{well_known_keys, ChildInfo},
ChangesTrieConfiguration,
};
use sp_database::Transaction;
use sp_runtime::{
generic::{BlockId, DigestItem},
generic::BlockId,
traits::{
Block as BlockT, Hash, HashFor, Header as HeaderT, NumberFor, One, SaturatedConversion,
Zero,
@@ -91,9 +85,8 @@ use sp_runtime::{
Justification, Justifications, Storage,
};
use sp_state_machine::{
backend::Backend as StateBackend, ChangesTrieCacheAction, ChangesTrieTransaction,
ChildStorageCollection, DBValue, IndexOperation, OffchainChangesCollection, StateMachineStats,
StorageCollection, UsageInfo as StateUsageInfo,
backend::Backend as StateBackend, ChildStorageCollection, DBValue, IndexOperation,
OffchainChangesCollection, StateMachineStats, StorageCollection, UsageInfo as StateUsageInfo,
};
use sp_trie::{prefixed_key, MemoryDB, PrefixedMemoryDB};
@@ -104,7 +97,6 @@ pub use sp_database::Database;
#[cfg(any(feature = "with-kvdb-rocksdb", test))]
pub use bench::BenchmarkingState;
const MIN_BLOCKS_TO_KEEP_CHANGES_TRIES_FOR: u32 = 32768;
const CACHE_HEADERS: usize = 8;
/// Default value for storage cache child ratio.
@@ -406,11 +398,9 @@ pub(crate) mod columns {
pub const HEADER: u32 = 4;
pub const BODY: u32 = 5;
pub const JUSTIFICATIONS: u32 = 6;
pub const CHANGES_TRIE: u32 = 7;
pub const AUX: u32 = 8;
/// Offchain workers local storage
pub const OFFCHAIN: u32 = 9;
pub const CACHE: u32 = 10;
/// Transactions
pub const TRANSACTION: u32 = 11;
}
@@ -506,13 +496,6 @@ impl<Block: BlockT> BlockchainDb<Block> {
let mut meta = self.meta.write();
meta.block_gap = gap;
}
// Get block changes trie root, if available.
fn changes_trie_root(&self, block: BlockId<Block>) -> ClientResult<Option<Block::Hash>> {
self.header(block).map(|header| {
header.and_then(|header| header.digest().log(DigestItem::as_changes_trie_root).cloned())
})
}
}
impl<Block: BlockT> sc_client_api::blockchain::HeaderBackend<Block> for BlockchainDb<Block> {
@@ -646,10 +629,6 @@ impl<Block: BlockT> sc_client_api::blockchain::Backend<Block> for BlockchainDb<B
Ok(self.meta.read().finalized_hash.clone())
}
fn cache(&self) -> Option<Arc<dyn sc_client_api::blockchain::Cache<Block>>> {
None
}
fn leaves(&self) -> ClientResult<Vec<Block::Hash>> {
Ok(self.leaves.read().hashes())
}
@@ -702,12 +681,6 @@ impl<Block: BlockT> sc_client_api::blockchain::Backend<Block> for BlockchainDb<B
}
}
impl<Block: BlockT> sc_client_api::blockchain::ProvideCache<Block> for BlockchainDb<Block> {
fn cache(&self) -> Option<Arc<dyn sc_client_api::blockchain::Cache<Block>>> {
None
}
}
impl<Block: BlockT> HeaderMetadata<Block> for BlockchainDb<Block> {
type Error = sp_blockchain::Error;
@@ -745,62 +718,6 @@ impl<Block: BlockT> HeaderMetadata<Block> for BlockchainDb<Block> {
}
}
impl<Block: BlockT> ProvideChtRoots<Block> for BlockchainDb<Block> {
fn header_cht_root(
&self,
cht_size: NumberFor<Block>,
block: NumberFor<Block>,
) -> sp_blockchain::Result<Option<Block::Hash>> {
let cht_number = match cht::block_to_cht_number(cht_size, block) {
Some(number) => number,
None => return Ok(None),
};
let cht_start: NumberFor<Block> = cht::start_number(cht::size(), cht_number);
let mut current_num = cht_start;
let cht_range = ::std::iter::from_fn(|| {
let old_current_num = current_num;
current_num = current_num + One::one();
Some(old_current_num)
});
cht::compute_root::<Block::Header, HashFor<Block>, _>(
cht::size(),
cht_number,
cht_range.map(|num| self.hash(num)),
)
.map(Some)
}
fn changes_trie_cht_root(
&self,
cht_size: NumberFor<Block>,
block: NumberFor<Block>,
) -> sp_blockchain::Result<Option<Block::Hash>> {
let cht_number = match cht::block_to_cht_number(cht_size, block) {
Some(number) => number,
None => return Ok(None),
};
let cht_start: NumberFor<Block> = cht::start_number(cht::size(), cht_number);
let mut current_num = cht_start;
let cht_range = ::std::iter::from_fn(|| {
let old_current_num = current_num;
current_num = current_num + One::one();
Some(old_current_num)
});
cht::compute_root::<Block::Header, HashFor<Block>, _>(
cht::size(),
cht_number,
cht_range.map(|num| self.changes_trie_root(BlockId::Number(num))),
)
.map(Some)
}
}
/// Database transaction
pub struct BlockImportOperation<Block: BlockT> {
old_state: SyncingCachingState<RefTrackingState<Block>, Block>,
@@ -808,9 +725,6 @@ pub struct BlockImportOperation<Block: BlockT> {
storage_updates: StorageCollection,
child_storage_updates: ChildStorageCollection,
offchain_storage_updates: OffchainChangesCollection,
changes_trie_updates: MemoryDB<HashFor<Block>>,
changes_trie_build_cache_update: Option<ChangesTrieCacheAction<Block::Hash, NumberFor<Block>>>,
changes_trie_config_update: Option<Option<ChangesTrieConfiguration>>,
pending_block: Option<PendingBlock<Block>>,
aux_ops: Vec<(Vec<u8>, Option<Vec<u8>>)>,
finalized_blocks: Vec<(BlockId<Block>, Option<Justification>)>,
@@ -858,25 +772,12 @@ impl<Block: BlockT> BlockImportOperation<Block> {
)
});
let mut changes_trie_config = None;
let (root, transaction) = self.old_state.full_storage_root(
storage.top.iter().map(|(k, v)| {
if &k[..] == well_known_keys::CHANGES_TRIE_CONFIG {
changes_trie_config = Some(Decode::decode(&mut &v[..]));
}
(&k[..], Some(&v[..]))
}),
storage.top.iter().map(|(k, v)| (&k[..], Some(&v[..]))),
child_delta,
);
let changes_trie_config = match changes_trie_config {
Some(Ok(c)) => Some(c),
Some(Err(_)) => return Err(sp_blockchain::Error::InvalidState.into()),
None => None,
};
self.db_updates = transaction;
self.changes_trie_config_update = Some(changes_trie_config);
Ok(root)
}
}
@@ -899,11 +800,6 @@ impl<Block: BlockT> sc_client_api::backend::BlockImportOperation<Block>
leaf_state: NewBlockState,
) -> ClientResult<()> {
assert!(self.pending_block.is_none(), "Only one block per operation is allowed");
if let Some(changes_trie_config_update) =
changes_tries_storage::extract_new_configuration(&header)
{
self.changes_trie_config_update = Some(changes_trie_config_update.clone());
}
self.pending_block =
Some(PendingBlock { header, body, indexed_body, justifications, leaf_state });
Ok(())
@@ -930,15 +826,6 @@ impl<Block: BlockT> sc_client_api::backend::BlockImportOperation<Block>
Ok(root)
}
fn update_changes_trie(
&mut self,
update: ChangesTrieTransaction<HashFor<Block>, NumberFor<Block>>,
) -> ClientResult<()> {
self.changes_trie_updates = update.0;
self.changes_trie_build_cache_update = Some(update.1);
Ok(())
}
fn insert_aux<I>(&mut self, ops: I) -> ClientResult<()>
where
I: IntoIterator<Item = (Vec<u8>, Option<Vec<u8>>)>,
@@ -1095,7 +982,6 @@ impl<T: Clone> FrozenForDuration<T> {
pub struct Backend<Block: BlockT> {
storage: Arc<StorageDb<Block>>,
offchain_storage: offchain::LocalStorage,
changes_tries_storage: DbChangesTrieStorage<Block>,
blockchain: BlockchainDb<Block>,
canonicalization_delay: u64,
shared_cache: SharedCache<Block>,
@@ -1155,7 +1041,6 @@ impl<Block: BlockT> Backend<Block> {
) -> ClientResult<Self> {
let is_archive_pruning = config.state_pruning.is_archive();
let blockchain = BlockchainDb::new(db.clone(), config.transaction_storage.clone())?;
let meta = blockchain.meta.clone();
let map_e = |e: sc_state_db::Error<io::Error>| sp_blockchain::Error::from_state_db(e);
let state_db: StateDb<_, _> = StateDb::new(
config.state_pruning.clone(),
@@ -1166,22 +1051,10 @@ impl<Block: BlockT> Backend<Block> {
let storage_db =
StorageDb { db: db.clone(), state_db, prefix_keys: !db.supports_ref_counting() };
let offchain_storage = offchain::LocalStorage::new(db.clone());
let changes_tries_storage = DbChangesTrieStorage::new(
db,
blockchain.header_metadata_cache.clone(),
columns::META,
columns::CHANGES_TRIE,
columns::KEY_LOOKUP,
columns::HEADER,
columns::CACHE,
meta,
if is_archive_pruning { None } else { Some(MIN_BLOCKS_TO_KEEP_CHANGES_TRIES_FOR) },
)?;
let backend = Backend {
storage: Arc::new(storage_db),
offchain_storage,
changes_tries_storage,
blockchain,
canonicalization_delay,
shared_cache: new_shared_cache(
@@ -1318,7 +1191,6 @@ impl<Block: BlockT> Backend<Block> {
header: &Block::Header,
last_finalized: Option<Block::Hash>,
justification: Option<Justification>,
changes_trie_cache_ops: &mut Option<DbChangesTrieStorageTransaction<Block>>,
finalization_displaced: &mut Option<FinalizationDisplaced<Block::Hash, NumberFor<Block>>>,
) -> ClientResult<MetaUpdate<Block>> {
// TODO: ensure best chain contains this block.
@@ -1326,15 +1198,7 @@ impl<Block: BlockT> Backend<Block> {
self.ensure_sequential_finalization(header, last_finalized)?;
let with_state = sc_client_api::Backend::have_state_at(self, &hash, number);
self.note_finalized(
transaction,
false,
header,
*hash,
changes_trie_cache_ops,
finalization_displaced,
with_state,
)?;
self.note_finalized(transaction, header, *hash, finalization_displaced, with_state)?;
if let Some(justification) = justification {
transaction.set_from_vec(
@@ -1400,7 +1264,6 @@ impl<Block: BlockT> Backend<Block> {
(meta.best_number, meta.finalized_hash, meta.finalized_number, meta.block_gap.clone())
};
let mut changes_trie_cache_ops = None;
for (block, justification) in operation.finalized_blocks {
let block_hash = self.blockchain.expect_block_hash_from_id(&block)?;
let block_header = self.blockchain.expect_header(BlockId::Hash(block_hash))?;
@@ -1410,7 +1273,6 @@ impl<Block: BlockT> Backend<Block> {
&block_header,
Some(last_finalized_hash),
justification,
&mut changes_trie_cache_ops,
&mut finalization_displaced_leaves,
)?);
last_finalized_hash = block_hash;
@@ -1475,11 +1337,6 @@ impl<Block: BlockT> Backend<Block> {
);
transaction.set(columns::META, meta_keys::GENESIS_HASH, hash.as_ref());
// for tests, because config is set from within the reset_storage
if operation.changes_trie_config_update.is_none() {
operation.changes_trie_config_update = Some(None);
}
if operation.commit_state {
transaction.set_from_vec(columns::META, meta_keys::FINALIZED_STATE, lookup_key);
} else {
@@ -1578,7 +1435,6 @@ impl<Block: BlockT> Backend<Block> {
let header = &pending_block.header;
let is_best = pending_block.leaf_state.is_best();
let changes_trie_updates = operation.changes_trie_updates;
debug!(target: "db",
"DB Commit {:?} ({}), best={}, state={}, existing={}",
hash, number, is_best, operation.commit_state, existing_header,
@@ -1593,10 +1449,8 @@ impl<Block: BlockT> Backend<Block> {
self.ensure_sequential_finalization(header, Some(last_finalized_hash))?;
self.note_finalized(
&mut transaction,
true,
header,
hash,
&mut changes_trie_cache_ops,
&mut finalization_displaced_leaves,
operation.commit_state,
)?;
@@ -1606,21 +1460,6 @@ impl<Block: BlockT> Backend<Block> {
}
if !existing_header {
let changes_trie_config_update = operation.changes_trie_config_update;
changes_trie_cache_ops = Some(self.changes_tries_storage.commit(
&mut transaction,
changes_trie_updates,
cache::ComplexBlockId::new(
*header.parent_hash(),
if number.is_zero() { Zero::zero() } else { number - One::one() },
),
cache::ComplexBlockId::new(hash, number),
header,
finalized,
changes_trie_config_update,
changes_trie_cache_ops,
)?);
{
let mut leaves = self.blockchain.leaves.write();
leaves.import(hash, number, parent_hash);
@@ -1747,11 +1586,6 @@ impl<Block: BlockT> Backend<Block> {
);
}
if let Some(changes_trie_build_cache_update) = operation.changes_trie_build_cache_update {
self.changes_tries_storage.commit_build_cache(changes_trie_build_cache_update);
}
self.changes_tries_storage.post_commit(changes_trie_cache_ops);
if let Some((enacted, retracted)) = cache_update {
self.shared_cache.write().sync(&enacted, &retracted);
}
@@ -1770,10 +1604,8 @@ impl<Block: BlockT> Backend<Block> {
fn note_finalized(
&self,
transaction: &mut Transaction<DbHash>,
is_inserted: bool,
f_header: &Block::Header,
f_hash: Block::Hash,
changes_trie_cache_ops: &mut Option<DbChangesTrieStorageTransaction<Block>>,
displaced: &mut Option<FinalizationDisplaced<Block::Hash, NumberFor<Block>>>,
with_state: bool,
) -> ClientResult<()> {
@@ -1798,18 +1630,6 @@ impl<Block: BlockT> Backend<Block> {
apply_state_commit(transaction, commit);
}
if !f_num.is_zero() {
let new_changes_trie_cache_ops = self.changes_tries_storage.finalize(
transaction,
*f_header.parent_hash(),
f_hash,
f_num,
if is_inserted { Some(&f_header) } else { None },
changes_trie_cache_ops.take(),
)?;
*changes_trie_cache_ops = Some(new_changes_trie_cache_ops);
}
let new_displaced = self.blockchain.leaves.write().finalize_height(f_num);
self.prune_blocks(transaction, f_num, &new_displaced)?;
match displaced {
@@ -2036,9 +1856,6 @@ impl<Block: BlockT> sc_client_api::backend::Backend<Block> for Backend<Block> {
storage_updates: Default::default(),
child_storage_updates: Default::default(),
offchain_storage_updates: Default::default(),
changes_trie_config_update: None,
changes_trie_updates: MemoryDB::default(),
changes_trie_build_cache_update: None,
aux_ops: Vec::new(),
finalized_blocks: Vec::new(),
set_head: None,
@@ -2089,19 +1906,16 @@ impl<Block: BlockT> sc_client_api::backend::Backend<Block> for Backend<Block> {
let header = self.blockchain.expect_header(block)?;
let mut displaced = None;
let mut changes_trie_cache_ops = None;
let m = self.finalize_block_with_transaction(
&mut transaction,
&hash,
&header,
None,
justification,
&mut changes_trie_cache_ops,
&mut displaced,
)?;
self.storage.db.commit(transaction)?;
self.blockchain.update_meta(m);
self.changes_tries_storage.post_commit(changes_trie_cache_ops);
Ok(())
}
@@ -2148,10 +1962,6 @@ impl<Block: BlockT> sc_client_api::backend::Backend<Block> for Backend<Block> {
Ok(())
}
fn changes_trie_storage(&self) -> Option<&dyn PrunableStateChangesTrieStorage<Block>> {
Some(&self.changes_tries_storage)
}
fn offchain_storage(&self) -> Option<Self::OffchainStorage> {
Some(self.offchain_storage.clone())
}
@@ -2208,7 +2018,6 @@ impl<Block: BlockT> sc_client_api::backend::Backend<Block> for Backend<Block> {
return Ok(c.saturated_into::<NumberFor<Block>>())
}
let mut transaction = Transaction::new();
let removed_number = best_number;
let removed =
self.blockchain.header(BlockId::Number(best_number))?.ok_or_else(|| {
sp_blockchain::Error::UnknownBlock(format!(
@@ -2241,10 +2050,6 @@ impl<Block: BlockT> sc_client_api::backend::Backend<Block> for Backend<Block> {
let key =
utils::number_and_hash_to_lookup_key(best_number.clone(), &best_hash)?;
let changes_trie_cache_ops = self.changes_tries_storage.revert(
&mut transaction,
&cache::ComplexBlockId::new(removed.hash(), removed_number),
)?;
if update_finalized {
transaction.set_from_vec(
columns::META,
@@ -2283,7 +2088,6 @@ impl<Block: BlockT> sc_client_api::backend::Backend<Block> for Backend<Block> {
best_hash,
);
self.storage.db.commit(transaction)?;
self.changes_tries_storage.post_commit(Some(changes_trie_cache_ops));
self.blockchain.update_meta(MetaUpdate {
hash: best_hash,
number: best_number,
@@ -2345,11 +2149,6 @@ impl<Block: BlockT> sc_client_api::backend::Backend<Block> for Backend<Block> {
apply_state_commit(&mut transaction, commit);
}
transaction.remove(columns::KEY_LOOKUP, hash.as_ref());
let changes_trie_cache_ops = self
.changes_tries_storage
.revert(&mut transaction, &cache::ComplexBlockId::new(*hash, hdr.number))?;
self.changes_tries_storage.post_commit(Some(changes_trie_cache_ops));
leaves.revert(hash.clone(), hdr.number);
leaves.prepare_transaction(&mut transaction, columns::META, meta_keys::LEAF_PREFIX);
self.storage.db.commit(transaction)?;
@@ -2461,32 +2260,16 @@ pub(crate) mod tests {
use sp_blockchain::{lowest_common_ancestor, tree_route};
use sp_core::H256;
use sp_runtime::{
generic::DigestItem,
testing::{Block as RawBlock, ExtrinsicWrapper, Header},
traits::{BlakeTwo256, Hash},
ConsensusEngineId,
};
use sp_state_machine::{TrieDBMut, TrieMut};
const CONS0_ENGINE_ID: ConsensusEngineId = *b"CON0";
const CONS1_ENGINE_ID: ConsensusEngineId = *b"CON1";
pub(crate) type Block = RawBlock<ExtrinsicWrapper<u64>>;
pub fn prepare_changes(changes: Vec<(Vec<u8>, Vec<u8>)>) -> (H256, MemoryDB<BlakeTwo256>) {
let mut changes_root = H256::default();
let mut changes_trie_update = MemoryDB::<BlakeTwo256>::default();
{
let mut trie =
TrieDBMut::<BlakeTwo256>::new(&mut changes_trie_update, &mut changes_root);
for (key, value) in changes {
trie.insert(&key, &value).unwrap();
}
}
(changes_root, changes_trie_update)
}
pub fn insert_header(
backend: &Backend<Block>,
number: u64,
@@ -2501,20 +2284,14 @@ pub(crate) mod tests {
backend: &Backend<Block>,
number: u64,
parent_hash: H256,
changes: Option<Vec<(Vec<u8>, Vec<u8>)>>,
_changes: Option<Vec<(Vec<u8>, Vec<u8>)>>,
extrinsics_root: H256,
body: Vec<ExtrinsicWrapper<u64>>,
transaction_index: Option<Vec<IndexOperation>>,
) -> H256 {
use sp_runtime::testing::Digest;
let mut digest = Digest::default();
let mut changes_trie_update = Default::default();
if let Some(changes) = changes {
let (root, update) = prepare_changes(changes);
digest.push(DigestItem::ChangesTrieRoot(root));
changes_trie_update = update;
}
let digest = Digest::default();
let header = Header {
number,
parent_hash,
@@ -2535,8 +2312,6 @@ pub(crate) mod tests {
if let Some(index) = transaction_index {
op.update_transaction_index(index).unwrap();
}
op.update_changes_trie((changes_trie_update, ChangesTrieCacheAction::Clear))
.unwrap();
backend.commit_operation(op).unwrap();
header_hash
@@ -3241,38 +3016,6 @@ pub(crate) mod tests {
}
}
#[test]
fn header_cht_root_works() {
use sc_client_api::ProvideChtRoots;
let backend = Backend::<Block>::new_test(10, 10);
// insert 1 + SIZE + SIZE + 1 blocks so that CHT#0 is created
let mut prev_hash =
insert_header(&backend, 0, Default::default(), None, Default::default());
let cht_size: u64 = cht::size();
for i in 1..1 + cht_size + cht_size + 1 {
prev_hash = insert_header(&backend, i, prev_hash, None, Default::default());
}
let blockchain = backend.blockchain();
let cht_root_1 = blockchain
.header_cht_root(cht_size, cht::start_number(cht_size, 0))
.unwrap()
.unwrap();
let cht_root_2 = blockchain
.header_cht_root(cht_size, cht::start_number(cht_size, 0) + cht_size / 2)
.unwrap()
.unwrap();
let cht_root_3 = blockchain
.header_cht_root(cht_size, cht::end_number(cht_size, 0))
.unwrap()
.unwrap();
assert_eq!(cht_root_1, cht_root_2);
assert_eq!(cht_root_2, cht_root_3);
}
#[test]
fn prune_blocks_on_finalize() {
for storage in &[TransactionStorageMode::BlockBody, TransactionStorageMode::StorageChain] {
substrate/client/db/src/light.rs
-1329
View File
File diff suppressed because it is too large Load Diff
substrate/client/db/src/parity_db.rs
+1 -5
View File
@@ -16,7 +16,7 @@
// You should have received a copy of the GNU General Public License
// along with this program. If not, see <https://www.gnu.org/licenses/>.
use crate::{
columns, light,
columns,
utils::{DatabaseType, NUM_COLUMNS},
};
/// A `Database` adapter for parity-db.
@@ -61,10 +61,6 @@ pub fn open<H: Clone + AsRef<[u8]>>(
state_col.preimage = true;
state_col.uniform = true;
},
DatabaseType::Light => {
config.columns[light::columns::HEADER as usize].compression =
parity_db::CompressionType::Lz4;
},
}
let db = if create {
substrate/client/db/src/utils.rs
+1 -78
View File
@@ -56,10 +56,6 @@ pub mod meta_keys {
pub const FINALIZED_STATE: &[u8; 6] = b"fstate";
/// Block gap.
pub const BLOCK_GAP: &[u8; 3] = b"gap";
/// Meta information prefix for list-based caches.
pub const CACHE_META_PREFIX: &[u8; 5] = b"cache";
/// Meta information for changes tries key.
pub const CHANGES_TRIES_META: &[u8; 5] = b"ctrie";
/// Genesis block hash.
pub const GENESIS_HASH: &[u8; 3] = b"gen";
/// Leaves prefix list key.
@@ -95,8 +91,6 @@ pub type NumberIndexKey = [u8; 4];
pub enum DatabaseType {
/// Full node database.
Full,
/// Light node database.
Light,
}
/// Convert block number into short lookup key (LE representation) for
@@ -124,19 +118,6 @@ where
Ok(lookup_key)
}
/// Convert block lookup key into block number.
/// all block lookup keys start with the block number.
pub fn lookup_key_to_number<N>(key: &[u8]) -> sp_blockchain::Result<N>
where
N: From<u32>,
{
if key.len() < 4 {
return Err(sp_blockchain::Error::Backend("Invalid block key".into()))
}
Ok((key[0] as u32) << 24 | (key[1] as u32) << 16 | (key[2] as u32) << 8 | (key[3] as u32))
.map(Into::into)
}
/// Delete number to hash mapping in DB transaction.
pub fn remove_number_to_key_mapping<N: TryInto<u32>>(
transaction: &mut Transaction<DbHash>,
@@ -147,18 +128,6 @@ pub fn remove_number_to_key_mapping<N: TryInto<u32>>(
Ok(())
}
/// Remove key mappings.
pub fn remove_key_mappings<N: TryInto<u32>, H: AsRef<[u8]>>(
transaction: &mut Transaction<DbHash>,
key_lookup_col: u32,
number: N,
hash: H,
) -> sp_blockchain::Result<()> {
remove_number_to_key_mapping(transaction, key_lookup_col, number)?;
transaction.remove(key_lookup_col, hash.as_ref());
Ok(())
}
/// Place a number mapping into the database. This maps number to current perceived
/// block hash at that position.
pub fn insert_number_to_key_mapping<N: TryInto<u32> + Clone, H: AsRef<[u8]>>(
@@ -357,18 +326,6 @@ fn open_kvdb_rocksdb<Block: BlockT>(
other_col_budget,
);
},
DatabaseType::Light => {
let col_budget = cache_size / (NUM_COLUMNS as usize);
for i in 0..NUM_COLUMNS {
memory_budget.insert(i, col_budget);
}
log::trace!(
target: "db",
"Open RocksDB light database at {:?}, column cache: {} MiB",
path,
col_budget,
);
},
}
db_config.memory_budget = memory_budget;
@@ -424,8 +381,7 @@ fn maybe_migrate_to_type_subdir<Block: BlockT>(
// See if there's a file identifying a rocksdb or paritydb folder in the parent dir and
// the target path ends in a role specific directory
if (basedir.join("db_version").exists() || basedir.join("metadata").exists()) &&
(p.ends_with(DatabaseType::Full.as_str()) ||
p.ends_with(DatabaseType::Light.as_str()))
(p.ends_with(DatabaseType::Full.as_str()))
{
// Try to open the database to check if the current `DatabaseType` matches the type of
// database stored in the target directory and close the database on success.
@@ -501,18 +457,6 @@ pub fn read_header<Block: BlockT>(
}
}
/// Required header from the database.
pub fn require_header<Block: BlockT>(
db: &dyn Database<DbHash>,
col_index: u32,
col: u32,
id: BlockId<Block>,
) -> sp_blockchain::Result<Block::Header> {
read_header(db, col_index, col, id).and_then(|header| {
header.ok_or_else(|| sp_blockchain::Error::UnknownBlock(format!("Require header: {}", id)))
})
}
/// Read meta from the database.
pub fn read_meta<Block>(
db: &dyn Database<DbHash>,
@@ -598,7 +542,6 @@ impl DatabaseType {
pub fn as_str(&self) -> &'static str {
match *self {
DatabaseType::Full => "full",
DatabaseType::Light => "light",
}
}
}
@@ -669,23 +612,12 @@ mod tests {
assert!(old_db_path.join(db_type.as_str()).join(db_check_file).exists());
}
check_dir_for_db_type(
DatabaseType::Light,
DatabaseSource::RocksDb { path: PathBuf::new(), cache_size: 128 },
"db_version",
);
check_dir_for_db_type(
DatabaseType::Full,
DatabaseSource::RocksDb { path: PathBuf::new(), cache_size: 128 },
"db_version",
);
#[cfg(feature = "with-parity-db")]
check_dir_for_db_type(
DatabaseType::Light,
DatabaseSource::ParityDb { path: PathBuf::new() },
"metadata",
);
#[cfg(feature = "with-parity-db")]
check_dir_for_db_type(
DatabaseType::Full,
@@ -709,16 +641,8 @@ mod tests {
assert!(!old_db_path.join("light/db_version").exists());
assert!(!old_db_path.join("full/db_version").exists());
}
let source = DatabaseSource::RocksDb {
path: old_db_path.join(DatabaseType::Light.as_str()),
cache_size: 128,
};
let settings = db_settings(source);
let db_res = open_database::<Block>(&settings, DatabaseType::Light);
assert!(db_res.is_err(), "Opening a light database in full role should fail");
// assert nothing was changed
assert!(old_db_path.join("db_version").exists());
assert!(!old_db_path.join("light/db_version").exists());
assert!(!old_db_path.join("full/db_version").exists());
}
}
@@ -735,7 +659,6 @@ mod tests {
#[test]
fn database_type_as_str_works() {
assert_eq!(DatabaseType::Full.as_str(), "full");
assert_eq!(DatabaseType::Light.as_str(), "light");
}
#[test]