pezkuwichain/pezkuwi-subxt
1
0
Fork 0
mirror of https://github.com/pezkuwichain/pezkuwi-subxt.git synced 2026-06-12 07:41:08 +00:00
Code Issues Packages Projects Releases Wiki Activity

Run cargo fmt on the whole code base (#9394)

Browse Source 
* Run cargo fmt on the whole code base

* Second run

* Add CI check

* Fix compilation

* More unnecessary braces

* Handle weights

* Use --all

* Use correct attributes...

* Fix UI tests

* AHHHHHHHHH

* 🤦

* Docs

* Fix compilation

* 🤷

* Please stop

* 🤦 x 2

* More

* make rustfmt.toml consistent with polkadot

Co-authored-by: André Silva <andrerfosilva@gmail.com>
This commit is contained in:
Bastian Köcher
2021-07-21 16:32:32 +02:00
committed by GitHub
parent d451c38c1c
commit 7b56ab15b4
1010 changed files with 53339 additions and 51208 deletions
Download Patch File Download Diff File Expand all files Collapse all files
substrate/client/db/src/bench.rs
+121 -80
View File
@@ -18,27 +18,31 @@
//! State backend that's useful for benchmarking
use std::sync::Arc;
use std::cell::{Cell, RefCell};
use std::collections::HashMap;
use std::{
cell::{Cell, RefCell},
collections::HashMap,
sync::Arc,
};
use hash_db::{Prefix, Hasher};
use sp_trie::{MemoryDB, prefixed_key};
use crate::storage_cache::{new_shared_cache, CachingState, SharedCache};
use hash_db::{Hasher, Prefix};
use kvdb::{DBTransaction, KeyValueDB};
use sp_core::{
hexdisplay::HexDisplay,
storage::{ChildInfo, TrackedStorageKey},
hexdisplay::HexDisplay
};
use sp_runtime::traits::{Block as BlockT, HashFor};
use sp_runtime::Storage;
use sp_runtime::{
traits::{Block as BlockT, HashFor},
Storage,
};
use sp_state_machine::{
DBValue, backend::Backend as StateBackend, StorageCollection, ChildStorageCollection, ProofRecorder,
backend::Backend as StateBackend, ChildStorageCollection, DBValue, ProofRecorder,
StorageCollection,
};
use kvdb::{KeyValueDB, DBTransaction};
use crate::storage_cache::{CachingState, SharedCache, new_shared_cache};
use sp_trie::{prefixed_key, MemoryDB};
type DbState<B> = sp_state_machine::TrieBackend<
Arc<dyn sp_state_machine::Storage<HashFor<B>>>, HashFor<B>
>;
type DbState<B> =
sp_state_machine::TrieBackend<Arc<dyn sp_state_machine::Storage<HashFor<B>>>, HashFor<B>>;
type State<B> = CachingState<DbState<B>, B>;
@@ -53,14 +57,17 @@ impl<Block: BlockT> sp_state_machine::Storage<HashFor<Block>> for StorageDb<Bloc
let prefixed_key = prefixed_key::<HashFor<Block>>(key, prefix);
if let Some(recorder) = &self.proof_recorder {
if let Some(v) = recorder.get(&key) {
return Ok(v.clone());
return Ok(v.clone())
}
let backend_value = self.db.get(0, &prefixed_key)
let backend_value = self
.db
.get(0, &prefixed_key)
.map_err(|e| format!("Database backend error: {:?}", e))?;
recorder.record(key.clone(), backend_value.clone());
Ok(backend_value)
} else {
self.db.get(0, &prefixed_key)
self.db
.get(0, &prefixed_key)
.map_err(|e| format!("Database backend error: {:?}", e))
}
}
@@ -91,7 +98,11 @@ pub struct BenchmarkingState<B: BlockT> {
impl<B: BlockT> BenchmarkingState<B> {
/// Create a new instance that creates a database in a temporary dir.
pub fn new(genesis: Storage, _cache_size_mb: Option<usize>, record_proof: bool) -> Result<Self, String> {
pub fn new(
genesis: Storage,
_cache_size_mb: Option<usize>,
record_proof: bool,
) -> Result<Self, String> {
let mut root = B::Hash::default();
let mut mdb = MemoryDB::<HashFor<B>>::default();
sp_state_machine::TrieDBMut::<HashFor<B>>::new(&mut mdb, &mut root);
@@ -114,14 +125,17 @@ impl<B: BlockT> BenchmarkingState<B> {
state.add_whitelist_to_tracker();
state.reopen()?;
let child_delta = genesis.children_default.iter().map(|(_storage_key, child_content)| (
&child_content.child_info,
child_content.data.iter().map(|(k, v)| (k.as_ref(), Some(v.as_ref()))),
));
let (root, transaction): (B::Hash, _) = state.state.borrow_mut().as_mut().unwrap().full_storage_root(
genesis.top.iter().map(|(k, v)| (k.as_ref(), Some(v.as_ref()))),
child_delta,
);
let child_delta = genesis.children_default.iter().map(|(_storage_key, child_content)| {
(
&child_content.child_info,
child_content.data.iter().map(|(k, v)| (k.as_ref(), Some(v.as_ref()))),
)
});
let (root, transaction): (B::Hash, _) =
state.state.borrow_mut().as_mut().unwrap().full_storage_root(
genesis.top.iter().map(|(k, v)| (k.as_ref(), Some(v.as_ref()))),
child_delta,
);
state.genesis = transaction.clone().drain();
state.genesis_root = root.clone();
state.commit(root, transaction, Vec::new(), Vec::new())?;
@@ -143,12 +157,12 @@ impl<B: BlockT> BenchmarkingState<B> {
let storage_db = Arc::new(StorageDb::<B> {
db,
proof_recorder: self.proof_recorder.clone(),
_block: Default::default()
_block: Default::default(),
});
*self.state.borrow_mut() = Some(State::new(
DbState::<B>::new(storage_db, self.root.get()),
self.shared_cache.clone(),
None
None,
));
Ok(())
}
@@ -178,7 +192,7 @@ impl<B: BlockT> BenchmarkingState<B> {
let key_tracker = if let Some(childtrie) = childtrie {
child_key_tracker.entry(childtrie.to_vec()).or_insert_with(|| HashMap::new())
} else {
} else {
&mut main_key_tracker
};
@@ -193,7 +207,7 @@ impl<B: BlockT> BenchmarkingState<B> {
let should_log = !tracker.has_been_read();
tracker.add_read();
should_log
}
},
};
if should_log {
@@ -215,7 +229,7 @@ impl<B: BlockT> BenchmarkingState<B> {
let key_tracker = if let Some(childtrie) = childtrie {
child_key_tracker.entry(childtrie.to_vec()).or_insert_with(|| HashMap::new())
} else {
} else {
&mut main_key_tracker
};
@@ -231,7 +245,7 @@ impl<B: BlockT> BenchmarkingState<B> {
let should_log = !tracker.has_been_written();
tracker.add_write();
should_log
}
},
};
if should_log {
@@ -269,7 +283,7 @@ fn state_err() -> String {
}
impl<B: BlockT> StateBackend<HashFor<B>> for BenchmarkingState<B> {
type Error = <DbState<B> as StateBackend<HashFor<B>>>::Error;
type Error = <DbState<B> as StateBackend<HashFor<B>>>::Error;
type Transaction = <DbState<B> as StateBackend<HashFor<B>>>::Transaction;
type TrieBackendStorage = <DbState<B> as StateBackend<HashFor<B>>>::TrieBackendStorage;
@@ -289,7 +303,11 @@ impl<B: BlockT> StateBackend<HashFor<B>> for BenchmarkingState<B> {
key: &[u8],
) -> Result<Option<Vec<u8>>, Self::Error> {
self.add_read_key(Some(child_info.storage_key()), key);
self.state.borrow().as_ref().ok_or_else(state_err)?.child_storage(child_info, key)
self.state
.borrow()
.as_ref()
.ok_or_else(state_err)?
.child_storage(child_info, key)
}
fn exists_storage(&self, key: &[u8]) -> Result<bool, Self::Error> {
@@ -303,7 +321,11 @@ impl<B: BlockT> StateBackend<HashFor<B>> for BenchmarkingState<B> {
key: &[u8],
) -> Result<bool, Self::Error> {
self.add_read_key(Some(child_info.storage_key()), key);
self.state.borrow().as_ref().ok_or_else(state_err)?.exists_child_storage(child_info, key)
self.state
.borrow()
.as_ref()
.ok_or_else(state_err)?
.exists_child_storage(child_info, key)
}
fn next_storage_key(&self, key: &[u8]) -> Result<Option<Vec<u8>>, Self::Error> {
@@ -317,7 +339,11 @@ impl<B: BlockT> StateBackend<HashFor<B>> for BenchmarkingState<B> {
key: &[u8],
) -> Result<Option<Vec<u8>>, Self::Error> {
self.add_read_key(Some(child_info.storage_key()), key);
self.state.borrow().as_ref().ok_or_else(state_err)?.next_child_storage_key(child_info, key)
self.state
.borrow()
.as_ref()
.ok_or_else(state_err)?
.next_child_storage_key(child_info, key)
}
fn for_keys_with_prefix<F: FnMut(&[u8])>(&self, prefix: &[u8], f: F) {
@@ -340,8 +366,13 @@ impl<B: BlockT> StateBackend<HashFor<B>> for BenchmarkingState<B> {
f: F,
allow_missing: bool,
) -> Result<bool, Self::Error> {
self.state.borrow().as_ref().ok_or_else(state_err)?
.apply_to_key_values_while(child_info, prefix, start_at, f, allow_missing)
self.state.borrow().as_ref().ok_or_else(state_err)?.apply_to_key_values_while(
child_info,
prefix,
start_at,
f,
allow_missing,
)
}
fn apply_to_keys_while<F: FnMut(&[u8]) -> bool>(
@@ -368,17 +399,29 @@ impl<B: BlockT> StateBackend<HashFor<B>> for BenchmarkingState<B> {
fn storage_root<'a>(
&self,
delta: impl Iterator<Item=(&'a [u8], Option<&'a [u8]>)>,
) -> (B::Hash, Self::Transaction) where B::Hash: Ord {
self.state.borrow().as_ref().map_or(Default::default(), |s| s.storage_root(delta))
delta: impl Iterator<Item = (&'a [u8], Option<&'a [u8]>)>,
) -> (B::Hash, Self::Transaction)
where
B::Hash: Ord,
{
self.state
.borrow()
.as_ref()
.map_or(Default::default(), |s| s.storage_root(delta))
}
fn child_storage_root<'a>(
&self,
child_info: &ChildInfo,
delta: impl Iterator<Item=(&'a [u8], Option<&'a [u8]>)>,
) -> (B::Hash, bool, Self::Transaction) where B::Hash: Ord {
self.state.borrow().as_ref().map_or(Default::default(), |s| s.child_storage_root(child_info, delta))
delta: impl Iterator<Item = (&'a [u8], Option<&'a [u8]>)>,
) -> (B::Hash, bool, Self::Transaction)
where
B::Hash: Ord,
{
self.state
.borrow()
.as_ref()
.map_or(Default::default(), |s| s.child_storage_root(child_info, delta))
}
fn pairs(&self) -> Vec<(Vec<u8>, Vec<u8>)> {
@@ -389,17 +432,16 @@ impl<B: BlockT> StateBackend<HashFor<B>> for BenchmarkingState<B> {
self.state.borrow().as_ref().map_or(Default::default(), |s| s.keys(prefix))
}
fn child_keys(
&self,
child_info: &ChildInfo,
prefix: &[u8],
) -> Vec<Vec<u8>> {
self.state.borrow().as_ref().map_or(Default::default(), |s| s.child_keys(child_info, prefix))
fn child_keys(&self, child_info: &ChildInfo, prefix: &[u8]) -> Vec<Vec<u8>> {
self.state
.borrow()
.as_ref()
.map_or(Default::default(), |s| s.child_keys(child_info, prefix))
}
fn as_trie_backend(&mut self)
-> Option<&sp_state_machine::TrieBackend<Self::TrieBackendStorage, HashFor<B>>>
{
fn as_trie_backend(
&mut self,
) -> Option<&sp_state_machine::TrieBackend<Self::TrieBackendStorage, HashFor<B>>> {
None
}
@@ -425,7 +467,8 @@ impl<B: BlockT> StateBackend<HashFor<B>> for BenchmarkingState<B> {
let mut record = self.record.take();
record.extend(keys);
self.record.set(record);
db.write(db_transaction).map_err(|_| String::from("Error committing transaction"))?;
db.write(db_transaction)
.map_err(|_| String::from("Error committing transaction"))?;
self.root.set(storage_root);
self.db.set(Some(db));
@@ -455,7 +498,8 @@ impl<B: BlockT> StateBackend<HashFor<B>> for BenchmarkingState<B> {
None => db_transaction.delete(0, &key),
}
}
db.write(db_transaction).map_err(|_| String::from("Error committing transaction"))?;
db.write(db_transaction)
.map_err(|_| String::from("Error committing transaction"))?;
self.db.set(Some(db));
}
@@ -519,24 +563,20 @@ impl<B: BlockT> StateBackend<HashFor<B>> for BenchmarkingState<B> {
let reads = tracker.reads.min(1);
let writes = tracker.writes.min(1);
if let Some(prefix_tracker) = prefix_key_tracker.get_mut(&prefix) {
prefix_tracker.0 += reads;
prefix_tracker.1 += writes;
prefix_tracker.0 += reads;
prefix_tracker.1 += writes;
} else {
prefix_key_tracker.insert(
prefix,
(
reads,
writes,
tracker.whitelisted,
),
);
prefix_key_tracker.insert(prefix, (reads, writes, tracker.whitelisted));
}
}
});
prefix_key_tracker.iter().map(|(key, tracker)| -> (Vec<u8>, u32, u32, bool) {
prefix_key_tracker
.iter()
.map(|(key, tracker)| -> (Vec<u8>, u32, u32, bool) {
(key.to_vec(), tracker.0, tracker.1, tracker.2)
}).collect::<Vec<_>>()
})
.collect::<Vec<_>>()
}
fn register_overlay_stats(&self, stats: &sp_state_machine::StateMachineStats) {
@@ -544,7 +584,10 @@ impl<B: BlockT> StateBackend<HashFor<B>> for BenchmarkingState<B> {
}
fn usage_info(&self) -> sp_state_machine::UsageInfo {
self.state.borrow().as_ref().map_or(sp_state_machine::UsageInfo::empty(), |s| s.usage_info())
self.state
.borrow()
.as_ref()
.map_or(sp_state_machine::UsageInfo::empty(), |s| s.usage_info())
}
fn proof_size(&self) -> Option<u32> {
@@ -585,8 +628,8 @@ mod test {
#[test]
fn read_to_main_and_child_tries() {
let bench_state = BenchmarkingState::<crate::tests::Block>::new(Default::default(), None, false)
.unwrap();
let bench_state =
BenchmarkingState::<crate::tests::Block>::new(Default::default(), None, false).unwrap();
for _ in 0..2 {
let child1 = sp_core::storage::ChildInfo::new_default(b"child1");
@@ -600,16 +643,14 @@ mod test {
bench_state.child_storage(&child1, b"bar").unwrap();
bench_state.child_storage(&child2, b"bar").unwrap();
bench_state.commit(
Default::default(),
Default::default(),
vec![
("foo".as_bytes().to_vec(), None)
],
vec![
("child1".as_bytes().to_vec(), vec![("foo".as_bytes().to_vec(), None)])
]
).unwrap();
bench_state
.commit(
Default::default(),
Default::default(),
vec![("foo".as_bytes().to_vec(), None)],
vec![("child1".as_bytes().to_vec(), vec![("foo".as_bytes().to_vec(), None)])],
)
.unwrap();
let rw_tracker = bench_state.read_write_count();
assert_eq!(rw_tracker.0, 6);
substrate/client/db/src/cache/list_cache.rs
Vendored
+1026 -457
View File
File diff suppressed because it is too large Load Diff
substrate/client/db/src/cache/list_entry.rs
Vendored
+54 -34
View File
@@ -18,12 +18,11 @@
//! List-cache storage entries.
use codec::{Decode, Encode};
use sp_blockchain::Result as ClientResult;
use sp_runtime::traits::{Block as BlockT, NumberFor};
use codec::{Encode, Decode};
use crate::cache::{CacheItemT, ComplexBlockId};
use crate::cache::list_storage::{Storage};
use crate::cache::{list_storage::Storage, CacheItemT, ComplexBlockId};
/// Single list-based cache entry.
#[derive(Debug)]
@@ -52,10 +51,8 @@ impl<Block: BlockT, T: CacheItemT> Entry<Block, T> {
match value {
Some(value) => match self.value == value {
true => None,
false => Some(StorageEntry {
prev_valid_from: Some(self.valid_from.clone()),
value,
}),
false =>
Some(StorageEntry { prev_valid_from: Some(self.valid_from.clone()), value }),
},
None => None,
}
@@ -67,7 +64,8 @@ impl<Block: BlockT, T: CacheItemT> Entry<Block, T> {
storage: &S,
block: NumberFor<Block>,
) -> ClientResult<Option<(ComplexBlockId<Block>, Option<ComplexBlockId<Block>>)>> {
Ok(self.search_best_before(storage, block)?
Ok(self
.search_best_before(storage, block)?
.map(|(entry, next)| (entry.valid_from, next)))
}
@@ -86,14 +84,14 @@ impl<Block: BlockT, T: CacheItemT> Entry<Block, T> {
let mut current = self.valid_from.clone();
if block >= self.valid_from.number {
let value = self.value.clone();
return Ok(Some((Entry { valid_from: current, value }, next)));
return Ok(Some((Entry { valid_from: current, value }, next)))
}
// else - travel back in time
loop {
let entry = storage.require_entry(&current)?;
if block >= current.number {
return Ok(Some((Entry { valid_from: current, value: entry.value }, next)));
return Ok(Some((Entry { valid_from: current, value: entry.value }, next)))
}
next = Some(current);
@@ -108,18 +106,15 @@ impl<Block: BlockT, T: CacheItemT> Entry<Block, T> {
impl<Block: BlockT, T: CacheItemT> StorageEntry<Block, T> {
/// Converts storage entry into an entry, valid from given block.
pub fn into_entry(self, valid_from: ComplexBlockId<Block>) -> Entry<Block, T> {
Entry {
valid_from,
value: self.value,
}
Entry { valid_from, value: self.value }
}
}
#[cfg(test)]
mod tests {
use crate::cache::list_storage::tests::{DummyStorage, FaultyStorage};
use substrate_test_runtime_client::runtime::{H256, Block};
use super::*;
use crate::cache::list_storage::tests::{DummyStorage, FaultyStorage};
use substrate_test_runtime_client::runtime::{Block, H256};
fn test_id(number: u64) -> ComplexBlockId<Block> {
ComplexBlockId::new(H256::from_low_u64_be(number), number)
@@ -132,36 +127,61 @@ mod tests {
// when trying to update with the same Some value
assert_eq!(Entry { valid_from: test_id(1), value: 1 }.try_update(Some(1)), None);
// when trying to update with different Some value
assert_eq!(Entry { valid_from: test_id(1), value: 1 }.try_update(Some(2)),
Some(StorageEntry { prev_valid_from: Some(test_id(1)), value: 2 }));
assert_eq!(
Entry { valid_from: test_id(1), value: 1 }.try_update(Some(2)),
Some(StorageEntry { prev_valid_from: Some(test_id(1)), value: 2 })
);
}
#[test]
fn entry_search_best_before_fails() {
// when storage returns error
assert!(Entry::<_, u64> { valid_from: test_id(100), value: 42 }
.search_best_before(&FaultyStorage, 50).is_err());
.search_best_before(&FaultyStorage, 50)
.is_err());
}
#[test]
fn entry_search_best_before_works() {
// when block is better than our best block
assert_eq!(Entry::<_, u64> { valid_from: test_id(100), value: 100 }
.search_best_before(&DummyStorage::new(), 150).unwrap(),
Some((Entry::<_, u64> { valid_from: test_id(100), value: 100 }, None)));
assert_eq!(
Entry::<_, u64> { valid_from: test_id(100), value: 100 }
.search_best_before(&DummyStorage::new(), 150)
.unwrap(),
Some((Entry::<_, u64> { valid_from: test_id(100), value: 100 }, None))
);
// when block is found between two entries
assert_eq!(Entry::<_, u64> { valid_from: test_id(100), value: 100 }
.search_best_before(&DummyStorage::new()
.with_entry(test_id(100), StorageEntry { prev_valid_from: Some(test_id(50)), value: 100 })
.with_entry(test_id(50), StorageEntry { prev_valid_from: Some(test_id(30)), value: 50 }),
75).unwrap(),
Some((Entry::<_, u64> { valid_from: test_id(50), value: 50 }, Some(test_id(100)))));
assert_eq!(
Entry::<_, u64> { valid_from: test_id(100), value: 100 }
.search_best_before(
&DummyStorage::new()
.with_entry(
test_id(100),
StorageEntry { prev_valid_from: Some(test_id(50)), value: 100 }
)
.with_entry(
test_id(50),
StorageEntry { prev_valid_from: Some(test_id(30)), value: 50 }
),
75
)
.unwrap(),
Some((Entry::<_, u64> { valid_from: test_id(50), value: 50 }, Some(test_id(100))))
);
// when block is not found
assert_eq!(Entry::<_, u64> { valid_from: test_id(100), value: 100 }
.search_best_before(&DummyStorage::new()
.with_entry(test_id(100), StorageEntry { prev_valid_from: Some(test_id(50)), value: 100 })
.with_entry(test_id(50), StorageEntry { prev_valid_from: None, value: 50 }),
30).unwrap(),
None);
assert_eq!(
Entry::<_, u64> { valid_from: test_id(100), value: 100 }
.search_best_before(
&DummyStorage::new()
.with_entry(
test_id(100),
StorageEntry { prev_valid_from: Some(test_id(50)), value: 100 }
)
.with_entry(test_id(50), StorageEntry { prev_valid_from: None, value: 50 }),
30
)
.unwrap(),
None
);
}
}
substrate/client/db/src/cache/list_storage.rs
Vendored
+100 -45
View File
@@ -20,17 +20,23 @@
use std::sync::Arc;
use sp_blockchain::{Error as ClientError, Result as ClientResult};
use codec::{Encode, Decode};
use sp_runtime::generic::BlockId;
use sp_runtime::traits::{Block as BlockT, Header as HeaderT, NumberFor};
use sp_database::{Database, Transaction};
use crate::utils::{self, meta_keys};
use codec::{Decode, Encode};
use sp_blockchain::{Error as ClientError, Result as ClientResult};
use sp_database::{Database, Transaction};
use sp_runtime::{
generic::BlockId,
traits::{Block as BlockT, Header as HeaderT, NumberFor},
};
use crate::cache::{CacheItemT, ComplexBlockId};
use crate::cache::list_cache::{CommitOperation, Fork};
use crate::cache::list_entry::{Entry, StorageEntry};
use crate::DbHash;
use crate::{
cache::{
list_cache::{CommitOperation, Fork},
list_entry::{Entry, StorageEntry},
CacheItemT, ComplexBlockId,
},
DbHash,
};
/// Single list-cache metadata.
#[derive(Debug)]
@@ -54,14 +60,21 @@ pub trait Storage<Block: BlockT, T: CacheItemT> {
fn read_meta(&self) -> ClientResult<Metadata<Block>>;
/// Reads cache entry from the storage.
fn read_entry(&self, at: &ComplexBlockId<Block>) -> ClientResult<Option<StorageEntry<Block, T>>>;
fn read_entry(
&self,
at: &ComplexBlockId<Block>,
) -> ClientResult<Option<StorageEntry<Block, T>>>;
/// Reads referenced (and thus existing) cache entry from the storage.
fn require_entry(&self, at: &ComplexBlockId<Block>) -> ClientResult<StorageEntry<Block, T>> {
self.read_entry(at)
.and_then(|entry| entry
.ok_or_else(|| ClientError::from(
ClientError::Backend(format!("Referenced cache entry at {:?} is not found", at)))))
self.read_entry(at).and_then(|entry| {
entry.ok_or_else(|| {
ClientError::from(ClientError::Backend(format!(
"Referenced cache entry at {:?} is not found",
at
)))
})
})
}
}
@@ -111,10 +124,14 @@ impl DbStorage {
}
/// Get reference to the database.
pub fn db(&self) -> &Arc<dyn Database<DbHash>> { &self.db }
pub fn db(&self) -> &Arc<dyn Database<DbHash>> {
&self.db
}
/// Get reference to the database columns.
pub fn columns(&self) -> &DbColumns { &self.columns }
pub fn columns(&self) -> &DbColumns {
&self.columns
}
/// Encode block id for storing as a key in cache column.
/// We append prefix to the actual encoding to allow several caches
@@ -128,25 +145,35 @@ impl DbStorage {
impl<Block: BlockT, T: CacheItemT> Storage<Block, T> for DbStorage {
fn read_id(&self, at: NumberFor<Block>) -> ClientResult<Option<Block::Hash>> {
utils::read_header::<Block>(&*self.db, self.columns.key_lookup, self.columns.header, BlockId::Number(at))
.map(|maybe_header| maybe_header.map(|header| header.hash()))
utils::read_header::<Block>(
&*self.db,
self.columns.key_lookup,
self.columns.header,
BlockId::Number(at),
)
.map(|maybe_header| maybe_header.map(|header| header.hash()))
}
fn read_header(&self, at: &Block::Hash) -> ClientResult<Option<Block::Header>> {
utils::read_header::<Block>(&*self.db, self.columns.key_lookup, self.columns.header, BlockId::Hash(*at))
utils::read_header::<Block>(
&*self.db,
self.columns.key_lookup,
self.columns.header,
BlockId::Hash(*at),
)
}
fn read_meta(&self) -> ClientResult<Metadata<Block>> {
match self.db.get(self.columns.meta, &self.meta_key) {
Some(meta) => meta::decode(&*meta),
None => Ok(Metadata {
finalized: None,
unfinalized: Vec::new(),
})
None => Ok(Metadata { finalized: None, unfinalized: Vec::new() }),
}
}
fn read_entry(&self, at: &ComplexBlockId<Block>) -> ClientResult<Option<StorageEntry<Block, T>>> {
fn read_entry(
&self,
at: &ComplexBlockId<Block>,
) -> ClientResult<Option<StorageEntry<Block, T>>> {
match self.db.get(self.columns.cache, &self.encode_block_id(at)) {
Some(entry) => StorageEntry::<Block, T>::decode(&mut &entry[..])
.map_err(|_| ClientError::Backend("Failed to decode cache entry".into()))
@@ -171,7 +198,11 @@ impl<'a> DbStorageTransaction<'a> {
impl<'a, Block: BlockT, T: CacheItemT> StorageTransaction<Block, T> for DbStorageTransaction<'a> {
fn insert_storage_entry(&mut self, at: &ComplexBlockId<Block>, entry: &StorageEntry<Block, T>) {
self.tx.set_from_vec(self.storage.columns.cache, &self.storage.encode_block_id(at), entry.encode());
self.tx.set_from_vec(
self.storage.columns.cache,
&self.storage.encode_block_id(at),
entry.encode(),
);
}
fn remove_storage_entry(&mut self, at: &ComplexBlockId<Block>) {
@@ -187,7 +218,8 @@ impl<'a, Block: BlockT, T: CacheItemT> StorageTransaction<Block, T> for DbStorag
self.tx.set_from_vec(
self.storage.columns.meta,
&self.storage.meta_key,
meta::encode(best_finalized_entry, unfinalized, operation));
meta::encode(best_finalized_entry, unfinalized, operation),
);
}
}
@@ -206,10 +238,11 @@ mod meta {
pub fn encode<Block: BlockT, T: CacheItemT>(
best_finalized_entry: Option<&Entry<Block, T>>,
unfinalized: &[Fork<Block, T>],
op: &CommitOperation<Block, T>
op: &CommitOperation<Block, T>,
) -> Vec<u8> {
let mut finalized = best_finalized_entry.as_ref().map(|entry| &entry.valid_from);
let mut unfinalized = unfinalized.iter().map(|fork| &fork.head().valid_from).collect::<Vec<_>>();
let mut unfinalized =
unfinalized.iter().map(|fork| &fork.head().valid_from).collect::<Vec<_>>();
match op {
CommitOperation::AppendNewBlock(_, _) => (),
@@ -230,8 +263,11 @@ mod meta {
CommitOperation::BlockReverted(ref forks) => {
for (fork_index, updated_fork) in forks.iter().rev() {
match updated_fork {
Some(updated_fork) => unfinalized[*fork_index] = &updated_fork.head().valid_from,
None => { unfinalized.remove(*fork_index); },
Some(updated_fork) =>
unfinalized[*fork_index] = &updated_fork.head().valid_from,
None => {
unfinalized.remove(*fork_index);
},
}
}
},
@@ -243,10 +279,12 @@ mod meta {
/// Decode meta information.
pub fn decode<Block: BlockT>(encoded: &[u8]) -> ClientResult<Metadata<Block>> {
let input = &mut &*encoded;
let finalized: Option<ComplexBlockId<Block>> = Decode::decode(input)
.map_err(|_| ClientError::from(ClientError::Backend("Error decoding cache meta".into())))?;
let unfinalized: Vec<ComplexBlockId<Block>> = Decode::decode(input)
.map_err(|_| ClientError::from(ClientError::Backend("Error decoding cache meta".into())))?;
let finalized: Option<ComplexBlockId<Block>> = Decode::decode(input).map_err(|_| {
ClientError::from(ClientError::Backend("Error decoding cache meta".into()))
})?;
let unfinalized: Vec<ComplexBlockId<Block>> = Decode::decode(input).map_err(|_| {
ClientError::from(ClientError::Backend("Error decoding cache meta".into()))
})?;
Ok(Metadata { finalized, unfinalized })
}
@@ -254,8 +292,8 @@ mod meta {
#[cfg(test)]
pub mod tests {
use std::collections::{HashMap, HashSet};
use super::*;
use std::collections::{HashMap, HashSet};
pub struct FaultyStorage;
@@ -272,7 +310,10 @@ pub mod tests {
Err(ClientError::Backend("TestError".into()))
}
fn read_entry(&self, _at: &ComplexBlockId<Block>) -> ClientResult<Option<StorageEntry<Block, T>>> {
fn read_entry(
&self,
_at: &ComplexBlockId<Block>,
) -> ClientResult<Option<StorageEntry<Block, T>>> {
Err(ClientError::Backend("TestError".into()))
}
}
@@ -287,17 +328,18 @@ pub mod tests {
impl<Block: BlockT, T: CacheItemT> DummyStorage<Block, T> {
pub fn new() -> Self {
DummyStorage {
meta: Metadata {
finalized: None,
unfinalized: Vec::new(),
},
meta: Metadata { finalized: None, unfinalized: Vec::new() },
ids: HashMap::new(),
headers: HashMap::new(),
entries: HashMap::new(),
}
}
pub fn with_meta(mut self, finalized: Option<ComplexBlockId<Block>>, unfinalized: Vec<ComplexBlockId<Block>>) -> Self {
pub fn with_meta(
mut self,
finalized: Option<ComplexBlockId<Block>>,
unfinalized: Vec<ComplexBlockId<Block>>,
) -> Self {
self.meta.finalized = finalized;
self.meta.unfinalized = unfinalized;
self
@@ -313,7 +355,11 @@ pub mod tests {
self
}
pub fn with_entry(mut self, at: ComplexBlockId<Block>, entry: StorageEntry<Block, T>) -> Self {
pub fn with_entry(
mut self,
at: ComplexBlockId<Block>,
entry: StorageEntry<Block, T>,
) -> Self {
self.entries.insert(at.hash, entry);
self
}
@@ -332,7 +378,10 @@ pub mod tests {
Ok(self.meta.clone())
}
fn read_entry(&self, at: &ComplexBlockId<Block>) -> ClientResult<Option<StorageEntry<Block, T>>> {
fn read_entry(
&self,
at: &ComplexBlockId<Block>,
) -> ClientResult<Option<StorageEntry<Block, T>>> {
Ok(self.entries.get(&at.hash).cloned())
}
}
@@ -366,7 +415,11 @@ pub mod tests {
}
impl<Block: BlockT, T: CacheItemT> StorageTransaction<Block, T> for DummyTransaction<Block> {
fn insert_storage_entry(&mut self, at: &ComplexBlockId<Block>, _entry: &StorageEntry<Block, T>) {
fn insert_storage_entry(
&mut self,
at: &ComplexBlockId<Block>,
_entry: &StorageEntry<Block, T>,
) {
self.inserted_entries.insert(at.hash);
}
@@ -380,7 +433,9 @@ pub mod tests {
unfinalized: &[Fork<Block, T>],
operation: &CommitOperation<Block, T>,
) {
self.updated_meta = Some(meta::decode(&meta::encode(best_finalized_entry, unfinalized, operation)).unwrap());
self.updated_meta = Some(
meta::decode(&meta::encode(best_finalized_entry, unfinalized, operation)).unwrap(),
);
}
}
}
substrate/client/db/src/cache/mod.rs
Vendored
+63 -59
View File
@@ -18,17 +18,27 @@
//! DB-backed cache of blockchain data.
use std::{sync::Arc, collections::{HashMap, hash_map::Entry}};
use parking_lot::RwLock;
use std::{
collections::{hash_map::Entry, HashMap},
sync::Arc,
};
use sc_client_api::blockchain::{well_known_cache_keys::{self, Id as CacheKeyId}, Cache as BlockchainCache};
use sp_blockchain::{Result as ClientResult, HeaderMetadataCache};
use crate::{
utils::{self, COLUMN_META},
DbHash,
};
use codec::{Decode, Encode};
use sc_client_api::blockchain::{
well_known_cache_keys::{self, Id as CacheKeyId},
Cache as BlockchainCache,
};
use sp_blockchain::{HeaderMetadataCache, Result as ClientResult};
use sp_database::{Database, Transaction};
use codec::{Encode, Decode};
use sp_runtime::generic::BlockId;
use sp_runtime::traits::{Block as BlockT, Header as HeaderT, NumberFor, Zero};
use crate::utils::{self, COLUMN_META};
use crate::DbHash;
use sp_runtime::{
generic::BlockId,
traits::{Block as BlockT, Header as HeaderT, NumberFor, Zero},
};
use self::list_cache::{ListCache, PruningStrategy};
@@ -118,7 +128,10 @@ impl<Block: BlockT> DbCache<Block> {
}
/// Begin cache transaction.
pub fn transaction<'a>(&'a mut self, tx: &'a mut Transaction<DbHash>) -> DbCacheTransaction<'a, Block> {
pub fn transaction<'a>(
&'a mut self,
tx: &'a mut Transaction<DbHash>,
) -> DbCacheTransaction<'a, Block> {
DbCacheTransaction {
cache: self,
tx,
@@ -164,7 +177,7 @@ impl<Block: BlockT> DbCache<Block> {
self.key_lookup_column,
self.header_column,
self.cache_column,
&self.best_finalized_block
&self.best_finalized_block,
)
}
}
@@ -184,19 +197,16 @@ fn get_cache_helper<'a, Block: BlockT>(
Entry::Occupied(entry) => Ok(entry.into_mut()),
Entry::Vacant(entry) => {
let cache = ListCache::new(
self::list_storage::DbStorage::new(name.to_vec(), db.clone(),
self::list_storage::DbColumns {
meta: COLUMN_META,
key_lookup,
header,
cache,
},
self::list_storage::DbStorage::new(
name.to_vec(),
db.clone(),
self::list_storage::DbColumns { meta: COLUMN_META, key_lookup, header, cache },
),
cache_pruning_strategy(name),
best_finalized_block.clone(),
)?;
Ok(entry.insert(cache))
}
},
}
}
@@ -210,10 +220,7 @@ pub struct DbCacheTransactionOps<Block: BlockT> {
impl<Block: BlockT> DbCacheTransactionOps<Block> {
/// Empty transaction ops.
pub fn empty() -> DbCacheTransactionOps<Block> {
DbCacheTransactionOps {
cache_at_ops: HashMap::new(),
best_finalized_block: None,
}
DbCacheTransactionOps { cache_at_ops: HashMap::new(), best_finalized_block: None }
}
}
@@ -244,19 +251,21 @@ impl<'a, Block: BlockT> DbCacheTransaction<'a, Block> {
) -> ClientResult<Self> {
// prepare list of caches that are not update
// (we might still need to do some cache maintenance in this case)
let missed_caches = self.cache.cache_at.keys()
let missed_caches = self
.cache
.cache_at
.keys()
.filter(|cache| !data_at.contains_key(*cache))
.cloned()
.collect::<Vec<_>>();
let mut insert_op = |name: CacheKeyId, value: Option<Vec<u8>>| -> Result<(), sp_blockchain::Error> {
let mut insert_op = |name: CacheKeyId,
value: Option<Vec<u8>>|
-> Result<(), sp_blockchain::Error> {
let cache = self.cache.get_cache(name)?;
let cache_ops = self.cache_at_ops.entry(name).or_default();
cache.on_block_insert(
&mut self::list_storage::DbStorageTransaction::new(
cache.storage(),
&mut self.tx,
),
&mut self::list_storage::DbStorageTransaction::new(cache.storage(), &mut self.tx),
parent.clone(),
block.clone(),
value,
@@ -271,8 +280,7 @@ impl<'a, Block: BlockT> DbCacheTransaction<'a, Block> {
missed_caches.into_iter().try_for_each(|name| insert_op(name, None))?;
match entry_type {
EntryType::Final | EntryType::Genesis =>
self.best_finalized_block = Some(block),
EntryType::Final | EntryType::Genesis => self.best_finalized_block = Some(block),
EntryType::NonFinal => (),
}
@@ -288,10 +296,7 @@ impl<'a, Block: BlockT> DbCacheTransaction<'a, Block> {
for (name, cache) in self.cache.cache_at.iter() {
let cache_ops = self.cache_at_ops.entry(*name).or_default();
cache.on_block_finalize(
&mut self::list_storage::DbStorageTransaction::new(
cache.storage(),
&mut self.tx
),
&mut self::list_storage::DbStorageTransaction::new(cache.storage(), &mut self.tx),
parent.clone(),
block.clone(),
cache_ops,
@@ -304,17 +309,11 @@ impl<'a, Block: BlockT> DbCacheTransaction<'a, Block> {
}
/// When block is reverted.
pub fn on_block_revert(
mut self,
reverted_block: &ComplexBlockId<Block>,
) -> ClientResult<Self> {
pub fn on_block_revert(mut self, reverted_block: &ComplexBlockId<Block>) -> ClientResult<Self> {
for (name, cache) in self.cache.cache_at.iter() {
let cache_ops = self.cache_at_ops.entry(*name).or_default();
cache.on_block_revert(
&mut self::list_storage::DbStorageTransaction::new(
cache.storage(),
&mut self.tx
),
&mut self::list_storage::DbStorageTransaction::new(cache.storage(), &mut self.tx),
reverted_block,
cache_ops,
)?;
@@ -352,7 +351,9 @@ impl<Block: BlockT> BlockchainCache<Block> for DbCacheSync<Block> {
&self,
key: &CacheKeyId,
at: &BlockId<Block>,
) -> ClientResult<Option<((NumberFor<Block>, Block::Hash), Option<(NumberFor<Block>, Block::Hash)>, Vec<u8>)>> {
) -> ClientResult<
Option<((NumberFor<Block>, Block::Hash), Option<(NumberFor<Block>, Block::Hash)>, Vec<u8>)>,
> {
let mut cache = self.0.write();
let header_metadata_cache = cache.header_metadata_cache.clone();
let cache = cache.get_cache(*key)?;
@@ -360,36 +361,39 @@ impl<Block: BlockT> BlockchainCache<Block> for DbCacheSync<Block> {
let db = storage.db();
let columns = storage.columns();
let at = match *at {
BlockId::Hash(hash) => {
match header_metadata_cache.header_metadata(hash) {
Some(metadata) => ComplexBlockId::new(hash, metadata.number),
None => {
let header = utils::require_header::<Block>(
&**db,
columns.key_lookup,
columns.header,
BlockId::Hash(hash.clone()))?;
ComplexBlockId::new(hash, *header.number())
}
}
BlockId::Hash(hash) => match header_metadata_cache.header_metadata(hash) {
Some(metadata) => ComplexBlockId::new(hash, metadata.number),
None => {
let header = utils::require_header::<Block>(
&**db,
columns.key_lookup,
columns.header,
BlockId::Hash(hash.clone()),
)?;
ComplexBlockId::new(hash, *header.number())
},
},
BlockId::Number(number) => {
let hash = utils::require_header::<Block>(
&**db,
columns.key_lookup,
columns.header,
BlockId::Number(number.clone()))?.hash();
BlockId::Number(number.clone()),
)?
.hash();
ComplexBlockId::new(hash, number)
},
};
cache.value_at_block(&at)
.map(|block_and_value| block_and_value.map(|(begin_block, end_block, value)|
cache.value_at_block(&at).map(|block_and_value| {
block_and_value.map(|(begin_block, end_block, value)| {
(
(begin_block.number, begin_block.hash),
end_block.map(|end_block| (end_block.number, end_block.hash)),
value,
)))
)
})
})
}
}
substrate/client/db/src/changes_tries_storage.rs
+325 -182
View File
@@ -18,33 +18,43 @@
//! DB-backed changes tries storage.
use std::collections::{HashMap, HashSet};
use std::sync::Arc;
use hash_db::Prefix;
use codec::{Decode, Encode};
use parking_lot::RwLock;
use sp_blockchain::{Error as ClientError, Result as ClientResult};
use sp_trie::MemoryDB;
use sc_client_api::backend::PrunableStateChangesTrieStorage;
use sp_blockchain::{well_known_cache_keys, Cache as BlockchainCache, HeaderMetadataCache};
use sp_core::{ChangesTrieConfiguration, ChangesTrieConfigurationRange, convert_hash};
use sp_core::storage::PrefixedStorageKey;
use sp_database::Transaction;
use sp_runtime::traits::{
Block as BlockT, Header as HeaderT, HashFor, NumberFor, One, Zero, CheckedSub,
use crate::{
cache::{
ComplexBlockId, DbCache, DbCacheSync, DbCacheTransactionOps, EntryType as CacheEntryType,
},
utils::{self, meta_keys, Meta},
Database, DbHash,
};
use codec::{Decode, Encode};
use hash_db::Prefix;
use parking_lot::RwLock;
use sc_client_api::backend::PrunableStateChangesTrieStorage;
use sp_blockchain::{
well_known_cache_keys, Cache as BlockchainCache, Error as ClientError, HeaderMetadataCache,
Result as ClientResult,
};
use sp_core::{
convert_hash, storage::PrefixedStorageKey, ChangesTrieConfiguration,
ChangesTrieConfigurationRange,
};
use sp_database::Transaction;
use sp_runtime::{
generic::{BlockId, ChangesTrieSignal, DigestItem},
traits::{Block as BlockT, CheckedSub, HashFor, Header as HeaderT, NumberFor, One, Zero},
};
use sp_runtime::generic::{BlockId, DigestItem, ChangesTrieSignal};
use sp_state_machine::{ChangesTrieBuildCache, ChangesTrieCacheAction};
use crate::{Database, DbHash};
use crate::utils::{self, Meta, meta_keys};
use crate::cache::{
DbCacheSync, DbCache, DbCacheTransactionOps,
ComplexBlockId, EntryType as CacheEntryType,
use sp_trie::MemoryDB;
use std::{
collections::{HashMap, HashSet},
sync::Arc,
};
/// Extract new changes trie configuration (if available) from the header.
pub fn extract_new_configuration<Header: HeaderT>(header: &Header) -> Option<&Option<ChangesTrieConfiguration>> {
header.digest()
pub fn extract_new_configuration<Header: HeaderT>(
header: &Header,
) -> Option<&Option<ChangesTrieConfiguration>> {
header
.digest()
.log(DigestItem::as_changes_trie_signal)
.and_then(ChangesTrieSignal::as_new_configuration)
}
@@ -68,10 +78,7 @@ impl<Block: BlockT> DbChangesTrieStorageTransaction<Block> {
impl<Block: BlockT> From<DbCacheTransactionOps<Block>> for DbChangesTrieStorageTransaction<Block> {
fn from(cache_ops: DbCacheTransactionOps<Block>) -> Self {
DbChangesTrieStorageTransaction {
cache_ops,
new_config: None,
}
DbChangesTrieStorageTransaction { cache_ops, new_config: None }
}
}
@@ -173,21 +180,25 @@ impl<Block: BlockT> DbChangesTrieStorage<Block> {
let new_configuration = match new_configuration {
Some(new_configuration) => new_configuration,
None if !finalized => return Ok(DbCacheTransactionOps::empty().into()),
None => return self.finalize(
tx,
parent_block.hash,
block.hash,
block.number,
Some(new_header),
cache_tx,
),
None =>
return self.finalize(
tx,
parent_block.hash,
block.hash,
block.number,
Some(new_header),
cache_tx,
),
};
// update configuration cache
let mut cache_at = HashMap::new();
cache_at.insert(well_known_cache_keys::CHANGES_TRIE_CONFIG, new_configuration.encode());
Ok(DbChangesTrieStorageTransaction::from(match cache_tx {
Some(cache_tx) => self.cache.0.write()
Some(cache_tx) => self
.cache
.0
.write()
.transaction_with_ops(tx, cache_tx.cache_ops)
.on_block_insert(
parent_block,
@@ -196,7 +207,10 @@ impl<Block: BlockT> DbChangesTrieStorage<Block> {
if finalized { CacheEntryType::Final } else { CacheEntryType::NonFinal },
)?
.into_ops(),
None => self.cache.0.write()
None => self
.cache
.0
.write()
.transaction(tx)
.on_block_insert(
parent_block,
@@ -205,7 +219,8 @@ impl<Block: BlockT> DbChangesTrieStorage<Block> {
if finalized { CacheEntryType::Final } else { CacheEntryType::NonFinal },
)?
.into_ops(),
}).with_new_config(Some(new_configuration)))
})
.with_new_config(Some(new_configuration)))
}
/// Called when block is finalized.
@@ -226,7 +241,7 @@ impl<Block: BlockT> DbChangesTrieStorage<Block> {
if cache_tx.is_some() {
if let Some(new_header) = new_header {
if new_header.hash() == block_hash {
return Ok(cache_tx.expect("guarded by cache_tx.is_some(); qed"));
return Ok(cache_tx.expect("guarded by cache_tx.is_some(); qed"))
}
}
}
@@ -237,22 +252,21 @@ impl<Block: BlockT> DbChangesTrieStorage<Block> {
let parent_block = ComplexBlockId::new(parent_block_hash, parent_block_num);
Ok(match cache_tx {
Some(cache_tx) => DbChangesTrieStorageTransaction::from(
self.cache.0.write()
self.cache
.0
.write()
.transaction_with_ops(tx, cache_tx.cache_ops)
.on_block_finalize(
parent_block,
block,
)?
.into_ops()
).with_new_config(cache_tx.new_config),
.on_block_finalize(parent_block, block)?
.into_ops(),
)
.with_new_config(cache_tx.new_config),
None => DbChangesTrieStorageTransaction::from(
self.cache.0.write()
self.cache
.0
.write()
.transaction(tx)
.on_block_finalize(
parent_block,
block,
)?
.into_ops()
.on_block_finalize(parent_block, block)?
.into_ops(),
),
})
}
@@ -263,23 +277,24 @@ impl<Block: BlockT> DbChangesTrieStorage<Block> {
tx: &mut Transaction<DbHash>,
block: &ComplexBlockId<Block>,
) -> ClientResult<DbChangesTrieStorageTransaction<Block>> {
Ok(self.cache.0.write().transaction(tx)
.on_block_revert(block)?
.into_ops()
.into())
Ok(self.cache.0.write().transaction(tx).on_block_revert(block)?.into_ops().into())
}
/// When transaction has been committed.
pub fn post_commit(&self, tx: Option<DbChangesTrieStorageTransaction<Block>>) {
if let Some(tx) = tx {
self.cache.0.write().commit(tx.cache_ops)
.expect("only fails if cache with given name isn't loaded yet;\
cache is already loaded because there is tx; qed");
self.cache.0.write().commit(tx.cache_ops).expect(
"only fails if cache with given name isn't loaded yet;\
cache is already loaded because there is tx; qed",
);
}
}
/// Commit changes into changes trie build cache.
pub fn commit_build_cache(&self, cache_update: ChangesTrieCacheAction<Block::Hash, NumberFor<Block>>) {
pub fn commit_build_cache(
&self,
cache_update: ChangesTrieCacheAction<Block::Hash, NumberFor<Block>>,
) {
self.build_cache.write().perform(cache_update);
}
@@ -307,7 +322,7 @@ impl<Block: BlockT> DbChangesTrieStorage<Block> {
// 2) or we are (or were) in period where changes tries are disabled
if let Some((begin, end)) = tries_meta.oldest_digest_range {
if block_num <= end || block_num - end <= min_blocks_to_keep.into() {
break;
break
}
tries_meta.oldest_pruned_digest_range_end = end;
@@ -333,7 +348,8 @@ impl<Block: BlockT> DbChangesTrieStorage<Block> {
self.key_lookup_column,
self.header_column,
BlockId::Number(next_digest_range_start),
)?.hash(),
)?
.hash(),
};
let config_for_new_block = new_header
@@ -341,21 +357,18 @@ impl<Block: BlockT> DbChangesTrieStorage<Block> {
.unwrap_or(false);
let next_config = match cache_tx {
Some(cache_tx) if config_for_new_block && cache_tx.new_config.is_some() => {
let config = cache_tx
.new_config
.clone()
.expect("guarded by is_some(); qed");
let config = cache_tx.new_config.clone().expect("guarded by is_some(); qed");
ChangesTrieConfigurationRange {
zero: (block_num, block_hash),
end: None,
config,
}
},
_ if config_for_new_block => {
self.configuration_at(&BlockId::Hash(*new_header.expect(
"config_for_new_block is only true when new_header is passed; qed"
).parent_hash()))?
},
_ if config_for_new_block => self.configuration_at(&BlockId::Hash(
*new_header
.expect("config_for_new_block is only true when new_header is passed; qed")
.parent_hash(),
))?,
_ => self.configuration_at(&BlockId::Hash(next_digest_range_start_hash))?,
};
if let Some(config) = next_config.config {
@@ -370,11 +383,11 @@ impl<Block: BlockT> DbChangesTrieStorage<Block> {
}
tries_meta.oldest_digest_range = Some(oldest_digest_range);
continue;
continue
}
tries_meta.oldest_digest_range = None;
break;
break
}
write_tries_meta(tx, self.meta_column, &*tries_meta);
@@ -383,17 +396,23 @@ impl<Block: BlockT> DbChangesTrieStorage<Block> {
}
impl<Block: BlockT> PrunableStateChangesTrieStorage<Block> for DbChangesTrieStorage<Block> {
fn storage(&self) -> &dyn sp_state_machine::ChangesTrieStorage<HashFor<Block>, NumberFor<Block>> {
fn storage(
&self,
) -> &dyn sp_state_machine::ChangesTrieStorage<HashFor<Block>, NumberFor<Block>> {
self
}
fn configuration_at(&self, at: &BlockId<Block>) -> ClientResult<
ChangesTrieConfigurationRange<NumberFor<Block>, Block::Hash>
> {
fn configuration_at(
&self,
at: &BlockId<Block>,
) -> ClientResult<ChangesTrieConfigurationRange<NumberFor<Block>, Block::Hash>> {
self.cache
.get_at(&well_known_cache_keys::CHANGES_TRIE_CONFIG, at)?
.and_then(|(zero, end, encoded)| Decode::decode(&mut &encoded[..]).ok()
.map(|config| ChangesTrieConfigurationRange { zero, end, config }))
.and_then(|(zero, end, encoded)| {
Decode::decode(&mut &encoded[..])
.ok()
.map(|config| ChangesTrieConfigurationRange { zero, end, config })
})
.ok_or_else(|| ClientError::ErrorReadingChangesTriesConfig)
}
@@ -409,14 +428,21 @@ impl<Block: BlockT> sp_state_machine::ChangesTrieRootsStorage<HashFor<Block>, Nu
&self,
hash: Block::Hash,
) -> Result<sp_state_machine::ChangesTrieAnchorBlockId<Block::Hash, NumberFor<Block>>, String> {
utils::read_header::<Block>(&*self.db, self.key_lookup_column, self.header_column, BlockId::Hash(hash))
.map_err(|e| e.to_string())
.and_then(|maybe_header| maybe_header.map(|header|
sp_state_machine::ChangesTrieAnchorBlockId {
utils::read_header::<Block>(
&*self.db,
self.key_lookup_column,
self.header_column,
BlockId::Hash(hash),
)
.map_err(|e| e.to_string())
.and_then(|maybe_header| {
maybe_header
.map(|header| sp_state_machine::ChangesTrieAnchorBlockId {
hash,
number: *header.number(),
}
).ok_or_else(|| format!("Unknown header: {}", hash)))
})
.ok_or_else(|| format!("Unknown header: {}", hash))
})
}
fn root(
@@ -426,7 +452,10 @@ impl<Block: BlockT> sp_state_machine::ChangesTrieRootsStorage<HashFor<Block>, Nu
) -> Result<Option<Block::Hash>, String> {
// check API requirement: we can't get NEXT block(s) based on anchor
if block > anchor.number {
return Err(format!("Can't get changes trie root at {} using anchor at {}", block, anchor.number));
return Err(format!(
"Can't get changes trie root at {} using anchor at {}",
block, anchor.number
))
}
// we need to get hash of the block to resolve changes trie root
@@ -438,8 +467,12 @@ impl<Block: BlockT> sp_state_machine::ChangesTrieRootsStorage<HashFor<Block>, Nu
let mut current_num = anchor.number;
let mut current_hash: Block::Hash = convert_hash(&anchor.hash);
let maybe_anchor_header: Block::Header = utils::require_header::<Block>(
&*self.db, self.key_lookup_column, self.header_column, BlockId::Number(current_num)
).map_err(|e| e.to_string())?;
&*self.db,
self.key_lookup_column,
self.header_column,
BlockId::Number(current_num),
)
.map_err(|e| e.to_string())?;
if maybe_anchor_header.hash() == current_hash {
// if anchor is canonicalized, then the block is also canonicalized
BlockId::Number(block)
@@ -449,8 +482,12 @@ impl<Block: BlockT> sp_state_machine::ChangesTrieRootsStorage<HashFor<Block>, Nu
// back from the anchor to the block with given number
while current_num != block {
let current_header: Block::Header = utils::require_header::<Block>(
&*self.db, self.key_lookup_column, self.header_column, BlockId::Hash(current_hash)
).map_err(|e| e.to_string())?;
&*self.db,
self.key_lookup_column,
self.header_column,
BlockId::Hash(current_hash),
)
.map_err(|e| e.to_string())?;
current_hash = *current_header.parent_hash();
current_num = current_num - One::one();
@@ -460,18 +497,16 @@ impl<Block: BlockT> sp_state_machine::ChangesTrieRootsStorage<HashFor<Block>, Nu
}
};
Ok(
utils::require_header::<Block>(
&*self.db,
self.key_lookup_column,
self.header_column,
block_id,
)
.map_err(|e| e.to_string())?
.digest()
.log(DigestItem::as_changes_trie_root)
.cloned()
Ok(utils::require_header::<Block>(
&*self.db,
self.key_lookup_column,
self.header_column,
block_id,
)
.map_err(|e| e.to_string())?
.digest()
.log(DigestItem::as_changes_trie_root)
.cloned())
}
}
@@ -480,7 +515,9 @@ impl<Block> sp_state_machine::ChangesTrieStorage<HashFor<Block>, NumberFor<Block
where
Block: BlockT,
{
fn as_roots_storage(&self) -> &dyn sp_state_machine::ChangesTrieRootsStorage<HashFor<Block>, NumberFor<Block>> {
fn as_roots_storage(
&self,
) -> &dyn sp_state_machine::ChangesTrieRootsStorage<HashFor<Block>, NumberFor<Block>> {
self
}
@@ -503,8 +540,9 @@ fn read_tries_meta<Block: BlockT>(
meta_column: u32,
) -> ClientResult<ChangesTriesMeta<Block>> {
match db.get(meta_column, meta_keys::CHANGES_TRIES_META) {
Some(h) => Decode::decode(&mut &h[..])
.map_err(|err| ClientError::Backend(format!("Error decoding changes tries metadata: {}", err))),
Some(h) => Decode::decode(&mut &h[..]).map_err(|err| {
ClientError::Backend(format!("Error decoding changes tries metadata: {}", err))
}),
None => Ok(ChangesTriesMeta {
oldest_digest_range: None,
oldest_pruned_digest_range_end: Zero::zero(),
@@ -523,18 +561,23 @@ fn write_tries_meta<Block: BlockT>(
#[cfg(test)]
mod tests {
use super::*;
use crate::{
tests::{insert_header, prepare_changes, Block},
Backend,
};
use hash_db::EMPTY_PREFIX;
use sc_client_api::backend::{
Backend as ClientBackend, NewBlockState, BlockImportOperation, PrunableStateChangesTrieStorage,
Backend as ClientBackend, BlockImportOperation, NewBlockState,
PrunableStateChangesTrieStorage,
};
use sp_blockchain::HeaderBackend as BlockchainHeaderBackend;
use sp_core::H256;
use sp_runtime::testing::{Digest, Header};
use sp_runtime::traits::{Hash, BlakeTwo256};
use sp_runtime::{
testing::{Digest, Header},
traits::{BlakeTwo256, Hash},
};
use sp_state_machine::{ChangesTrieRootsStorage, ChangesTrieStorage};
use crate::Backend;
use crate::tests::{Block, insert_header, prepare_changes};
use super::*;
fn changes(number: u64) -> Option<Vec<(Vec<u8>, Vec<u8>)>> {
Some(vec![(number.to_le_bytes().to_vec(), number.to_le_bytes().to_vec())])
@@ -554,7 +597,9 @@ mod tests {
digest.push(DigestItem::ChangesTrieRoot(root));
changes_trie_update = update;
}
digest.push(DigestItem::ChangesTrieSignal(ChangesTrieSignal::NewConfiguration(new_configuration)));
digest.push(DigestItem::ChangesTrieSignal(ChangesTrieSignal::NewConfiguration(
new_configuration,
)));
let header = Header {
number,
@@ -573,7 +618,8 @@ mod tests {
let mut op = backend.begin_operation().unwrap();
backend.begin_state_operation(&mut op, block_id).unwrap();
op.set_block_data(header, None, None, None, NewBlockState::Best).unwrap();
op.update_changes_trie((changes_trie_update, ChangesTrieCacheAction::Clear)).unwrap();
op.update_changes_trie((changes_trie_update, ChangesTrieCacheAction::Clear))
.unwrap();
backend.commit_operation(op).unwrap();
header_hash
@@ -584,11 +630,13 @@ mod tests {
let backend = Backend::<Block>::new_test(1000, 100);
backend.changes_tries_storage.meta.write().finalized_number = 1000;
let check_changes = |backend: &Backend<Block>, block: u64, changes: Vec<(Vec<u8>, Vec<u8>)>| {
let check_changes = |backend: &Backend<Block>,
block: u64,
changes: Vec<(Vec<u8>, Vec<u8>)>| {
let (changes_root, mut changes_trie_update) = prepare_changes(changes);
let anchor = sp_state_machine::ChangesTrieAnchorBlockId {
hash: backend.blockchain().header(BlockId::Number(block)).unwrap().unwrap().hash(),
number: block
number: block,
};
assert_eq!(backend.changes_tries_storage.root(&anchor, block), Ok(Some(changes_root)));
@@ -605,7 +653,13 @@ mod tests {
];
let changes2 = vec![(b"key_at_2".to_vec(), b"val_at_2".to_vec())];
let block0 = insert_header(&backend, 0, Default::default(), Some(changes0.clone()), Default::default());
let block0 = insert_header(
&backend,
0,
Default::default(),
Some(changes0.clone()),
Default::default(),
);
let block1 = insert_header(&backend, 1, block0, Some(changes1.clone()), Default::default());
let _ = insert_header(&backend, 2, block1, Some(changes2.clone()), Default::default());
@@ -622,19 +676,29 @@ mod tests {
let changes0 = vec![(b"k0".to_vec(), b"v0".to_vec())];
let changes1 = vec![(b"k1".to_vec(), b"v1".to_vec())];
let changes2 = vec![(b"k2".to_vec(), b"v2".to_vec())];
let block0 = insert_header(&backend, 0, Default::default(), Some(changes0.clone()), Default::default());
let block0 = insert_header(
&backend,
0,
Default::default(),
Some(changes0.clone()),
Default::default(),
);
let block1 = insert_header(&backend, 1, block0, Some(changes1.clone()), Default::default());
let block2 = insert_header(&backend, 2, block1, Some(changes2.clone()), Default::default());
let changes2_1_0 = vec![(b"k3".to_vec(), b"v3".to_vec())];
let changes2_1_1 = vec![(b"k4".to_vec(), b"v4".to_vec())];
let block2_1_0 = insert_header(&backend, 3, block2, Some(changes2_1_0.clone()), Default::default());
let block2_1_1 = insert_header(&backend, 4, block2_1_0, Some(changes2_1_1.clone()), Default::default());
let block2_1_0 =
insert_header(&backend, 3, block2, Some(changes2_1_0.clone()), Default::default());
let block2_1_1 =
insert_header(&backend, 4, block2_1_0, Some(changes2_1_1.clone()), Default::default());
let changes2_2_0 = vec![(b"k5".to_vec(), b"v5".to_vec())];
let changes2_2_1 = vec![(b"k6".to_vec(), b"v6".to_vec())];
let block2_2_0 = insert_header(&backend, 3, block2, Some(changes2_2_0.clone()), Default::default());
let block2_2_1 = insert_header(&backend, 4, block2_2_0, Some(changes2_2_1.clone()), Default::default());
let block2_2_0 =
insert_header(&backend, 3, block2, Some(changes2_2_0.clone()), Default::default());
let block2_2_1 =
insert_header(&backend, 4, block2_2_0, Some(changes2_2_1.clone()), Default::default());
// finalize block1
backend.changes_tries_storage.meta.write().finalized_number = 1;
@@ -680,7 +744,12 @@ mod tests {
if number == 0 {
Default::default()
} else {
backend.blockchain().header(BlockId::Number(number - 1)).unwrap().unwrap().hash()
backend
.blockchain()
.header(BlockId::Number(number - 1))
.unwrap()
.unwrap()
.hash()
}
};
@@ -698,12 +767,14 @@ mod tests {
let trie_root = backend
.blockchain()
.header(BlockId::Number(number))
.unwrap().unwrap()
.unwrap()
.unwrap()
.digest()
.log(DigestItem::as_changes_trie_root)
.cloned();
match trie_root {
Some(trie_root) => backend.changes_tries_storage.get(&trie_root, EMPTY_PREFIX).unwrap().is_none(),
Some(trie_root) =>
backend.changes_tries_storage.get(&trie_root, EMPTY_PREFIX).unwrap().is_none(),
None => true,
}
};
@@ -711,14 +782,10 @@ mod tests {
let finalize_block = |number| {
let header = backend.blockchain().header(BlockId::Number(number)).unwrap().unwrap();
let mut tx = Transaction::new();
let cache_ops = backend.changes_tries_storage.finalize(
&mut tx,
*header.parent_hash(),
header.hash(),
number,
None,
None,
).unwrap();
let cache_ops = backend
.changes_tries_storage
.finalize(&mut tx, *header.parent_hash(), header.hash(), number, None, None)
.unwrap();
backend.storage.db.commit(tx).unwrap();
backend.changes_tries_storage.post_commit(Some(cache_ops));
};
@@ -737,11 +804,23 @@ mod tests {
(0..6).for_each(|number| insert_regular_header(false, number));
insert_header_with_configuration_change(&backend, 6, parent_hash(6), None, config_at_6);
(7..17).for_each(|number| insert_regular_header(true, number));
insert_header_with_configuration_change(&backend, 17, parent_hash(17), changes(17), config_at_17);
insert_header_with_configuration_change(
&backend,
17,
parent_hash(17),
changes(17),
config_at_17,
);
(18..21).for_each(|number| insert_regular_header(false, number));
insert_header_with_configuration_change(&backend, 21, parent_hash(21), None, config_at_21);
(22..32).for_each(|number| insert_regular_header(true, number));
insert_header_with_configuration_change(&backend, 32, parent_hash(32), changes(32), config_at_32);
insert_header_with_configuration_change(
&backend,
32,
parent_hash(32),
changes(32),
config_at_32,
);
(33..50).for_each(|number| insert_regular_header(true, number));
// when only genesis is finalized, nothing is pruned
@@ -826,29 +905,24 @@ mod tests {
let backend = Backend::<Block>::new_test(1000, 100);
// configurations at blocks
let config_at_1 = Some(ChangesTrieConfiguration {
digest_interval: 4,
digest_levels: 2,
});
let config_at_3 = Some(ChangesTrieConfiguration {
digest_interval: 8,
digest_levels: 1,
});
let config_at_1 = Some(ChangesTrieConfiguration { digest_interval: 4, digest_levels: 2 });
let config_at_3 = Some(ChangesTrieConfiguration { digest_interval: 8, digest_levels: 1 });
let config_at_5 = None;
let config_at_7 = Some(ChangesTrieConfiguration {
digest_interval: 8,
digest_levels: 1,
});
let config_at_7 = Some(ChangesTrieConfiguration { digest_interval: 8, digest_levels: 1 });
// insert some blocks
let block0 = insert_header(&backend, 0, Default::default(), None, Default::default());
let block1 = insert_header_with_configuration_change(&backend, 1, block0, None, config_at_1.clone());
let block1 =
insert_header_with_configuration_change(&backend, 1, block0, None, config_at_1.clone());
let block2 = insert_header(&backend, 2, block1, None, Default::default());
let block3 = insert_header_with_configuration_change(&backend, 3, block2, None, config_at_3.clone());
let block3 =
insert_header_with_configuration_change(&backend, 3, block2, None, config_at_3.clone());
let block4 = insert_header(&backend, 4, block3, None, Default::default());
let block5 = insert_header_with_configuration_change(&backend, 5, block4, None, config_at_5.clone());
let block5 =
insert_header_with_configuration_change(&backend, 5, block4, None, config_at_5.clone());
let block6 = insert_header(&backend, 6, block5, None, Default::default());
let block7 = insert_header_with_configuration_change(&backend, 7, block6, None, config_at_7.clone());
let block7 =
insert_header_with_configuration_change(&backend, 7, block6, None, config_at_7.clone());
// test configuration cache
let storage = &backend.changes_tries_storage;
@@ -887,17 +961,48 @@ mod tests {
let mut backend = Backend::<Block>::new_test(10, 10);
backend.changes_tries_storage.min_blocks_to_keep = Some(8);
let configs = (0..=7).map(|i| Some(ChangesTrieConfiguration::new(2, i))).collect::<Vec<_>>();
let configs =
(0..=7).map(|i| Some(ChangesTrieConfiguration::new(2, i))).collect::<Vec<_>>();
// insert unfinalized headers
let block0 = insert_header_with_configuration_change(&backend, 0, Default::default(), None, configs[0].clone());
let block1 = insert_header_with_configuration_change(&backend, 1, block0, changes(1), configs[1].clone());
let block2 = insert_header_with_configuration_change(&backend, 2, block1, changes(2), configs[2].clone());
let block0 = insert_header_with_configuration_change(
&backend,
0,
Default::default(),
None,
configs[0].clone(),
);
let block1 = insert_header_with_configuration_change(
&backend,
1,
block0,
changes(1),
configs[1].clone(),
);
let block2 = insert_header_with_configuration_change(
&backend,
2,
block1,
changes(2),
configs[2].clone(),
);
let side_config2_1 = Some(ChangesTrieConfiguration::new(3, 2));
let side_config2_2 = Some(ChangesTrieConfiguration::new(3, 3));
let block2_1 = insert_header_with_configuration_change(&backend, 2, block1, changes(8), side_config2_1.clone());
let _ = insert_header_with_configuration_change(&backend, 3, block2_1, changes(9), side_config2_2.clone());
let block2_1 = insert_header_with_configuration_change(
&backend,
2,
block1,
changes(8),
side_config2_1.clone(),
);
let _ = insert_header_with_configuration_change(
&backend,
3,
block2_1,
changes(9),
side_config2_2.clone(),
);
// insert finalized header => 4 headers are finalized at once
let header3 = Header {
@@ -905,9 +1010,9 @@ mod tests {
parent_hash: block2,
state_root: Default::default(),
digest: Digest {
logs: vec![
DigestItem::ChangesTrieSignal(ChangesTrieSignal::NewConfiguration(configs[3].clone())),
],
logs: vec![DigestItem::ChangesTrieSignal(ChangesTrieSignal::NewConfiguration(
configs[3].clone(),
))],
},
extrinsics_root: Default::default(),
};
@@ -920,9 +1025,27 @@ mod tests {
backend.commit_operation(op).unwrap();
// insert more unfinalized headers
let block4 = insert_header_with_configuration_change(&backend, 4, block3, changes(4), configs[4].clone());
let block5 = insert_header_with_configuration_change(&backend, 5, block4, changes(5), configs[5].clone());
let block6 = insert_header_with_configuration_change(&backend, 6, block5, changes(6), configs[6].clone());
let block4 = insert_header_with_configuration_change(
&backend,
4,
block3,
changes(4),
configs[4].clone(),
);
let block5 = insert_header_with_configuration_change(
&backend,
5,
block4,
changes(5),
configs[5].clone(),
);
let block6 = insert_header_with_configuration_change(
&backend,
6,
block5,
changes(6),
configs[6].clone(),
);
// insert finalized header => 4 headers are finalized at once
let header7 = Header {
@@ -930,9 +1053,9 @@ mod tests {
parent_hash: block6,
state_root: Default::default(),
digest: Digest {
logs: vec![
DigestItem::ChangesTrieSignal(ChangesTrieSignal::NewConfiguration(configs[7].clone())),
],
logs: vec![DigestItem::ChangesTrieSignal(ChangesTrieSignal::NewConfiguration(
configs[7].clone(),
))],
},
extrinsics_root: Default::default(),
};
@@ -950,23 +1073,33 @@ mod tests {
let backend = Backend::<Block>::new_test(10, 10);
let config0 = Some(ChangesTrieConfiguration::new(2, 5));
let block0 = insert_header_with_configuration_change(&backend, 0, Default::default(), None, config0);
let block0 =
insert_header_with_configuration_change(&backend, 0, Default::default(), None, config0);
let config1 = Some(ChangesTrieConfiguration::new(2, 6));
let block1 = insert_header_with_configuration_change(&backend, 1, block0, changes(0), config1);
let block1 =
insert_header_with_configuration_change(&backend, 1, block0, changes(0), config1);
let just1 = Some((*b"TEST", vec![42]));
backend.finalize_block(BlockId::Number(1), just1).unwrap();
let config2 = Some(ChangesTrieConfiguration::new(2, 7));
let block2 = insert_header_with_configuration_change(&backend, 2, block1, changes(1), config2);
let block2 =
insert_header_with_configuration_change(&backend, 2, block1, changes(1), config2);
let config2_1 = Some(ChangesTrieConfiguration::new(2, 8));
let _ = insert_header_with_configuration_change(&backend, 3, block2, changes(10), config2_1);
let _ =
insert_header_with_configuration_change(&backend, 3, block2, changes(10), config2_1);
let config2_2 = Some(ChangesTrieConfiguration::new(2, 9));
let block2_2 = insert_header_with_configuration_change(&backend, 3, block2, changes(20), config2_2);
let block2_2 =
insert_header_with_configuration_change(&backend, 3, block2, changes(20), config2_2);
let config2_3 = Some(ChangesTrieConfiguration::new(2, 10));
let _ = insert_header_with_configuration_change(&backend, 4, block2_2, changes(30), config2_3);
let _ =
insert_header_with_configuration_change(&backend, 4, block2_2, changes(30), config2_3);
// before truncate there are 2 unfinalized forks - block2_1+block2_3
assert_eq!(
backend.changes_tries_storage.cache.0.write()
backend
.changes_tries_storage
.cache
.0
.write()
.get_cache(well_known_cache_keys::CHANGES_TRIE_CONFIG)
.unwrap()
.unfinalized()
@@ -979,7 +1112,11 @@ mod tests {
// after truncating block2_3 - there are 2 unfinalized forks - block2_1+block2_2
backend.revert(1, false).unwrap();
assert_eq!(
backend.changes_tries_storage.cache.0.write()
backend
.changes_tries_storage
.cache
.0
.write()
.get_cache(well_known_cache_keys::CHANGES_TRIE_CONFIG)
.unwrap()
.unfinalized()
@@ -993,7 +1130,11 @@ mod tests {
// the 1st one points to the block #3 because it isn't truncated
backend.revert(1, false).unwrap();
assert_eq!(
backend.changes_tries_storage.cache.0.write()
backend
.changes_tries_storage
.cache
.0
.write()
.get_cache(well_known_cache_keys::CHANGES_TRIE_CONFIG)
.unwrap()
.unfinalized()
@@ -1005,15 +1146,17 @@ mod tests {
// after truncating block2 - there are no unfinalized forks
backend.revert(1, false).unwrap();
assert!(
backend.changes_tries_storage.cache.0.write()
.get_cache(well_known_cache_keys::CHANGES_TRIE_CONFIG)
.unwrap()
.unfinalized()
.iter()
.map(|fork| fork.head().valid_from.number)
.collect::<Vec<_>>()
.is_empty(),
);
assert!(backend
.changes_tries_storage
.cache
.0
.write()
.get_cache(well_known_cache_keys::CHANGES_TRIE_CONFIG)
.unwrap()
.unfinalized()
.iter()
.map(|fork| fork.head().valid_from.number)
.collect::<Vec<_>>()
.is_empty(),);
}
}
substrate/client/db/src/children.rs
+11 -9
View File
@@ -18,17 +18,22 @@
//! Functionality for reading and storing children hashes from db.
use codec::{Encode, Decode};
use sp_blockchain;
use std::hash::Hash;
use sp_database::{Database, Transaction};
use crate::DbHash;
use codec::{Decode, Encode};
use sp_blockchain;
use sp_database::{Database, Transaction};
use std::hash::Hash;
/// Returns the hashes of the children blocks of the block with `parent_hash`.
pub fn read_children<
K: Eq + Hash + Clone + Encode + Decode,
V: Eq + Hash + Clone + Encode + Decode,
>(db: &dyn Database<DbHash>, column: u32, prefix: &[u8], parent_hash: K) -> sp_blockchain::Result<Vec<V>> {
>(
db: &dyn Database<DbHash>,
column: u32,
prefix: &[u8],
parent_hash: K,
) -> sp_blockchain::Result<Vec<V>> {
let mut buf = prefix.to_vec();
parent_hash.using_encoded(|s| buf.extend(s));
@@ -65,9 +70,7 @@ pub fn write_children<
}
/// Prepare transaction to remove the children of `parent_hash`.
pub fn remove_children<
K: Eq + Hash + Clone + Encode + Decode,
>(
pub fn remove_children<K: Eq + Hash + Clone + Encode + Decode>(
tx: &mut Transaction<DbHash>,
column: u32,
prefix: &[u8],
@@ -78,7 +81,6 @@ pub fn remove_children<
tx.remove(column, &key);
}
#[cfg(test)]
mod tests {
use super::*;
substrate/client/db/src/lib.rs
+655 -631
View File
File diff suppressed because it is too large Load Diff
substrate/client/db/src/light.rs
+294 -203
View File
@@ -18,31 +18,31 @@
//! RocksDB-based light client blockchain storage.
use std::{sync::Arc, collections::HashMap};
use std::convert::TryInto;
use parking_lot::RwLock;
use std::{collections::HashMap, convert::TryInto, sync::Arc};
use crate::{
cache::{ComplexBlockId, DbCache, DbCacheSync, EntryType as CacheEntryType},
utils::{self, block_id_to_lookup_key, meta_keys, read_db, read_meta, DatabaseType, Meta},
DatabaseSettings, DbHash, FrozenForDuration,
};
use codec::{Decode, Encode};
use log::{debug, trace, warn};
use sc_client_api::{
cht, backend::{AuxStore, NewBlockState, ProvideChtRoots}, UsageInfo,
blockchain::{
BlockStatus, Cache as BlockchainCache, Info as BlockchainInfo,
},
Storage,
backend::{AuxStore, NewBlockState, ProvideChtRoots},
blockchain::{BlockStatus, Cache as BlockchainCache, Info as BlockchainInfo},
cht, Storage, UsageInfo,
};
use sp_blockchain::{
CachedHeaderMetadata, HeaderMetadata, HeaderMetadataCache,
Error as ClientError, Result as ClientResult,
HeaderBackend as BlockchainHeaderBackend,
well_known_cache_keys,
well_known_cache_keys, CachedHeaderMetadata, Error as ClientError,
HeaderBackend as BlockchainHeaderBackend, HeaderMetadata, HeaderMetadataCache,
Result as ClientResult,
};
use sp_database::{Database, Transaction};
use codec::{Decode, Encode};
use sp_runtime::generic::{DigestItem, BlockId};
use sp_runtime::traits::{Block as BlockT, Header as HeaderT, Zero, One, NumberFor, HashFor};
use crate::cache::{DbCacheSync, DbCache, ComplexBlockId, EntryType as CacheEntryType};
use crate::utils::{self, meta_keys, DatabaseType, Meta, read_db, block_id_to_lookup_key, read_meta};
use crate::{DatabaseSettings, FrozenForDuration, DbHash};
use log::{trace, warn, debug};
use sp_runtime::{
generic::{BlockId, DigestItem},
traits::{Block as BlockT, HashFor, Header as HeaderT, NumberFor, One, Zero},
};
pub(crate) mod columns {
pub const META: u32 = crate::utils::COLUMN_META;
@@ -139,8 +139,8 @@ impl<Block: BlockT> LightStorage<Block> {
}
impl<Block> BlockchainHeaderBackend<Block> for LightStorage<Block>
where
Block: BlockT,
where
Block: BlockT,
{
fn header(&self, id: BlockId<Block>) -> ClientResult<Option<Block::Header>> {
utils::read_header(&*self.db, columns::KEY_LOOKUP, columns::HEADER, id)
@@ -165,12 +165,8 @@ impl<Block> BlockchainHeaderBackend<Block> for LightStorage<Block>
fn status(&self, id: BlockId<Block>) -> ClientResult<BlockStatus> {
let exists = match id {
BlockId::Hash(_) => read_db(
&*self.db,
columns::KEY_LOOKUP,
columns::HEADER,
id
)?.is_some(),
BlockId::Hash(_) =>
read_db(&*self.db, columns::KEY_LOOKUP, columns::HEADER, id)?.is_some(),
BlockId::Number(n) => n <= self.meta.read().best_number,
};
match exists {
@@ -180,7 +176,9 @@ impl<Block> BlockchainHeaderBackend<Block> for LightStorage<Block>
}
fn number(&self, hash: Block::Hash) -> ClientResult<Option<NumberFor<Block>>> {
if let Some(lookup_key) = block_id_to_lookup_key::<Block>(&*self.db, columns::KEY_LOOKUP, BlockId::Hash(hash))? {
if let Some(lookup_key) =
block_id_to_lookup_key::<Block>(&*self.db, columns::KEY_LOOKUP, BlockId::Hash(hash))?
{
let number = utils::lookup_key_to_number(&lookup_key)?;
Ok(Some(number))
} else {
@@ -196,17 +194,25 @@ impl<Block> BlockchainHeaderBackend<Block> for LightStorage<Block>
impl<Block: BlockT> HeaderMetadata<Block> for LightStorage<Block> {
type Error = ClientError;
fn header_metadata(&self, hash: Block::Hash) -> Result<CachedHeaderMetadata<Block>, Self::Error> {
self.header_metadata_cache.header_metadata(hash).map_or_else(|| {
self.header(BlockId::hash(hash))?.map(|header| {
let header_metadata = CachedHeaderMetadata::from(&header);
self.header_metadata_cache.insert_header_metadata(
header_metadata.hash,
header_metadata.clone(),
);
header_metadata
}).ok_or_else(|| ClientError::UnknownBlock(format!("header not found in db: {}", hash)))
}, Ok)
fn header_metadata(
&self,
hash: Block::Hash,
) -> Result<CachedHeaderMetadata<Block>, Self::Error> {
self.header_metadata_cache.header_metadata(hash).map_or_else(
|| {
self.header(BlockId::hash(hash))?
.map(|header| {
let header_metadata = CachedHeaderMetadata::from(&header);
self.header_metadata_cache
.insert_header_metadata(header_metadata.hash, header_metadata.clone());
header_metadata
})
.ok_or_else(|| {
ClientError::UnknownBlock(format!("header not found in db: {}", hash))
})
},
Ok,
)
}
fn insert_header_metadata(&self, hash: Block::Hash, metadata: CachedHeaderMetadata<Block>) {
@@ -221,10 +227,9 @@ impl<Block: BlockT> HeaderMetadata<Block> for LightStorage<Block> {
impl<Block: BlockT> LightStorage<Block> {
// Get block changes trie root, if available.
fn changes_trie_root(&self, block: BlockId<Block>) -> ClientResult<Option<Block::Hash>> {
self.header(block)
.map(|header| header.and_then(|header|
header.digest().log(DigestItem::as_changes_trie_root)
.cloned()))
self.header(block).map(|header| {
header.and_then(|header| header.digest().log(DigestItem::as_changes_trie_root).cloned())
})
}
/// Handle setting head within a transaction. `route_to` should be the last
@@ -251,14 +256,16 @@ impl<Block: BlockT> LightStorage<Block> {
for retracted in tree_route.retracted() {
if retracted.hash == meta.finalized_hash {
// TODO: can we recover here?
warn!("Safety failure: reverting finalized block {:?}",
(&retracted.number, &retracted.hash));
warn!(
"Safety failure: reverting finalized block {:?}",
(&retracted.number, &retracted.hash)
);
}
utils::remove_number_to_key_mapping(
transaction,
columns::KEY_LOOKUP,
retracted.number
retracted.number,
)?;
}
@@ -267,7 +274,7 @@ impl<Block: BlockT> LightStorage<Block> {
transaction,
columns::KEY_LOOKUP,
enacted.number,
enacted.hash
enacted.hash,
)?;
}
}
@@ -292,10 +299,11 @@ impl<Block: BlockT> LightStorage<Block> {
) -> ClientResult<()> {
let meta = self.meta.read();
if &meta.finalized_hash != header.parent_hash() {
return Err(::sp_blockchain::Error::NonSequentialFinalization(
format!("Last finalized {:?} not parent of {:?}",
meta.finalized_hash, hash),
).into())
return Err(::sp_blockchain::Error::NonSequentialFinalization(format!(
"Last finalized {:?} not parent of {:?}",
meta.finalized_hash, hash
))
.into())
}
let lookup_key = utils::number_and_hash_to_lookup_key(header.number().clone(), hash)?;
@@ -313,12 +321,14 @@ impl<Block: BlockT> LightStorage<Block> {
});
let new_header_cht_root = cht::compute_root::<Block::Header, HashFor<Block>, _>(
cht::size(), new_cht_number, cht_range.map(|num| self.hash(num))
cht::size(),
new_cht_number,
cht_range.map(|num| self.hash(num)),
)?;
transaction.set(
columns::CHT,
&cht_key(HEADER_CHT_PREFIX, new_cht_start)?,
new_header_cht_root.as_ref()
new_header_cht_root.as_ref(),
);
// if the header includes changes trie root, let's build a changes tries roots CHT
@@ -329,14 +339,16 @@ impl<Block: BlockT> LightStorage<Block> {
current_num = current_num + One::one();
Some(old_current_num)
});
let new_changes_trie_cht_root = cht::compute_root::<Block::Header, HashFor<Block>, _>(
cht::size(), new_cht_number, cht_range
.map(|num| self.changes_trie_root(BlockId::Number(num)))
)?;
let new_changes_trie_cht_root =
cht::compute_root::<Block::Header, HashFor<Block>, _>(
cht::size(),
new_cht_number,
cht_range.map(|num| self.changes_trie_root(BlockId::Number(num))),
)?;
transaction.set(
columns::CHT,
&cht_key(CHANGES_TRIE_CHT_PREFIX, new_cht_start)?,
new_changes_trie_cht_root.as_ref()
new_changes_trie_cht_root.as_ref(),
);
}
@@ -354,7 +366,7 @@ impl<Block: BlockT> LightStorage<Block> {
transaction,
columns::KEY_LOOKUP,
prune_block,
hash
hash,
)?;
transaction.remove(columns::HEADER, &lookup_key);
}
@@ -370,7 +382,7 @@ impl<Block: BlockT> LightStorage<Block> {
&self,
cht_type: u8,
cht_size: NumberFor<Block>,
block: NumberFor<Block>
block: NumberFor<Block>,
) -> ClientResult<Option<Block::Hash>> {
let no_cht_for_block = || ClientError::Backend(format!("Missing CHT for block {}", block));
@@ -383,7 +395,8 @@ impl<Block: BlockT> LightStorage<Block> {
}
let cht_start = cht::start_number(cht_size, cht_number);
self.db.get(columns::CHT, &cht_key(cht_type, cht_start)?)
self.db
.get(columns::CHT, &cht_key(cht_type, cht_start)?)
.ok_or_else(no_cht_for_block)
.and_then(|hash| Block::Hash::decode(&mut &*hash).map_err(|_| no_cht_for_block()))
.map(Some)
@@ -391,15 +404,20 @@ impl<Block: BlockT> LightStorage<Block> {
}
impl<Block> AuxStore for LightStorage<Block>
where Block: BlockT,
where
Block: BlockT,
{
fn insert_aux<
'a,
'b: 'a,
'c: 'a,
I: IntoIterator<Item=&'a(&'c [u8], &'c [u8])>,
D: IntoIterator<Item=&'a &'b [u8]>,
>(&self, insert: I, delete: D) -> ClientResult<()> {
I: IntoIterator<Item = &'a (&'c [u8], &'c [u8])>,
D: IntoIterator<Item = &'a &'b [u8]>,
>(
&self,
insert: I,
delete: D,
) -> ClientResult<()> {
let mut transaction = Transaction::new();
for (k, v) in insert {
transaction.set(columns::AUX, k, v);
@@ -418,7 +436,8 @@ impl<Block> AuxStore for LightStorage<Block>
}
impl<Block> Storage<Block> for LightStorage<Block>
where Block: BlockT,
where
Block: BlockT,
{
fn import_header(
&self,
@@ -447,19 +466,12 @@ impl<Block> Storage<Block> for LightStorage<Block>
self.set_head_with_transaction(&mut transaction, parent_hash, (number, hash))?;
}
utils::insert_hash_to_key_mapping(
&mut transaction,
columns::KEY_LOOKUP,
number,
hash,
)?;
utils::insert_hash_to_key_mapping(&mut transaction, columns::KEY_LOOKUP, number, hash)?;
transaction.set_from_vec(columns::HEADER, &lookup_key, header.encode());
let header_metadata = CachedHeaderMetadata::from(&header);
self.header_metadata_cache.insert_header_metadata(
header.hash().clone(),
header_metadata,
);
self.header_metadata_cache
.insert_header_metadata(header.hash().clone(), header_metadata);
let is_genesis = number.is_zero();
if is_genesis {
@@ -474,25 +486,28 @@ impl<Block> Storage<Block> for LightStorage<Block>
};
if finalized {
self.note_finalized(
&mut transaction,
&header,
hash,
)?;
self.note_finalized(&mut transaction, &header, hash)?;
}
// update changes trie configuration cache
if !cache_at.contains_key(&well_known_cache_keys::CHANGES_TRIE_CONFIG) {
if let Some(new_configuration) = crate::changes_tries_storage::extract_new_configuration(&header) {
cache_at.insert(well_known_cache_keys::CHANGES_TRIE_CONFIG, new_configuration.encode());
if let Some(new_configuration) =
crate::changes_tries_storage::extract_new_configuration(&header)
{
cache_at
.insert(well_known_cache_keys::CHANGES_TRIE_CONFIG, new_configuration.encode());
}
}
{
let mut cache = self.cache.0.write();
let cache_ops = cache.transaction(&mut transaction)
let cache_ops = cache
.transaction(&mut transaction)
.on_block_insert(
ComplexBlockId::new(*header.parent_hash(), if number.is_zero() { Zero::zero() } else { number - One::one() }),
ComplexBlockId::new(
*header.parent_hash(),
if number.is_zero() { Zero::zero() } else { number - One::one() },
),
ComplexBlockId::new(hash, number),
cache_at,
if finalized { CacheEntryType::Final } else { CacheEntryType::NonFinal },
@@ -502,9 +517,10 @@ impl<Block> Storage<Block> for LightStorage<Block>
debug!("Light DB Commit {:?} ({})", hash, number);
self.db.commit(transaction)?;
cache.commit(cache_ops)
.expect("only fails if cache with given name isn't loaded yet;\
cache is already loaded because there are cache_ops; qed");
cache.commit(cache_ops).expect(
"only fails if cache with given name isn't loaded yet;\
cache is already loaded because there are cache_ops; qed",
);
}
self.update_meta(hash, number, leaf_state.is_best(), finalized);
@@ -518,7 +534,11 @@ impl<Block> Storage<Block> for LightStorage<Block>
let number = header.number();
let mut transaction = Transaction::new();
self.set_head_with_transaction(&mut transaction, hash.clone(), (number.clone(), hash.clone()))?;
self.set_head_with_transaction(
&mut transaction,
hash.clone(),
(number.clone(), hash.clone()),
)?;
self.db.commit(transaction)?;
self.update_meta(hash, header.number().clone(), true, false);
@@ -536,17 +556,22 @@ impl<Block> Storage<Block> for LightStorage<Block>
self.note_finalized(&mut transaction, &header, hash.clone())?;
{
let mut cache = self.cache.0.write();
let cache_ops = cache.transaction(&mut transaction)
let cache_ops = cache
.transaction(&mut transaction)
.on_block_finalize(
ComplexBlockId::new(*header.parent_hash(), if number.is_zero() { Zero::zero() } else { number - One::one() }),
ComplexBlockId::new(hash, number)
ComplexBlockId::new(
*header.parent_hash(),
if number.is_zero() { Zero::zero() } else { number - One::one() },
),
ComplexBlockId::new(hash, number),
)?
.into_ops();
self.db.commit(transaction)?;
cache.commit(cache_ops)
.expect("only fails if cache with given name isn't loaded yet;\
cache is already loaded because there are cache_ops; qed");
cache.commit(cache_ops).expect(
"only fails if cache with given name isn't loaded yet;\
cache is already loaded because there are cache_ops; qed",
);
}
self.update_meta(hash, header.number().clone(), false, true);
@@ -566,7 +591,7 @@ impl<Block> Storage<Block> for LightStorage<Block>
#[cfg(not(target_os = "unknown"))]
fn usage_info(&self) -> Option<UsageInfo> {
use sc_client_api::{MemoryInfo, IoInfo, MemorySize};
use sc_client_api::{IoInfo, MemoryInfo, MemorySize};
// TODO: reimplement IO stats
let database_cache = MemorySize::from_bytes(0);
@@ -591,7 +616,7 @@ impl<Block> Storage<Block> for LightStorage<Block>
state_reads_cache: 0,
state_writes_cache: 0,
state_writes_nodes: 0,
}
},
})
}
@@ -602,7 +627,8 @@ impl<Block> Storage<Block> for LightStorage<Block>
}
impl<Block> ProvideChtRoots<Block> for LightStorage<Block>
where Block: BlockT,
where
Block: BlockT,
{
fn header_cht_root(
&self,
@@ -630,12 +656,14 @@ fn cht_key<N: TryInto<u32>>(cht_type: u8, block: N) -> ClientResult<[u8; 5]> {
#[cfg(test)]
pub(crate) mod tests {
use sc_client_api::cht;
use sp_core::ChangesTrieConfiguration;
use sp_runtime::generic::{DigestItem, ChangesTrieSignal};
use sp_runtime::testing::{H256 as Hash, Header, Block as RawBlock, ExtrinsicWrapper};
use sp_blockchain::{lowest_common_ancestor, tree_route};
use super::*;
use sc_client_api::cht;
use sp_blockchain::{lowest_common_ancestor, tree_route};
use sp_core::ChangesTrieConfiguration;
use sp_runtime::{
generic::{ChangesTrieSignal, DigestItem},
testing::{Block as RawBlock, ExtrinsicWrapper, Header, H256 as Hash},
};
type Block = RawBlock<ExtrinsicWrapper<u32>>;
type AuthorityId = sp_core::ed25519::Public;
@@ -652,7 +680,10 @@ pub(crate) mod tests {
fn header_with_changes_trie(parent: &Hash, number: u64) -> Header {
let mut header = default_header(parent, number);
header.digest.logs.push(DigestItem::ChangesTrieRoot([(number % 256) as u8; 32].into()));
header
.digest
.logs
.push(DigestItem::ChangesTrieRoot([(number % 256) as u8; 32].into()));
header
}
@@ -698,7 +729,8 @@ pub(crate) mod tests {
#[test]
fn returns_known_header() {
let db = LightStorage::new_test();
let known_hash = insert_block(&db, HashMap::new(), || default_header(&Default::default(), 0));
let known_hash =
insert_block(&db, HashMap::new(), || default_header(&Default::default(), 0));
let header_by_hash = db.header(BlockId::Hash(known_hash)).unwrap().unwrap();
let header_by_number = db.header(BlockId::Number(0)).unwrap().unwrap();
assert_eq!(header_by_hash, header_by_number);
@@ -714,7 +746,8 @@ pub(crate) mod tests {
#[test]
fn returns_info() {
let db = LightStorage::new_test();
let genesis_hash = insert_block(&db, HashMap::new(), || default_header(&Default::default(), 0));
let genesis_hash =
insert_block(&db, HashMap::new(), || default_header(&Default::default(), 0));
let info = db.info();
assert_eq!(info.best_hash, genesis_hash);
assert_eq!(info.best_number, 0);
@@ -729,17 +762,22 @@ pub(crate) mod tests {
#[test]
fn returns_block_status() {
let db = LightStorage::new_test();
let genesis_hash = insert_block(&db, HashMap::new(), || default_header(&Default::default(), 0));
let genesis_hash =
insert_block(&db, HashMap::new(), || default_header(&Default::default(), 0));
assert_eq!(db.status(BlockId::Hash(genesis_hash)).unwrap(), BlockStatus::InChain);
assert_eq!(db.status(BlockId::Number(0)).unwrap(), BlockStatus::InChain);
assert_eq!(db.status(BlockId::Hash(Hash::from_low_u64_be(1))).unwrap(), BlockStatus::Unknown);
assert_eq!(
db.status(BlockId::Hash(Hash::from_low_u64_be(1))).unwrap(),
BlockStatus::Unknown
);
assert_eq!(db.status(BlockId::Number(1)).unwrap(), BlockStatus::Unknown);
}
#[test]
fn returns_block_hash() {
let db = LightStorage::new_test();
let genesis_hash = insert_block(&db, HashMap::new(), || default_header(&Default::default(), 0));
let genesis_hash =
insert_block(&db, HashMap::new(), || default_header(&Default::default(), 0));
assert_eq!(db.hash(0).unwrap(), Some(genesis_hash));
assert_eq!(db.hash(1).unwrap(), None);
}
@@ -749,7 +787,8 @@ pub(crate) mod tests {
let raw_db = Arc::new(sp_database::MemDb::default());
let db = LightStorage::from_kvdb(raw_db.clone()).unwrap();
let genesis_hash = insert_block(&db, HashMap::new(), || default_header(&Default::default(), 0));
let genesis_hash =
insert_block(&db, HashMap::new(), || default_header(&Default::default(), 0));
assert_eq!(raw_db.count(columns::HEADER), 1);
assert_eq!(raw_db.count(columns::KEY_LOOKUP), 2);
@@ -760,43 +799,41 @@ pub(crate) mod tests {
#[test]
fn finalized_ancient_headers_are_replaced_with_cht() {
fn insert_headers<F: Fn(&Hash, u64) -> Header>(header_producer: F) ->
(Arc<sp_database::MemDb>, LightStorage<Block>)
{
fn insert_headers<F: Fn(&Hash, u64) -> Header>(
header_producer: F,
) -> (Arc<sp_database::MemDb>, LightStorage<Block>) {
let raw_db = Arc::new(sp_database::MemDb::default());
let db = LightStorage::from_kvdb(raw_db.clone()).unwrap();
let cht_size: u64 = cht::size();
let ucht_size: usize = cht_size as _;
// insert genesis block header (never pruned)
let mut prev_hash = insert_final_block(&db, HashMap::new(), || header_producer(&Default::default(), 0));
let mut prev_hash =
insert_final_block(&db, HashMap::new(), || header_producer(&Default::default(), 0));
// insert SIZE blocks && ensure that nothing is pruned
for number in 0..cht::size() {
prev_hash = insert_block(&db, HashMap::new(), || header_producer(&prev_hash, 1 + number));
prev_hash =
insert_block(&db, HashMap::new(), || header_producer(&prev_hash, 1 + number));
}
assert_eq!(raw_db.count(columns::HEADER), 1 + ucht_size);
assert_eq!(raw_db.count(columns::CHT), 0);
// insert next SIZE blocks && ensure that nothing is pruned
for number in 0..(cht_size as _) {
prev_hash = insert_block(
&db,
HashMap::new(),
|| header_producer(&prev_hash, 1 + cht_size + number),
);
prev_hash = insert_block(&db, HashMap::new(), || {
header_producer(&prev_hash, 1 + cht_size + number)
});
}
assert_eq!(raw_db.count(columns::HEADER), 1 + ucht_size + ucht_size);
assert_eq!(raw_db.count(columns::CHT), 0);
// insert block #{2 * cht::size() + 1} && check that new CHT is created + headers of this CHT are pruned
// nothing is yet finalized, so nothing is pruned.
prev_hash = insert_block(
&db,
HashMap::new(),
|| header_producer(&prev_hash, 1 + cht_size + cht_size),
);
prev_hash = insert_block(&db, HashMap::new(), || {
header_producer(&prev_hash, 1 + cht_size + cht_size)
});
assert_eq!(raw_db.count(columns::HEADER), 2 + ucht_size + ucht_size);
assert_eq!(raw_db.count(columns::CHT), 0);
@@ -839,7 +876,10 @@ pub(crate) mod tests {
#[test]
fn get_cht_fails_for_non_existent_cht() {
let cht_size: u64 = cht::size();
assert!(LightStorage::<Block>::new_test().header_cht_root(cht_size, cht_size / 2).unwrap().is_none());
assert!(LightStorage::<Block>::new_test()
.header_cht_root(cht_size, cht_size / 2)
.unwrap()
.is_none());
}
#[test]
@@ -847,26 +887,41 @@ pub(crate) mod tests {
let db = LightStorage::new_test();
// insert 1 + SIZE + SIZE + 1 blocks so that CHT#0 is created
let mut prev_hash = insert_final_block(&db, HashMap::new(), || header_with_changes_trie(&Default::default(), 0));
let mut prev_hash = insert_final_block(&db, HashMap::new(), || {
header_with_changes_trie(&Default::default(), 0)
});
let cht_size: u64 = cht::size();
let ucht_size: usize = cht_size as _;
for i in 1..1 + ucht_size + ucht_size + 1 {
prev_hash = insert_block(&db, HashMap::new(), || header_with_changes_trie(&prev_hash, i as u64));
prev_hash = insert_block(&db, HashMap::new(), || {
header_with_changes_trie(&prev_hash, i as u64)
});
db.finalize_header(BlockId::Hash(prev_hash)).unwrap();
}
let cht_root_1 = db.header_cht_root(cht_size, cht::start_number(cht_size, 0)).unwrap().unwrap();
let cht_root_2 = db.header_cht_root(cht_size, cht::start_number(cht_size, 0) + cht_size / 2).unwrap().unwrap();
let cht_root_3 = db.header_cht_root(cht_size, cht::end_number(cht_size, 0)).unwrap().unwrap();
let cht_root_1 =
db.header_cht_root(cht_size, cht::start_number(cht_size, 0)).unwrap().unwrap();
let cht_root_2 = db
.header_cht_root(cht_size, cht::start_number(cht_size, 0) + cht_size / 2)
.unwrap()
.unwrap();
let cht_root_3 =
db.header_cht_root(cht_size, cht::end_number(cht_size, 0)).unwrap().unwrap();
assert_eq!(cht_root_1, cht_root_2);
assert_eq!(cht_root_2, cht_root_3);
let cht_root_1 = db.changes_trie_cht_root(cht_size, cht::start_number(cht_size, 0)).unwrap().unwrap();
let cht_root_2 = db.changes_trie_cht_root(
cht_size,
cht::start_number(cht_size, 0) + cht_size / 2,
).unwrap().unwrap();
let cht_root_3 = db.changes_trie_cht_root(cht_size, cht::end_number(cht_size, 0)).unwrap().unwrap();
let cht_root_1 = db
.changes_trie_cht_root(cht_size, cht::start_number(cht_size, 0))
.unwrap()
.unwrap();
let cht_root_2 = db
.changes_trie_cht_root(cht_size, cht::start_number(cht_size, 0) + cht_size / 2)
.unwrap()
.unwrap();
let cht_root_3 = db
.changes_trie_cht_root(cht_size, cht::end_number(cht_size, 0))
.unwrap()
.unwrap();
assert_eq!(cht_root_1, cht_root_2);
assert_eq!(cht_root_2, cht_root_3);
}
@@ -882,15 +937,23 @@ pub(crate) mod tests {
let a3 = insert_block(&db, HashMap::new(), || default_header(&a2, 3));
// fork from genesis: 2 prong.
let b1 = insert_block(&db, HashMap::new(), || header_with_extrinsics_root(&block0, 1, Hash::from([1; 32])));
let b1 = insert_block(&db, HashMap::new(), || {
header_with_extrinsics_root(&block0, 1, Hash::from([1; 32]))
});
let b2 = insert_block(&db, HashMap::new(), || default_header(&b1, 2));
{
let tree_route = tree_route(&db, a3, b2).unwrap();
assert_eq!(tree_route.common_block().hash, block0);
assert_eq!(tree_route.retracted().iter().map(|r| r.hash).collect::<Vec<_>>(), vec![a3, a2, a1]);
assert_eq!(tree_route.enacted().iter().map(|r| r.hash).collect::<Vec<_>>(), vec![b1, b2]);
assert_eq!(
tree_route.retracted().iter().map(|r| r.hash).collect::<Vec<_>>(),
vec![a3, a2, a1]
);
assert_eq!(
tree_route.enacted().iter().map(|r| r.hash).collect::<Vec<_>>(),
vec![b1, b2]
);
}
{
@@ -898,14 +961,20 @@ pub(crate) mod tests {
assert_eq!(tree_route.common_block().hash, a1);
assert!(tree_route.retracted().is_empty());
assert_eq!(tree_route.enacted().iter().map(|r| r.hash).collect::<Vec<_>>(), vec![a2, a3]);
assert_eq!(
tree_route.enacted().iter().map(|r| r.hash).collect::<Vec<_>>(),
vec![a2, a3]
);
}
{
let tree_route = tree_route(&db, a3, a1).unwrap();
assert_eq!(tree_route.common_block().hash, a1);
assert_eq!(tree_route.retracted().iter().map(|r| r.hash).collect::<Vec<_>>(), vec![a3, a2]);
assert_eq!(
tree_route.retracted().iter().map(|r| r.hash).collect::<Vec<_>>(),
vec![a3, a2]
);
assert!(tree_route.enacted().is_empty());
}
@@ -929,7 +998,9 @@ pub(crate) mod tests {
let a3 = insert_block(&db, HashMap::new(), || default_header(&a2, 3));
// fork from genesis: 2 prong.
let b1 = insert_block(&db, HashMap::new(), || header_with_extrinsics_root(&block0, 1, Hash::from([1; 32])));
let b1 = insert_block(&db, HashMap::new(), || {
header_with_extrinsics_root(&block0, 1, Hash::from([1; 32]))
});
let b2 = insert_block(&db, HashMap::new(), || default_header(&b1, 2));
{
@@ -979,7 +1050,11 @@ pub(crate) mod tests {
fn authorities_are_cached() {
let db = LightStorage::new_test();
fn run_checks(db: &LightStorage<Block>, max: u64, checks: &[(u64, Option<Vec<AuthorityId>>)]) {
fn run_checks(
db: &LightStorage<Block>,
max: u64,
checks: &[(u64, Option<Vec<AuthorityId>>)],
) {
for (at, expected) in checks.iter().take_while(|(at, _)| *at <= max) {
let actual = authorities(db.cache(), BlockId::Number(*at));
assert_eq!(*expected, actual);
@@ -990,14 +1065,21 @@ pub(crate) mod tests {
HashMap::new()
}
fn make_authorities(authorities: Vec<AuthorityId>) -> HashMap<well_known_cache_keys::Id, Vec<u8>> {
fn make_authorities(
authorities: Vec<AuthorityId>,
) -> HashMap<well_known_cache_keys::Id, Vec<u8>> {
let mut map = HashMap::new();
map.insert(well_known_cache_keys::AUTHORITIES, authorities.encode());
map
}
fn authorities(cache: &dyn BlockchainCache<Block>, at: BlockId<Block>) -> Option<Vec<AuthorityId>> {
cache.get_at(&well_known_cache_keys::AUTHORITIES, &at).unwrap_or(None)
fn authorities(
cache: &dyn BlockchainCache<Block>,
at: BlockId<Block>,
) -> Option<Vec<AuthorityId>> {
cache
.get_at(&well_known_cache_keys::AUTHORITIES, &at)
.unwrap_or(None)
.and_then(|(_, _, val)| Decode::decode(&mut &val[..]).ok())
}
@@ -1021,17 +1103,27 @@ pub(crate) mod tests {
(6, Some(vec![auth1(), auth2()])),
];
let hash0 = insert_final_block(&db, same_authorities(), || default_header(&Default::default(), 0));
let hash0 = insert_final_block(&db, same_authorities(), || {
default_header(&Default::default(), 0)
});
run_checks(&db, 0, &checks);
let hash1 = insert_final_block(&db, same_authorities(), || default_header(&hash0, 1));
run_checks(&db, 1, &checks);
let hash2 = insert_final_block(&db, make_authorities(vec![auth1()]), || default_header(&hash1, 2));
let hash2 = insert_final_block(&db, make_authorities(vec![auth1()]), || {
default_header(&hash1, 2)
});
run_checks(&db, 2, &checks);
let hash3 = insert_final_block(&db, make_authorities(vec![auth1()]), || default_header(&hash2, 3));
let hash3 = insert_final_block(&db, make_authorities(vec![auth1()]), || {
default_header(&hash2, 3)
});
run_checks(&db, 3, &checks);
let hash4 = insert_final_block(&db, make_authorities(vec![auth1(), auth2()]), || default_header(&hash3, 4));
let hash4 = insert_final_block(&db, make_authorities(vec![auth1(), auth2()]), || {
default_header(&hash3, 4)
});
run_checks(&db, 4, &checks);
let hash5 = insert_final_block(&db, make_authorities(vec![auth1(), auth2()]), || default_header(&hash4, 5));
let hash5 = insert_final_block(&db, make_authorities(vec![auth1(), auth2()]), || {
default_header(&hash4, 5)
});
run_checks(&db, 5, &checks);
let hash6 = insert_final_block(&db, same_authorities(), || default_header(&hash5, 6));
run_checks(&db, 6, &checks);
@@ -1043,9 +1135,14 @@ pub(crate) mod tests {
// some older non-best blocks are inserted
// ... -> B2(1) -> B2_1(1) -> B2_2(2)
// => the cache ignores all writes before best finalized block
let hash2_1 = insert_non_best_block(&db, make_authorities(vec![auth1()]), || default_header(&hash2, 3));
let hash2_1 = insert_non_best_block(&db, make_authorities(vec![auth1()]), || {
default_header(&hash2, 3)
});
assert_eq!(None, authorities(db.cache(), BlockId::Hash(hash2_1)));
let hash2_2 = insert_non_best_block(&db, make_authorities(vec![auth1(), auth2()]), || default_header(&hash2_1, 4));
let hash2_2 =
insert_non_best_block(&db, make_authorities(vec![auth1(), auth2()]), || {
default_header(&hash2_1, 4)
});
assert_eq!(None, authorities(db.cache(), BlockId::Hash(hash2_2)));
}
@@ -1056,51 +1153,41 @@ pub(crate) mod tests {
// \> B6_1_1(5)
// \> B6_1_2(6) -> B6_1_3(7)
let hash7 = insert_block(&db, make_authorities(vec![auth3()]), || default_header(&hash6, 7));
assert_eq!(
authorities(db.cache(), BlockId::Hash(hash6)),
Some(vec![auth1(), auth2()]),
);
let hash7 =
insert_block(&db, make_authorities(vec![auth3()]), || default_header(&hash6, 7));
assert_eq!(authorities(db.cache(), BlockId::Hash(hash6)), Some(vec![auth1(), auth2()]),);
assert_eq!(authorities(db.cache(), BlockId::Hash(hash7)), Some(vec![auth3()]));
let hash8 = insert_block(&db, make_authorities(vec![auth3()]), || default_header(&hash7, 8));
assert_eq!(
authorities(db.cache(), BlockId::Hash(hash6)),
Some(vec![auth1(), auth2()]),
);
let hash8 =
insert_block(&db, make_authorities(vec![auth3()]), || default_header(&hash7, 8));
assert_eq!(authorities(db.cache(), BlockId::Hash(hash6)), Some(vec![auth1(), auth2()]),);
assert_eq!(authorities(db.cache(), BlockId::Hash(hash7)), Some(vec![auth3()]));
assert_eq!(authorities(db.cache(), BlockId::Hash(hash8)), Some(vec![auth3()]));
let hash6_1 = insert_block(&db, make_authorities(vec![auth4()]), || default_header(&hash6, 7));
assert_eq!(
authorities(db.cache(), BlockId::Hash(hash6)),
Some(vec![auth1(), auth2()]),
);
let hash6_1 =
insert_block(&db, make_authorities(vec![auth4()]), || default_header(&hash6, 7));
assert_eq!(authorities(db.cache(), BlockId::Hash(hash6)), Some(vec![auth1(), auth2()]),);
assert_eq!(authorities(db.cache(), BlockId::Hash(hash7)), Some(vec![auth3()]));
assert_eq!(authorities(db.cache(), BlockId::Hash(hash8)), Some(vec![auth3()]));
assert_eq!(authorities(db.cache(), BlockId::Hash(hash6_1)), Some(vec![auth4()]));
let hash6_1_1 = insert_non_best_block(&db, make_authorities(vec![auth5()]), || default_header(&hash6_1, 8));
assert_eq!(
authorities(db.cache(), BlockId::Hash(hash6)),
Some(vec![auth1(), auth2()]),
);
let hash6_1_1 = insert_non_best_block(&db, make_authorities(vec![auth5()]), || {
default_header(&hash6_1, 8)
});
assert_eq!(authorities(db.cache(), BlockId::Hash(hash6)), Some(vec![auth1(), auth2()]),);
assert_eq!(authorities(db.cache(), BlockId::Hash(hash7)), Some(vec![auth3()]));
assert_eq!(authorities(db.cache(), BlockId::Hash(hash8)), Some(vec![auth3()]));
assert_eq!(authorities(db.cache(), BlockId::Hash(hash6_1)), Some(vec![auth4()]));
assert_eq!(authorities(db.cache(), BlockId::Hash(hash6_1_1)), Some(vec![auth5()]));
let hash6_1_2 = insert_non_best_block(&db, make_authorities(vec![auth6()]), || default_header(&hash6_1, 8));
assert_eq!(
authorities(db.cache(), BlockId::Hash(hash6)),
Some(vec![auth1(), auth2()]),
);
let hash6_1_2 = insert_non_best_block(&db, make_authorities(vec![auth6()]), || {
default_header(&hash6_1, 8)
});
assert_eq!(authorities(db.cache(), BlockId::Hash(hash6)), Some(vec![auth1(), auth2()]),);
assert_eq!(authorities(db.cache(), BlockId::Hash(hash7)), Some(vec![auth3()]));
assert_eq!(authorities(db.cache(), BlockId::Hash(hash8)), Some(vec![auth3()]));
assert_eq!(authorities(db.cache(), BlockId::Hash(hash6_1)), Some(vec![auth4()]));
assert_eq!(authorities(db.cache(), BlockId::Hash(hash6_1_1)), Some(vec![auth5()]));
assert_eq!(authorities(db.cache(), BlockId::Hash(hash6_1_2)), Some(vec![auth6()]));
let hash6_2 = insert_block(&db, make_authorities(vec![auth4()]), || default_header(&hash6_1, 8));
assert_eq!(
authorities(db.cache(), BlockId::Hash(hash6)),
Some(vec![auth1(), auth2()]),
);
let hash6_2 =
insert_block(&db, make_authorities(vec![auth4()]), || default_header(&hash6_1, 8));
assert_eq!(authorities(db.cache(), BlockId::Hash(hash6)), Some(vec![auth1(), auth2()]),);
assert_eq!(authorities(db.cache(), BlockId::Hash(hash7)), Some(vec![auth3()]));
assert_eq!(authorities(db.cache(), BlockId::Hash(hash8)), Some(vec![auth3()]));
assert_eq!(authorities(db.cache(), BlockId::Hash(hash6_1)), Some(vec![auth4()]));
@@ -1114,10 +1201,7 @@ pub(crate) mod tests {
{
// finalize block hash6_1
db.finalize_header(BlockId::Hash(hash6_1)).unwrap();
assert_eq!(
authorities(db.cache(), BlockId::Hash(hash6)),
Some(vec![auth1(), auth2()]),
);
assert_eq!(authorities(db.cache(), BlockId::Hash(hash6)), Some(vec![auth1(), auth2()]),);
assert_eq!(authorities(db.cache(), BlockId::Hash(hash7)), None);
assert_eq!(authorities(db.cache(), BlockId::Hash(hash8)), None);
assert_eq!(authorities(db.cache(), BlockId::Hash(hash6_1)), Some(vec![auth4()]));
@@ -1126,10 +1210,7 @@ pub(crate) mod tests {
assert_eq!(authorities(db.cache(), BlockId::Hash(hash6_2)), Some(vec![auth4()]));
// finalize block hash6_2
db.finalize_header(BlockId::Hash(hash6_2)).unwrap();
assert_eq!(
authorities(db.cache(), BlockId::Hash(hash6)),
Some(vec![auth1(), auth2()]),
);
assert_eq!(authorities(db.cache(), BlockId::Hash(hash6)), Some(vec![auth1(), auth2()]),);
assert_eq!(authorities(db.cache(), BlockId::Hash(hash7)), None);
assert_eq!(authorities(db.cache(), BlockId::Hash(hash8)), None);
assert_eq!(authorities(db.cache(), BlockId::Hash(hash6_1)), Some(vec![auth4()]));
@@ -1142,7 +1223,8 @@ pub(crate) mod tests {
#[test]
fn database_is_reopened() {
let db = LightStorage::new_test();
let hash0 = insert_final_block(&db, HashMap::new(), || default_header(&Default::default(), 0));
let hash0 =
insert_final_block(&db, HashMap::new(), || default_header(&Default::default(), 0));
assert_eq!(db.info().best_hash, hash0);
assert_eq!(db.header(BlockId::Hash(hash0)).unwrap().unwrap().hash(), hash0);
@@ -1157,7 +1239,8 @@ pub(crate) mod tests {
let db = LightStorage::<Block>::new_test();
// insert aux1 + aux2 using direct store access
db.insert_aux(&[(&[1][..], &[101][..]), (&[2][..], &[102][..])], ::std::iter::empty()).unwrap();
db.insert_aux(&[(&[1][..], &[101][..]), (&[2][..], &[102][..])], ::std::iter::empty())
.unwrap();
// check aux values
assert_eq!(db.get_aux(&[1]).unwrap(), Some(vec![101]));
@@ -1165,10 +1248,13 @@ pub(crate) mod tests {
assert_eq!(db.get_aux(&[3]).unwrap(), None);
// delete aux1 + insert aux3 using import operation
db.import_header(default_header(&Default::default(), 0), HashMap::new(), NewBlockState::Best, vec![
(vec![3], Some(vec![103])),
(vec![1], None),
]).unwrap();
db.import_header(
default_header(&Default::default(), 0),
HashMap::new(),
NewBlockState::Best,
vec![(vec![3], Some(vec![103])), (vec![1], None)],
)
.unwrap();
// check aux values
assert_eq!(db.get_aux(&[1]).unwrap(), None);
@@ -1208,7 +1294,8 @@ pub(crate) mod tests {
};
// restart && check that after restart value is read from the cache
let db = LightStorage::<Block>::from_kvdb(storage as Arc<_>).expect("failed to create test-db");
let db =
LightStorage::<Block>::from_kvdb(storage as Arc<_>).expect("failed to create test-db");
assert_eq!(
db.cache().get_at(b"test", &BlockId::Number(0)).unwrap(),
Some(((0, genesis_hash.unwrap()), None, vec![42])),
@@ -1224,7 +1311,9 @@ pub(crate) mod tests {
// insert block#0 && block#1 (no value for cache is provided)
let hash0 = insert_block(&db, HashMap::new(), || default_header(&Default::default(), 0));
assert_eq!(
db.cache().get_at(&well_known_cache_keys::CHANGES_TRIE_CONFIG, &BlockId::Number(0)).unwrap()
db.cache()
.get_at(&well_known_cache_keys::CHANGES_TRIE_CONFIG, &BlockId::Number(0))
.unwrap()
.map(|(_, _, v)| ChangesTrieConfiguration::decode(&mut &v[..]).unwrap()),
None,
);
@@ -1232,13 +1321,15 @@ pub(crate) mod tests {
// insert configuration at block#1 (starts from block#2)
insert_block(&db, HashMap::new(), || {
let mut header = default_header(&hash0, 1);
header.digest_mut().push(
DigestItem::ChangesTrieSignal(ChangesTrieSignal::NewConfiguration(new_config.clone()))
);
header.digest_mut().push(DigestItem::ChangesTrieSignal(
ChangesTrieSignal::NewConfiguration(new_config.clone()),
));
header
});
assert_eq!(
db.cache().get_at(&well_known_cache_keys::CHANGES_TRIE_CONFIG, &BlockId::Number(1)).unwrap()
db.cache()
.get_at(&well_known_cache_keys::CHANGES_TRIE_CONFIG, &BlockId::Number(1))
.unwrap()
.map(|(_, _, v)| Option::<ChangesTrieConfiguration>::decode(&mut &v[..]).unwrap()),
Some(new_config),
);
substrate/client/db/src/offchain.rs
+4 -13
View File
@@ -21,8 +21,8 @@
use std::{collections::HashMap, sync::Arc};
use crate::{columns, Database, DbHash, Transaction};
use parking_lot::Mutex;
use log::error;
use parking_lot::Mutex;
/// Offchain local storage
#[derive(Clone)]
@@ -33,8 +33,7 @@ pub struct LocalStorage {
impl std::fmt::Debug for LocalStorage {
fn fmt(&self, fmt: &mut std::fmt::Formatter) -> std::fmt::Result {
fmt.debug_struct("LocalStorage")
.finish()
fmt.debug_struct("LocalStorage").finish()
}
}
@@ -49,10 +48,7 @@ impl LocalStorage {
/// Create offchain local storage with given `KeyValueDB` backend.
pub fn new(db: Arc<dyn Database<DbHash>>) -> Self {
Self {
db,
locks: Default::default(),
}
Self { db, locks: Default::default() }
}
}
@@ -118,11 +114,7 @@ impl sp_core::offchain::OffchainStorage for LocalStorage {
/// Concatenate the prefix and key to create an offchain key in the db.
pub(crate) fn concatenate_prefix_and_key(prefix: &[u8], key: &[u8]) -> Vec<u8> {
prefix
.iter()
.chain(key.into_iter())
.cloned()
.collect()
prefix.iter().chain(key.into_iter()).cloned().collect()
}
#[cfg(test)]
@@ -155,5 +147,4 @@ mod tests {
assert_eq!(storage.get(prefix, key), Some(b"asd".to_vec()));
assert!(storage.locks.lock().is_empty(), "Locks map should be empty!");
}
}
substrate/client/db/src/parity_db.rs
+16 -16
View File
@@ -15,27 +15,29 @@
// You should have received a copy of the GNU General Public License
// along with this program. If not, see <https://www.gnu.org/licenses/>.
use crate::{
columns,
utils::{DatabaseType, NUM_COLUMNS},
};
/// A `Database` adapter for parity-db.
use sp_database::{Database, Change, ColumnId, Transaction, error::DatabaseError};
use crate::utils::{DatabaseType, NUM_COLUMNS};
use crate::columns;
use sp_database::{error::DatabaseError, Change, ColumnId, Database, Transaction};
struct DbAdapter(parity_db::Db);
fn handle_err<T>(result: parity_db::Result<T>) -> T {
match result {
Ok(r) => r,
Err(e) => {
Err(e) => {
panic!("Critical database error: {:?}", e);
}
},
}
}
/// Wrap parity-db database into a trait object that implements `sp_database::Database`
pub fn open<H: Clone + AsRef<[u8]>>(path: &std::path::Path, db_type: DatabaseType)
-> parity_db::Result<std::sync::Arc<dyn Database<H>>>
{
pub fn open<H: Clone + AsRef<[u8]>>(
path: &std::path::Path,
db_type: DatabaseType,
) -> parity_db::Result<std::sync::Arc<dyn Database<H>>> {
let mut config = parity_db::Options::with_columns(path, NUM_COLUMNS as u8);
config.sync = true; // Flush each commit
if db_type == DatabaseType::Full {
@@ -50,13 +52,11 @@ pub fn open<H: Clone + AsRef<[u8]>>(path: &std::path::Path, db_type: DatabaseTyp
impl<H: Clone + AsRef<[u8]>> Database<H> for DbAdapter {
fn commit(&self, transaction: Transaction<H>) -> Result<(), DatabaseError> {
handle_err(self.0.commit(transaction.0.into_iter().map(|change|
match change {
Change::Set(col, key, value) => (col as u8, key, Some(value)),
Change::Remove(col, key) => (col as u8, key, None),
_ => unimplemented!(),
}))
);
handle_err(self.0.commit(transaction.0.into_iter().map(|change| match change {
Change::Set(col, key, value) => (col as u8, key, Some(value)),
Change::Remove(col, key) => (col as u8, key, None),
_ => unimplemented!(),
})));
Ok(())
}
substrate/client/db/src/stats.rs
+8 -3
View File
@@ -65,7 +65,10 @@ impl StateUsageStats {
/// Tally one key read.
pub fn tally_key_read(&self, key: &[u8], val: Option<&Vec<u8>>, cache: bool) {
self.tally_read(key.len() as u64 + val.as_ref().map(|x| x.len() as u64).unwrap_or(0), cache);
self.tally_read(
key.len() as u64 + val.as_ref().map(|x| x.len() as u64).unwrap_or(0),
cache,
);
}
/// Tally one child key read.
@@ -103,9 +106,11 @@ impl StateUsageStats {
self.reads.fetch_add(info.reads.ops, AtomicOrdering::Relaxed);
self.bytes_read.fetch_add(info.reads.bytes, AtomicOrdering::Relaxed);
self.writes_nodes.fetch_add(info.nodes_writes.ops, AtomicOrdering::Relaxed);
self.bytes_written_nodes.fetch_add(info.nodes_writes.bytes, AtomicOrdering::Relaxed);
self.bytes_written_nodes
.fetch_add(info.nodes_writes.bytes, AtomicOrdering::Relaxed);
self.removed_nodes.fetch_add(info.removed_nodes.ops, AtomicOrdering::Relaxed);
self.bytes_removed_nodes.fetch_add(info.removed_nodes.bytes, AtomicOrdering::Relaxed);
self.bytes_removed_nodes
.fetch_add(info.removed_nodes.bytes, AtomicOrdering::Relaxed);
self.reads_cache.fetch_add(info.cache_reads.ops, AtomicOrdering::Relaxed);
self.bytes_read_cache.fetch_add(info.cache_reads.bytes, AtomicOrdering::Relaxed);
}
substrate/client/db/src/storage_cache.rs
+289 -345
View File
File diff suppressed because it is too large Load Diff
substrate/client/db/src/upgrade.rs
+47 -23
View File
@@ -18,14 +18,16 @@
//! Database upgrade logic.
use std::fs;
use std::io::{Read, Write, ErrorKind};
use std::path::{Path, PathBuf};
use std::{
fs,
io::{ErrorKind, Read, Write},
path::{Path, PathBuf},
};
use sp_runtime::traits::Block as BlockT;
use crate::{columns, utils::DatabaseType};
use kvdb_rocksdb::{Database, DatabaseConfig};
use codec::{Decode, Encode};
use kvdb_rocksdb::{Database, DatabaseConfig};
use sp_runtime::traits::Block as BlockT;
/// Version file name.
const VERSION_FILE_NAME: &'static str = "db_version";
@@ -38,19 +40,28 @@ const V1_NUM_COLUMNS: u32 = 11;
const V2_NUM_COLUMNS: u32 = 12;
/// Upgrade database to current version.
pub fn upgrade_db<Block: BlockT>(db_path: &Path, db_type: DatabaseType) -> sp_blockchain::Result<()> {
pub fn upgrade_db<Block: BlockT>(
db_path: &Path,
db_type: DatabaseType,
) -> sp_blockchain::Result<()> {
let is_empty = db_path.read_dir().map_or(true, |mut d| d.next().is_none());
if !is_empty {
let db_version = current_version(db_path)?;
match db_version {
0 => Err(sp_blockchain::Error::Backend(format!("Unsupported database version: {}", db_version)))?,
0 => Err(sp_blockchain::Error::Backend(format!(
"Unsupported database version: {}",
db_version
)))?,
1 => {
migrate_1_to_2::<Block>(db_path, db_type)?;
migrate_2_to_3::<Block>(db_path, db_type)?
},
2 => migrate_2_to_3::<Block>(db_path, db_type)?,
CURRENT_VERSION => (),
_ => Err(sp_blockchain::Error::Backend(format!("Future database version: {}", db_version)))?,
_ => Err(sp_blockchain::Error::Backend(format!(
"Future database version: {}",
db_version
)))?,
}
}
@@ -60,8 +71,12 @@ pub fn upgrade_db<Block: BlockT>(db_path: &Path, db_type: DatabaseType) -> sp_bl
/// Migration from version1 to version2:
/// 1) the number of columns has changed from 11 to 12;
/// 2) transactions column is added;
fn migrate_1_to_2<Block: BlockT>(db_path: &Path, _db_type: DatabaseType) -> sp_blockchain::Result<()> {
let db_path = db_path.to_str()
fn migrate_1_to_2<Block: BlockT>(
db_path: &Path,
_db_type: DatabaseType,
) -> sp_blockchain::Result<()> {
let db_path = db_path
.to_str()
.ok_or_else(|| sp_blockchain::Error::Backend("Invalid database path".into()))?;
let db_cfg = DatabaseConfig::with_columns(V1_NUM_COLUMNS);
let db = Database::open(&db_cfg, db_path).map_err(db_err)?;
@@ -70,8 +85,12 @@ fn migrate_1_to_2<Block: BlockT>(db_path: &Path, _db_type: DatabaseType) -> sp_b
/// Migration from version2 to version3:
/// - The format of the stored Justification changed to support multiple Justifications.
fn migrate_2_to_3<Block: BlockT>(db_path: &Path, _db_type: DatabaseType) -> sp_blockchain::Result<()> {
let db_path = db_path.to_str()
fn migrate_2_to_3<Block: BlockT>(
db_path: &Path,
_db_type: DatabaseType,
) -> sp_blockchain::Result<()> {
let db_path = db_path
.to_str()
.ok_or_else(|| sp_blockchain::Error::Backend("Invalid database path".into()))?;
let db_cfg = DatabaseConfig::with_columns(V2_NUM_COLUMNS);
let db = Database::open(&db_cfg, db_path).map_err(db_err)?;
@@ -137,10 +156,11 @@ fn version_file_path(path: &Path) -> PathBuf {
#[cfg(test)]
mod tests {
use sc_state_db::PruningMode;
use crate::{DatabaseSettings, DatabaseSettingsSrc, KeepBlocks, TransactionStorageMode};
use crate::tests::Block;
use super::*;
use crate::{
tests::Block, DatabaseSettings, DatabaseSettingsSrc, KeepBlocks, TransactionStorageMode,
};
use sc_state_db::PruningMode;
fn create_db(db_path: &Path, version: Option<u32>) {
if let Some(version) = version {
@@ -151,14 +171,18 @@ mod tests {
}
fn open_database(db_path: &Path) -> sp_blockchain::Result<()> {
crate::utils::open_database::<Block>(&DatabaseSettings {
state_cache_size: 0,
state_cache_child_ratio: None,
state_pruning: PruningMode::ArchiveAll,
source: DatabaseSettingsSrc::RocksDb { path: db_path.to_owned(), cache_size: 128 },
keep_blocks: KeepBlocks::All,
transaction_storage: TransactionStorageMode::BlockBody,
}, DatabaseType::Full).map(|_| ())
crate::utils::open_database::<Block>(
&DatabaseSettings {
state_cache_size: 0,
state_cache_child_ratio: None,
state_pruning: PruningMode::ArchiveAll,
source: DatabaseSettingsSrc::RocksDb { path: db_path.to_owned(), cache_size: 128 },
keep_blocks: KeepBlocks::All,
transaction_storage: TransactionStorageMode::BlockBody,
},
DatabaseType::Full,
)
.map(|_| ())
}
#[test]
substrate/client/db/src/utils.rs
+86 -90
View File
@@ -19,24 +19,27 @@
//! Db-based backend utility structures and functions, used by both
//! full and light storages.
use std::sync::Arc;
use std::convert::TryInto;
use std::{convert::TryInto, sync::Arc};
use log::debug;
use crate::{Database, DatabaseSettings, DatabaseSettingsSrc, DbHash};
use codec::Decode;
use sp_trie::DBValue;
use sp_database::Transaction;
use sp_runtime::generic::BlockId;
use sp_runtime::traits::{
Block as BlockT, Header as HeaderT, Zero,
UniqueSaturatedFrom, UniqueSaturatedInto,
use sp_runtime::{
generic::BlockId,
traits::{Block as BlockT, Header as HeaderT, UniqueSaturatedFrom, UniqueSaturatedInto, Zero},
};
use crate::{DatabaseSettings, DatabaseSettingsSrc, Database, DbHash};
use sp_trie::DBValue;
/// Number of columns in the db. Must be the same for both full && light dbs.
/// Otherwise RocksDb will fail to open database && check its type.
#[cfg(any(feature = "with-kvdb-rocksdb", feature = "with-parity-db", feature = "test-helpers", test))]
#[cfg(any(
feature = "with-kvdb-rocksdb",
feature = "with-parity-db",
feature = "test-helpers",
test
))]
pub const NUM_COLUMNS: u32 = 12;
/// Meta column. The set of keys in the column is shared by full && light storages.
pub const COLUMN_META: u32 = 0;
@@ -98,24 +101,17 @@ pub enum DatabaseType {
/// In the current database schema, this kind of key is only used for
/// lookups into an index, NOT for storing header data or others.
pub fn number_index_key<N: TryInto<u32>>(n: N) -> sp_blockchain::Result<NumberIndexKey> {
let n = n.try_into().map_err(|_|
let n = n.try_into().map_err(|_| {
sp_blockchain::Error::Backend("Block number cannot be converted to u32".into())
)?;
})?;
Ok([
(n >> 24) as u8,
((n >> 16) & 0xff) as u8,
((n >> 8) & 0xff) as u8,
(n & 0xff) as u8
])
Ok([(n >> 24) as u8, ((n >> 16) & 0xff) as u8, ((n >> 8) & 0xff) as u8, (n & 0xff) as u8])
}
/// Convert number and hash into long lookup key for blocks that are
/// not in the canonical chain.
pub fn number_and_hash_to_lookup_key<N, H>(
number: N,
hash: H,
) -> sp_blockchain::Result<Vec<u8>> where
pub fn number_and_hash_to_lookup_key<N, H>(number: N, hash: H) -> sp_blockchain::Result<Vec<u8>>
where
N: TryInto<u32>,
H: AsRef<[u8]>,
{
@@ -126,16 +122,15 @@ pub fn number_and_hash_to_lookup_key<N, H>(
/// Convert block lookup key into block number.
/// all block lookup keys start with the block number.
pub fn lookup_key_to_number<N>(key: &[u8]) -> sp_blockchain::Result<N> where
N: From<u32>
pub fn lookup_key_to_number<N>(key: &[u8]) -> sp_blockchain::Result<N>
where
N: From<u32>,
{
if key.len() < 4 {
return Err(sp_blockchain::Error::Backend("Invalid block key".into()));
return Err(sp_blockchain::Error::Backend("Invalid block key".into()))
}
Ok((key[0] as u32) << 24
| (key[1] as u32) << 16
| (key[2] as u32) << 8
| (key[3] as u32)).map(Into::into)
Ok((key[0] as u32) << 24 | (key[1] as u32) << 16 | (key[2] as u32) << 8 | (key[3] as u32))
.map(Into::into)
}
/// Delete number to hash mapping in DB transaction.
@@ -197,17 +192,15 @@ pub fn insert_hash_to_key_mapping<N: TryInto<u32>, H: AsRef<[u8]> + Clone>(
pub fn block_id_to_lookup_key<Block>(
db: &dyn Database<DbHash>,
key_lookup_col: u32,
id: BlockId<Block>
) -> Result<Option<Vec<u8>>, sp_blockchain::Error> where
id: BlockId<Block>,
) -> Result<Option<Vec<u8>>, sp_blockchain::Error>
where
Block: BlockT,
::sp_runtime::traits::NumberFor<Block>: UniqueSaturatedFrom<u64> + UniqueSaturatedInto<u64>,
{
Ok(match id {
BlockId::Number(n) => db.get(
key_lookup_col,
number_index_key(n)?.as_ref(),
),
BlockId::Hash(h) => db.get(key_lookup_col, h.as_ref())
BlockId::Number(n) => db.get(key_lookup_col, number_index_key(n)?.as_ref()),
BlockId::Hash(h) => db.get(key_lookup_col, h.as_ref()),
})
}
@@ -218,9 +211,10 @@ pub fn open_database<Block: BlockT>(
) -> sp_blockchain::Result<Arc<dyn Database<DbHash>>> {
#[allow(unused)]
fn db_open_error(feat: &'static str) -> sp_blockchain::Error {
sp_blockchain::Error::Backend(
format!("`{}` feature not enabled, database can not be opened", feat),
)
sp_blockchain::Error::Backend(format!(
"`{}` feature not enabled, database can not be opened",
feat
))
}
let db: Arc<dyn Database<DbHash>> = match &config.source {
@@ -231,14 +225,16 @@ pub fn open_database<Block: BlockT>(
// and now open database assuming that it has the latest version
let mut db_config = kvdb_rocksdb::DatabaseConfig::with_columns(NUM_COLUMNS);
let path = path.to_str()
let path = path
.to_str()
.ok_or_else(|| sp_blockchain::Error::Backend("Invalid database path".into()))?;
let mut memory_budget = std::collections::HashMap::new();
match db_type {
DatabaseType::Full => {
let state_col_budget = (*cache_size as f64 * 0.9) as usize;
let other_col_budget = (cache_size - state_col_budget) / (NUM_COLUMNS as usize - 1);
let other_col_budget =
(cache_size - state_col_budget) / (NUM_COLUMNS as usize - 1);
for i in 0..NUM_COLUMNS {
if i == crate::columns::STATE {
@@ -267,7 +263,7 @@ pub fn open_database<Block: BlockT>(
path,
col_budget,
);
}
},
}
db_config.memory_budget = memory_budget;
@@ -276,18 +272,12 @@ pub fn open_database<Block: BlockT>(
sp_database::as_database(db)
},
#[cfg(not(any(feature = "with-kvdb-rocksdb", test)))]
DatabaseSettingsSrc::RocksDb { .. } => {
return Err(db_open_error("with-kvdb-rocksdb"));
},
DatabaseSettingsSrc::RocksDb { .. } => return Err(db_open_error("with-kvdb-rocksdb")),
#[cfg(feature = "with-parity-db")]
DatabaseSettingsSrc::ParityDb { path } => {
crate::parity_db::open(&path, db_type)
.map_err(|e| sp_blockchain::Error::Backend(format!("{}", e)))?
},
DatabaseSettingsSrc::ParityDb { path } => crate::parity_db::open(&path, db_type)
.map_err(|e| sp_blockchain::Error::Backend(format!("{}", e)))?,
#[cfg(not(feature = "with-parity-db"))]
DatabaseSettingsSrc::ParityDb { .. } => {
return Err(db_open_error("with-parity-db"))
},
DatabaseSettingsSrc::ParityDb { .. } => return Err(db_open_error("with-parity-db")),
DatabaseSettingsSrc::Custom(db) => db.clone(),
};
@@ -297,14 +287,19 @@ pub fn open_database<Block: BlockT>(
}
/// Check database type.
pub fn check_database_type(db: &dyn Database<DbHash>, db_type: DatabaseType) -> sp_blockchain::Result<()> {
pub fn check_database_type(
db: &dyn Database<DbHash>,
db_type: DatabaseType,
) -> sp_blockchain::Result<()> {
match db.get(COLUMN_META, meta_keys::TYPE) {
Some(stored_type) => {
Some(stored_type) =>
if db_type.as_str().as_bytes() != &*stored_type {
return Err(sp_blockchain::Error::Backend(
format!("Unexpected database type. Expected: {}", db_type.as_str())).into());
}
},
return Err(sp_blockchain::Error::Backend(format!(
"Unexpected database type. Expected: {}",
db_type.as_str()
))
.into())
},
None => {
let mut transaction = Transaction::new();
transaction.set(COLUMN_META, meta_keys::TYPE, db_type.as_str().as_bytes());
@@ -320,10 +315,10 @@ pub fn read_db<Block>(
db: &dyn Database<DbHash>,
col_index: u32,
col: u32,
id: BlockId<Block>
id: BlockId<Block>,
) -> sp_blockchain::Result<Option<DBValue>>
where
Block: BlockT,
where
Block: BlockT,
{
block_id_to_lookup_key(db, col_index, id).and_then(|key| match key {
Some(key) => Ok(db.get(col, key.as_ref())),
@@ -358,10 +353,8 @@ pub fn read_header<Block: BlockT>(
match read_db(db, col_index, col, id)? {
Some(header) => match Block::Header::decode(&mut &header[..]) {
Ok(header) => Ok(Some(header)),
Err(_) => return Err(
sp_blockchain::Error::Backend("Error decoding header".into())
),
}
Err(_) => return Err(sp_blockchain::Error::Backend("Error decoding header".into())),
},
None => Ok(None),
}
}
@@ -373,34 +366,35 @@ pub fn require_header<Block: BlockT>(
col: u32,
id: BlockId<Block>,
) -> sp_blockchain::Result<Block::Header> {
read_header(db, col_index, col, id)
.and_then(|header| header.ok_or_else(||
sp_blockchain::Error::UnknownBlock(format!("Require header: {}", id))
))
read_header(db, col_index, col, id).and_then(|header| {
header.ok_or_else(|| sp_blockchain::Error::UnknownBlock(format!("Require header: {}", id)))
})
}
/// Read meta from the database.
pub fn read_meta<Block>(db: &dyn Database<DbHash>, col_header: u32) -> Result<
Meta<<<Block as BlockT>::Header as HeaderT>::Number, Block::Hash>,
sp_blockchain::Error,
>
where
Block: BlockT,
pub fn read_meta<Block>(
db: &dyn Database<DbHash>,
col_header: u32,
) -> Result<Meta<<<Block as BlockT>::Header as HeaderT>::Number, Block::Hash>, sp_blockchain::Error>
where
Block: BlockT,
{
let genesis_hash: Block::Hash = match read_genesis_hash(db)? {
Some(genesis_hash) => genesis_hash,
None => return Ok(Meta {
best_hash: Default::default(),
best_number: Zero::zero(),
finalized_hash: Default::default(),
finalized_number: Zero::zero(),
genesis_hash: Default::default(),
finalized_state: None,
}),
None =>
return Ok(Meta {
best_hash: Default::default(),
best_number: Zero::zero(),
finalized_hash: Default::default(),
finalized_number: Zero::zero(),
genesis_hash: Default::default(),
finalized_state: None,
}),
};
let load_meta_block = |desc, key| -> Result<_, sp_blockchain::Error> {
if let Some(Some(header)) = db.get(COLUMN_META, key)
if let Some(Some(header)) = db
.get(COLUMN_META, key)
.and_then(|id| db.get(col_header, &id).map(|b| Block::Header::decode(&mut &b[..]).ok()))
{
let hash = header.hash();
@@ -419,7 +413,8 @@ pub fn read_meta<Block>(db: &dyn Database<DbHash>, col_header: u32) -> Result<
let (best_hash, best_number) = load_meta_block("best", meta_keys::BEST_BLOCK)?;
let (finalized_hash, finalized_number) = load_meta_block("final", meta_keys::FINALIZED_BLOCK)?;
let (finalized_state_hash, finalized_state_number) = load_meta_block("final_state", meta_keys::FINALIZED_STATE)?;
let (finalized_state_hash, finalized_state_number) =
load_meta_block("final_state", meta_keys::FINALIZED_STATE)?;
let finalized_state = if finalized_state_hash != Default::default() {
Some((finalized_state_hash, finalized_state_number))
} else {
@@ -437,13 +432,14 @@ pub fn read_meta<Block>(db: &dyn Database<DbHash>, col_header: u32) -> Result<
}
/// Read genesis hash from database.
pub fn read_genesis_hash<Hash: Decode>(db: &dyn Database<DbHash>) -> sp_blockchain::Result<Option<Hash>> {
pub fn read_genesis_hash<Hash: Decode>(
db: &dyn Database<DbHash>,
) -> sp_blockchain::Result<Option<Hash>> {
match db.get(COLUMN_META, meta_keys::GENESIS_HASH) {
Some(h) => match Decode::decode(&mut &h[..]) {
Ok(h) => Ok(Some(h)),
Err(err) => Err(sp_blockchain::Error::Backend(
format!("Error decoding genesis hash: {}", err)
)),
Err(err) =>
Err(sp_blockchain::Error::Backend(format!("Error decoding genesis hash: {}", err))),
},
None => Ok(None),
}
@@ -461,7 +457,7 @@ impl DatabaseType {
pub(crate) struct JoinInput<'a, 'b>(&'a [u8], &'b [u8]);
pub(crate) fn join_input<'a, 'b>(i1: &'a[u8], i2: &'b [u8]) -> JoinInput<'a, 'b> {
pub(crate) fn join_input<'a, 'b>(i1: &'a [u8], i2: &'b [u8]) -> JoinInput<'a, 'b> {
JoinInput(i1, i2)
}
@@ -486,8 +482,8 @@ impl<'a, 'b> codec::Input for JoinInput<'a, 'b> {
#[cfg(test)]
mod tests {
use super::*;
use sp_runtime::testing::{Block as RawBlock, ExtrinsicWrapper};
use codec::Input;
use sp_runtime::testing::{Block as RawBlock, ExtrinsicWrapper};
type Block = RawBlock<ExtrinsicWrapper<u32>>;
#[test]