pezkuwichain/pezkuwi-subxt
1
Fork 0
mirror of https://github.com/pezkuwichain/pezkuwi-subxt.git synced 2026-10-29 15:09:49 +00:00
Code Releases Activity

Allow updating configuration of changes tries (#3201)

Browse Source 
* DigestItem::ChangesTrieSignal

* introduce changes_trie::State

* introduce config activation block

* ChangesTrieSignal::as_new_configuration

* moved well_known_cache_keys to client

* extracted DbChangesTrieStorage to separate file

* change meaning of none in blockchain cache

* changes trie config (FULL) cache draft

* eliminating const ChangesTrieConfiguration

* delay pruning

* continue elimination

* do not prune CT config from cache

* removed redundant code

* fix some TODOs

* introduce ConfigurationRange

* use Configuration range in build

* build skewed digest

* remove debug print

* extracted surface iterator

* key_changes works with skewed digests

* fix client build

* add test for NeverPrune

* fix TODO

* fixed some TODOs

* more tests

* fixing TODOs

* fixed compilation

* update runtime version

* git rid of large tuple

* too long lines

* config_activation_block -> zero

* obsolete TODO

* removed unjustified expect

* update TODOs with issue number

* new CT pruning algorithm

fixed cache + multiple blocks finalization

track CT configuraiton on light clients

support CT configuration change revert

revert CT config test

new CT pruning algorithm

fixed cache + multiple blocks finalization

track CT configuraiton on light clients

support CT configuration change revert

revert CT config test

* BlockIdOrHeader isn't really required

* removed debug leftovers + some docs

* more docs

* more post-merge fixes

* more post-merge fixes

* revertes some unnecessary changes

* reverted unnecessary changes

* fix compilation + unnecessary changes

* (restart CI)

* fix cache update when finalizing multiple blocks

* fixed tests

* collect_extrinsics -> set_collect_extrinsics

* restore lost test

* do not calculate block number twice

* Update primitives/blockchain/src/error.rs

Co-Authored-By: cheme <emericchevalier.pro@gmail.com>

* map_err -> unwrap_or

* document get_at Result

* delete abandoned file

* added weight for set_changes_trie_config

* prefer_configs -> fail_if_disabled

* Update client/api/src/backend.rs

Co-Authored-By: cheme <emericchevalier.pro@gmail.com>

* Update client/db/src/changes_tries_storage.rs

Co-Authored-By: cheme <emericchevalier.pro@gmail.com>

* CommitOperation+merge -> CommitOperations

* fixed test compilation

* merged two different CTRange structs

* lost file

* uggrade db from v0 to v1 (init CT cache + add column)

* fix after merge

Co-authored-by: cheme <emericchevalier.pro@gmail.com>
Co-authored-by: Gavin Wood <github@gavwood.com>
This commit is contained in:
Svyatoslav NikolskychemeGavin Wood
 authored and Gavin Wood committed 2020-01-16 17:38:24 +01:00
1 parent 45fbf09dac
commit febf29390a
48 files changed
+2743 -1548
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

substrate/client/db/src/cache/list_cache.rs
Vendored
+392 -157
View File
@@ -94,6 +94,12 @@ pub enum CommitOperation<Block: BlockT, T: CacheItemT> {
BlockReverted(BTreeMap<usize, Option<Fork<Block, T>>>),
}
/// A set of commit operations.
#[derive(Debug)]
pub struct CommitOperations<Block: BlockT, T: CacheItemT> {
operations: Vec<CommitOperation<Block, T>>,
}
/// Single fork of list-based cache.
#[derive(Debug)]
#[cfg_attr(test, derive(PartialEq))]
@@ -123,21 +129,17 @@ impl<Block: BlockT, T: CacheItemT, S: Storage<Block, T>> ListCache<Block, T, S>
storage: S,
pruning_strategy: PruningStrategy<NumberFor<Block>>,
best_finalized_block: ComplexBlockId<Block>,
) -> Self {
) -> ClientResult<Self> {
let (best_finalized_entry, unfinalized) = storage.read_meta()
.and_then(|meta| read_forks(&storage, meta))
.unwrap_or_else(|error| {
warn!(target: "db", "Unable to initialize list cache: {}. Restarting", error);
(None, Vec::new())
});
.and_then(|meta| read_forks(&storage, meta))?;
ListCache {
Ok(ListCache {
storage,
pruning_strategy,
best_finalized_block,
best_finalized_entry,
unfinalized,
}
})
}
/// Get reference to the storage.
@@ -145,6 +147,12 @@ impl<Block: BlockT, T: CacheItemT, S: Storage<Block, T>> ListCache<Block, T, S>
&self.storage
}
/// Get unfinalized forks reference.
#[cfg(test)]
pub fn unfinalized(&self) -> &[Fork<Block, T>] {
&self.unfinalized
}
/// Get value valid at block.
pub fn value_at_block(
&self,
@@ -156,8 +164,8 @@ impl<Block: BlockT, T: CacheItemT, S: Storage<Block, T>> ListCache<Block, T, S>
// BUT since we're not guaranteeing to provide correct values for forks
// behind the finalized block, check if the block is finalized first
if !chain::is_finalized_block(&self.storage, at, Bounded::max_value())? {
return Ok(None);
if !chain::is_finalized_block(&self.storage, &at, Bounded::max_value())? {
return Err(ClientError::NotInFinalizedChain);
}
self.best_finalized_entry.as_ref()
@@ -171,11 +179,14 @@ impl<Block: BlockT, T: CacheItemT, S: Storage<Block, T>> ListCache<Block, T, S>
// IF there's no matching fork, ensure that this isn't a block from a fork that has forked
// behind the best finalized block and search at finalized fork
match self.find_unfinalized_fork(at)? {
match self.find_unfinalized_fork(&at)? {
Some(fork) => Some(&fork.head),
None => match self.best_finalized_entry.as_ref() {
Some(best_finalized_entry) if chain::is_connected_to_block(&self.storage, &best_finalized_entry.valid_from, at)? =>
Some(best_finalized_entry),
Some(best_finalized_entry) if chain::is_connected_to_block(
&self.storage,
&at,
&best_finalized_entry.valid_from,
)? => Some(best_finalized_entry),
_ => None,
},
}
@@ -198,9 +209,98 @@ impl<Block: BlockT, T: CacheItemT, S: Storage<Block, T>> ListCache<Block, T, S>
block: ComplexBlockId<Block>,
value: Option<T>,
entry_type: EntryType,
operations: &mut CommitOperations<Block, T>,
) -> ClientResult<()> {
Ok(operations.append(self.do_on_block_insert(tx, parent, block, value, entry_type, operations)?))
}
/// When previously inserted block is finalized.
pub fn on_block_finalize<Tx: StorageTransaction<Block, T>>(
&self,
tx: &mut Tx,
parent: ComplexBlockId<Block>,
block: ComplexBlockId<Block>,
operations: &mut CommitOperations<Block, T>,
) -> ClientResult<()> {
Ok(operations.append(self.do_on_block_finalize(tx, parent, block, operations)?))
}
/// When block is reverted.
pub fn on_block_revert<Tx: StorageTransaction<Block, T>>(
&self,
tx: &mut Tx,
reverted_block: &ComplexBlockId<Block>,
operations: &mut CommitOperations<Block, T>,
) -> ClientResult<()> {
Ok(operations.append(Some(self.do_on_block_revert(tx, reverted_block)?)))
}
/// When transaction is committed.
pub fn on_transaction_commit(&mut self, ops: CommitOperations<Block, T>) {
for op in ops.operations {
match op {
CommitOperation::AppendNewBlock(index, best_block) => {
let mut fork = self.unfinalized.get_mut(index)
.expect("ListCache is a crate-private type;
internal clients of ListCache are committing transaction while cache is locked;
CommitOperation holds valid references while cache is locked; qed");
fork.best_block = Some(best_block);
},
CommitOperation::AppendNewEntry(index, entry) => {
let mut fork = self.unfinalized.get_mut(index)
.expect("ListCache is a crate-private type;
internal clients of ListCache are committing transaction while cache is locked;
CommitOperation holds valid references while cache is locked; qed");
fork.best_block = Some(entry.valid_from.clone());
fork.head = entry;
},
CommitOperation::AddNewFork(entry) => {
self.unfinalized.push(Fork {
best_block: Some(entry.valid_from.clone()),
head: entry,
});
},
CommitOperation::BlockFinalized(block, finalizing_entry, forks) => {
self.best_finalized_block = block;
if let Some(finalizing_entry) = finalizing_entry {
self.best_finalized_entry = Some(finalizing_entry);
}
for fork_index in forks.iter().rev() {
self.unfinalized.remove(*fork_index);
}
},
CommitOperation::BlockReverted(forks) => {
for (fork_index, updated_fork) in forks.into_iter().rev() {
match updated_fork {
Some(updated_fork) => self.unfinalized[fork_index] = updated_fork,
None => { self.unfinalized.remove(fork_index); },
}
}
},
}
}
}
fn do_on_block_insert<Tx: StorageTransaction<Block, T>>(
&self,
tx: &mut Tx,
parent: ComplexBlockId<Block>,
block: ComplexBlockId<Block>,
value: Option<T>,
entry_type: EntryType,
operations: &CommitOperations<Block, T>,
) -> ClientResult<Option<CommitOperation<Block, T>>> {
// this guarantee is currently provided by LightStorage && we're relying on it here
debug_assert!(entry_type != EntryType::Final || self.best_finalized_block.hash == parent.hash);
let prev_operation = operations.operations.last();
debug_assert!(
entry_type != EntryType::Final ||
self.best_finalized_block.hash == parent.hash ||
match prev_operation {
Some(&CommitOperation::BlockFinalized(ref best_finalized_block, _, _))
=> best_finalized_block.hash == parent.hash,
_ => false,
}
);
// we do not store any values behind finalized
if block.number != Zero::zero() && self.best_finalized_block.number >= block.number {
@@ -296,7 +396,7 @@ impl<Block: BlockT, T: CacheItemT, S: Storage<Block, T>> ListCache<Block, T, S>
}
// cleanup database from abandoned unfinalized forks and obsolete finalized entries
let abandoned_forks = self.destroy_abandoned_forks(tx, &block);
let abandoned_forks = self.destroy_abandoned_forks(tx, &block, prev_operation);
self.prune_finalized_entries(tx, &block);
match new_storage_entry {
@@ -310,34 +410,39 @@ impl<Block: BlockT, T: CacheItemT, S: Storage<Block, T>> ListCache<Block, T, S>
}
}
/// When previously inserted block is finalized.
pub fn on_block_finalize<Tx: StorageTransaction<Block, T>>(
fn do_on_block_finalize<Tx: StorageTransaction<Block, T>>(
&self,
tx: &mut Tx,
parent: ComplexBlockId<Block>,
block: ComplexBlockId<Block>,
operations: &CommitOperations<Block, T>,
) -> ClientResult<Option<CommitOperation<Block, T>>> {
// this guarantee is currently provided by LightStorage && we're relying on it here
debug_assert_eq!(self.best_finalized_block.hash, parent.hash);
// this guarantee is currently provided by db backend && we're relying on it here
let prev_operation = operations.operations.last();
debug_assert!(
self.best_finalized_block.hash == parent.hash ||
match prev_operation {
Some(&CommitOperation::BlockFinalized(ref best_finalized_block, _, _))
=> best_finalized_block.hash == parent.hash,
_ => false,
}
);
// there could be at most one entry that is finalizing
let finalizing_entry = self.storage.read_entry(&block)?
.map(|entry| entry.into_entry(block.clone()));
// cleanup database from abandoned unfinalized forks and obsolete finalized entries
let abandoned_forks = self.destroy_abandoned_forks(tx, &block);
let abandoned_forks = self.destroy_abandoned_forks(tx, &block, prev_operation);
self.prune_finalized_entries(tx, &block);
let update_meta = finalizing_entry.is_some();
let operation = CommitOperation::BlockFinalized(block, finalizing_entry, abandoned_forks);
if update_meta {
tx.update_meta(self.best_finalized_entry.as_ref(), &self.unfinalized, &operation);
}
tx.update_meta(self.best_finalized_entry.as_ref(), &self.unfinalized, &operation);
Ok(Some(operation))
}
/// When block is reverted.
pub fn on_block_revert<Tx: StorageTransaction<Block, T>>(
fn do_on_block_revert<Tx: StorageTransaction<Block, T>>(
&self,
tx: &mut Tx,
reverted_block: &ComplexBlockId<Block>,
@@ -374,50 +479,6 @@ impl<Block: BlockT, T: CacheItemT, S: Storage<Block, T>> ListCache<Block, T, S>
Ok(operation)
}
/// When transaction is committed.
pub fn on_transaction_commit(&mut self, op: CommitOperation<Block, T>) {
match op {
CommitOperation::AppendNewBlock(index, best_block) => {
let mut fork = self.unfinalized.get_mut(index)
.expect("ListCache is a crate-private type;
internal clients of ListCache are committing transaction while cache is locked;
CommitOperation holds valid references while cache is locked; qed");
fork.best_block = Some(best_block);
},
CommitOperation::AppendNewEntry(index, entry) => {
let mut fork = self.unfinalized.get_mut(index)
.expect("ListCache is a crate-private type;
internal clients of ListCache are committing transaction while cache is locked;
CommitOperation holds valid references while cache is locked; qed");
fork.best_block = Some(entry.valid_from.clone());
fork.head = entry;
},
CommitOperation::AddNewFork(entry) => {
self.unfinalized.push(Fork {
best_block: Some(entry.valid_from.clone()),
head: entry,
});
},
CommitOperation::BlockFinalized(block, finalizing_entry, forks) => {
self.best_finalized_block = block;
if let Some(finalizing_entry) = finalizing_entry {
self.best_finalized_entry = Some(finalizing_entry);
}
for fork_index in forks.iter().rev() {
self.unfinalized.remove(*fork_index);
}
},
CommitOperation::BlockReverted(forks) => {
for (fork_index, updated_fork) in forks.into_iter().rev() {
match updated_fork {
Some(updated_fork) => self.unfinalized[fork_index] = updated_fork,
None => { self.unfinalized.remove(fork_index); },
}
}
},
}
}
/// Prune old finalized entries.
fn prune_finalized_entries<Tx: StorageTransaction<Block, T>>(
&self,
@@ -475,10 +536,22 @@ impl<Block: BlockT, T: CacheItemT, S: Storage<Block, T>> ListCache<Block, T, S>
fn destroy_abandoned_forks<Tx: StorageTransaction<Block, T>>(
&self,
tx: &mut Tx,
block: &ComplexBlockId<Block>
block: &ComplexBlockId<Block>,
prev_operation: Option<&CommitOperation<Block, T>>,
) -> BTreeSet<usize> {
let mut destroyed = BTreeSet::new();
for (index, fork) in self.unfinalized.iter().enumerate() {
// if some block has been finalized already => take it into account
let prev_abandoned_forks = match prev_operation {
Some(&CommitOperation::BlockFinalized(_, _, ref abandoned_forks)) => Some(abandoned_forks),
_ => None,
};
let mut destroyed = prev_abandoned_forks.cloned().unwrap_or_else(|| BTreeSet::new());
let live_unfinalized = self.unfinalized.iter()
.enumerate()
.filter(|(idx, _)| prev_abandoned_forks
.map(|prev_abandoned_forks| !prev_abandoned_forks.contains(idx))
.unwrap_or(true));
for (index, fork) in live_unfinalized {
if fork.head.valid_from.number == block.number {
destroyed.insert(index);
if fork.head.valid_from.hash != block.hash {
@@ -493,7 +566,10 @@ impl<Block: BlockT, T: CacheItemT, S: Storage<Block, T>> ListCache<Block, T, S>
}
/// Search unfinalized fork where given block belongs.
fn find_unfinalized_fork(&self, block: &ComplexBlockId<Block>) -> ClientResult<Option<&Fork<Block, T>>> {
fn find_unfinalized_fork(
&self,
block: &ComplexBlockId<Block>,
) -> ClientResult<Option<&Fork<Block, T>>> {
for unfinalized in &self.unfinalized {
if unfinalized.matches(&self.storage, block)? {
return Ok(Some(&unfinalized));
@@ -549,7 +625,7 @@ impl<Block: BlockT, T: CacheItemT> Fork<Block, T> {
};
// check if the parent is connected to the beginning of the range
if !chain::is_connected_to_block(storage, &parent, &begin)? {
if !chain::is_connected_to_block(storage, parent, &begin)? {
return Ok(None);
}
@@ -621,6 +697,65 @@ impl<Block: BlockT, T: CacheItemT> Fork<Block, T> {
}
}
impl<Block: BlockT, T: CacheItemT> Default for CommitOperations<Block, T> {
fn default() -> Self {
CommitOperations { operations: Vec::new() }
}
}
// This should never be allowed for non-test code to avoid revealing its internals.
#[cfg(test)]
impl<Block: BlockT, T: CacheItemT> From<Vec<CommitOperation<Block, T>>> for CommitOperations<Block, T> {
fn from(operations: Vec<CommitOperation<Block, T>>) -> Self {
CommitOperations { operations }
}
}
impl<Block: BlockT, T: CacheItemT> CommitOperations<Block, T> {
/// Append operation to the set.
fn append(&mut self, new_operation: Option<CommitOperation<Block, T>>) {
let new_operation = match new_operation {
Some(new_operation) => new_operation,
None => return,
};
let last_operation = match self.operations.pop() {
Some(last_operation) => last_operation,
None => {
self.operations.push(new_operation);
return;
},
};
// we are able (and obliged to) to merge two consequent block finalization operations
match last_operation {
CommitOperation::BlockFinalized(old_finalized_block, old_finalized_entry, old_abandoned_forks) => {
match new_operation {
CommitOperation::BlockFinalized(new_finalized_block, new_finalized_entry, new_abandoned_forks) => {
self.operations.push(CommitOperation::BlockFinalized(
new_finalized_block,
new_finalized_entry,
new_abandoned_forks,
));
},
_ => {
self.operations.push(CommitOperation::BlockFinalized(
old_finalized_block,
old_finalized_entry,
old_abandoned_forks,
));
self.operations.push(new_operation);
},
}
},
_ => {
self.operations.push(last_operation);
self.operations.push(new_operation);
},
}
}
}
/// Destroy fork by deleting all unfinalized entries.
pub fn destroy_fork<Block: BlockT, T: CacheItemT, S: Storage<Block, T>, Tx: StorageTransaction<Block, T>>(
head_valid_from: ComplexBlockId<Block>,
@@ -674,7 +809,7 @@ mod chain {
block1: &ComplexBlockId<Block>,
block2: &ComplexBlockId<Block>,
) -> ClientResult<bool> {
let (begin, end) = if block1 > block2 { (block2, block1) } else { (block1, block2) };
let (begin, end) = if *block1 > *block2 { (block2, block1) } else { (block1, block2) };
let mut current = storage.read_header(&end.hash)?
.ok_or_else(|| ClientError::UnknownBlock(format!("{}", end.hash)))?;
while *current.number() > begin.number {
@@ -773,8 +908,8 @@ pub mod tests {
// when block is earlier than best finalized block AND it is not finalized
// --- 50 ---
// ----------> [100]
assert_eq!(ListCache::<_, u64, _>::new(DummyStorage::new(), PruningStrategy::ByDepth(1024), test_id(100))
.value_at_block(&test_id(50)).unwrap(), None);
assert!(ListCache::<_, u64, _>::new(DummyStorage::new(), PruningStrategy::ByDepth(1024), test_id(100))
.unwrap().value_at_block(&test_id(50)).is_err());
// when block is earlier than best finalized block AND it is finalized AND value is some
// [30] ---- 50 ---> [100]
assert_eq!(ListCache::new(
@@ -784,7 +919,7 @@ pub mod tests {
.with_entry(test_id(100), StorageEntry { prev_valid_from: Some(test_id(30)), value: 100 })
.with_entry(test_id(30), StorageEntry { prev_valid_from: None, value: 30 }),
PruningStrategy::ByDepth(1024), test_id(100)
).value_at_block(&test_id(50)).unwrap(), Some((test_id(30), Some(test_id(100)), 30)));
).unwrap().value_at_block(&test_id(50)).unwrap(), Some((test_id(30), Some(test_id(100)), 30)));
// when block is the best finalized block AND value is some
// ---> [100]
assert_eq!(ListCache::new(
@@ -794,18 +929,18 @@ pub mod tests {
.with_entry(test_id(100), StorageEntry { prev_valid_from: Some(test_id(30)), value: 100 })
.with_entry(test_id(30), StorageEntry { prev_valid_from: None, value: 30 }),
PruningStrategy::ByDepth(1024), test_id(100)
).value_at_block(&test_id(100)).unwrap(), Some((test_id(100), None, 100)));
).unwrap().value_at_block(&test_id(100)).unwrap(), Some((test_id(100), None, 100)));
// when block is parallel to the best finalized block
// ---- 100
// ---> [100]
assert_eq!(ListCache::new(
assert!(ListCache::new(
DummyStorage::new()
.with_meta(Some(test_id(100)), Vec::new())
.with_id(50, H256::from_low_u64_be(50))
.with_entry(test_id(100), StorageEntry { prev_valid_from: Some(test_id(30)), value: 100 })
.with_entry(test_id(30), StorageEntry { prev_valid_from: None, value: 30 }),
PruningStrategy::ByDepth(1024), test_id(100)
).value_at_block(&ComplexBlockId::new(H256::from_low_u64_be(2), 100)).unwrap(), None);
).unwrap().value_at_block(&ComplexBlockId::new(H256::from_low_u64_be(2), 100)).is_err());
// when block is later than last finalized block AND there are no forks AND finalized value is Some
// ---> [100] --- 200
@@ -815,7 +950,7 @@ pub mod tests {
.with_id(50, H256::from_low_u64_be(50))
.with_entry(test_id(100), StorageEntry { prev_valid_from: Some(test_id(30)), value: 100 }),
PruningStrategy::ByDepth(1024), test_id(100)
).value_at_block(&test_id(200)).unwrap(), Some((test_id(100), None, 100)));
).unwrap().value_at_block(&test_id(200)).unwrap(), Some((test_id(100), None, 100)));
// when block is later than last finalized block AND there are no matching forks
// AND block is connected to finalized block AND finalized value is Some
@@ -831,7 +966,7 @@ pub mod tests {
.with_header(test_header(4))
.with_header(fork_header(0, 2, 3)),
PruningStrategy::ByDepth(1024), test_id(2)
).value_at_block(&fork_id(0, 2, 3)).unwrap(), Some((correct_id(2), None, 2)));
).unwrap().value_at_block(&fork_id(0, 2, 3)).unwrap(), Some((correct_id(2), None, 2)));
// when block is later than last finalized block AND there are no matching forks
// AND block is not connected to finalized block
// --- 2 --- 3
@@ -848,7 +983,7 @@ pub mod tests {
.with_header(fork_header(0, 1, 3))
.with_header(fork_header(0, 1, 2)),
PruningStrategy::ByDepth(1024), test_id(2)
).value_at_block(&fork_id(0, 1, 3)).unwrap(), None);
).unwrap().value_at_block(&fork_id(0, 1, 3)).unwrap(), None);
// when block is later than last finalized block AND it appends to unfinalized fork from the end
// AND unfinalized value is Some
@@ -861,7 +996,7 @@ pub mod tests {
.with_header(test_header(4))
.with_header(test_header(5)),
PruningStrategy::ByDepth(1024), test_id(2)
).value_at_block(&correct_id(5)).unwrap(), Some((correct_id(4), None, 4)));
).unwrap().value_at_block(&correct_id(5)).unwrap(), Some((correct_id(4), None, 4)));
// when block is later than last finalized block AND it does not fits unfinalized fork
// AND it is connected to the finalized block AND finalized value is Some
// ---> [2] ----------> [4]
@@ -876,7 +1011,7 @@ pub mod tests {
.with_header(test_header(4))
.with_header(fork_header(0, 2, 3)),
PruningStrategy::ByDepth(1024), test_id(2)
).value_at_block(&fork_id(0, 2, 3)).unwrap(), Some((correct_id(2), None, 2)));
).unwrap().value_at_block(&fork_id(0, 2, 3)).unwrap(), Some((correct_id(2), None, 2)));
}
#[test]
@@ -885,22 +1020,25 @@ pub mod tests {
let fin = EntryType::Final;
// when trying to insert block < finalized number
assert!(ListCache::new(DummyStorage::new(), PruningStrategy::ByDepth(1024), test_id(100))
.on_block_insert(
let mut ops = Default::default();
assert!(ListCache::new(DummyStorage::new(), PruningStrategy::ByDepth(1024), test_id(100)).unwrap()
.do_on_block_insert(
&mut DummyTransaction::new(),
test_id(49),
test_id(50),
Some(50),
nfin,
&mut ops,
).unwrap().is_none());
// when trying to insert block @ finalized number
assert!(ListCache::new(DummyStorage::new(), PruningStrategy::ByDepth(1024), test_id(100))
.on_block_insert(
assert!(ListCache::new(DummyStorage::new(), PruningStrategy::ByDepth(1024), test_id(100)).unwrap()
.do_on_block_insert(
&mut DummyTransaction::new(),
test_id(99),
test_id(100),
Some(100),
nfin,
&Default::default(),
).unwrap().is_none());
// when trying to insert non-final block AND it appends to the best block of unfinalized fork
@@ -910,19 +1048,23 @@ pub mod tests {
.with_meta(None, vec![test_id(4)])
.with_entry(test_id(4), StorageEntry { prev_valid_from: None, value: 4 }),
PruningStrategy::ByDepth(1024), test_id(2)
);
).unwrap();
cache.unfinalized[0].best_block = Some(test_id(4));
let mut tx = DummyTransaction::new();
assert_eq!(cache.on_block_insert(&mut tx, test_id(4), test_id(5), Some(4), nfin).unwrap(),
Some(CommitOperation::AppendNewBlock(0, test_id(5))));
assert_eq!(
cache.do_on_block_insert(&mut tx, test_id(4), test_id(5), Some(4), nfin, &Default::default()).unwrap(),
Some(CommitOperation::AppendNewBlock(0, test_id(5))),
);
assert!(tx.inserted_entries().is_empty());
assert!(tx.removed_entries().is_empty());
assert!(tx.updated_meta().is_none());
// when trying to insert non-final block AND it appends to the best block of unfinalized fork
// AND new value is the same as in the fork' best block
let mut tx = DummyTransaction::new();
assert_eq!(cache.on_block_insert(&mut tx, test_id(4), test_id(5), Some(5), nfin).unwrap(),
Some(CommitOperation::AppendNewEntry(0, Entry { valid_from: test_id(5), value: 5 })));
assert_eq!(
cache.do_on_block_insert(&mut tx, test_id(4), test_id(5), Some(5), nfin, &Default::default()).unwrap(),
Some(CommitOperation::AppendNewEntry(0, Entry { valid_from: test_id(5), value: 5 })),
);
assert_eq!(*tx.inserted_entries(), vec![test_id(5).hash].into_iter().collect());
assert!(tx.removed_entries().is_empty());
assert_eq!(*tx.updated_meta(), Some(Metadata { finalized: None, unfinalized: vec![test_id(5)] }));
@@ -935,18 +1077,36 @@ pub mod tests {
.with_entry(correct_id(4), StorageEntry { prev_valid_from: None, value: 4 })
.with_header(test_header(4)),
PruningStrategy::ByDepth(1024), test_id(2)
);
).unwrap();
let mut tx = DummyTransaction::new();
assert_eq!(cache.on_block_insert(&mut tx, correct_id(4), correct_id(5), Some(4), nfin).unwrap(),
Some(CommitOperation::AppendNewBlock(0, correct_id(5))));
assert_eq!(
cache.do_on_block_insert(
&mut tx,
correct_id(4),
correct_id(5),
Some(4),
nfin,
&Default::default(),
).unwrap(),
Some(CommitOperation::AppendNewBlock(0, correct_id(5))),
);
assert!(tx.inserted_entries().is_empty());
assert!(tx.removed_entries().is_empty());
assert!(tx.updated_meta().is_none());
// when trying to insert non-final block AND it is the first block that appends to the best block of unfinalized fork
// AND new value is the same as in the fork' best block
let mut tx = DummyTransaction::new();
assert_eq!(cache.on_block_insert(&mut tx, correct_id(4), correct_id(5), Some(5), nfin).unwrap(),
Some(CommitOperation::AppendNewEntry(0, Entry { valid_from: correct_id(5), value: 5 })));
assert_eq!(
cache.do_on_block_insert(
&mut tx,
correct_id(4),
correct_id(5),
Some(5),
nfin,
&Default::default(),
).unwrap(),
Some(CommitOperation::AppendNewEntry(0, Entry { valid_from: correct_id(5), value: 5 })),
);
assert_eq!(*tx.inserted_entries(), vec![correct_id(5).hash].into_iter().collect());
assert!(tx.removed_entries().is_empty());
assert_eq!(*tx.updated_meta(), Some(Metadata { finalized: None, unfinalized: vec![correct_id(5)] }));
@@ -961,10 +1121,13 @@ pub mod tests {
.with_header(test_header(3))
.with_header(test_header(4)),
PruningStrategy::ByDepth(1024), correct_id(2)
);
).unwrap();
let mut tx = DummyTransaction::new();
assert_eq!(cache.on_block_insert(&mut tx, correct_id(3), fork_id(0, 3, 4), Some(14), nfin).unwrap(),
Some(CommitOperation::AddNewFork(Entry { valid_from: fork_id(0, 3, 4), value: 14 })));
assert_eq!(
cache.do_on_block_insert(&mut tx, correct_id(3), fork_id(0, 3, 4), Some(14), nfin, &Default::default())
.unwrap(),
Some(CommitOperation::AddNewFork(Entry { valid_from: fork_id(0, 3, 4), value: 14 })),
);
assert_eq!(*tx.inserted_entries(), vec![fork_id(0, 3, 4).hash].into_iter().collect());
assert!(tx.removed_entries().is_empty());
assert_eq!(*tx.updated_meta(), Some(Metadata { finalized: Some(correct_id(2)), unfinalized: vec![correct_id(4), fork_id(0, 3, 4)] }));
@@ -976,9 +1139,13 @@ pub mod tests {
.with_meta(Some(correct_id(2)), vec![])
.with_entry(correct_id(2), StorageEntry { prev_valid_from: None, value: 2 }),
PruningStrategy::ByDepth(1024), correct_id(2)
);
).unwrap();
let mut tx = DummyTransaction::new();
assert_eq!(cache.on_block_insert(&mut tx, correct_id(2), correct_id(3), Some(2), nfin).unwrap(), None);
assert_eq!(
cache.do_on_block_insert(&mut tx, correct_id(2), correct_id(3), Some(2), nfin, &Default::default())
.unwrap(),
None,
);
assert!(tx.inserted_entries().is_empty());
assert!(tx.removed_entries().is_empty());
assert!(tx.updated_meta().is_none());
@@ -989,19 +1156,23 @@ pub mod tests {
.with_meta(Some(correct_id(2)), vec![])
.with_entry(correct_id(2), StorageEntry { prev_valid_from: None, value: 2 }),
PruningStrategy::ByDepth(1024), correct_id(2)
);
).unwrap();
let mut tx = DummyTransaction::new();
assert_eq!(cache.on_block_insert(&mut tx, correct_id(2), correct_id(3), Some(3), nfin).unwrap(),
Some(CommitOperation::AddNewFork(Entry { valid_from: correct_id(3), value: 3 })));
assert_eq!(
cache.do_on_block_insert(&mut tx, correct_id(2), correct_id(3), Some(3), nfin, &Default::default())
.unwrap(),
Some(CommitOperation::AddNewFork(Entry { valid_from: correct_id(3), value: 3 })),
);
assert_eq!(*tx.inserted_entries(), vec![correct_id(3).hash].into_iter().collect());
assert!(tx.removed_entries().is_empty());
assert_eq!(*tx.updated_meta(), Some(Metadata { finalized: Some(correct_id(2)), unfinalized: vec![correct_id(3)] }));
// when inserting finalized entry AND there are no previous finalized entries
let cache = ListCache::new(DummyStorage::new(), PruningStrategy::ByDepth(1024), correct_id(2));
let cache = ListCache::new(DummyStorage::new(), PruningStrategy::ByDepth(1024), correct_id(2)).unwrap();
let mut tx = DummyTransaction::new();
assert_eq!(
cache.on_block_insert(&mut tx, correct_id(2), correct_id(3), Some(3), fin).unwrap(),
cache.do_on_block_insert(&mut tx, correct_id(2), correct_id(3), Some(3), fin, &Default::default())
.unwrap(),
Some(CommitOperation::BlockFinalized(
correct_id(3),
Some(Entry { valid_from: correct_id(3), value: 3 }),
@@ -1017,17 +1188,19 @@ pub mod tests {
.with_meta(Some(correct_id(2)), vec![])
.with_entry(correct_id(2), StorageEntry { prev_valid_from: None, value: 2 }),
PruningStrategy::ByDepth(1024), correct_id(2)
);
).unwrap();
let mut tx = DummyTransaction::new();
assert_eq!(cache.on_block_insert(&mut tx, correct_id(2), correct_id(3), Some(2), fin).unwrap(),
Some(CommitOperation::BlockFinalized(correct_id(3), None, Default::default())));
assert_eq!(
cache.do_on_block_insert(&mut tx, correct_id(2), correct_id(3), Some(2), fin, &Default::default()).unwrap(),
Some(CommitOperation::BlockFinalized(correct_id(3), None, Default::default())),
);
assert!(tx.inserted_entries().is_empty());
assert!(tx.removed_entries().is_empty());
assert!(tx.updated_meta().is_none());
// when inserting finalized entry AND value differs from previous finalized
let mut tx = DummyTransaction::new();
assert_eq!(
cache.on_block_insert(&mut tx, correct_id(2), correct_id(3), Some(3), fin).unwrap(),
cache.do_on_block_insert(&mut tx, correct_id(2), correct_id(3), Some(3), fin, &Default::default()).unwrap(),
Some(CommitOperation::BlockFinalized(
correct_id(3),
Some(Entry { valid_from: correct_id(3), value: 3 }),
@@ -1045,10 +1218,12 @@ pub mod tests {
.with_entry(correct_id(2), StorageEntry { prev_valid_from: None, value: 2 })
.with_entry(fork_id(0, 1, 3), StorageEntry { prev_valid_from: None, value: 13 }),
PruningStrategy::ByDepth(1024), correct_id(2)
);
).unwrap();
let mut tx = DummyTransaction::new();
assert_eq!(cache.on_block_insert(&mut tx, correct_id(2), correct_id(3), Some(2), fin).unwrap(),
Some(CommitOperation::BlockFinalized(correct_id(3), None, vec![0].into_iter().collect())));
assert_eq!(
cache.do_on_block_insert(&mut tx, correct_id(2), correct_id(3), Some(2), fin, &Default::default()).unwrap(),
Some(CommitOperation::BlockFinalized(correct_id(3), None, vec![0].into_iter().collect())),
);
}
#[test]
@@ -1060,13 +1235,18 @@ pub mod tests {
.with_entry(correct_id(2), StorageEntry { prev_valid_from: None, value: 2 })
.with_entry(correct_id(5), StorageEntry { prev_valid_from: Some(correct_id(2)), value: 5 }),
PruningStrategy::ByDepth(1024), correct_id(2)
);
).unwrap();
let mut tx = DummyTransaction::new();
assert_eq!(cache.on_block_finalize(&mut tx, correct_id(2), correct_id(3)).unwrap(),
Some(CommitOperation::BlockFinalized(correct_id(3), None, Default::default())));
assert_eq!(
cache.do_on_block_finalize(&mut tx, correct_id(2), correct_id(3), &Default::default()).unwrap(),
Some(CommitOperation::BlockFinalized(correct_id(3), None, Default::default())),
);
assert!(tx.inserted_entries().is_empty());
assert!(tx.removed_entries().is_empty());
assert!(tx.updated_meta().is_none());
assert_eq!(
*tx.updated_meta(),
Some(Metadata { finalized: Some(correct_id(2)), unfinalized: vec![correct_id(5)] }),
);
// finalization finalizes entry
let cache = ListCache::new(
DummyStorage::new()
@@ -1074,10 +1254,10 @@ pub mod tests {
.with_entry(correct_id(2), StorageEntry { prev_valid_from: None, value: 2 })
.with_entry(correct_id(5), StorageEntry { prev_valid_from: Some(correct_id(2)), value: 5 }),
PruningStrategy::ByDepth(1024), correct_id(4)
);
).unwrap();
let mut tx = DummyTransaction::new();
assert_eq!(
cache.on_block_finalize(&mut tx, correct_id(4), correct_id(5)).unwrap(),
cache.do_on_block_finalize(&mut tx, correct_id(4), correct_id(5), &Default::default()).unwrap(),
Some(CommitOperation::BlockFinalized(
correct_id(5),
Some(Entry { valid_from: correct_id(5), value: 5 }),
@@ -1094,10 +1274,12 @@ pub mod tests {
.with_entry(correct_id(2), StorageEntry { prev_valid_from: None, value: 2 })
.with_entry(fork_id(0, 1, 3), StorageEntry { prev_valid_from: None, value: 13 }),
PruningStrategy::ByDepth(1024), correct_id(2)
);
).unwrap();
let mut tx = DummyTransaction::new();
assert_eq!(cache.on_block_finalize(&mut tx, correct_id(2), correct_id(3)).unwrap(),
Some(CommitOperation::BlockFinalized(correct_id(3), None, vec![0].into_iter().collect())));
assert_eq!(
cache.do_on_block_finalize(&mut tx, correct_id(2), correct_id(3), &Default::default()).unwrap(),
Some(CommitOperation::BlockFinalized(correct_id(3), None, vec![0].into_iter().collect())),
);
}
#[test]
@@ -1109,25 +1291,29 @@ pub mod tests {
.with_entry(correct_id(5), StorageEntry { prev_valid_from: Some(correct_id(2)), value: 5 })
.with_entry(correct_id(6), StorageEntry { prev_valid_from: Some(correct_id(5)), value: 6 }),
PruningStrategy::ByDepth(1024), correct_id(2)
);
).unwrap();
// when new block is appended to unfinalized fork
cache.on_transaction_commit(CommitOperation::AppendNewBlock(0, correct_id(6)));
cache.on_transaction_commit(vec![CommitOperation::AppendNewBlock(0, correct_id(6))].into());
assert_eq!(cache.unfinalized[0].best_block, Some(correct_id(6)));
// when new entry is appended to unfinalized fork
cache.on_transaction_commit(CommitOperation::AppendNewEntry(0, Entry { valid_from: correct_id(7), value: 7 }));
cache.on_transaction_commit(vec![
CommitOperation::AppendNewEntry(0, Entry { valid_from: correct_id(7), value: 7 }),
].into());
assert_eq!(cache.unfinalized[0].best_block, Some(correct_id(7)));
assert_eq!(cache.unfinalized[0].head, Entry { valid_from: correct_id(7), value: 7 });
// when new fork is added
cache.on_transaction_commit(CommitOperation::AddNewFork(Entry { valid_from: correct_id(10), value: 10 }));
cache.on_transaction_commit(vec![
CommitOperation::AddNewFork(Entry { valid_from: correct_id(10), value: 10 }),
].into());
assert_eq!(cache.unfinalized[2].best_block, Some(correct_id(10)));
assert_eq!(cache.unfinalized[2].head, Entry { valid_from: correct_id(10), value: 10 });
// when block is finalized + entry is finalized + unfinalized forks are deleted
cache.on_transaction_commit(CommitOperation::BlockFinalized(
cache.on_transaction_commit(vec![CommitOperation::BlockFinalized(
correct_id(20),
Some(Entry { valid_from: correct_id(20), value: 20 }),
vec![0, 1, 2].into_iter().collect(),
));
)].into());
assert_eq!(cache.best_finalized_block, correct_id(20));
assert_eq!(cache.best_finalized_entry, Some(Entry { valid_from: correct_id(20), value: 20 }));
assert!(cache.unfinalized.is_empty());
@@ -1148,7 +1334,7 @@ pub mod tests {
.with_header(test_header(4))
.with_header(test_header(5)),
PruningStrategy::ByDepth(1024), correct_id(0)
).find_unfinalized_fork(&correct_id(4)).unwrap().unwrap().head.valid_from, correct_id(5));
).unwrap().find_unfinalized_fork((&correct_id(4)).into()).unwrap().unwrap().head.valid_from, correct_id(5));
// --- [2] ---------------> [5]
// ----------> [3] ---> 4
assert_eq!(ListCache::new(
@@ -1165,7 +1351,8 @@ pub mod tests {
.with_header(fork_header(0, 1, 3))
.with_header(fork_header(0, 1, 4)),
PruningStrategy::ByDepth(1024), correct_id(0)
).find_unfinalized_fork(&fork_id(0, 1, 4)).unwrap().unwrap().head.valid_from, fork_id(0, 1, 3));
).unwrap()
.find_unfinalized_fork((&fork_id(0, 1, 4)).into()).unwrap().unwrap().head.valid_from, fork_id(0, 1, 3));
// --- [2] ---------------> [5]
// ----------> [3]
// -----------------> 4
@@ -1185,7 +1372,7 @@ pub mod tests {
.with_header(fork_header(1, 1, 3))
.with_header(fork_header(1, 1, 4)),
PruningStrategy::ByDepth(1024), correct_id(0)
).find_unfinalized_fork(&fork_id(1, 1, 4)).unwrap().is_none());
).unwrap().find_unfinalized_fork((&fork_id(1, 1, 4)).into()).unwrap().is_none());
}
#[test]
@@ -1195,7 +1382,7 @@ pub mod tests {
.with_entry(test_id(100), StorageEntry { prev_valid_from: Some(test_id(50)), value: 100 })
.with_entry(test_id(50), StorageEntry { prev_valid_from: None, value: 50 });
assert_eq!(Fork::<_, u64> { best_block: None, head: Entry { valid_from: test_id(100), value: 0 } }
.matches(&storage, &test_id(20)).unwrap(), false);
.matches(&storage, (&test_id(20)).into()).unwrap(), false);
// when block is not connected to the begin block
let storage = DummyStorage::new()
.with_entry(correct_id(5), StorageEntry { prev_valid_from: Some(correct_id(3)), value: 100 })
@@ -1206,7 +1393,7 @@ pub mod tests {
.with_header(fork_header(0, 2, 4))
.with_header(fork_header(0, 2, 3));
assert_eq!(Fork::<_, u64> { best_block: None, head: Entry { valid_from: correct_id(5), value: 100 } }
.matches(&storage, &fork_id(0, 2, 4)).unwrap(), false);
.matches(&storage, (&fork_id(0, 2, 4)).into()).unwrap(), false);
// when block is not connected to the end block
let storage = DummyStorage::new()
.with_entry(correct_id(5), StorageEntry { prev_valid_from: Some(correct_id(3)), value: 100 })
@@ -1216,14 +1403,14 @@ pub mod tests {
.with_header(test_header(3))
.with_header(fork_header(0, 3, 4));
assert_eq!(Fork::<_, u64> { best_block: None, head: Entry { valid_from: correct_id(5), value: 100 } }
.matches(&storage, &fork_id(0, 3, 4)).unwrap(), false);
.matches(&storage, (&fork_id(0, 3, 4)).into()).unwrap(), false);
// when block is connected to the begin block AND end is open
let storage = DummyStorage::new()
.with_entry(correct_id(5), StorageEntry { prev_valid_from: None, value: 100 })
.with_header(test_header(5))
.with_header(test_header(6));
assert_eq!(Fork::<_, u64> { best_block: None, head: Entry { valid_from: correct_id(5), value: 100 } }
.matches(&storage, &correct_id(6)).unwrap(), true);
.matches(&storage, (&correct_id(6)).into()).unwrap(), true);
// when block is connected to the begin block AND to the end block
let storage = DummyStorage::new()
.with_entry(correct_id(5), StorageEntry { prev_valid_from: Some(correct_id(3)), value: 100 })
@@ -1232,7 +1419,7 @@ pub mod tests {
.with_header(test_header(4))
.with_header(test_header(3));
assert_eq!(Fork::<_, u64> { best_block: None, head: Entry { valid_from: correct_id(5), value: 100 } }
.matches(&storage, &correct_id(4)).unwrap(), true);
.matches(&storage, (&correct_id(4)).into()).unwrap(), true);
}
#[test]
@@ -1338,9 +1525,21 @@ pub mod tests {
#[test]
fn is_connected_to_block_fails() {
// when storage returns error
assert!(chain::is_connected_to_block::<_, u64, _>(&FaultyStorage, &test_id(1), &test_id(100)).is_err());
assert!(
chain::is_connected_to_block::<_, u64, _>(
&FaultyStorage,
(&test_id(1)).into(),
&test_id(100),
).is_err(),
);
// when there's no header in the storage
assert!(chain::is_connected_to_block::<_, u64, _>(&DummyStorage::new(), &test_id(1), &test_id(100)).is_err());
assert!(
chain::is_connected_to_block::<_, u64, _>(
&DummyStorage::new(),
(&test_id(1)).into(),
&test_id(100),
).is_err(),
);
}
#[test]
@@ -1348,35 +1547,35 @@ pub mod tests {
// when without iterations we end up with different block
assert_eq!(chain::is_connected_to_block::<_, u64, _>(&DummyStorage::new()
.with_header(test_header(1)),
&test_id(1), &correct_id(1)).unwrap(), false);
(&test_id(1)).into(), &correct_id(1)).unwrap(), false);
// when with ASC iterations we end up with different block
assert_eq!(chain::is_connected_to_block::<_, u64, _>(&DummyStorage::new()
.with_header(test_header(0))
.with_header(test_header(1))
.with_header(test_header(2)),
&test_id(0), &correct_id(2)).unwrap(), false);
(&test_id(0)).into(), &correct_id(2)).unwrap(), false);
// when with DESC iterations we end up with different block
assert_eq!(chain::is_connected_to_block::<_, u64, _>(&DummyStorage::new()
.with_header(test_header(0))
.with_header(test_header(1))
.with_header(test_header(2)),
&correct_id(2), &test_id(0)).unwrap(), false);
(&correct_id(2)).into(), &test_id(0)).unwrap(), false);
// when without iterations we end up with the same block
assert_eq!(chain::is_connected_to_block::<_, u64, _>(&DummyStorage::new()
.with_header(test_header(1)),
&correct_id(1), &correct_id(1)).unwrap(), true);
(&correct_id(1)).into(), &correct_id(1)).unwrap(), true);
// when with ASC iterations we end up with the same block
assert_eq!(chain::is_connected_to_block::<_, u64, _>(&DummyStorage::new()
.with_header(test_header(0))
.with_header(test_header(1))
.with_header(test_header(2)),
&correct_id(0), &correct_id(2)).unwrap(), true);
(&correct_id(0)).into(), &correct_id(2)).unwrap(), true);
// when with DESC iterations we end up with the same block
assert_eq!(chain::is_connected_to_block::<_, u64, _>(&DummyStorage::new()
.with_header(test_header(0))
.with_header(test_header(1))
.with_header(test_header(2)),
&correct_id(2), &correct_id(0)).unwrap(), true);
(&correct_id(2)).into(), &correct_id(0)).unwrap(), true);
}
#[test]
@@ -1454,7 +1653,7 @@ pub mod tests {
.with_entry(test_id(10), StorageEntry { prev_valid_from: None, value: 10 })
.with_entry(test_id(20), StorageEntry { prev_valid_from: Some(test_id(10)), value: 20 })
.with_entry(test_id(30), StorageEntry { prev_valid_from: Some(test_id(20)), value: 30 }),
strategy, test_id(9));
strategy, test_id(9)).unwrap();
let mut tx = DummyTransaction::new();
// when finalizing entry #10: no entries pruned
@@ -1515,28 +1714,64 @@ pub mod tests {
.with_header(fork_header(1, 2, 5))
.with_header(fork_header(2, 4, 5)),
PruningStrategy::ByDepth(1024), correct_id(1)
);
).unwrap();
// when 5 is reverted: entry 5 is truncated
let op = cache.on_block_revert(&mut DummyTransaction::new(), &correct_id(5)).unwrap();
let op = cache.do_on_block_revert(&mut DummyTransaction::new(), &correct_id(5)).unwrap();
assert_eq!(op, CommitOperation::BlockReverted(vec![
(0, Some(Fork { best_block: None, head: Entry { valid_from: correct_id(4), value: 4 } })),
].into_iter().collect()));
cache.on_transaction_commit(op);
cache.on_transaction_commit(vec![op].into());
// when 3 is reverted: entries 4+5' are truncated
let op = cache.on_block_revert(&mut DummyTransaction::new(), &correct_id(3)).unwrap();
let op = cache.do_on_block_revert(&mut DummyTransaction::new(), &correct_id(3)).unwrap();
assert_eq!(op, CommitOperation::BlockReverted(vec![
(0, None),
(2, None),
].into_iter().collect()));
cache.on_transaction_commit(op);
cache.on_transaction_commit(vec![op].into());
// when 2 is reverted: entries 4'+5' are truncated
let op = cache.on_block_revert(&mut DummyTransaction::new(), &correct_id(2)).unwrap();
let op = cache.do_on_block_revert(&mut DummyTransaction::new(), &correct_id(2)).unwrap();
assert_eq!(op, CommitOperation::BlockReverted(vec![
(0, None),
].into_iter().collect()));
cache.on_transaction_commit(op);
cache.on_transaction_commit(vec![op].into());
}
#[test]
fn append_commit_operation_works() {
let mut ops = CommitOperations::default();
ops.append(None);
assert_eq!(ops.operations, Vec::new());
ops.append(Some(CommitOperation::BlockFinalized(
test_id(10),
Some(Entry { valid_from: test_id(10), value: 10 }),
vec![5].into_iter().collect(),
)));
assert_eq!(
ops.operations,
vec![CommitOperation::BlockFinalized(
test_id(10),
Some(Entry { valid_from: test_id(10), value: 10 }),
vec![5].into_iter().collect(),
)],
);
ops.append(Some(CommitOperation::BlockFinalized(
test_id(20),
Some(Entry { valid_from: test_id(20), value: 20 }),
vec![5, 6].into_iter().collect(),
)));
assert_eq!(
ops.operations,
vec![CommitOperation::BlockFinalized(
test_id(20),
Some(Entry { valid_from: test_id(20), value: 20 }),
vec![5, 6].into_iter().collect(),
)],
);
}
}
substrate/client/db/src/cache/list_storage.rs
Vendored
+3 -1
View File
@@ -222,7 +222,9 @@ mod meta {
unfinalized.push(&entry.valid_from);
},
CommitOperation::BlockFinalized(_, ref finalizing_entry, ref forks) => {
finalized = finalizing_entry.as_ref().map(|entry| &entry.valid_from);
if let Some(finalizing_entry) = finalizing_entry.as_ref() {
finalized = Some(&finalizing_entry.valid_from);
}
for fork_index in forks.iter().rev() {
unfinalized.remove(*fork_index);
}
substrate/client/db/src/cache/mod.rs
Vendored
+88 -62
View File
@@ -16,7 +16,7 @@
//! DB-backed cache of blockchain data.
use std::{sync::Arc, collections::HashMap};
use std::{sync::Arc, collections::{HashMap, hash_map::Entry}};
use parking_lot::RwLock;
use kvdb::{KeyValueDB, DBTransaction};
@@ -51,8 +51,10 @@ pub enum EntryType {
/// Block identifier that holds both hash and number.
#[derive(Clone, Debug, Encode, Decode, PartialEq)]
pub struct ComplexBlockId<Block: BlockT> {
hash: Block::Hash,
number: NumberFor<Block>,
/// Hash of the block.
pub(crate) hash: Block::Hash,
/// Number of the block.
pub(crate) number: NumberFor<Block>,
}
impl<Block: BlockT> ComplexBlockId<Block> {
@@ -79,7 +81,7 @@ pub struct DbCache<Block: BlockT> {
db: Arc<dyn KeyValueDB>,
key_lookup_column: u32,
header_column: u32,
authorities_column: u32,
cache_column: u32,
genesis_hash: Block::Hash,
best_finalized_block: ComplexBlockId<Block>,
}
@@ -90,7 +92,7 @@ impl<Block: BlockT> DbCache<Block> {
db: Arc<dyn KeyValueDB>,
key_lookup_column: u32,
header_column: u32,
authorities_column: u32,
cache_column: u32,
genesis_hash: Block::Hash,
best_finalized_block: ComplexBlockId<Block>,
) -> Self {
@@ -99,7 +101,7 @@ impl<Block: BlockT> DbCache<Block> {
db,
key_lookup_column,
header_column,
authorities_column,
cache_column,
genesis_hash,
best_finalized_block,
}
@@ -115,30 +117,48 @@ impl<Block: BlockT> DbCache<Block> {
DbCacheTransaction {
cache: self,
tx,
cache_at_op: HashMap::new(),
cache_at_ops: HashMap::new(),
best_finalized_block: None,
}
}
/// Begin cache transaction with given ops.
pub fn transaction_with_ops<'a>(
&'a mut self,
tx: &'a mut DBTransaction,
ops: DbCacheTransactionOps<Block>,
) -> DbCacheTransaction<'a, Block> {
DbCacheTransaction {
cache: self,
tx,
cache_at_ops: ops.cache_at_ops,
best_finalized_block: ops.best_finalized_block,
}
}
/// Run post-commit cache operations.
pub fn commit(&mut self, ops: DbCacheTransactionOps<Block>) {
for (name, op) in ops.cache_at_op.into_iter() {
self.get_cache(name).on_transaction_commit(op);
pub fn commit(&mut self, ops: DbCacheTransactionOps<Block>) -> ClientResult<()> {
for (name, ops) in ops.cache_at_ops.into_iter() {
self.get_cache(name)?.on_transaction_commit(ops);
}
if let Some(best_finalized_block) = ops.best_finalized_block {
self.best_finalized_block = best_finalized_block;
}
Ok(())
}
/// Creates `ListCache` with the given name or returns a reference to the existing.
fn get_cache(&mut self, name: CacheKeyId) -> &mut ListCache<Block, Vec<u8>, self::list_storage::DbStorage> {
pub(crate) fn get_cache(
&mut self,
name: CacheKeyId,
) -> ClientResult<&mut ListCache<Block, Vec<u8>, self::list_storage::DbStorage>> {
get_cache_helper(
&mut self.cache_at,
name,
&self.db,
self.key_lookup_column,
self.header_column,
self.authorities_column,
self.cache_column,
&self.best_finalized_block
)
}
@@ -154,34 +174,49 @@ fn get_cache_helper<'a, Block: BlockT>(
header: u32,
cache: u32,
best_finalized_block: &ComplexBlockId<Block>,
) -> &'a mut ListCache<Block, Vec<u8>, self::list_storage::DbStorage> {
cache_at.entry(name).or_insert_with(|| {
ListCache::new(
self::list_storage::DbStorage::new(name.to_vec(), db.clone(),
self::list_storage::DbColumns {
meta: COLUMN_META,
key_lookup,
header,
cache,
},
),
cache_pruning_strategy(name),
best_finalized_block.clone(),
)
})
) -> ClientResult<&'a mut ListCache<Block, Vec<u8>, self::list_storage::DbStorage>> {
match cache_at.entry(name) {
Entry::Occupied(entry) => Ok(entry.into_mut()),
Entry::Vacant(entry) => {
let cache = ListCache::new(
self::list_storage::DbStorage::new(name.to_vec(), db.clone(),
self::list_storage::DbColumns {
meta: COLUMN_META,
key_lookup,
header,
cache,
},
),
cache_pruning_strategy(name),
best_finalized_block.clone(),
)?;
Ok(entry.insert(cache))
}
}
}
/// Cache operations that are to be committed after database transaction is committed.
#[derive(Default)]
pub struct DbCacheTransactionOps<Block: BlockT> {
cache_at_op: HashMap<CacheKeyId, self::list_cache::CommitOperation<Block, Vec<u8>>>,
cache_at_ops: HashMap<CacheKeyId, self::list_cache::CommitOperations<Block, Vec<u8>>>,
best_finalized_block: Option<ComplexBlockId<Block>>,
}
impl<Block: BlockT> DbCacheTransactionOps<Block> {
/// Empty transaction ops.
pub fn empty() -> DbCacheTransactionOps<Block> {
DbCacheTransactionOps {
cache_at_ops: HashMap::new(),
best_finalized_block: None,
}
}
}
/// Database-backed blockchain data cache transaction valid for single block import.
pub struct DbCacheTransaction<'a, Block: BlockT> {
cache: &'a mut DbCache<Block>,
tx: &'a mut DBTransaction,
cache_at_op: HashMap<CacheKeyId, self::list_cache::CommitOperation<Block, Vec<u8>>>,
cache_at_ops: HashMap<CacheKeyId, self::list_cache::CommitOperations<Block, Vec<u8>>>,
best_finalized_block: Option<ComplexBlockId<Block>>,
}
@@ -189,7 +224,7 @@ impl<'a, Block: BlockT> DbCacheTransaction<'a, Block> {
/// Convert transaction into post-commit operations set.
pub fn into_ops(self) -> DbCacheTransactionOps<Block> {
DbCacheTransactionOps {
cache_at_op: self.cache_at_op,
cache_at_ops: self.cache_at_ops,
best_finalized_block: self.best_finalized_block,
}
}
@@ -202,8 +237,6 @@ impl<'a, Block: BlockT> DbCacheTransaction<'a, Block> {
data_at: HashMap<CacheKeyId, Vec<u8>>,
entry_type: EntryType,
) -> ClientResult<Self> {
assert!(self.cache_at_op.is_empty());
// prepare list of caches that are not update
// (we might still need to do some cache maintenance in this case)
let missed_caches = self.cache.cache_at.keys()
@@ -212,8 +245,9 @@ impl<'a, Block: BlockT> DbCacheTransaction<'a, Block> {
.collect::<Vec<_>>();
let mut insert_op = |name: CacheKeyId, value: Option<Vec<u8>>| -> Result<(), sp_blockchain::Error> {
let cache = self.cache.get_cache(name);
let op = cache.on_block_insert(
let cache = self.cache.get_cache(name)?;
let cache_ops = self.cache_at_ops.entry(name).or_default();
cache.on_block_insert(
&mut self::list_storage::DbStorageTransaction::new(
cache.storage(),
&mut self.tx,
@@ -222,10 +256,9 @@ impl<'a, Block: BlockT> DbCacheTransaction<'a, Block> {
block.clone(),
value,
entry_type,
cache_ops,
)?;
if let Some(op) = op {
self.cache_at_op.insert(name, op);
}
Ok(())
};
@@ -245,23 +278,19 @@ impl<'a, Block: BlockT> DbCacheTransaction<'a, Block> {
pub fn on_block_finalize(
mut self,
parent: ComplexBlockId<Block>,
block: ComplexBlockId<Block>
block: ComplexBlockId<Block>,
) -> ClientResult<Self> {
assert!(self.cache_at_op.is_empty());
for (name, cache_at) in self.cache.cache_at.iter() {
let op = cache_at.on_block_finalize(
for (name, cache) in self.cache.cache_at.iter() {
let cache_ops = self.cache_at_ops.entry(*name).or_default();
cache.on_block_finalize(
&mut self::list_storage::DbStorageTransaction::new(
cache_at.storage(),
cache.storage(),
&mut self.tx
),
parent.clone(),
block.clone(),
cache_ops,
)?;
if let Some(op) = op {
self.cache_at_op.insert(name.to_owned(), op);
}
}
self.best_finalized_block = Some(block);
@@ -275,16 +304,15 @@ impl<'a, Block: BlockT> DbCacheTransaction<'a, Block> {
reverted_block: &ComplexBlockId<Block>,
) -> ClientResult<Self> {
for (name, cache) in self.cache.cache_at.iter() {
let op = cache.on_block_revert(
let cache_ops = self.cache_at_ops.entry(*name).or_default();
cache.on_block_revert(
&mut self::list_storage::DbStorageTransaction::new(
cache.storage(),
&mut self.tx
),
reverted_block,
cache_ops,
)?;
assert!(!self.cache_at_op.contains_key(name));
self.cache_at_op.insert(name.to_owned(), op);
}
Ok(self)
@@ -310,7 +338,7 @@ impl<Block: BlockT> BlockchainCache<Block> for DbCacheSync<Block> {
)?;
let tx_ops = tx.into_ops();
db.write(dbtx).map_err(db_err)?;
cache.commit(tx_ops);
cache.commit(tx_ops)?;
Ok(())
}
@@ -318,40 +346,38 @@ impl<Block: BlockT> BlockchainCache<Block> for DbCacheSync<Block> {
&self,
key: &CacheKeyId,
at: &BlockId<Block>,
) -> Option<((NumberFor<Block>, Block::Hash), Option<(NumberFor<Block>, Block::Hash)>, Vec<u8>)> {
) -> ClientResult<Option<((NumberFor<Block>, Block::Hash), Option<(NumberFor<Block>, Block::Hash)>, Vec<u8>)>> {
let mut cache = self.0.write();
let storage = cache.get_cache(*key).storage();
let cache = cache.get_cache(*key)?;
let storage = cache.storage();
let db = storage.db();
let columns = storage.columns();
let at = match *at {
BlockId::Hash(hash) => {
let header = utils::read_header::<Block>(
let header = utils::require_header::<Block>(
&**db,
columns.key_lookup,
columns.header,
BlockId::Hash(hash.clone())).ok()??;
BlockId::Hash(hash.clone()))?;
ComplexBlockId::new(hash, *header.number())
},
BlockId::Number(number) => {
let hash = utils::read_header::<Block>(
let hash = utils::require_header::<Block>(
&**db,
columns.key_lookup,
columns.header,
BlockId::Number(number.clone())).ok()??.hash();
BlockId::Number(number.clone()))?.hash();
ComplexBlockId::new(hash, number)
},
};
cache.cache_at
.get(key)?
.value_at_block(&at)
cache.value_at_block(&at)
.map(|block_and_value| block_and_value.map(|(begin_block, end_block, value)|
(
(begin_block.number, begin_block.hash),
end_block.map(|end_block| (end_block.number, end_block.hash)),
value,
)))
.ok()?
}
}
substrate/client/db/src/changes_tries_storage.rs
+1015
View File
File diff suppressed because it is too large. Load diff
substrate/client/db/src/lib.rs
+162 -491
View File
@@ -31,18 +31,21 @@ pub mod offchain;
mod children;
mod cache;
mod changes_tries_storage;
mod storage_cache;
#[cfg(any(feature = "kvdb-rocksdb", test))]
mod upgrade;
mod utils;
mod stats;
use std::sync::Arc;
use std::path::PathBuf;
use std::io;
use std::collections::{HashMap, HashSet};
use std::collections::HashMap;
use sc_client_api::{execution_extensions::ExecutionExtensions, ForkBlocks, UsageInfo, MemoryInfo, BadBlocks, IoInfo};
use sc_client_api::backend::NewBlockState;
use sc_client_api::backend::{StorageCollection, ChildStorageCollection};
use sc_client_api::backend::{PrunableStateChangesTrieStorage, StorageCollection, ChildStorageCollection};
use sp_blockchain::{
Result as ClientResult, Error as ClientError,
well_known_cache_keys, HeaderBackend,
@@ -51,11 +54,11 @@ use codec::{Decode, Encode};
use hash_db::Prefix;
use kvdb::{KeyValueDB, DBTransaction};
use sp_trie::{MemoryDB, PrefixedMemoryDB, prefixed_key};
use parking_lot::{Mutex, RwLock};
use sp_core::{ChangesTrieConfiguration, convert_hash, traits::CodeExecutor};
use parking_lot::RwLock;
use sp_core::{ChangesTrieConfiguration, traits::CodeExecutor};
use sp_core::storage::{well_known_keys, ChildInfo};
use sp_runtime::{
generic::{BlockId, DigestItem}, Justification, Storage,
generic::BlockId, Justification, Storage,
BuildStorage,
};
use sp_runtime::traits::{
@@ -63,10 +66,11 @@ use sp_runtime::traits::{
};
use sc_executor::RuntimeInfo;
use sp_state_machine::{
DBValue, ChangesTrieTransaction, ChangesTrieCacheAction, ChangesTrieBuildCache,
DBValue, ChangesTrieTransaction, ChangesTrieCacheAction,
backend::Backend as StateBackend, UsageInfo as StateUsageInfo,
};
use crate::utils::{Meta, db_err, meta_keys, read_db, read_meta};
use crate::utils::{DatabaseType, Meta, db_err, meta_keys, read_db, read_meta};
use crate::changes_tries_storage::{DbChangesTrieStorage, DbChangesTrieStorageTransaction};
use sc_client::leaves::{LeafSet, FinalizationDisplaced};
use sc_state_db::StateDb;
use sp_blockchain::{CachedHeaderMetadata, HeaderMetadata, HeaderMetadataCache};
@@ -322,6 +326,7 @@ pub(crate) mod columns {
pub const AUX: u32 = 8;
/// Offchain workers local storage
pub const OFFCHAIN: u32 = 9;
pub const CACHE: u32 = 10;
}
struct PendingBlock<Block: BlockT> {
@@ -352,7 +357,7 @@ pub struct BlockchainDb<Block: BlockT> {
impl<Block: BlockT> BlockchainDb<Block> {
fn new(db: Arc<dyn KeyValueDB>) -> ClientResult<Self> {
let meta = read_meta::<Block>(&*db, columns::META, columns::HEADER)?;
let meta = read_meta::<Block>(&*db, columns::HEADER)?;
let leaves = LeafSet::read_from_db(&*db, columns::META, meta_keys::LEAF_PREFIX)?;
Ok(BlockchainDb {
db,
@@ -511,7 +516,8 @@ pub struct BlockImportOperation<Block: BlockT> {
storage_updates: StorageCollection,
child_storage_updates: ChildStorageCollection,
changes_trie_updates: MemoryDB<HasherFor<Block>>,
changes_trie_cache_update: Option<ChangesTrieCacheAction<Block::Hash, NumberFor<Block>>>,
changes_trie_build_cache_update: Option<ChangesTrieCacheAction<Block::Hash, NumberFor<Block>>>,
changes_trie_config_update: Option<Option<ChangesTrieConfiguration>>,
pending_block: Option<PendingBlock<Block>>,
aux_ops: Vec<(Vec<u8>, Option<Vec<u8>>)>,
finalized_blocks: Vec<(BlockId<Block>, Option<Justification>)>,
@@ -545,6 +551,9 @@ impl<Block: BlockT> sc_client_api::backend::BlockImportOperation<Block> for Bloc
leaf_state: NewBlockState,
) -> ClientResult<()> {
assert!(self.pending_block.is_none(), "Only one block per operation is allowed");
if let Some(changes_trie_config_update) = changes_tries_storage::extract_new_configuration(&header) {
self.changes_trie_config_update = Some(changes_trie_config_update.clone());
}
self.pending_block = Some(PendingBlock {
header,
body,
@@ -583,12 +592,22 @@ impl<Block: BlockT> sc_client_api::backend::BlockImportOperation<Block> for Bloc
child_content.data.into_iter().map(|(k, v)| (k, Some(v))), child_content.child_info),
);
let mut changes_trie_config: Option<ChangesTrieConfiguration> = None;
let (root, transaction) = self.old_state.full_storage_root(
storage.top.into_iter().map(|(k, v)| (k, Some(v))),
storage.top.into_iter().map(|(k, v)| {
if k == well_known_keys::CHANGES_TRIE_CONFIG {
changes_trie_config = Some(
Decode::decode(&mut &v[..])
.expect("changes trie configuration is encoded properly at genesis")
);
}
(k, Some(v))
}),
child_delta
);
self.db_updates = transaction;
self.changes_trie_config_update = Some(changes_trie_config);
self.commit_state = true;
Ok(root)
}
@@ -598,7 +617,7 @@ impl<Block: BlockT> sc_client_api::backend::BlockImportOperation<Block> for Bloc
update: ChangesTrieTransaction<HasherFor<Block>, NumberFor<Block>>,
) -> ClientResult<()> {
self.changes_trie_updates = update.0;
self.changes_trie_cache_update = Some(update.1);
self.changes_trie_build_cache_update = Some(update.1);
Ok(())
}
@@ -674,174 +693,6 @@ impl<Block: BlockT> sp_state_machine::Storage<HasherFor<Block>> for DbGenesisSto
}
}
/// A database wrapper for changes tries.
pub struct DbChangesTrieStorage<Block: BlockT> {
db: Arc<dyn KeyValueDB>,
meta: Arc<RwLock<Meta<NumberFor<Block>, Block::Hash>>>,
min_blocks_to_keep: Option<u32>,
cache: RwLock<ChangesTrieBuildCache<Block::Hash, NumberFor<Block>>>,
}
impl<Block: BlockT> DbChangesTrieStorage<Block> {
/// Commit new changes trie.
pub fn commit(&self, tx: &mut DBTransaction, mut changes_trie: MemoryDB<HasherFor<Block>>) {
for (key, (val, _)) in changes_trie.drain() {
tx.put(columns::CHANGES_TRIE, key.as_ref(), &val);
}
}
/// Commit changes into changes trie build cache.
pub fn commit_cache(&self, cache_update: ChangesTrieCacheAction<Block::Hash, NumberFor<Block>>) {
self.cache.write().perform(cache_update);
}
/// Prune obsolete changes tries.
pub fn prune(
&self,
config: &ChangesTrieConfiguration,
tx: &mut DBTransaction,
block_hash: Block::Hash,
block_num: NumberFor<Block>,
) {
// never prune on archive nodes
let min_blocks_to_keep = match self.min_blocks_to_keep {
Some(min_blocks_to_keep) => min_blocks_to_keep,
None => return,
};
sp_state_machine::prune_changes_tries(
config,
&*self,
min_blocks_to_keep.into(),
&sp_state_machine::ChangesTrieAnchorBlockId {
hash: convert_hash(&block_hash),
number: block_num,
},
|node| tx.delete(columns::CHANGES_TRIE, node.as_ref()));
}
}
impl<Block: BlockT> sc_client_api::backend::PrunableStateChangesTrieStorage<Block>
for DbChangesTrieStorage<Block>
{
fn oldest_changes_trie_block(
&self,
config: &ChangesTrieConfiguration,
best_finalized_block: NumberFor<Block>,
) -> NumberFor<Block> {
match self.min_blocks_to_keep {
Some(min_blocks_to_keep) => sp_state_machine::oldest_non_pruned_changes_trie(
config,
min_blocks_to_keep.into(),
best_finalized_block,
),
None => One::one(),
}
}
}
impl<Block: BlockT> sp_state_machine::ChangesTrieRootsStorage<HasherFor<Block>, NumberFor<Block>>
for DbChangesTrieStorage<Block>
{
fn build_anchor(
&self,
hash: Block::Hash,
) -> Result<sp_state_machine::ChangesTrieAnchorBlockId<Block::Hash, NumberFor<Block>>, String> {
utils::read_header::<Block>(
&*self.db,
columns::KEY_LOOKUP,
columns::HEADER,
BlockId::Hash(hash),
)
.map_err(|e| e.to_string())
.and_then(|maybe_header| maybe_header.map(|header|
sp_state_machine::ChangesTrieAnchorBlockId {
hash,
number: *header.number(),
}
).ok_or_else(|| format!("Unknown header: {}", hash)))
}
fn root(
&self,
anchor: &sp_state_machine::ChangesTrieAnchorBlockId<Block::Hash, NumberFor<Block>>,
block: NumberFor<Block>,
) -> Result<Option<Block::Hash>, String> {
// check API requirement: we can't get NEXT block(s) based on anchor
if block > anchor.number {
return Err(
format!("Can't get changes trie root at {} using anchor at {}", block, anchor.number)
)
}
// we need to get hash of the block to resolve changes trie root
let block_id = if block <= self.meta.read().finalized_number {
// if block is finalized, we could just read canonical hash
BlockId::Number(block)
} else {
// the block is not finalized
let mut current_num = anchor.number;
let mut current_hash: Block::Hash = convert_hash(&anchor.hash);
let maybe_anchor_header: Block::Header = utils::require_header::<Block>(
&*self.db, columns::KEY_LOOKUP, columns::HEADER, BlockId::Number(current_num)
).map_err(|e| e.to_string())?;
if maybe_anchor_header.hash() == current_hash {
// if anchor is canonicalized, then the block is also canonicalized
BlockId::Number(block)
} else {
// else (block is not finalized + anchor is not canonicalized):
// => we should find the required block hash by traversing
// back from the anchor to the block with given number
while current_num != block {
let current_header: Block::Header = utils::require_header::<Block>(
&*self.db, columns::KEY_LOOKUP, columns::HEADER, BlockId::Hash(current_hash)
).map_err(|e| e.to_string())?;
current_hash = *current_header.parent_hash();
current_num = current_num - One::one();
}
BlockId::Hash(current_hash)
}
};
Ok(
utils::require_header::<Block>(
&*self.db,
columns::KEY_LOOKUP,
columns::HEADER,
block_id,
)
.map_err(|e| e.to_string())?
.digest()
.log(DigestItem::as_changes_trie_root)
.cloned()
)
}
}
impl<Block: BlockT> sp_state_machine::ChangesTrieStorage<HasherFor<Block>, NumberFor<Block>>
for DbChangesTrieStorage<Block>
{
fn as_roots_storage(&self)
-> &dyn sp_state_machine::ChangesTrieRootsStorage<HasherFor<Block>, NumberFor<Block>>
{
self
}
fn with_cached_changed_keys(
&self,
root: &Block::Hash,
functor: &mut dyn FnMut(&HashMap<Option<Vec<u8>>, HashSet<Vec<u8>>>),
) -> bool {
self.cache.read().with_changed_keys(root, functor)
}
fn get(&self, key: &Block::Hash, _prefix: Prefix) -> Result<Option<DBValue>, String> {
self.db.get(columns::CHANGES_TRIE, key.as_ref()).map_err(|err| format!("{}", err))
}
}
/// Frozen `value` at time `at`.
///
/// Used as inner structure under lock in `FrozenForDuration`.
@@ -889,9 +740,6 @@ pub struct Backend<Block: BlockT> {
storage: Arc<StorageDb<Block>>,
offchain_storage: offchain::LocalStorage,
changes_tries_storage: DbChangesTrieStorage<Block>,
/// None<*> means that the value hasn't been cached yet. Some(*) means that the value (either None or
/// Some(*)) has been cached and is valid.
changes_trie_config: Mutex<Option<Option<ChangesTrieConfiguration>>>,
blockchain: BlockchainDb<Block>,
canonicalization_delay: u64,
shared_cache: SharedCache<Block>,
@@ -906,7 +754,7 @@ impl<Block: BlockT> Backend<Block> {
///
/// The pruning window is how old a block must be before the state is pruned.
pub fn new(config: DatabaseSettings, canonicalization_delay: u64) -> ClientResult<Self> {
let db = crate::utils::open_database(&config, columns::META, "full")?;
let db = crate::utils::open_database::<Block>(&config, DatabaseType::Full)?;
Self::from_kvdb(db as Arc<_>, canonicalization_delay, &config)
}
@@ -942,22 +790,25 @@ impl<Block: BlockT> Backend<Block> {
state_db,
};
let offchain_storage = offchain::LocalStorage::new(db.clone());
let changes_tries_storage = DbChangesTrieStorage {
let changes_tries_storage = DbChangesTrieStorage::new(
db,
columns::META,
columns::CHANGES_TRIE,
columns::KEY_LOOKUP,
columns::HEADER,
columns::CACHE,
meta,
min_blocks_to_keep: if is_archive_pruning {
if is_archive_pruning {
None
} else {
Some(MIN_BLOCKS_TO_KEEP_CHANGES_TRIES_FOR)
},
cache: RwLock::new(ChangesTrieBuildCache::new()),
};
)?;
Ok(Backend {
storage: Arc::new(storage_db),
offchain_storage,
changes_tries_storage,
changes_trie_config: Mutex::new(None),
blockchain,
canonicalization_delay,
shared_cache: new_shared_cache(
@@ -1026,26 +877,6 @@ impl<Block: BlockT> Backend<Block> {
self.blockchain.db.iter(columns::HEADER).count() as u64
}
/// Read (from storage or cache) changes trie config.
///
/// Currently changes tries configuration is set up once (at genesis) and could not
/// be changed. Thus, we'll actually read value once and then just use cached value.
fn changes_trie_config(&self, block: Block::Hash) -> ClientResult<Option<ChangesTrieConfiguration>> {
let mut cached_changes_trie_config = self.changes_trie_config.lock();
match cached_changes_trie_config.clone() {
Some(cached_changes_trie_config) => Ok(cached_changes_trie_config),
None => {
use sc_client_api::backend::Backend;
let changes_trie_config = self
.state_at(BlockId::Hash(block))?
.storage(well_known_keys::CHANGES_TRIE_CONFIG)?
.and_then(|v| Decode::decode(&mut &*v).ok());
*cached_changes_trie_config = Some(changes_trie_config.clone());
Ok(changes_trie_config)
},
}
}
/// Handle setting head within a transaction. `route_to` should be the last
/// block that existed in the database. `best_to` should be the best block
/// to be set.
@@ -1137,6 +968,7 @@ impl<Block: BlockT> Backend<Block> {
header: &Block::Header,
last_finalized: Option<Block::Hash>,
justification: Option<Justification>,
changes_trie_cache_ops: &mut Option<DbChangesTrieStorageTransaction<Block>>,
finalization_displaced: &mut Option<FinalizationDisplaced<Block::Hash, NumberFor<Block>>>,
) -> ClientResult<(Block::Hash, <Block::Header as HeaderT>::Number, bool, bool)> {
// TODO: ensure best chain contains this block.
@@ -1144,8 +976,10 @@ impl<Block: BlockT> Backend<Block> {
self.ensure_sequential_finalization(header, last_finalized)?;
self.note_finalized(
transaction,
false,
header,
*hash,
changes_trie_cache_ops,
finalization_displaced,
)?;
@@ -1204,6 +1038,7 @@ impl<Block: BlockT> Backend<Block> {
let mut meta_updates = Vec::with_capacity(operation.finalized_blocks.len());
let mut last_finalized_hash = self.blockchain.meta.read().finalized_hash;
let mut changes_trie_cache_ops = None;
for (block, justification) in operation.finalized_blocks {
let block_hash = self.blockchain.expect_block_hash_from_id(&block)?;
let block_header = self.blockchain.expect_header(BlockId::Hash(block_hash))?;
@@ -1214,6 +1049,7 @@ impl<Block: BlockT> Backend<Block> {
&block_header,
Some(last_finalized_hash),
justification,
&mut changes_trie_cache_ops,
&mut finalization_displaced_leaves,
)?);
last_finalized_hash = block_hash;
@@ -1257,6 +1093,11 @@ impl<Block: BlockT> Backend<Block> {
if number.is_zero() {
transaction.put(columns::META, meta_keys::FINALIZED_BLOCK, &lookup_key);
transaction.put(columns::META, meta_keys::GENESIS_HASH, hash.as_ref());
// for tests, because config is set from within the reset_storage
if operation.changes_trie_config_update.is_none() {
operation.changes_trie_config_update = Some(None);
}
}
let finalized = if operation.commit_state {
@@ -1293,8 +1134,20 @@ impl<Block: BlockT> Backend<Block> {
let header = &pending_block.header;
let is_best = pending_block.leaf_state.is_best();
let changes_trie_updates = operation.changes_trie_updates;
self.changes_tries_storage.commit(&mut transaction, changes_trie_updates);
let changes_trie_config_update = operation.changes_trie_config_update;
changes_trie_cache_ops = Some(self.changes_tries_storage.commit(
&mut transaction,
changes_trie_updates,
cache::ComplexBlockId::new(
*header.parent_hash(),
if number.is_zero() { Zero::zero() } else { number - One::one() },
),
cache::ComplexBlockId::new(hash, number),
header,
finalized,
changes_trie_config_update,
changes_trie_cache_ops,
)?);
let cache = operation.old_state.release(); // release state reference so that it can be finalized
if finalized {
@@ -1302,8 +1155,10 @@ impl<Block: BlockT> Backend<Block> {
self.ensure_sequential_finalization(header, Some(last_finalized_hash))?;
self.note_finalized(
&mut transaction,
true,
header,
hash,
&mut changes_trie_cache_ops,
&mut finalization_displaced_leaves,
)?;
} else {
@@ -1353,11 +1208,15 @@ impl<Block: BlockT> Backend<Block> {
let write_result = self.storage.db.write(transaction).map_err(db_err);
if let Some(changes_trie_cache_update) = operation.changes_trie_cache_update {
self.changes_tries_storage.commit_cache(changes_trie_cache_update);
}
if let Some((number, hash, enacted, retracted, displaced_leaf, is_best, mut cache)) = imported {
if let Some((
number,
hash,
enacted,
retracted,
displaced_leaf,
is_best,
mut cache,
)) = imported {
if let Err(e) = write_result {
let mut leaves = self.blockchain.leaves.write();
let mut undo = leaves.undo();
@@ -1383,6 +1242,11 @@ impl<Block: BlockT> Backend<Block> {
);
}
if let Some(changes_trie_build_cache_update) = operation.changes_trie_build_cache_update {
self.changes_tries_storage.commit_build_cache(changes_trie_build_cache_update);
}
self.changes_tries_storage.post_commit(changes_trie_cache_ops);
if let Some((enacted, retracted)) = cache_update {
self.shared_cache.lock().sync(&enacted, &retracted);
}
@@ -1401,15 +1265,15 @@ impl<Block: BlockT> Backend<Block> {
fn note_finalized(
&self,
transaction: &mut DBTransaction,
is_inserted: bool,
f_header: &Block::Header,
f_hash: Block::Hash,
changes_trie_cache_ops: &mut Option<DbChangesTrieStorageTransaction<Block>>,
displaced: &mut Option<FinalizationDisplaced<Block::Hash, NumberFor<Block>>>
) -> ClientResult<()> {
let f_num = f_header.number().clone();
if self.storage.state_db.best_canonical().map(|c| f_num.saturated_into::<u64>() > c).unwrap_or(true) {
let parent_hash = f_header.parent_hash().clone();
let lookup_key = utils::number_and_hash_to_lookup_key(f_num, f_hash.clone())?;
transaction.put(columns::META, meta_keys::FINALIZED_BLOCK, &lookup_key);
@@ -1417,9 +1281,16 @@ impl<Block: BlockT> Backend<Block> {
.map_err(|e: sc_state_db::Error<io::Error>| sp_blockchain::Error::from(format!("State database error: {:?}", e)))?;
apply_state_commit(transaction, commit);
let changes_trie_config = self.changes_trie_config(parent_hash)?;
if let Some(changes_trie_config) = changes_trie_config {
self.changes_tries_storage.prune(&changes_trie_config, transaction, f_hash, f_num);
if !f_num.is_zero() {
let new_changes_trie_cache_ops = self.changes_tries_storage.finalize(
transaction,
*f_header.parent_hash(),
f_hash,
f_num,
if is_inserted { Some(&f_header) } else { None },
changes_trie_cache_ops.take(),
)?;
*changes_trie_cache_ops = Some(new_changes_trie_cache_ops);
}
}
@@ -1476,7 +1347,6 @@ impl<Block: BlockT> sc_client_api::backend::Backend<Block> for Backend<Block> {
type BlockImportOperation = BlockImportOperation<Block>;
type Blockchain = BlockchainDb<Block>;
type State = CachingState<RefTrackingState<Block>, Block>;
type ChangesTrieStorage = DbChangesTrieStorage<Block>;
type OffchainStorage = offchain::LocalStorage;
fn begin_operation(&self) -> ClientResult<Self::BlockImportOperation> {
@@ -1487,8 +1357,9 @@ impl<Block: BlockT> sc_client_api::backend::Backend<Block> for Backend<Block> {
db_updates: PrefixedMemoryDB::default(),
storage_updates: Default::default(),
child_storage_updates: Default::default(),
changes_trie_config_update: None,
changes_trie_updates: MemoryDB::default(),
changes_trie_cache_update: None,
changes_trie_build_cache_update: None,
aux_ops: Vec::new(),
finalized_blocks: Vec::new(),
set_head: None,
@@ -1532,16 +1403,19 @@ impl<Block: BlockT> sc_client_api::backend::Backend<Block> for Backend<Block> {
let header = self.blockchain.expect_header(block)?;
let mut displaced = None;
let commit = |displaced| {
let mut changes_trie_cache_ops = None;
let (hash, number, is_best, is_finalized) = self.finalize_block_with_transaction(
&mut transaction,
&hash,
&header,
None,
justification,
&mut changes_trie_cache_ops,
displaced,
)?;
self.storage.db.write(transaction).map_err(db_err)?;
self.blockchain.update_meta(hash, number, is_best, is_finalized);
self.changes_tries_storage.post_commit(changes_trie_cache_ops);
Ok(())
};
match commit(&mut displaced) {
@@ -1557,7 +1431,7 @@ impl<Block: BlockT> sc_client_api::backend::Backend<Block> for Backend<Block> {
Ok(())
}
fn changes_trie_storage(&self) -> Option<&Self::ChangesTrieStorage> {
fn changes_trie_storage(&self) -> Option<&dyn PrunableStateChangesTrieStorage<Block>> {
Some(&self.changes_tries_storage)
}
@@ -1616,6 +1490,7 @@ impl<Block: BlockT> sc_client_api::backend::Backend<Block> for Backend<Block> {
match self.storage.state_db.revert_one() {
Some(commit) => {
apply_state_commit(&mut transaction, commit);
let removed_number = best_number;
let removed = self.blockchain.header(BlockId::Number(best_number))?.ok_or_else(
|| sp_blockchain::Error::UnknownBlock(
format!("Error reverting to {}. Block hash not found.", best_number)))?;
@@ -1628,6 +1503,13 @@ impl<Block: BlockT> sc_client_api::backend::Backend<Block> for Backend<Block> {
let update_finalized = best_number < finalized;
let key = utils::number_and_hash_to_lookup_key(best_number.clone(), &best_hash)?;
let changes_trie_cache_ops = self.changes_tries_storage.revert(
&mut transaction,
&cache::ComplexBlockId::new(
removed.hash(),
removed_number,
),
)?;
transaction.put(columns::META, meta_keys::BEST_BLOCK, &key);
if update_finalized {
transaction.put(columns::META, meta_keys::FINALIZED_BLOCK, &key);
@@ -1635,6 +1517,7 @@ impl<Block: BlockT> sc_client_api::backend::Backend<Block> for Backend<Block> {
transaction.delete(columns::KEY_LOOKUP, removed.hash().as_ref());
children::remove_children(&mut transaction, columns::META, meta_keys::CHILDREN_PREFIX, best_hash);
self.storage.db.write(transaction).map_err(db_err)?;
self.changes_tries_storage.post_commit(Some(changes_trie_cache_ops));
self.blockchain.update_meta(best_hash, best_number, true, update_finalized);
}
None => return Ok(c.saturated_into::<NumberFor<Block>>())
@@ -1749,14 +1632,8 @@ impl<Block: BlockT> sc_client_api::backend::Backend<Block> for Backend<Block> {
impl<Block: BlockT> sc_client_api::backend::LocalBackend<Block> for Backend<Block> {}
/// TODO: remove me in #3201
pub fn unused_sink<Block: BlockT>(cache_tx: crate::cache::DbCacheTransaction<Block>) {
cache_tx.on_block_revert(&crate::cache::ComplexBlockId::new(Default::default(), 0.into())).unwrap();
unimplemented!()
}
#[cfg(test)]
mod tests {
pub(crate) mod tests {
use hash_db::{HashDB, EMPTY_PREFIX};
use super::*;
use crate::columns;
@@ -1765,12 +1642,13 @@ mod tests {
use sc_client::blockchain::Backend as BLBTrait;
use sp_runtime::testing::{Header, Block as RawBlock, ExtrinsicWrapper};
use sp_runtime::traits::{Hash, BlakeTwo256};
use sp_state_machine::{TrieMut, TrieDBMut, ChangesTrieRootsStorage, ChangesTrieStorage};
use sp_runtime::generic::DigestItem;
use sp_state_machine::{TrieMut, TrieDBMut};
use sp_blockchain::{lowest_common_ancestor, tree_route};
type Block = RawBlock<ExtrinsicWrapper<u64>>;
pub(crate) type Block = RawBlock<ExtrinsicWrapper<u64>>;
fn prepare_changes(changes: Vec<(Vec<u8>, Vec<u8>)>) -> (H256, MemoryDB<Blake2Hasher>) {
pub fn prepare_changes(changes: Vec<(Vec<u8>, Vec<u8>)>) -> (H256, MemoryDB<Blake2Hasher>) {
let mut changes_root = H256::default();
let mut changes_trie_update = MemoryDB::<Blake2Hasher>::default();
{
@@ -1786,20 +1664,22 @@ mod tests {
(changes_root, changes_trie_update)
}
fn insert_header(
pub fn insert_header(
backend: &Backend<Block>,
number: u64,
parent_hash: H256,
changes: Vec<(Vec<u8>, Vec<u8>)>,
changes: Option<Vec<(Vec<u8>, Vec<u8>)>>,
extrinsics_root: H256,
) -> H256 {
use sp_runtime::testing::Digest;
let (changes_root, changes_trie_update) = prepare_changes(changes);
let digest = Digest {
logs: vec![
DigestItem::ChangesTrieRoot(changes_root),
],
};
let mut digest = Digest::default();
let mut changes_trie_update = Default::default();
if let Some(changes) = changes {
let (root, update) = prepare_changes(changes);
digest.push(DigestItem::ChangesTrieRoot(root));
changes_trie_update = update;
}
let header = Header {
number,
parent_hash,
@@ -2126,238 +2006,20 @@ mod tests {
).unwrap().is_none());
}
#[test]
fn changes_trie_storage_works() {
let backend = Backend::<Block>::new_test(1000, 100);
backend.changes_tries_storage.meta.write().finalized_number = 1000;
let check_changes = |backend: &Backend<Block>, block: u64, changes: Vec<(Vec<u8>, Vec<u8>)>| {
let (changes_root, mut changes_trie_update) = prepare_changes(changes);
let anchor = sp_state_machine::ChangesTrieAnchorBlockId {
hash: backend.blockchain().header(BlockId::Number(block)).unwrap().unwrap().hash(),
number: block
};
assert_eq!(backend.changes_tries_storage.root(&anchor, block), Ok(Some(changes_root)));
for (key, (val, _)) in changes_trie_update.drain() {
assert_eq!(backend.changes_trie_storage().unwrap().get(&key, EMPTY_PREFIX), Ok(Some(val)));
}
};
let changes0 = vec![(b"key_at_0".to_vec(), b"val_at_0".to_vec())];
let changes1 = vec![
(b"key_at_1".to_vec(), b"val_at_1".to_vec()),
(b"another_key_at_1".to_vec(), b"another_val_at_1".to_vec()),
];
let changes2 = vec![(b"key_at_2".to_vec(), b"val_at_2".to_vec())];
let block0 = insert_header(&backend, 0, Default::default(), changes0.clone(), Default::default());
let block1 = insert_header(&backend, 1, block0, changes1.clone(), Default::default());
let _ = insert_header(&backend, 2, block1, changes2.clone(), Default::default());
// check that the storage contains tries for all blocks
check_changes(&backend, 0, changes0);
check_changes(&backend, 1, changes1);
check_changes(&backend, 2, changes2);
}
#[test]
fn changes_trie_storage_works_with_forks() {
let backend = Backend::<Block>::new_test(1000, 100);
let changes0 = vec![(b"k0".to_vec(), b"v0".to_vec())];
let changes1 = vec![(b"k1".to_vec(), b"v1".to_vec())];
let changes2 = vec![(b"k2".to_vec(), b"v2".to_vec())];
let block0 = insert_header(&backend, 0, Default::default(), changes0.clone(), Default::default());
let block1 = insert_header(&backend, 1, block0, changes1.clone(), Default::default());
let block2 = insert_header(&backend, 2, block1, changes2.clone(), Default::default());
let changes2_1_0 = vec![(b"k3".to_vec(), b"v3".to_vec())];
let changes2_1_1 = vec![(b"k4".to_vec(), b"v4".to_vec())];
let block2_1_0 = insert_header(&backend, 3, block2, changes2_1_0.clone(), Default::default());
let block2_1_1 = insert_header(&backend, 4, block2_1_0, changes2_1_1.clone(), Default::default());
let changes2_2_0 = vec![(b"k5".to_vec(), b"v5".to_vec())];
let changes2_2_1 = vec![(b"k6".to_vec(), b"v6".to_vec())];
let block2_2_0 = insert_header(&backend, 3, block2, changes2_2_0.clone(), Default::default());
let block2_2_1 = insert_header(&backend, 4, block2_2_0, changes2_2_1.clone(), Default::default());
// finalize block1
backend.changes_tries_storage.meta.write().finalized_number = 1;
// branch1: when asking for finalized block hash
let (changes1_root, _) = prepare_changes(changes1);
let anchor = sp_state_machine::ChangesTrieAnchorBlockId { hash: block2_1_1, number: 4 };
assert_eq!(backend.changes_tries_storage.root(&anchor, 1), Ok(Some(changes1_root)));
// branch2: when asking for finalized block hash
let anchor = sp_state_machine::ChangesTrieAnchorBlockId { hash: block2_2_1, number: 4 };
assert_eq!(backend.changes_tries_storage.root(&anchor, 1), Ok(Some(changes1_root)));
// branch1: when asking for non-finalized block hash (search by traversal)
let (changes2_1_0_root, _) = prepare_changes(changes2_1_0);
let anchor = sp_state_machine::ChangesTrieAnchorBlockId { hash: block2_1_1, number: 4 };
assert_eq!(backend.changes_tries_storage.root(&anchor, 3), Ok(Some(changes2_1_0_root)));
// branch2: when asking for non-finalized block hash (search using canonicalized hint)
let (changes2_2_0_root, _) = prepare_changes(changes2_2_0);
let anchor = sp_state_machine::ChangesTrieAnchorBlockId { hash: block2_2_1, number: 4 };
assert_eq!(backend.changes_tries_storage.root(&anchor, 3), Ok(Some(changes2_2_0_root)));
// finalize first block of branch2 (block2_2_0)
backend.changes_tries_storage.meta.write().finalized_number = 3;
// branch2: when asking for finalized block of this branch
assert_eq!(backend.changes_tries_storage.root(&anchor, 3), Ok(Some(changes2_2_0_root)));
// branch1: when asking for finalized block of other branch
// => result is incorrect (returned for the block of branch1), but this is expected,
// because the other fork is abandoned (forked before finalized header)
let anchor = sp_state_machine::ChangesTrieAnchorBlockId { hash: block2_1_1, number: 4 };
assert_eq!(backend.changes_tries_storage.root(&anchor, 3), Ok(Some(changes2_2_0_root)));
}
#[test]
fn changes_tries_with_digest_are_pruned_on_finalization() {
let mut backend = Backend::<Block>::new_test(1000, 100);
backend.changes_tries_storage.min_blocks_to_keep = Some(8);
let config = ChangesTrieConfiguration {
digest_interval: 2,
digest_levels: 2,
};
// insert some blocks
let block0 = insert_header(&backend, 0, Default::default(), vec![(b"key_at_0".to_vec(), b"val_at_0".to_vec())], Default::default());
let block1 = insert_header(&backend, 1, block0, vec![(b"key_at_1".to_vec(), b"val_at_1".to_vec())], Default::default());
let block2 = insert_header(&backend, 2, block1, vec![(b"key_at_2".to_vec(), b"val_at_2".to_vec())], Default::default());
let block3 = insert_header(&backend, 3, block2, vec![(b"key_at_3".to_vec(), b"val_at_3".to_vec())], Default::default());
let block4 = insert_header(&backend, 4, block3, vec![(b"key_at_4".to_vec(), b"val_at_4".to_vec())], Default::default());
let block5 = insert_header(&backend, 5, block4, vec![(b"key_at_5".to_vec(), b"val_at_5".to_vec())], Default::default());
let block6 = insert_header(&backend, 6, block5, vec![(b"key_at_6".to_vec(), b"val_at_6".to_vec())], Default::default());
let block7 = insert_header(&backend, 7, block6, vec![(b"key_at_7".to_vec(), b"val_at_7".to_vec())], Default::default());
let block8 = insert_header(&backend, 8, block7, vec![(b"key_at_8".to_vec(), b"val_at_8".to_vec())], Default::default());
let block9 = insert_header(&backend, 9, block8, vec![(b"key_at_9".to_vec(), b"val_at_9".to_vec())], Default::default());
let block10 = insert_header(&backend, 10, block9, vec![(b"key_at_10".to_vec(), b"val_at_10".to_vec())], Default::default());
let block11 = insert_header(&backend, 11, block10, vec![(b"key_at_11".to_vec(), b"val_at_11".to_vec())], Default::default());
let block12 = insert_header(&backend, 12, block11, vec![(b"key_at_12".to_vec(), b"val_at_12".to_vec())], Default::default());
let block13 = insert_header(&backend, 13, block12, vec![(b"key_at_13".to_vec(), b"val_at_13".to_vec())], Default::default());
backend.changes_tries_storage.meta.write().finalized_number = 13;
// check that roots of all tries are in the columns::CHANGES_TRIE
let anchor = sp_state_machine::ChangesTrieAnchorBlockId { hash: block13, number: 13 };
fn read_changes_trie_root(backend: &Backend<Block>, num: u64) -> H256 {
backend.blockchain().header(BlockId::Number(num)).unwrap().unwrap().digest().logs().iter()
.find(|i| i.as_changes_trie_root().is_some()).unwrap().as_changes_trie_root().unwrap().clone()
}
let root1 = read_changes_trie_root(&backend, 1); assert_eq!(backend.changes_tries_storage.root(&anchor, 1).unwrap(), Some(root1));
let root2 = read_changes_trie_root(&backend, 2); assert_eq!(backend.changes_tries_storage.root(&anchor, 2).unwrap(), Some(root2));
let root3 = read_changes_trie_root(&backend, 3); assert_eq!(backend.changes_tries_storage.root(&anchor, 3).unwrap(), Some(root3));
let root4 = read_changes_trie_root(&backend, 4); assert_eq!(backend.changes_tries_storage.root(&anchor, 4).unwrap(), Some(root4));
let root5 = read_changes_trie_root(&backend, 5); assert_eq!(backend.changes_tries_storage.root(&anchor, 5).unwrap(), Some(root5));
let root6 = read_changes_trie_root(&backend, 6); assert_eq!(backend.changes_tries_storage.root(&anchor, 6).unwrap(), Some(root6));
let root7 = read_changes_trie_root(&backend, 7); assert_eq!(backend.changes_tries_storage.root(&anchor, 7).unwrap(), Some(root7));
let root8 = read_changes_trie_root(&backend, 8); assert_eq!(backend.changes_tries_storage.root(&anchor, 8).unwrap(), Some(root8));
let root9 = read_changes_trie_root(&backend, 9); assert_eq!(backend.changes_tries_storage.root(&anchor, 9).unwrap(), Some(root9));
let root10 = read_changes_trie_root(&backend, 10); assert_eq!(backend.changes_tries_storage.root(&anchor, 10).unwrap(), Some(root10));
let root11 = read_changes_trie_root(&backend, 11); assert_eq!(backend.changes_tries_storage.root(&anchor, 11).unwrap(), Some(root11));
let root12 = read_changes_trie_root(&backend, 12); assert_eq!(backend.changes_tries_storage.root(&anchor, 12).unwrap(), Some(root12));
// now simulate finalization of block#12, causing prune of tries at #1..#4
let mut tx = DBTransaction::new();
backend.changes_tries_storage.prune(&config, &mut tx, Default::default(), 12);
backend.storage.db.write(tx).unwrap();
assert!(backend.changes_tries_storage.get(&root1, EMPTY_PREFIX).unwrap().is_none());
assert!(backend.changes_tries_storage.get(&root2, EMPTY_PREFIX).unwrap().is_none());
assert!(backend.changes_tries_storage.get(&root3, EMPTY_PREFIX).unwrap().is_none());
assert!(backend.changes_tries_storage.get(&root4, EMPTY_PREFIX).unwrap().is_none());
assert!(backend.changes_tries_storage.get(&root5, EMPTY_PREFIX).unwrap().is_some());
assert!(backend.changes_tries_storage.get(&root6, EMPTY_PREFIX).unwrap().is_some());
assert!(backend.changes_tries_storage.get(&root7, EMPTY_PREFIX).unwrap().is_some());
assert!(backend.changes_tries_storage.get(&root8, EMPTY_PREFIX).unwrap().is_some());
// now simulate finalization of block#16, causing prune of tries at #5..#8
let mut tx = DBTransaction::new();
backend.changes_tries_storage.prune(&config, &mut tx, Default::default(), 16);
backend.storage.db.write(tx).unwrap();
assert!(backend.changes_tries_storage.get(&root5, EMPTY_PREFIX).unwrap().is_none());
assert!(backend.changes_tries_storage.get(&root6, EMPTY_PREFIX).unwrap().is_none());
assert!(backend.changes_tries_storage.get(&root7, EMPTY_PREFIX).unwrap().is_none());
assert!(backend.changes_tries_storage.get(&root8, EMPTY_PREFIX).unwrap().is_none());
// now "change" pruning mode to archive && simulate finalization of block#20
// => no changes tries are pruned, because we never prune in archive mode
backend.changes_tries_storage.min_blocks_to_keep = None;
let mut tx = DBTransaction::new();
backend.changes_tries_storage.prune(&config, &mut tx, Default::default(), 20);
backend.storage.db.write(tx).unwrap();
assert!(backend.changes_tries_storage.get(&root9, EMPTY_PREFIX).unwrap().is_some());
assert!(backend.changes_tries_storage.get(&root10, EMPTY_PREFIX).unwrap().is_some());
assert!(backend.changes_tries_storage.get(&root11, EMPTY_PREFIX).unwrap().is_some());
assert!(backend.changes_tries_storage.get(&root12, EMPTY_PREFIX).unwrap().is_some());
}
#[test]
fn changes_tries_without_digest_are_pruned_on_finalization() {
let mut backend = Backend::<Block>::new_test(1000, 100);
backend.changes_tries_storage.min_blocks_to_keep = Some(4);
let config = ChangesTrieConfiguration {
digest_interval: 0,
digest_levels: 0,
};
// insert some blocks
let block0 = insert_header(&backend, 0, Default::default(), vec![(b"key_at_0".to_vec(), b"val_at_0".to_vec())], Default::default());
let block1 = insert_header(&backend, 1, block0, vec![(b"key_at_1".to_vec(), b"val_at_1".to_vec())], Default::default());
let block2 = insert_header(&backend, 2, block1, vec![(b"key_at_2".to_vec(), b"val_at_2".to_vec())], Default::default());
let block3 = insert_header(&backend, 3, block2, vec![(b"key_at_3".to_vec(), b"val_at_3".to_vec())], Default::default());
let block4 = insert_header(&backend, 4, block3, vec![(b"key_at_4".to_vec(), b"val_at_4".to_vec())], Default::default());
let block5 = insert_header(&backend, 5, block4, vec![(b"key_at_5".to_vec(), b"val_at_5".to_vec())], Default::default());
let block6 = insert_header(&backend, 6, block5, vec![(b"key_at_6".to_vec(), b"val_at_6".to_vec())], Default::default());
// check that roots of all tries are in the columns::CHANGES_TRIE
let anchor = sp_state_machine::ChangesTrieAnchorBlockId { hash: block6, number: 6 };
fn read_changes_trie_root(backend: &Backend<Block>, num: u64) -> H256 {
backend.blockchain().header(BlockId::Number(num)).unwrap().unwrap().digest().logs().iter()
.find(|i| i.as_changes_trie_root().is_some()).unwrap().as_changes_trie_root().unwrap().clone()
}
let root1 = read_changes_trie_root(&backend, 1); assert_eq!(backend.changes_tries_storage.root(&anchor, 1).unwrap(), Some(root1));
let root2 = read_changes_trie_root(&backend, 2); assert_eq!(backend.changes_tries_storage.root(&anchor, 2).unwrap(), Some(root2));
let root3 = read_changes_trie_root(&backend, 3); assert_eq!(backend.changes_tries_storage.root(&anchor, 3).unwrap(), Some(root3));
let root4 = read_changes_trie_root(&backend, 4); assert_eq!(backend.changes_tries_storage.root(&anchor, 4).unwrap(), Some(root4));
let root5 = read_changes_trie_root(&backend, 5); assert_eq!(backend.changes_tries_storage.root(&anchor, 5).unwrap(), Some(root5));
let root6 = read_changes_trie_root(&backend, 6); assert_eq!(backend.changes_tries_storage.root(&anchor, 6).unwrap(), Some(root6));
// now simulate finalization of block#5, causing prune of trie at #1
let mut tx = DBTransaction::new();
backend.changes_tries_storage.prune(&config, &mut tx, block5, 5);
backend.storage.db.write(tx).unwrap();
assert!(backend.changes_tries_storage.get(&root1, EMPTY_PREFIX).unwrap().is_none());
assert!(backend.changes_tries_storage.get(&root2, EMPTY_PREFIX).unwrap().is_some());
// now simulate finalization of block#6, causing prune of tries at #2
let mut tx = DBTransaction::new();
backend.changes_tries_storage.prune(&config, &mut tx, block6, 6);
backend.storage.db.write(tx).unwrap();
assert!(backend.changes_tries_storage.get(&root2, EMPTY_PREFIX).unwrap().is_none());
assert!(backend.changes_tries_storage.get(&root3, EMPTY_PREFIX).unwrap().is_some());
}
#[test]
fn tree_route_works() {
let backend = Backend::<Block>::new_test(1000, 100);
let blockchain = backend.blockchain();
let block0 = insert_header(&backend, 0, Default::default(), Vec::new(), Default::default());
let block0 = insert_header(&backend, 0, Default::default(), None, Default::default());
// fork from genesis: 3 prong.
let a1 = insert_header(&backend, 1, block0, Vec::new(), Default::default());
let a2 = insert_header(&backend, 2, a1, Vec::new(), Default::default());
let a3 = insert_header(&backend, 3, a2, Vec::new(), Default::default());
let a1 = insert_header(&backend, 1, block0, None, Default::default());
let a2 = insert_header(&backend, 2, a1, None, Default::default());
let a3 = insert_header(&backend, 3, a2, None, Default::default());
// fork from genesis: 2 prong.
let b1 = insert_header(&backend, 1, block0, Vec::new(), H256::from([1; 32]));
let b2 = insert_header(&backend, 2, b1, Vec::new(), Default::default());
let b1 = insert_header(&backend, 1, block0, None, H256::from([1; 32]));
let b2 = insert_header(&backend, 2, b1, None, Default::default());
{
let tree_route = tree_route(blockchain, a3, b2).unwrap();
@@ -2397,8 +2059,8 @@ mod tests {
let backend = Backend::<Block>::new_test(1000, 100);
let blockchain = backend.blockchain();
let block0 = insert_header(&backend, 0, Default::default(), Vec::new(), Default::default());
let block1 = insert_header(&backend, 1, block0, Vec::new(), Default::default());
let block0 = insert_header(&backend, 0, Default::default(), None, Default::default());
let block1 = insert_header(&backend, 1, block0, None, Default::default());
{
let tree_route = tree_route(blockchain, block0, block1).unwrap();
@@ -2413,16 +2075,16 @@ mod tests {
fn lowest_common_ancestor_works() {
let backend = Backend::<Block>::new_test(1000, 100);
let blockchain = backend.blockchain();
let block0 = insert_header(&backend, 0, Default::default(), Vec::new(), Default::default());
let block0 = insert_header(&backend, 0, Default::default(), None, Default::default());
// fork from genesis: 3 prong.
let a1 = insert_header(&backend, 1, block0, Vec::new(), Default::default());
let a2 = insert_header(&backend, 2, a1, Vec::new(), Default::default());
let a3 = insert_header(&backend, 3, a2, Vec::new(), Default::default());
let a1 = insert_header(&backend, 1, block0, None, Default::default());
let a2 = insert_header(&backend, 2, a1, None, Default::default());
let a3 = insert_header(&backend, 3, a2, None, Default::default());
// fork from genesis: 2 prong.
let b1 = insert_header(&backend, 1, block0, Vec::new(), H256::from([1; 32]));
let b2 = insert_header(&backend, 2, b1, Vec::new(), Default::default());
let b1 = insert_header(&backend, 1, block0, None, H256::from([1; 32]));
let b2 = insert_header(&backend, 2, b1, None, Default::default());
{
let lca = lowest_common_ancestor(blockchain, a3, b2).unwrap();
@@ -2479,14 +2141,14 @@ mod tests {
let backend = Backend::<Block>::new_test(10000, 10000);
let blockchain = backend.blockchain();
let genesis = insert_header(&backend, 0, Default::default(), Vec::new(), Default::default());
let genesis = insert_header(&backend, 0, Default::default(), None, Default::default());
let block100 = (1..=100).fold(genesis, |parent, n| {
insert_header(&backend, n, parent, Vec::new(), Default::default())
insert_header(&backend, n, parent, None, Default::default())
});
let block7000 = (101..=7000).fold(block100, |parent, n| {
insert_header(&backend, n, parent, Vec::new(), Default::default())
insert_header(&backend, n, parent, None, Default::default())
});
// This will cause the ancestor of `block100` to be set to `genesis` as a side-effect.
@@ -2522,17 +2184,17 @@ mod tests {
#[test]
fn test_leaves_pruned_on_finality() {
let backend: Backend<Block> = Backend::new_test(10, 10);
let block0 = insert_header(&backend, 0, Default::default(), Default::default(), Default::default());
let block0 = insert_header(&backend, 0, Default::default(), None, Default::default());
let block1_a = insert_header(&backend, 1, block0, Default::default(), Default::default());
let block1_b = insert_header(&backend, 1, block0, Default::default(), [1; 32].into());
let block1_c = insert_header(&backend, 1, block0, Default::default(), [2; 32].into());
let block1_a = insert_header(&backend, 1, block0, None, Default::default());
let block1_b = insert_header(&backend, 1, block0, None, [1; 32].into());
let block1_c = insert_header(&backend, 1, block0, None, [2; 32].into());
assert_eq!(backend.blockchain().leaves().unwrap(), vec![block1_a, block1_b, block1_c]);
let block2_a = insert_header(&backend, 2, block1_a, Default::default(), Default::default());
let block2_b = insert_header(&backend, 2, block1_b, Default::default(), Default::default());
let block2_c = insert_header(&backend, 2, block1_b, Default::default(), [1; 32].into());
let block2_a = insert_header(&backend, 2, block1_a, None, Default::default());
let block2_b = insert_header(&backend, 2, block1_b, None, Default::default());
let block2_c = insert_header(&backend, 2, block1_b, None, [1; 32].into());
assert_eq!(backend.blockchain().leaves().unwrap(), vec![block2_a, block2_b, block2_c, block1_c]);
@@ -2559,8 +2221,8 @@ mod tests {
let backend = Backend::<Block>::new_test(10, 10);
let block0 = insert_header(&backend, 0, Default::default(), Default::default(), Default::default());
let _ = insert_header(&backend, 1, block0, Default::default(), Default::default());
let block0 = insert_header(&backend, 0, Default::default(), None, Default::default());
let _ = insert_header(&backend, 1, block0, None, Default::default());
let justification = Some(vec![1, 2, 3]);
backend.finalize_block(BlockId::Number(1), justification.clone()).unwrap();
@@ -2575,9 +2237,11 @@ mod tests {
fn test_finalize_multiple_blocks_in_single_op() {
let backend = Backend::<Block>::new_test(10, 10);
let block0 = insert_header(&backend, 0, Default::default(), Default::default(), Default::default());
let block1 = insert_header(&backend, 1, block0, Default::default(), Default::default());
let block2 = insert_header(&backend, 2, block1, Default::default(), Default::default());
let block0 = insert_header(&backend, 0, Default::default(), None, Default::default());
let block1 = insert_header(&backend, 1, block0, None, Default::default());
let block2 = insert_header(&backend, 2, block1, None, Default::default());
let block3 = insert_header(&backend, 3, block2, None, Default::default());
let block4 = insert_header(&backend, 4, block3, None, Default::default());
{
let mut op = backend.begin_operation().unwrap();
backend.begin_state_operation(&mut op, BlockId::Hash(block0)).unwrap();
@@ -2585,15 +2249,22 @@ mod tests {
op.mark_finalized(BlockId::Hash(block2), None).unwrap();
backend.commit_operation(op).unwrap();
}
{
let mut op = backend.begin_operation().unwrap();
backend.begin_state_operation(&mut op, BlockId::Hash(block2)).unwrap();
op.mark_finalized(BlockId::Hash(block3), None).unwrap();
op.mark_finalized(BlockId::Hash(block4), None).unwrap();
backend.commit_operation(op).unwrap();
}
}
#[test]
fn test_finalize_non_sequential() {
let backend = Backend::<Block>::new_test(10, 10);
let block0 = insert_header(&backend, 0, Default::default(), Default::default(), Default::default());
let block1 = insert_header(&backend, 1, block0, Default::default(), Default::default());
let block2 = insert_header(&backend, 2, block1, Default::default(), Default::default());
let block0 = insert_header(&backend, 0, Default::default(), None, Default::default());
let block1 = insert_header(&backend, 1, block0, None, Default::default());
let block2 = insert_header(&backend, 2, block1, None, Default::default());
{
let mut op = backend.begin_operation().unwrap();
backend.begin_state_operation(&mut op, BlockId::Hash(block0)).unwrap();
substrate/client/db/src/light.rs
+56 -17
View File
@@ -38,7 +38,7 @@ use codec::{Decode, Encode};
use sp_runtime::generic::{DigestItem, BlockId};
use sp_runtime::traits::{Block as BlockT, Header as HeaderT, Zero, One, NumberFor, HasherFor};
use crate::cache::{DbCacheSync, DbCache, ComplexBlockId, EntryType as CacheEntryType};
use crate::utils::{self, meta_keys, Meta, db_err, read_db, block_id_to_lookup_key, read_meta};
use crate::utils::{self, meta_keys, DatabaseType, Meta, db_err, read_db, block_id_to_lookup_key, read_meta};
use crate::{DatabaseSettings, FrozenForDuration};
use log::{trace, warn, debug};
@@ -68,13 +68,10 @@ pub struct LightStorage<Block: BlockT> {
io_stats: FrozenForDuration<kvdb::IoStats>,
}
impl<Block> LightStorage<Block>
where
Block: BlockT,
{
impl<Block: BlockT> LightStorage<Block> {
/// Create new storage with given settings.
pub fn new(config: DatabaseSettings) -> ClientResult<Self> {
let db = crate::utils::open_database(&config, columns::META, "light")?;
let db = crate::utils::open_database::<Block>(&config, DatabaseType::Light)?;
Self::from_kvdb(db as Arc<_>)
}
@@ -89,7 +86,7 @@ impl<Block> LightStorage<Block>
}
fn from_kvdb(db: Arc<dyn KeyValueDB>) -> ClientResult<Self> {
let meta = read_meta::<Block>(&*db, columns::META, columns::HEADER)?;
let meta = read_meta::<Block>(&*db, columns::HEADER)?;
let cache = DbCache::new(
db.clone(),
columns::KEY_LOOKUP,
@@ -417,7 +414,7 @@ impl<Block> LightBlockchainStorage<Block> for LightStorage<Block>
fn import_header(
&self,
header: Block::Header,
cache_at: HashMap<well_known_cache_keys::Id, Vec<u8>>,
mut cache_at: HashMap<well_known_cache_keys::Id, Vec<u8>>,
leaf_state: NewBlockState,
aux_ops: Vec<(Vec<u8>, Option<Vec<u8>>)>,
) -> ClientResult<()> {
@@ -475,6 +472,13 @@ impl<Block> LightBlockchainStorage<Block> for LightStorage<Block>
)?;
}
// update changes trie configuration cache
if !cache_at.contains_key(&well_known_cache_keys::CHANGES_TRIE_CONFIG) {
if let Some(new_configuration) = crate::changes_tries_storage::extract_new_configuration(&header) {
cache_at.insert(well_known_cache_keys::CHANGES_TRIE_CONFIG, new_configuration.encode());
}
}
{
let mut cache = self.cache.0.write();
let cache_ops = cache.transaction(&mut transaction)
@@ -487,8 +491,11 @@ impl<Block> LightBlockchainStorage<Block> for LightStorage<Block>
.into_ops();
debug!("Light DB Commit {:?} ({})", hash, number);
self.db.write(transaction).map_err(db_err)?;
cache.commit(cache_ops);
cache.commit(cache_ops)
.expect("only fails if cache with given name isn't loaded yet;\
cache is already loaded because there are cache_ops; qed");
}
self.update_meta(hash, number, leaf_state.is_best(), finalized);
@@ -543,7 +550,9 @@ impl<Block> LightBlockchainStorage<Block> for LightStorage<Block>
.into_ops();
self.db.write(transaction).map_err(db_err)?;
cache.commit(cache_ops);
cache.commit(cache_ops)
.expect("only fails if cache with given name isn't loaded yet;\
cache is already loaded because there are cache_ops; qed");
}
self.update_meta(hash, header.number().clone(), false, true);
@@ -603,7 +612,8 @@ fn cht_key<N: TryInto<u32>>(cht_type: u8, block: N) -> ClientResult<[u8; 5]> {
#[cfg(test)]
pub(crate) mod tests {
use sc_client::cht;
use sp_runtime::generic::DigestItem;
use sp_core::ChangesTrieConfiguration;
use sp_runtime::generic::{DigestItem, ChangesTrieSignal};
use sp_runtime::testing::{H256 as Hash, Header, Block as RawBlock, ExtrinsicWrapper};
use sp_blockchain::{lowest_common_ancestor, tree_route};
use super::*;
@@ -964,7 +974,7 @@ pub(crate) mod tests {
}
fn get_authorities(cache: &dyn BlockchainCache<Block>, at: BlockId<Block>) -> Option<Vec<AuthorityId>> {
cache.get_at(&well_known_cache_keys::AUTHORITIES, &at)
cache.get_at(&well_known_cache_keys::AUTHORITIES, &at).unwrap_or(None)
.and_then(|(_, _, val)| Decode::decode(&mut &val[..]).ok())
}
@@ -1148,8 +1158,8 @@ pub(crate) mod tests {
let (genesis_hash, storage) = {
let db = LightStorage::<Block>::new_test();
// before cache is initialized => None
assert_eq!(db.cache().get_at(b"test", &BlockId::Number(0)), None);
// before cache is initialized => Err
assert!(db.cache().get_at(b"test", &BlockId::Number(0)).is_err());
// insert genesis block (no value for cache is provided)
let mut genesis_hash = None;
@@ -1160,14 +1170,14 @@ pub(crate) mod tests {
});
// after genesis is inserted => None
assert_eq!(db.cache().get_at(b"test", &BlockId::Number(0)), None);
assert_eq!(db.cache().get_at(b"test", &BlockId::Number(0)).unwrap(), None);
// initialize cache
db.cache().initialize(b"test", vec![42]).unwrap();
// after genesis is inserted + cache is initialized => Some
assert_eq!(
db.cache().get_at(b"test", &BlockId::Number(0)),
db.cache().get_at(b"test", &BlockId::Number(0)).unwrap(),
Some(((0, genesis_hash.unwrap()), None, vec![42])),
);
@@ -1177,8 +1187,37 @@ pub(crate) mod tests {
// restart && check that after restart value is read from the cache
let db = LightStorage::<Block>::from_kvdb(storage as Arc<_>).expect("failed to create test-db");
assert_eq!(
db.cache().get_at(b"test", &BlockId::Number(0)),
db.cache().get_at(b"test", &BlockId::Number(0)).unwrap(),
Some(((0, genesis_hash.unwrap()), None, vec![42])),
);
}
#[test]
fn changes_trie_configuration_is_tracked_on_light_client() {
let db = LightStorage::<Block>::new_test();
let new_config = Some(ChangesTrieConfiguration::new(2, 2));
// insert block#0 && block#1 (no value for cache is provided)
let hash0 = insert_block(&db, HashMap::new(), || default_header(&Default::default(), 0));
assert_eq!(
db.cache().get_at(&well_known_cache_keys::CHANGES_TRIE_CONFIG, &BlockId::Number(0)).unwrap()
.map(|(_, _, v)| ChangesTrieConfiguration::decode(&mut &v[..]).unwrap()),
None,
);
// insert configuration at block#1 (starts from block#2)
insert_block(&db, HashMap::new(), || {
let mut header = default_header(&hash0, 1);
header.digest_mut().push(
DigestItem::ChangesTrieSignal(ChangesTrieSignal::NewConfiguration(new_config.clone()))
);
header
});
assert_eq!(
db.cache().get_at(&well_known_cache_keys::CHANGES_TRIE_CONFIG, &BlockId::Number(1)).unwrap()
.map(|(_, _, v)| Option::<ChangesTrieConfiguration>::decode(&mut &v[..]).unwrap()),
Some(new_config),
);
}
}
substrate/client/db/src/upgrade.rs
+198
View File
@@ -0,0 +1,198 @@
// Copyright 2019-2020 Parity Technologies (UK) Ltd.
// This file is part of Substrate.
// Substrate is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
// Substrate is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
// You should have received a copy of the GNU General Public License
// along with Substrate. If not, see <http://www.gnu.org/licenses/>.
//! Database upgrade logic.
use std::fs;
use std::io::{Read, Write, ErrorKind};
use std::path::{Path, PathBuf};
use std::sync::Arc;
use codec::Encode;
use kvdb_rocksdb::{Database, DatabaseConfig};
use parking_lot::RwLock;
use sp_blockchain::{well_known_cache_keys, Cache};
use sp_core::ChangesTrieConfiguration;
use sp_runtime::traits::Block as BlockT;
use crate::{
cache::{ComplexBlockId, DbCache, DbCacheSync},
utils::{DatabaseType, check_database_type, db_err, read_genesis_hash},
};
/// Version file name.
const VERSION_FILE_NAME: &'static str = "db_version";
/// Current db version.
const CURRENT_VERSION: u32 = 1;
/// Number of columns in v0.
const V0_NUM_COLUMNS: u32 = 10;
/// Upgrade database to current version.
pub fn upgrade_db<Block: BlockT>(db_path: &Path, db_type: DatabaseType) -> sp_blockchain::Result<()> {
let db_version = current_version(db_path)?;
match db_version {
0 => migrate_0_to_1::<Block>(db_path, db_type)?,
1 => (),
_ => Err(sp_blockchain::Error::Backend(format!("Future database version: {}", db_version)))?,
}
update_version(db_path)
}
/// Migration from version0 to version1:
/// 1) the number of columns has changed from 10 to 11;
/// 2) changes tries configuration are now cached.
fn migrate_0_to_1<Block: BlockT>(db_path: &Path, db_type: DatabaseType) -> sp_blockchain::Result<()> {
{
let db = open_database(db_path, db_type, V0_NUM_COLUMNS)?;
db.add_column().map_err(db_err)?;
db.flush().map_err(db_err)?;
}
let db = open_database(db_path, db_type, V0_NUM_COLUMNS + 1)?;
const V0_FULL_KEY_LOOKUP_COLUMN: u32 = 3;
const V0_FULL_HEADER_COLUMN: u32 = 4;
const V0_FULL_CACHE_COLUMN: u32 = 10; // that's the column we have just added
const V0_LIGHT_KEY_LOOKUP_COLUMN: u32 = 1;
const V0_LIGHT_HEADER_COLUMN: u32 = 2;
const V0_LIGHT_CACHE_COLUMN: u32 = 3;
let (key_lookup_column, header_column, cache_column) = match db_type {
DatabaseType::Full => (
V0_FULL_KEY_LOOKUP_COLUMN,
V0_FULL_HEADER_COLUMN,
V0_FULL_CACHE_COLUMN,
),
DatabaseType::Light => (
V0_LIGHT_KEY_LOOKUP_COLUMN,
V0_LIGHT_HEADER_COLUMN,
V0_LIGHT_CACHE_COLUMN,
),
};
let genesis_hash: Option<Block::Hash> = read_genesis_hash(&db)?;
if let Some(genesis_hash) = genesis_hash {
let cache: DbCacheSync<Block> = DbCacheSync(RwLock::new(DbCache::new(
Arc::new(db),
key_lookup_column,
header_column,
cache_column,
genesis_hash,
ComplexBlockId::new(genesis_hash, 0.into()),
)));
let changes_trie_config: Option<ChangesTrieConfiguration> = None;
cache.initialize(&well_known_cache_keys::CHANGES_TRIE_CONFIG, changes_trie_config.encode())?;
}
Ok(())
}
/// Reads current database version from the file at given path.
/// If the file does not exist returns 0.
fn current_version(path: &Path) -> sp_blockchain::Result<u32> {
let unknown_version_err = || sp_blockchain::Error::Backend("Unknown database version".into());
match fs::File::open(version_file_path(path)) {
Err(ref err) if err.kind() == ErrorKind::NotFound => Ok(0),
Err(_) => Err(unknown_version_err()),
Ok(mut file) => {
let mut s = String::new();
file.read_to_string(&mut s).map_err(|_| unknown_version_err())?;
u32::from_str_radix(&s, 10).map_err(|_| unknown_version_err())
},
}
}
/// Opens database of givent type with given number of columns.
fn open_database(db_path: &Path, db_type: DatabaseType, db_columns: u32) -> sp_blockchain::Result<Database> {
let db_path = db_path.to_str()
.ok_or_else(|| sp_blockchain::Error::Backend("Invalid database path".into()))?;
let db_cfg = DatabaseConfig::with_columns(db_columns);
let db = Database::open(&db_cfg, db_path).map_err(db_err)?;
check_database_type(&db, db_type)?;
Ok(db)
}
/// Writes current database version to the file.
/// Creates a new file if the version file does not exist yet.
fn update_version(path: &Path) -> sp_blockchain::Result<()> {
fs::create_dir_all(path).map_err(db_err)?;
let mut file = fs::File::create(version_file_path(path)).map_err(db_err)?;
file.write_all(format!("{}", CURRENT_VERSION).as_bytes()).map_err(db_err)?;
Ok(())
}
/// Returns the version file path.
fn version_file_path(path: &Path) -> PathBuf {
let mut file_path = path.to_owned();
file_path.push(VERSION_FILE_NAME);
file_path
}
#[cfg(test)]
mod tests {
use sc_state_db::PruningMode;
use crate::{DatabaseSettings, DatabaseSettingsSrc};
use crate::tests::Block;
use super::*;
fn create_db(db_path: &Path, version: Option<u32>) {
let db_cfg = DatabaseConfig::with_columns(V0_NUM_COLUMNS);
Database::open(&db_cfg, db_path.to_str().unwrap()).unwrap();
if let Some(version) = version {
fs::create_dir_all(db_path).unwrap();
let mut file = fs::File::create(version_file_path(db_path)).unwrap();
file.write_all(format!("{}", version).as_bytes()).unwrap();
}
}
fn open_database(db_path: &Path) -> sp_blockchain::Result<()> {
crate::utils::open_database::<Block>(&DatabaseSettings {
state_cache_size: 0,
state_cache_child_ratio: None,
pruning: PruningMode::ArchiveAll,
source: DatabaseSettingsSrc::Path { path: db_path.to_owned(), cache_size: None },
}, DatabaseType::Full).map(|_| ())
}
#[test]
fn downgrade_never_happens() {
let db_dir = tempdir::TempDir::new("").unwrap();
create_db(db_dir.path(), Some(CURRENT_VERSION + 1));
assert!(open_database(db_dir.path()).is_err());
}
#[test]
fn open_empty_database_works() {
let db_dir = tempdir::TempDir::new("").unwrap();
open_database(db_dir.path()).unwrap();
open_database(db_dir.path()).unwrap();
assert_eq!(current_version(db_dir.path()).unwrap(), CURRENT_VERSION);
}
#[test]
fn upgrade_from_0_to_1_works() {
for version_from_file in &[None, Some(0)] {
let db_dir = tempdir::TempDir::new("").unwrap();
let db_path = db_dir.path();
create_db(db_path, *version_from_file);
open_database(db_path).unwrap();
assert_eq!(current_version(db_path).unwrap(), CURRENT_VERSION);
}
}
}
substrate/client/db/src/utils.rs
+66 -22
View File
@@ -21,7 +21,7 @@ use std::sync::Arc;
use std::{io, convert::TryInto};
use kvdb::{KeyValueDB, DBTransaction};
#[cfg(feature = "kvdb-rocksdb")]
#[cfg(any(feature = "kvdb-rocksdb", test))]
use kvdb_rocksdb::{Database, DatabaseConfig};
use log::debug;
@@ -36,7 +36,7 @@ use crate::{DatabaseSettings, DatabaseSettingsSrc};
/// Number of columns in the db. Must be the same for both full && light dbs.
/// Otherwise RocksDb will fail to open database && check its type.
pub const NUM_COLUMNS: u32 = 10;
pub const NUM_COLUMNS: u32 = 11;
/// Meta column. The set of keys in the column is shared by full && light storages.
pub const COLUMN_META: u32 = 0;
@@ -50,6 +50,8 @@ pub mod meta_keys {
pub const FINALIZED_BLOCK: &[u8; 5] = b"final";
/// Meta information prefix for list-based caches.
pub const CACHE_META_PREFIX: &[u8; 5] = b"cache";
/// Meta information for changes tries key.
pub const CHANGES_TRIES_META: &[u8; 5] = b"ctrie";
/// Genesis block hash.
pub const GENESIS_HASH: &[u8; 3] = b"gen";
/// Leaves prefix list key.
@@ -76,6 +78,15 @@ pub struct Meta<N, H> {
/// A block lookup key: used for canonical lookup from block number to hash
pub type NumberIndexKey = [u8; 4];
/// Database type.
#[derive(Clone, Copy, Debug, PartialEq)]
pub enum DatabaseType {
/// Full node database.
Full,
/// Light node database.
Light,
}
/// Convert block number into short lookup key (LE representation) for
/// blocks that are in the canonical chain.
///
@@ -203,14 +214,17 @@ pub fn db_err(err: io::Error) -> sp_blockchain::Error {
}
/// Open RocksDB database.
pub fn open_database(
pub fn open_database<Block: BlockT>(
config: &DatabaseSettings,
col_meta: u32,
db_type: &str
db_type: DatabaseType,
) -> sp_blockchain::Result<Arc<dyn KeyValueDB>> {
let db: Arc<dyn KeyValueDB> = match &config.source {
#[cfg(feature = "kvdb-rocksdb")]
#[cfg(any(feature = "kvdb-rocksdb", test))]
DatabaseSettingsSrc::Path { path, cache_size } => {
// first upgrade database to required version
crate::upgrade::upgrade_db::<Block>(&path, db_type)?;
// and now open database assuming that it has the latest version
let mut db_config = DatabaseConfig::with_columns(NUM_COLUMNS);
if let Some(cache_size) = cache_size {
@@ -232,7 +246,7 @@ pub fn open_database(
.ok_or_else(|| sp_blockchain::Error::Backend("Invalid database path".into()))?;
Arc::new(Database::open(&db_config, &path).map_err(db_err)?)
},
#[cfg(not(feature = "kvdb-rocksdb"))]
#[cfg(not(any(feature = "kvdb-rocksdb", test)))]
DatabaseSettingsSrc::Path { .. } => {
let msg = "Try to open RocksDB database with RocksDB disabled".into();
return Err(sp_blockchain::Error::Backend(msg));
@@ -240,22 +254,28 @@ pub fn open_database(
DatabaseSettingsSrc::Custom(db) => db.clone(),
};
// check database type
match db.get(col_meta, meta_keys::TYPE).map_err(db_err)? {
check_database_type(&*db, db_type)?;
Ok(db)
}
/// Check database type.
pub fn check_database_type(db: &dyn KeyValueDB, db_type: DatabaseType) -> sp_blockchain::Result<()> {
match db.get(COLUMN_META, meta_keys::TYPE).map_err(db_err)? {
Some(stored_type) => {
if db_type.as_bytes() != &*stored_type {
if db_type.as_str().as_bytes() != &*stored_type {
return Err(sp_blockchain::Error::Backend(
format!("Unexpected database type. Expected: {}", db_type)).into());
format!("Unexpected database type. Expected: {}", db_type.as_str())).into());
}
},
None => {
let mut transaction = DBTransaction::new();
transaction.put(col_meta, meta_keys::TYPE, db_type.as_bytes());
transaction.put(COLUMN_META, meta_keys::TYPE, db_type.as_str().as_bytes());
db.write(transaction).map_err(db_err)?;
},
}
Ok(db)
Ok(())
}
/// Read database column entry for the given block.
@@ -304,20 +324,15 @@ pub fn require_header<Block: BlockT>(
}
/// Read meta from the database.
pub fn read_meta<Block>(db: &dyn KeyValueDB, col_meta: u32, col_header: u32) -> Result<
pub fn read_meta<Block>(db: &dyn KeyValueDB, col_header: u32) -> Result<
Meta<<<Block as BlockT>::Header as HeaderT>::Number, Block::Hash>,
sp_blockchain::Error,
>
where
Block: BlockT,
{
let genesis_hash: Block::Hash = match db.get(col_meta, meta_keys::GENESIS_HASH).map_err(db_err)? {
Some(h) => match Decode::decode(&mut &h[..]) {
Ok(h) => h,
Err(err) => return Err(sp_blockchain::Error::Backend(
format!("Error decoding genesis hash: {}", err)
)),
},
let genesis_hash: Block::Hash = match read_genesis_hash(db)? {
Some(genesis_hash) => genesis_hash,
None => return Ok(Meta {
best_hash: Default::default(),
best_number: Zero::zero(),
@@ -328,7 +343,7 @@ pub fn read_meta<Block>(db: &dyn KeyValueDB, col_meta: u32, col_header: u32) ->
};
let load_meta_block = |desc, key| -> Result<_, sp_blockchain::Error> {
if let Some(Some(header)) = db.get(col_meta, key).and_then(|id|
if let Some(Some(header)) = db.get(COLUMN_META, key).and_then(|id|
match id {
Some(id) => db.get(col_header, &id).map(|h| h.map(|b| Block::Header::decode(&mut &b[..]).ok())),
None => Ok(None),
@@ -354,6 +369,29 @@ pub fn read_meta<Block>(db: &dyn KeyValueDB, col_meta: u32, col_header: u32) ->
})
}
/// Read genesis hash from database.
pub fn read_genesis_hash<Hash: Decode>(db: &dyn KeyValueDB) -> sp_blockchain::Result<Option<Hash>> {
match db.get(COLUMN_META, meta_keys::GENESIS_HASH).map_err(db_err)? {
Some(h) => match Decode::decode(&mut &h[..]) {
Ok(h) => Ok(Some(h)),
Err(err) => Err(sp_blockchain::Error::Backend(
format!("Error decoding genesis hash: {}", err)
)),
},
None => Ok(None),
}
}
impl DatabaseType {
/// Returns str representation of the type.
pub fn as_str(&self) -> &'static str {
match *self {
DatabaseType::Full => "full",
DatabaseType::Light => "light",
}
}
}
#[cfg(test)]
mod tests {
use super::*;
@@ -368,4 +406,10 @@ mod tests {
_ => unreachable!(),
};
}
#[test]
fn database_type_as_str_works() {
assert_eq!(DatabaseType::Full.as_str(), "full");
assert_eq!(DatabaseType::Light.as_str(), "light");
}
}