pezkuwichain/pezkuwi-subxt
1
0
Fork 0
mirror of https://github.com/pezkuwichain/pezkuwi-subxt.git synced 2026-06-10 17:11:03 +00:00
Code Issues Packages Projects Releases Wiki Activity

Allow updating configuration of changes tries (#3201)

Browse Source 
* DigestItem::ChangesTrieSignal

* introduce changes_trie::State

* introduce config activation block

* ChangesTrieSignal::as_new_configuration

* moved well_known_cache_keys to client

* extracted DbChangesTrieStorage to separate file

* change meaning of none in blockchain cache

* changes trie config (FULL) cache draft

* eliminating const ChangesTrieConfiguration

* delay pruning

* continue elimination

* do not prune CT config from cache

* removed redundant code

* fix some TODOs

* introduce ConfigurationRange

* use Configuration range in build

* build skewed digest

* remove debug print

* extracted surface iterator

* key_changes works with skewed digests

* fix client build

* add test for NeverPrune

* fix TODO

* fixed some TODOs

* more tests

* fixing TODOs

* fixed compilation

* update runtime version

* git rid of large tuple

* too long lines

* config_activation_block -> zero

* obsolete TODO

* removed unjustified expect

* update TODOs with issue number

* new CT pruning algorithm

fixed cache + multiple blocks finalization

track CT configuraiton on light clients

support CT configuration change revert

revert CT config test

new CT pruning algorithm

fixed cache + multiple blocks finalization

track CT configuraiton on light clients

support CT configuration change revert

revert CT config test

* BlockIdOrHeader isn't really required

* removed debug leftovers + some docs

* more docs

* more post-merge fixes

* more post-merge fixes

* revertes some unnecessary changes

* reverted unnecessary changes

* fix compilation + unnecessary changes

* (restart CI)

* fix cache update when finalizing multiple blocks

* fixed tests

* collect_extrinsics -> set_collect_extrinsics

* restore lost test

* do not calculate block number twice

* Update primitives/blockchain/src/error.rs

Co-Authored-By: cheme <emericchevalier.pro@gmail.com>

* map_err -> unwrap_or

* document get_at Result

* delete abandoned file

* added weight for set_changes_trie_config

* prefer_configs -> fail_if_disabled

* Update client/api/src/backend.rs

Co-Authored-By: cheme <emericchevalier.pro@gmail.com>

* Update client/db/src/changes_tries_storage.rs

Co-Authored-By: cheme <emericchevalier.pro@gmail.com>

* CommitOperation+merge -> CommitOperations

* fixed test compilation

* merged two different CTRange structs

* lost file

* uggrade db from v0 to v1 (init CT cache + add column)

* fix after merge

Co-authored-by: cheme <emericchevalier.pro@gmail.com>
Co-authored-by: Gavin Wood <github@gavwood.com>
This commit is contained in:
Svyatoslav Nikolsky
2020-01-16 19:38:24 +03:00
committed by Gavin Wood
parent 45fbf09dac
commit febf29390a
48 changed files with 2743 additions and 1548 deletions
Download Patch File Download Diff File Expand all files Collapse all files
substrate/client/db/src/cache/list_cache.rs
Vendored
+392 -157
View File
@@ -94,6 +94,12 @@ pub enum CommitOperation<Block: BlockT, T: CacheItemT> {
BlockReverted(BTreeMap<usize, Option<Fork<Block, T>>>),
}
/// A set of commit operations.
#[derive(Debug)]
pub struct CommitOperations<Block: BlockT, T: CacheItemT> {
operations: Vec<CommitOperation<Block, T>>,
}
/// Single fork of list-based cache.
#[derive(Debug)]
#[cfg_attr(test, derive(PartialEq))]
@@ -123,21 +129,17 @@ impl<Block: BlockT, T: CacheItemT, S: Storage<Block, T>> ListCache<Block, T, S>
storage: S,
pruning_strategy: PruningStrategy<NumberFor<Block>>,
best_finalized_block: ComplexBlockId<Block>,
) -> Self {
) -> ClientResult<Self> {
let (best_finalized_entry, unfinalized) = storage.read_meta()
.and_then(|meta| read_forks(&storage, meta))
.unwrap_or_else(|error| {
warn!(target: "db", "Unable to initialize list cache: {}. Restarting", error);
(None, Vec::new())
});
.and_then(|meta| read_forks(&storage, meta))?;
ListCache {
Ok(ListCache {
storage,
pruning_strategy,
best_finalized_block,
best_finalized_entry,
unfinalized,
}
})
}
/// Get reference to the storage.
@@ -145,6 +147,12 @@ impl<Block: BlockT, T: CacheItemT, S: Storage<Block, T>> ListCache<Block, T, S>
&self.storage
}
/// Get unfinalized forks reference.
#[cfg(test)]
pub fn unfinalized(&self) -> &[Fork<Block, T>] {
&self.unfinalized
}
/// Get value valid at block.
pub fn value_at_block(
&self,
@@ -156,8 +164,8 @@ impl<Block: BlockT, T: CacheItemT, S: Storage<Block, T>> ListCache<Block, T, S>
// BUT since we're not guaranteeing to provide correct values for forks
// behind the finalized block, check if the block is finalized first
if !chain::is_finalized_block(&self.storage, at, Bounded::max_value())? {
return Ok(None);
if !chain::is_finalized_block(&self.storage, &at, Bounded::max_value())? {
return Err(ClientError::NotInFinalizedChain);
}
self.best_finalized_entry.as_ref()
@@ -171,11 +179,14 @@ impl<Block: BlockT, T: CacheItemT, S: Storage<Block, T>> ListCache<Block, T, S>
// IF there's no matching fork, ensure that this isn't a block from a fork that has forked
// behind the best finalized block and search at finalized fork
match self.find_unfinalized_fork(at)? {
match self.find_unfinalized_fork(&at)? {
Some(fork) => Some(&fork.head),
None => match self.best_finalized_entry.as_ref() {
Some(best_finalized_entry) if chain::is_connected_to_block(&self.storage, &best_finalized_entry.valid_from, at)? =>
Some(best_finalized_entry),
Some(best_finalized_entry) if chain::is_connected_to_block(
&self.storage,
&at,
&best_finalized_entry.valid_from,
)? => Some(best_finalized_entry),
_ => None,
},
}
@@ -198,9 +209,98 @@ impl<Block: BlockT, T: CacheItemT, S: Storage<Block, T>> ListCache<Block, T, S>
block: ComplexBlockId<Block>,
value: Option<T>,
entry_type: EntryType,
operations: &mut CommitOperations<Block, T>,
) -> ClientResult<()> {
Ok(operations.append(self.do_on_block_insert(tx, parent, block, value, entry_type, operations)?))
}
/// When previously inserted block is finalized.
pub fn on_block_finalize<Tx: StorageTransaction<Block, T>>(
&self,
tx: &mut Tx,
parent: ComplexBlockId<Block>,
block: ComplexBlockId<Block>,
operations: &mut CommitOperations<Block, T>,
) -> ClientResult<()> {
Ok(operations.append(self.do_on_block_finalize(tx, parent, block, operations)?))
}
/// When block is reverted.
pub fn on_block_revert<Tx: StorageTransaction<Block, T>>(
&self,
tx: &mut Tx,
reverted_block: &ComplexBlockId<Block>,
operations: &mut CommitOperations<Block, T>,
) -> ClientResult<()> {
Ok(operations.append(Some(self.do_on_block_revert(tx, reverted_block)?)))
}
/// When transaction is committed.
pub fn on_transaction_commit(&mut self, ops: CommitOperations<Block, T>) {
for op in ops.operations {
match op {
CommitOperation::AppendNewBlock(index, best_block) => {
let mut fork = self.unfinalized.get_mut(index)
.expect("ListCache is a crate-private type;
internal clients of ListCache are committing transaction while cache is locked;
CommitOperation holds valid references while cache is locked; qed");
fork.best_block = Some(best_block);
},
CommitOperation::AppendNewEntry(index, entry) => {
let mut fork = self.unfinalized.get_mut(index)
.expect("ListCache is a crate-private type;
internal clients of ListCache are committing transaction while cache is locked;
CommitOperation holds valid references while cache is locked; qed");
fork.best_block = Some(entry.valid_from.clone());
fork.head = entry;
},
CommitOperation::AddNewFork(entry) => {
self.unfinalized.push(Fork {
best_block: Some(entry.valid_from.clone()),
head: entry,
});
},
CommitOperation::BlockFinalized(block, finalizing_entry, forks) => {
self.best_finalized_block = block;
if let Some(finalizing_entry) = finalizing_entry {
self.best_finalized_entry = Some(finalizing_entry);
}
for fork_index in forks.iter().rev() {
self.unfinalized.remove(*fork_index);
}
},
CommitOperation::BlockReverted(forks) => {
for (fork_index, updated_fork) in forks.into_iter().rev() {
match updated_fork {
Some(updated_fork) => self.unfinalized[fork_index] = updated_fork,
None => { self.unfinalized.remove(fork_index); },
}
}
},
}
}
}
fn do_on_block_insert<Tx: StorageTransaction<Block, T>>(
&self,
tx: &mut Tx,
parent: ComplexBlockId<Block>,
block: ComplexBlockId<Block>,
value: Option<T>,
entry_type: EntryType,
operations: &CommitOperations<Block, T>,
) -> ClientResult<Option<CommitOperation<Block, T>>> {
// this guarantee is currently provided by LightStorage && we're relying on it here
debug_assert!(entry_type != EntryType::Final || self.best_finalized_block.hash == parent.hash);
let prev_operation = operations.operations.last();
debug_assert!(
entry_type != EntryType::Final ||
self.best_finalized_block.hash == parent.hash ||
match prev_operation {
Some(&CommitOperation::BlockFinalized(ref best_finalized_block, _, _))
=> best_finalized_block.hash == parent.hash,
_ => false,
}
);
// we do not store any values behind finalized
if block.number != Zero::zero() && self.best_finalized_block.number >= block.number {
@@ -296,7 +396,7 @@ impl<Block: BlockT, T: CacheItemT, S: Storage<Block, T>> ListCache<Block, T, S>
}
// cleanup database from abandoned unfinalized forks and obsolete finalized entries
let abandoned_forks = self.destroy_abandoned_forks(tx, &block);
let abandoned_forks = self.destroy_abandoned_forks(tx, &block, prev_operation);
self.prune_finalized_entries(tx, &block);
match new_storage_entry {
@@ -310,34 +410,39 @@ impl<Block: BlockT, T: CacheItemT, S: Storage<Block, T>> ListCache<Block, T, S>
}
}
/// When previously inserted block is finalized.
pub fn on_block_finalize<Tx: StorageTransaction<Block, T>>(
fn do_on_block_finalize<Tx: StorageTransaction<Block, T>>(
&self,
tx: &mut Tx,
parent: ComplexBlockId<Block>,
block: ComplexBlockId<Block>,
operations: &CommitOperations<Block, T>,
) -> ClientResult<Option<CommitOperation<Block, T>>> {
// this guarantee is currently provided by LightStorage && we're relying on it here
debug_assert_eq!(self.best_finalized_block.hash, parent.hash);
// this guarantee is currently provided by db backend && we're relying on it here
let prev_operation = operations.operations.last();
debug_assert!(
self.best_finalized_block.hash == parent.hash ||
match prev_operation {
Some(&CommitOperation::BlockFinalized(ref best_finalized_block, _, _))
=> best_finalized_block.hash == parent.hash,
_ => false,
}
);
// there could be at most one entry that is finalizing
let finalizing_entry = self.storage.read_entry(&block)?
.map(|entry| entry.into_entry(block.clone()));
// cleanup database from abandoned unfinalized forks and obsolete finalized entries
let abandoned_forks = self.destroy_abandoned_forks(tx, &block);
let abandoned_forks = self.destroy_abandoned_forks(tx, &block, prev_operation);
self.prune_finalized_entries(tx, &block);
let update_meta = finalizing_entry.is_some();
let operation = CommitOperation::BlockFinalized(block, finalizing_entry, abandoned_forks);
if update_meta {
tx.update_meta(self.best_finalized_entry.as_ref(), &self.unfinalized, &operation);
}
tx.update_meta(self.best_finalized_entry.as_ref(), &self.unfinalized, &operation);
Ok(Some(operation))
}
/// When block is reverted.
pub fn on_block_revert<Tx: StorageTransaction<Block, T>>(
fn do_on_block_revert<Tx: StorageTransaction<Block, T>>(
&self,
tx: &mut Tx,
reverted_block: &ComplexBlockId<Block>,
@@ -374,50 +479,6 @@ impl<Block: BlockT, T: CacheItemT, S: Storage<Block, T>> ListCache<Block, T, S>
Ok(operation)
}
/// When transaction is committed.
pub fn on_transaction_commit(&mut self, op: CommitOperation<Block, T>) {
match op {
CommitOperation::AppendNewBlock(index, best_block) => {
let mut fork = self.unfinalized.get_mut(index)
.expect("ListCache is a crate-private type;
internal clients of ListCache are committing transaction while cache is locked;
CommitOperation holds valid references while cache is locked; qed");
fork.best_block = Some(best_block);
},
CommitOperation::AppendNewEntry(index, entry) => {
let mut fork = self.unfinalized.get_mut(index)
.expect("ListCache is a crate-private type;
internal clients of ListCache are committing transaction while cache is locked;
CommitOperation holds valid references while cache is locked; qed");
fork.best_block = Some(entry.valid_from.clone());
fork.head = entry;
},
CommitOperation::AddNewFork(entry) => {
self.unfinalized.push(Fork {
best_block: Some(entry.valid_from.clone()),
head: entry,
});
},
CommitOperation::BlockFinalized(block, finalizing_entry, forks) => {
self.best_finalized_block = block;
if let Some(finalizing_entry) = finalizing_entry {
self.best_finalized_entry = Some(finalizing_entry);
}
for fork_index in forks.iter().rev() {
self.unfinalized.remove(*fork_index);
}
},
CommitOperation::BlockReverted(forks) => {
for (fork_index, updated_fork) in forks.into_iter().rev() {
match updated_fork {
Some(updated_fork) => self.unfinalized[fork_index] = updated_fork,
None => { self.unfinalized.remove(fork_index); },
}
}
},
}
}
/// Prune old finalized entries.
fn prune_finalized_entries<Tx: StorageTransaction<Block, T>>(
&self,
@@ -475,10 +536,22 @@ impl<Block: BlockT, T: CacheItemT, S: Storage<Block, T>> ListCache<Block, T, S>
fn destroy_abandoned_forks<Tx: StorageTransaction<Block, T>>(
&self,
tx: &mut Tx,
block: &ComplexBlockId<Block>
block: &ComplexBlockId<Block>,
prev_operation: Option<&CommitOperation<Block, T>>,
) -> BTreeSet<usize> {
let mut destroyed = BTreeSet::new();
for (index, fork) in self.unfinalized.iter().enumerate() {
// if some block has been finalized already => take it into account
let prev_abandoned_forks = match prev_operation {
Some(&CommitOperation::BlockFinalized(_, _, ref abandoned_forks)) => Some(abandoned_forks),
_ => None,
};
let mut destroyed = prev_abandoned_forks.cloned().unwrap_or_else(|| BTreeSet::new());
let live_unfinalized = self.unfinalized.iter()
.enumerate()
.filter(|(idx, _)| prev_abandoned_forks
.map(|prev_abandoned_forks| !prev_abandoned_forks.contains(idx))
.unwrap_or(true));
for (index, fork) in live_unfinalized {
if fork.head.valid_from.number == block.number {
destroyed.insert(index);
if fork.head.valid_from.hash != block.hash {
@@ -493,7 +566,10 @@ impl<Block: BlockT, T: CacheItemT, S: Storage<Block, T>> ListCache<Block, T, S>
}
/// Search unfinalized fork where given block belongs.
fn find_unfinalized_fork(&self, block: &ComplexBlockId<Block>) -> ClientResult<Option<&Fork<Block, T>>> {
fn find_unfinalized_fork(
&self,
block: &ComplexBlockId<Block>,
) -> ClientResult<Option<&Fork<Block, T>>> {
for unfinalized in &self.unfinalized {
if unfinalized.matches(&self.storage, block)? {
return Ok(Some(&unfinalized));
@@ -549,7 +625,7 @@ impl<Block: BlockT, T: CacheItemT> Fork<Block, T> {
};
// check if the parent is connected to the beginning of the range
if !chain::is_connected_to_block(storage, &parent, &begin)? {
if !chain::is_connected_to_block(storage, parent, &begin)? {
return Ok(None);
}
@@ -621,6 +697,65 @@ impl<Block: BlockT, T: CacheItemT> Fork<Block, T> {
}
}
impl<Block: BlockT, T: CacheItemT> Default for CommitOperations<Block, T> {
fn default() -> Self {
CommitOperations { operations: Vec::new() }
}
}
// This should never be allowed for non-test code to avoid revealing its internals.
#[cfg(test)]
impl<Block: BlockT, T: CacheItemT> From<Vec<CommitOperation<Block, T>>> for CommitOperations<Block, T> {
fn from(operations: Vec<CommitOperation<Block, T>>) -> Self {
CommitOperations { operations }
}
}
impl<Block: BlockT, T: CacheItemT> CommitOperations<Block, T> {
/// Append operation to the set.
fn append(&mut self, new_operation: Option<CommitOperation<Block, T>>) {
let new_operation = match new_operation {
Some(new_operation) => new_operation,
None => return,
};
let last_operation = match self.operations.pop() {
Some(last_operation) => last_operation,
None => {
self.operations.push(new_operation);
return;
},
};
// we are able (and obliged to) to merge two consequent block finalization operations
match last_operation {
CommitOperation::BlockFinalized(old_finalized_block, old_finalized_entry, old_abandoned_forks) => {
match new_operation {
CommitOperation::BlockFinalized(new_finalized_block, new_finalized_entry, new_abandoned_forks) => {
self.operations.push(CommitOperation::BlockFinalized(
new_finalized_block,
new_finalized_entry,
new_abandoned_forks,
));
},
_ => {
self.operations.push(CommitOperation::BlockFinalized(
old_finalized_block,
old_finalized_entry,
old_abandoned_forks,
));
self.operations.push(new_operation);
},
}
},
_ => {
self.operations.push(last_operation);
self.operations.push(new_operation);
},
}
}
}
/// Destroy fork by deleting all unfinalized entries.
pub fn destroy_fork<Block: BlockT, T: CacheItemT, S: Storage<Block, T>, Tx: StorageTransaction<Block, T>>(
head_valid_from: ComplexBlockId<Block>,
@@ -674,7 +809,7 @@ mod chain {
block1: &ComplexBlockId<Block>,
block2: &ComplexBlockId<Block>,
) -> ClientResult<bool> {
let (begin, end) = if block1 > block2 { (block2, block1) } else { (block1, block2) };
let (begin, end) = if *block1 > *block2 { (block2, block1) } else { (block1, block2) };
let mut current = storage.read_header(&end.hash)?
.ok_or_else(|| ClientError::UnknownBlock(format!("{}", end.hash)))?;
while *current.number() > begin.number {
@@ -773,8 +908,8 @@ pub mod tests {
// when block is earlier than best finalized block AND it is not finalized
// --- 50 ---
// ----------> [100]
assert_eq!(ListCache::<_, u64, _>::new(DummyStorage::new(), PruningStrategy::ByDepth(1024), test_id(100))
.value_at_block(&test_id(50)).unwrap(), None);
assert!(ListCache::<_, u64, _>::new(DummyStorage::new(), PruningStrategy::ByDepth(1024), test_id(100))
.unwrap().value_at_block(&test_id(50)).is_err());
// when block is earlier than best finalized block AND it is finalized AND value is some
// [30] ---- 50 ---> [100]
assert_eq!(ListCache::new(
@@ -784,7 +919,7 @@ pub mod tests {
.with_entry(test_id(100), StorageEntry { prev_valid_from: Some(test_id(30)), value: 100 })
.with_entry(test_id(30), StorageEntry { prev_valid_from: None, value: 30 }),
PruningStrategy::ByDepth(1024), test_id(100)
).value_at_block(&test_id(50)).unwrap(), Some((test_id(30), Some(test_id(100)), 30)));
).unwrap().value_at_block(&test_id(50)).unwrap(), Some((test_id(30), Some(test_id(100)), 30)));
// when block is the best finalized block AND value is some
// ---> [100]
assert_eq!(ListCache::new(
@@ -794,18 +929,18 @@ pub mod tests {
.with_entry(test_id(100), StorageEntry { prev_valid_from: Some(test_id(30)), value: 100 })
.with_entry(test_id(30), StorageEntry { prev_valid_from: None, value: 30 }),
PruningStrategy::ByDepth(1024), test_id(100)
).value_at_block(&test_id(100)).unwrap(), Some((test_id(100), None, 100)));
).unwrap().value_at_block(&test_id(100)).unwrap(), Some((test_id(100), None, 100)));
// when block is parallel to the best finalized block
// ---- 100
// ---> [100]
assert_eq!(ListCache::new(
assert!(ListCache::new(
DummyStorage::new()
.with_meta(Some(test_id(100)), Vec::new())
.with_id(50, H256::from_low_u64_be(50))
.with_entry(test_id(100), StorageEntry { prev_valid_from: Some(test_id(30)), value: 100 })
.with_entry(test_id(30), StorageEntry { prev_valid_from: None, value: 30 }),
PruningStrategy::ByDepth(1024), test_id(100)
).value_at_block(&ComplexBlockId::new(H256::from_low_u64_be(2), 100)).unwrap(), None);
).unwrap().value_at_block(&ComplexBlockId::new(H256::from_low_u64_be(2), 100)).is_err());
// when block is later than last finalized block AND there are no forks AND finalized value is Some
// ---> [100] --- 200
@@ -815,7 +950,7 @@ pub mod tests {
.with_id(50, H256::from_low_u64_be(50))
.with_entry(test_id(100), StorageEntry { prev_valid_from: Some(test_id(30)), value: 100 }),
PruningStrategy::ByDepth(1024), test_id(100)
).value_at_block(&test_id(200)).unwrap(), Some((test_id(100), None, 100)));
).unwrap().value_at_block(&test_id(200)).unwrap(), Some((test_id(100), None, 100)));
// when block is later than last finalized block AND there are no matching forks
// AND block is connected to finalized block AND finalized value is Some
@@ -831,7 +966,7 @@ pub mod tests {
.with_header(test_header(4))
.with_header(fork_header(0, 2, 3)),
PruningStrategy::ByDepth(1024), test_id(2)
).value_at_block(&fork_id(0, 2, 3)).unwrap(), Some((correct_id(2), None, 2)));
).unwrap().value_at_block(&fork_id(0, 2, 3)).unwrap(), Some((correct_id(2), None, 2)));
// when block is later than last finalized block AND there are no matching forks
// AND block is not connected to finalized block
// --- 2 --- 3
@@ -848,7 +983,7 @@ pub mod tests {
.with_header(fork_header(0, 1, 3))
.with_header(fork_header(0, 1, 2)),
PruningStrategy::ByDepth(1024), test_id(2)
).value_at_block(&fork_id(0, 1, 3)).unwrap(), None);
).unwrap().value_at_block(&fork_id(0, 1, 3)).unwrap(), None);
// when block is later than last finalized block AND it appends to unfinalized fork from the end
// AND unfinalized value is Some
@@ -861,7 +996,7 @@ pub mod tests {
.with_header(test_header(4))
.with_header(test_header(5)),
PruningStrategy::ByDepth(1024), test_id(2)
).value_at_block(&correct_id(5)).unwrap(), Some((correct_id(4), None, 4)));
).unwrap().value_at_block(&correct_id(5)).unwrap(), Some((correct_id(4), None, 4)));
// when block is later than last finalized block AND it does not fits unfinalized fork
// AND it is connected to the finalized block AND finalized value is Some
// ---> [2] ----------> [4]
@@ -876,7 +1011,7 @@ pub mod tests {
.with_header(test_header(4))
.with_header(fork_header(0, 2, 3)),
PruningStrategy::ByDepth(1024), test_id(2)
).value_at_block(&fork_id(0, 2, 3)).unwrap(), Some((correct_id(2), None, 2)));
).unwrap().value_at_block(&fork_id(0, 2, 3)).unwrap(), Some((correct_id(2), None, 2)));
}
#[test]
@@ -885,22 +1020,25 @@ pub mod tests {
let fin = EntryType::Final;
// when trying to insert block < finalized number
assert!(ListCache::new(DummyStorage::new(), PruningStrategy::ByDepth(1024), test_id(100))
.on_block_insert(
let mut ops = Default::default();
assert!(ListCache::new(DummyStorage::new(), PruningStrategy::ByDepth(1024), test_id(100)).unwrap()
.do_on_block_insert(
&mut DummyTransaction::new(),
test_id(49),
test_id(50),
Some(50),
nfin,
&mut ops,
).unwrap().is_none());
// when trying to insert block @ finalized number
assert!(ListCache::new(DummyStorage::new(), PruningStrategy::ByDepth(1024), test_id(100))
.on_block_insert(
assert!(ListCache::new(DummyStorage::new(), PruningStrategy::ByDepth(1024), test_id(100)).unwrap()
.do_on_block_insert(
&mut DummyTransaction::new(),
test_id(99),
test_id(100),
Some(100),
nfin,
&Default::default(),
).unwrap().is_none());
// when trying to insert non-final block AND it appends to the best block of unfinalized fork
@@ -910,19 +1048,23 @@ pub mod tests {
.with_meta(None, vec![test_id(4)])
.with_entry(test_id(4), StorageEntry { prev_valid_from: None, value: 4 }),
PruningStrategy::ByDepth(1024), test_id(2)
);
).unwrap();
cache.unfinalized[0].best_block = Some(test_id(4));
let mut tx = DummyTransaction::new();
assert_eq!(cache.on_block_insert(&mut tx, test_id(4), test_id(5), Some(4), nfin).unwrap(),
Some(CommitOperation::AppendNewBlock(0, test_id(5))));
assert_eq!(
cache.do_on_block_insert(&mut tx, test_id(4), test_id(5), Some(4), nfin, &Default::default()).unwrap(),
Some(CommitOperation::AppendNewBlock(0, test_id(5))),
);
assert!(tx.inserted_entries().is_empty());
assert!(tx.removed_entries().is_empty());
assert!(tx.updated_meta().is_none());
// when trying to insert non-final block AND it appends to the best block of unfinalized fork
// AND new value is the same as in the fork' best block
let mut tx = DummyTransaction::new();
assert_eq!(cache.on_block_insert(&mut tx, test_id(4), test_id(5), Some(5), nfin).unwrap(),
Some(CommitOperation::AppendNewEntry(0, Entry { valid_from: test_id(5), value: 5 })));
assert_eq!(
cache.do_on_block_insert(&mut tx, test_id(4), test_id(5), Some(5), nfin, &Default::default()).unwrap(),
Some(CommitOperation::AppendNewEntry(0, Entry { valid_from: test_id(5), value: 5 })),
);
assert_eq!(*tx.inserted_entries(), vec![test_id(5).hash].into_iter().collect());
assert!(tx.removed_entries().is_empty());
assert_eq!(*tx.updated_meta(), Some(Metadata { finalized: None, unfinalized: vec![test_id(5)] }));
@@ -935,18 +1077,36 @@ pub mod tests {
.with_entry(correct_id(4), StorageEntry { prev_valid_from: None, value: 4 })
.with_header(test_header(4)),
PruningStrategy::ByDepth(1024), test_id(2)
);
).unwrap();
let mut tx = DummyTransaction::new();
assert_eq!(cache.on_block_insert(&mut tx, correct_id(4), correct_id(5), Some(4), nfin).unwrap(),
Some(CommitOperation::AppendNewBlock(0, correct_id(5))));
assert_eq!(
cache.do_on_block_insert(
&mut tx,
correct_id(4),
correct_id(5),
Some(4),
nfin,
&Default::default(),
).unwrap(),
Some(CommitOperation::AppendNewBlock(0, correct_id(5))),
);
assert!(tx.inserted_entries().is_empty());
assert!(tx.removed_entries().is_empty());
assert!(tx.updated_meta().is_none());
// when trying to insert non-final block AND it is the first block that appends to the best block of unfinalized fork
// AND new value is the same as in the fork' best block
let mut tx = DummyTransaction::new();
assert_eq!(cache.on_block_insert(&mut tx, correct_id(4), correct_id(5), Some(5), nfin).unwrap(),
Some(CommitOperation::AppendNewEntry(0, Entry { valid_from: correct_id(5), value: 5 })));
assert_eq!(
cache.do_on_block_insert(
&mut tx,
correct_id(4),
correct_id(5),
Some(5),
nfin,
&Default::default(),
).unwrap(),
Some(CommitOperation::AppendNewEntry(0, Entry { valid_from: correct_id(5), value: 5 })),
);
assert_eq!(*tx.inserted_entries(), vec![correct_id(5).hash].into_iter().collect());
assert!(tx.removed_entries().is_empty());
assert_eq!(*tx.updated_meta(), Some(Metadata { finalized: None, unfinalized: vec![correct_id(5)] }));
@@ -961,10 +1121,13 @@ pub mod tests {
.with_header(test_header(3))
.with_header(test_header(4)),
PruningStrategy::ByDepth(1024), correct_id(2)
);
).unwrap();
let mut tx = DummyTransaction::new();
assert_eq!(cache.on_block_insert(&mut tx, correct_id(3), fork_id(0, 3, 4), Some(14), nfin).unwrap(),
Some(CommitOperation::AddNewFork(Entry { valid_from: fork_id(0, 3, 4), value: 14 })));
assert_eq!(
cache.do_on_block_insert(&mut tx, correct_id(3), fork_id(0, 3, 4), Some(14), nfin, &Default::default())
.unwrap(),
Some(CommitOperation::AddNewFork(Entry { valid_from: fork_id(0, 3, 4), value: 14 })),
);
assert_eq!(*tx.inserted_entries(), vec![fork_id(0, 3, 4).hash].into_iter().collect());
assert!(tx.removed_entries().is_empty());
assert_eq!(*tx.updated_meta(), Some(Metadata { finalized: Some(correct_id(2)), unfinalized: vec![correct_id(4), fork_id(0, 3, 4)] }));
@@ -976,9 +1139,13 @@ pub mod tests {
.with_meta(Some(correct_id(2)), vec![])
.with_entry(correct_id(2), StorageEntry { prev_valid_from: None, value: 2 }),
PruningStrategy::ByDepth(1024), correct_id(2)
);
).unwrap();
let mut tx = DummyTransaction::new();
assert_eq!(cache.on_block_insert(&mut tx, correct_id(2), correct_id(3), Some(2), nfin).unwrap(), None);
assert_eq!(
cache.do_on_block_insert(&mut tx, correct_id(2), correct_id(3), Some(2), nfin, &Default::default())
.unwrap(),
None,
);
assert!(tx.inserted_entries().is_empty());
assert!(tx.removed_entries().is_empty());
assert!(tx.updated_meta().is_none());
@@ -989,19 +1156,23 @@ pub mod tests {
.with_meta(Some(correct_id(2)), vec![])
.with_entry(correct_id(2), StorageEntry { prev_valid_from: None, value: 2 }),
PruningStrategy::ByDepth(1024), correct_id(2)
);
).unwrap();
let mut tx = DummyTransaction::new();
assert_eq!(cache.on_block_insert(&mut tx, correct_id(2), correct_id(3), Some(3), nfin).unwrap(),
Some(CommitOperation::AddNewFork(Entry { valid_from: correct_id(3), value: 3 })));
assert_eq!(
cache.do_on_block_insert(&mut tx, correct_id(2), correct_id(3), Some(3), nfin, &Default::default())
.unwrap(),
Some(CommitOperation::AddNewFork(Entry { valid_from: correct_id(3), value: 3 })),
);
assert_eq!(*tx.inserted_entries(), vec![correct_id(3).hash].into_iter().collect());
assert!(tx.removed_entries().is_empty());
assert_eq!(*tx.updated_meta(), Some(Metadata { finalized: Some(correct_id(2)), unfinalized: vec![correct_id(3)] }));
// when inserting finalized entry AND there are no previous finalized entries
let cache = ListCache::new(DummyStorage::new(), PruningStrategy::ByDepth(1024), correct_id(2));
let cache = ListCache::new(DummyStorage::new(), PruningStrategy::ByDepth(1024), correct_id(2)).unwrap();
let mut tx = DummyTransaction::new();
assert_eq!(
cache.on_block_insert(&mut tx, correct_id(2), correct_id(3), Some(3), fin).unwrap(),
cache.do_on_block_insert(&mut tx, correct_id(2), correct_id(3), Some(3), fin, &Default::default())
.unwrap(),
Some(CommitOperation::BlockFinalized(
correct_id(3),
Some(Entry { valid_from: correct_id(3), value: 3 }),
@@ -1017,17 +1188,19 @@ pub mod tests {
.with_meta(Some(correct_id(2)), vec![])
.with_entry(correct_id(2), StorageEntry { prev_valid_from: None, value: 2 }),
PruningStrategy::ByDepth(1024), correct_id(2)
);
).unwrap();
let mut tx = DummyTransaction::new();
assert_eq!(cache.on_block_insert(&mut tx, correct_id(2), correct_id(3), Some(2), fin).unwrap(),
Some(CommitOperation::BlockFinalized(correct_id(3), None, Default::default())));
assert_eq!(
cache.do_on_block_insert(&mut tx, correct_id(2), correct_id(3), Some(2), fin, &Default::default()).unwrap(),
Some(CommitOperation::BlockFinalized(correct_id(3), None, Default::default())),
);
assert!(tx.inserted_entries().is_empty());
assert!(tx.removed_entries().is_empty());
assert!(tx.updated_meta().is_none());
// when inserting finalized entry AND value differs from previous finalized
let mut tx = DummyTransaction::new();
assert_eq!(
cache.on_block_insert(&mut tx, correct_id(2), correct_id(3), Some(3), fin).unwrap(),
cache.do_on_block_insert(&mut tx, correct_id(2), correct_id(3), Some(3), fin, &Default::default()).unwrap(),
Some(CommitOperation::BlockFinalized(
correct_id(3),
Some(Entry { valid_from: correct_id(3), value: 3 }),
@@ -1045,10 +1218,12 @@ pub mod tests {
.with_entry(correct_id(2), StorageEntry { prev_valid_from: None, value: 2 })
.with_entry(fork_id(0, 1, 3), StorageEntry { prev_valid_from: None, value: 13 }),
PruningStrategy::ByDepth(1024), correct_id(2)
);
).unwrap();
let mut tx = DummyTransaction::new();
assert_eq!(cache.on_block_insert(&mut tx, correct_id(2), correct_id(3), Some(2), fin).unwrap(),
Some(CommitOperation::BlockFinalized(correct_id(3), None, vec![0].into_iter().collect())));
assert_eq!(
cache.do_on_block_insert(&mut tx, correct_id(2), correct_id(3), Some(2), fin, &Default::default()).unwrap(),
Some(CommitOperation::BlockFinalized(correct_id(3), None, vec![0].into_iter().collect())),
);
}
#[test]
@@ -1060,13 +1235,18 @@ pub mod tests {
.with_entry(correct_id(2), StorageEntry { prev_valid_from: None, value: 2 })
.with_entry(correct_id(5), StorageEntry { prev_valid_from: Some(correct_id(2)), value: 5 }),
PruningStrategy::ByDepth(1024), correct_id(2)
);
).unwrap();
let mut tx = DummyTransaction::new();
assert_eq!(cache.on_block_finalize(&mut tx, correct_id(2), correct_id(3)).unwrap(),
Some(CommitOperation::BlockFinalized(correct_id(3), None, Default::default())));
assert_eq!(
cache.do_on_block_finalize(&mut tx, correct_id(2), correct_id(3), &Default::default()).unwrap(),
Some(CommitOperation::BlockFinalized(correct_id(3), None, Default::default())),
);
assert!(tx.inserted_entries().is_empty());
assert!(tx.removed_entries().is_empty());
assert!(tx.updated_meta().is_none());
assert_eq!(
*tx.updated_meta(),
Some(Metadata { finalized: Some(correct_id(2)), unfinalized: vec![correct_id(5)] }),
);
// finalization finalizes entry
let cache = ListCache::new(
DummyStorage::new()
@@ -1074,10 +1254,10 @@ pub mod tests {
.with_entry(correct_id(2), StorageEntry { prev_valid_from: None, value: 2 })
.with_entry(correct_id(5), StorageEntry { prev_valid_from: Some(correct_id(2)), value: 5 }),
PruningStrategy::ByDepth(1024), correct_id(4)
);
).unwrap();
let mut tx = DummyTransaction::new();
assert_eq!(
cache.on_block_finalize(&mut tx, correct_id(4), correct_id(5)).unwrap(),
cache.do_on_block_finalize(&mut tx, correct_id(4), correct_id(5), &Default::default()).unwrap(),
Some(CommitOperation::BlockFinalized(
correct_id(5),
Some(Entry { valid_from: correct_id(5), value: 5 }),
@@ -1094,10 +1274,12 @@ pub mod tests {
.with_entry(correct_id(2), StorageEntry { prev_valid_from: None, value: 2 })
.with_entry(fork_id(0, 1, 3), StorageEntry { prev_valid_from: None, value: 13 }),
PruningStrategy::ByDepth(1024), correct_id(2)
);
).unwrap();
let mut tx = DummyTransaction::new();
assert_eq!(cache.on_block_finalize(&mut tx, correct_id(2), correct_id(3)).unwrap(),
Some(CommitOperation::BlockFinalized(correct_id(3), None, vec![0].into_iter().collect())));
assert_eq!(
cache.do_on_block_finalize(&mut tx, correct_id(2), correct_id(3), &Default::default()).unwrap(),
Some(CommitOperation::BlockFinalized(correct_id(3), None, vec![0].into_iter().collect())),
);
}
#[test]
@@ -1109,25 +1291,29 @@ pub mod tests {
.with_entry(correct_id(5), StorageEntry { prev_valid_from: Some(correct_id(2)), value: 5 })
.with_entry(correct_id(6), StorageEntry { prev_valid_from: Some(correct_id(5)), value: 6 }),
PruningStrategy::ByDepth(1024), correct_id(2)
);
).unwrap();
// when new block is appended to unfinalized fork
cache.on_transaction_commit(CommitOperation::AppendNewBlock(0, correct_id(6)));
cache.on_transaction_commit(vec![CommitOperation::AppendNewBlock(0, correct_id(6))].into());
assert_eq!(cache.unfinalized[0].best_block, Some(correct_id(6)));
// when new entry is appended to unfinalized fork
cache.on_transaction_commit(CommitOperation::AppendNewEntry(0, Entry { valid_from: correct_id(7), value: 7 }));
cache.on_transaction_commit(vec![
CommitOperation::AppendNewEntry(0, Entry { valid_from: correct_id(7), value: 7 }),
].into());
assert_eq!(cache.unfinalized[0].best_block, Some(correct_id(7)));
assert_eq!(cache.unfinalized[0].head, Entry { valid_from: correct_id(7), value: 7 });
// when new fork is added
cache.on_transaction_commit(CommitOperation::AddNewFork(Entry { valid_from: correct_id(10), value: 10 }));
cache.on_transaction_commit(vec![
CommitOperation::AddNewFork(Entry { valid_from: correct_id(10), value: 10 }),
].into());
assert_eq!(cache.unfinalized[2].best_block, Some(correct_id(10)));
assert_eq!(cache.unfinalized[2].head, Entry { valid_from: correct_id(10), value: 10 });
// when block is finalized + entry is finalized + unfinalized forks are deleted
cache.on_transaction_commit(CommitOperation::BlockFinalized(
cache.on_transaction_commit(vec![CommitOperation::BlockFinalized(
correct_id(20),
Some(Entry { valid_from: correct_id(20), value: 20 }),
vec![0, 1, 2].into_iter().collect(),
));
)].into());
assert_eq!(cache.best_finalized_block, correct_id(20));
assert_eq!(cache.best_finalized_entry, Some(Entry { valid_from: correct_id(20), value: 20 }));
assert!(cache.unfinalized.is_empty());
@@ -1148,7 +1334,7 @@ pub mod tests {
.with_header(test_header(4))
.with_header(test_header(5)),
PruningStrategy::ByDepth(1024), correct_id(0)
).find_unfinalized_fork(&correct_id(4)).unwrap().unwrap().head.valid_from, correct_id(5));
).unwrap().find_unfinalized_fork((&correct_id(4)).into()).unwrap().unwrap().head.valid_from, correct_id(5));
// --- [2] ---------------> [5]
// ----------> [3] ---> 4
assert_eq!(ListCache::new(
@@ -1165,7 +1351,8 @@ pub mod tests {
.with_header(fork_header(0, 1, 3))
.with_header(fork_header(0, 1, 4)),
PruningStrategy::ByDepth(1024), correct_id(0)
).find_unfinalized_fork(&fork_id(0, 1, 4)).unwrap().unwrap().head.valid_from, fork_id(0, 1, 3));
).unwrap()
.find_unfinalized_fork((&fork_id(0, 1, 4)).into()).unwrap().unwrap().head.valid_from, fork_id(0, 1, 3));
// --- [2] ---------------> [5]
// ----------> [3]
// -----------------> 4
@@ -1185,7 +1372,7 @@ pub mod tests {
.with_header(fork_header(1, 1, 3))
.with_header(fork_header(1, 1, 4)),
PruningStrategy::ByDepth(1024), correct_id(0)
).find_unfinalized_fork(&fork_id(1, 1, 4)).unwrap().is_none());
).unwrap().find_unfinalized_fork((&fork_id(1, 1, 4)).into()).unwrap().is_none());
}
#[test]
@@ -1195,7 +1382,7 @@ pub mod tests {
.with_entry(test_id(100), StorageEntry { prev_valid_from: Some(test_id(50)), value: 100 })
.with_entry(test_id(50), StorageEntry { prev_valid_from: None, value: 50 });
assert_eq!(Fork::<_, u64> { best_block: None, head: Entry { valid_from: test_id(100), value: 0 } }
.matches(&storage, &test_id(20)).unwrap(), false);
.matches(&storage, (&test_id(20)).into()).unwrap(), false);
// when block is not connected to the begin block
let storage = DummyStorage::new()
.with_entry(correct_id(5), StorageEntry { prev_valid_from: Some(correct_id(3)), value: 100 })
@@ -1206,7 +1393,7 @@ pub mod tests {
.with_header(fork_header(0, 2, 4))
.with_header(fork_header(0, 2, 3));
assert_eq!(Fork::<_, u64> { best_block: None, head: Entry { valid_from: correct_id(5), value: 100 } }
.matches(&storage, &fork_id(0, 2, 4)).unwrap(), false);
.matches(&storage, (&fork_id(0, 2, 4)).into()).unwrap(), false);
// when block is not connected to the end block
let storage = DummyStorage::new()
.with_entry(correct_id(5), StorageEntry { prev_valid_from: Some(correct_id(3)), value: 100 })
@@ -1216,14 +1403,14 @@ pub mod tests {
.with_header(test_header(3))
.with_header(fork_header(0, 3, 4));
assert_eq!(Fork::<_, u64> { best_block: None, head: Entry { valid_from: correct_id(5), value: 100 } }
.matches(&storage, &fork_id(0, 3, 4)).unwrap(), false);
.matches(&storage, (&fork_id(0, 3, 4)).into()).unwrap(), false);
// when block is connected to the begin block AND end is open
let storage = DummyStorage::new()
.with_entry(correct_id(5), StorageEntry { prev_valid_from: None, value: 100 })
.with_header(test_header(5))
.with_header(test_header(6));
assert_eq!(Fork::<_, u64> { best_block: None, head: Entry { valid_from: correct_id(5), value: 100 } }
.matches(&storage, &correct_id(6)).unwrap(), true);
.matches(&storage, (&correct_id(6)).into()).unwrap(), true);
// when block is connected to the begin block AND to the end block
let storage = DummyStorage::new()
.with_entry(correct_id(5), StorageEntry { prev_valid_from: Some(correct_id(3)), value: 100 })
@@ -1232,7 +1419,7 @@ pub mod tests {
.with_header(test_header(4))
.with_header(test_header(3));
assert_eq!(Fork::<_, u64> { best_block: None, head: Entry { valid_from: correct_id(5), value: 100 } }
.matches(&storage, &correct_id(4)).unwrap(), true);
.matches(&storage, (&correct_id(4)).into()).unwrap(), true);
}
#[test]
@@ -1338,9 +1525,21 @@ pub mod tests {
#[test]
fn is_connected_to_block_fails() {
// when storage returns error
assert!(chain::is_connected_to_block::<_, u64, _>(&FaultyStorage, &test_id(1), &test_id(100)).is_err());
assert!(
chain::is_connected_to_block::<_, u64, _>(
&FaultyStorage,
(&test_id(1)).into(),
&test_id(100),
).is_err(),
);
// when there's no header in the storage
assert!(chain::is_connected_to_block::<_, u64, _>(&DummyStorage::new(), &test_id(1), &test_id(100)).is_err());
assert!(
chain::is_connected_to_block::<_, u64, _>(
&DummyStorage::new(),
(&test_id(1)).into(),
&test_id(100),
).is_err(),
);
}
#[test]
@@ -1348,35 +1547,35 @@ pub mod tests {
// when without iterations we end up with different block
assert_eq!(chain::is_connected_to_block::<_, u64, _>(&DummyStorage::new()
.with_header(test_header(1)),
&test_id(1), &correct_id(1)).unwrap(), false);
(&test_id(1)).into(), &correct_id(1)).unwrap(), false);
// when with ASC iterations we end up with different block
assert_eq!(chain::is_connected_to_block::<_, u64, _>(&DummyStorage::new()
.with_header(test_header(0))
.with_header(test_header(1))
.with_header(test_header(2)),
&test_id(0), &correct_id(2)).unwrap(), false);
(&test_id(0)).into(), &correct_id(2)).unwrap(), false);
// when with DESC iterations we end up with different block
assert_eq!(chain::is_connected_to_block::<_, u64, _>(&DummyStorage::new()
.with_header(test_header(0))
.with_header(test_header(1))
.with_header(test_header(2)),
&correct_id(2), &test_id(0)).unwrap(), false);
(&correct_id(2)).into(), &test_id(0)).unwrap(), false);
// when without iterations we end up with the same block
assert_eq!(chain::is_connected_to_block::<_, u64, _>(&DummyStorage::new()
.with_header(test_header(1)),
&correct_id(1), &correct_id(1)).unwrap(), true);
(&correct_id(1)).into(), &correct_id(1)).unwrap(), true);
// when with ASC iterations we end up with the same block
assert_eq!(chain::is_connected_to_block::<_, u64, _>(&DummyStorage::new()
.with_header(test_header(0))
.with_header(test_header(1))
.with_header(test_header(2)),
&correct_id(0), &correct_id(2)).unwrap(), true);
(&correct_id(0)).into(), &correct_id(2)).unwrap(), true);
// when with DESC iterations we end up with the same block
assert_eq!(chain::is_connected_to_block::<_, u64, _>(&DummyStorage::new()
.with_header(test_header(0))
.with_header(test_header(1))
.with_header(test_header(2)),
&correct_id(2), &correct_id(0)).unwrap(), true);
(&correct_id(2)).into(), &correct_id(0)).unwrap(), true);
}
#[test]
@@ -1454,7 +1653,7 @@ pub mod tests {
.with_entry(test_id(10), StorageEntry { prev_valid_from: None, value: 10 })
.with_entry(test_id(20), StorageEntry { prev_valid_from: Some(test_id(10)), value: 20 })
.with_entry(test_id(30), StorageEntry { prev_valid_from: Some(test_id(20)), value: 30 }),
strategy, test_id(9));
strategy, test_id(9)).unwrap();
let mut tx = DummyTransaction::new();
// when finalizing entry #10: no entries pruned
@@ -1515,28 +1714,64 @@ pub mod tests {
.with_header(fork_header(1, 2, 5))
.with_header(fork_header(2, 4, 5)),
PruningStrategy::ByDepth(1024), correct_id(1)
);
).unwrap();
// when 5 is reverted: entry 5 is truncated
let op = cache.on_block_revert(&mut DummyTransaction::new(), &correct_id(5)).unwrap();
let op = cache.do_on_block_revert(&mut DummyTransaction::new(), &correct_id(5)).unwrap();
assert_eq!(op, CommitOperation::BlockReverted(vec![
(0, Some(Fork { best_block: None, head: Entry { valid_from: correct_id(4), value: 4 } })),
].into_iter().collect()));
cache.on_transaction_commit(op);
cache.on_transaction_commit(vec![op].into());
// when 3 is reverted: entries 4+5' are truncated
let op = cache.on_block_revert(&mut DummyTransaction::new(), &correct_id(3)).unwrap();
let op = cache.do_on_block_revert(&mut DummyTransaction::new(), &correct_id(3)).unwrap();
assert_eq!(op, CommitOperation::BlockReverted(vec![
(0, None),
(2, None),
].into_iter().collect()));
cache.on_transaction_commit(op);
cache.on_transaction_commit(vec![op].into());
// when 2 is reverted: entries 4'+5' are truncated
let op = cache.on_block_revert(&mut DummyTransaction::new(), &correct_id(2)).unwrap();
let op = cache.do_on_block_revert(&mut DummyTransaction::new(), &correct_id(2)).unwrap();
assert_eq!(op, CommitOperation::BlockReverted(vec![
(0, None),
].into_iter().collect()));
cache.on_transaction_commit(op);
cache.on_transaction_commit(vec![op].into());
}
#[test]
fn append_commit_operation_works() {
let mut ops = CommitOperations::default();
ops.append(None);
assert_eq!(ops.operations, Vec::new());
ops.append(Some(CommitOperation::BlockFinalized(
test_id(10),
Some(Entry { valid_from: test_id(10), value: 10 }),
vec![5].into_iter().collect(),
)));
assert_eq!(
ops.operations,
vec![CommitOperation::BlockFinalized(
test_id(10),
Some(Entry { valid_from: test_id(10), value: 10 }),
vec![5].into_iter().collect(),
)],
);
ops.append(Some(CommitOperation::BlockFinalized(
test_id(20),
Some(Entry { valid_from: test_id(20), value: 20 }),
vec![5, 6].into_iter().collect(),
)));
assert_eq!(
ops.operations,
vec![CommitOperation::BlockFinalized(
test_id(20),
Some(Entry { valid_from: test_id(20), value: 20 }),
vec![5, 6].into_iter().collect(),
)],
);
}
}
substrate/client/db/src/cache/list_storage.rs
Vendored
+3 -1
View File
@@ -222,7 +222,9 @@ mod meta {
unfinalized.push(&entry.valid_from);
},
CommitOperation::BlockFinalized(_, ref finalizing_entry, ref forks) => {
finalized = finalizing_entry.as_ref().map(|entry| &entry.valid_from);
if let Some(finalizing_entry) = finalizing_entry.as_ref() {
finalized = Some(&finalizing_entry.valid_from);
}
for fork_index in forks.iter().rev() {
unfinalized.remove(*fork_index);
}
substrate/client/db/src/cache/mod.rs
Vendored
+88 -62
View File
@@ -16,7 +16,7 @@
//! DB-backed cache of blockchain data.
use std::{sync::Arc, collections::HashMap};
use std::{sync::Arc, collections::{HashMap, hash_map::Entry}};
use parking_lot::RwLock;
use kvdb::{KeyValueDB, DBTransaction};
@@ -51,8 +51,10 @@ pub enum EntryType {
/// Block identifier that holds both hash and number.
#[derive(Clone, Debug, Encode, Decode, PartialEq)]
pub struct ComplexBlockId<Block: BlockT> {
hash: Block::Hash,
number: NumberFor<Block>,
/// Hash of the block.
pub(crate) hash: Block::Hash,
/// Number of the block.
pub(crate) number: NumberFor<Block>,
}
impl<Block: BlockT> ComplexBlockId<Block> {
@@ -79,7 +81,7 @@ pub struct DbCache<Block: BlockT> {
db: Arc<dyn KeyValueDB>,
key_lookup_column: u32,
header_column: u32,
authorities_column: u32,
cache_column: u32,
genesis_hash: Block::Hash,
best_finalized_block: ComplexBlockId<Block>,
}
@@ -90,7 +92,7 @@ impl<Block: BlockT> DbCache<Block> {
db: Arc<dyn KeyValueDB>,
key_lookup_column: u32,
header_column: u32,
authorities_column: u32,
cache_column: u32,
genesis_hash: Block::Hash,
best_finalized_block: ComplexBlockId<Block>,
) -> Self {
@@ -99,7 +101,7 @@ impl<Block: BlockT> DbCache<Block> {
db,
key_lookup_column,
header_column,
authorities_column,
cache_column,
genesis_hash,
best_finalized_block,
}
@@ -115,30 +117,48 @@ impl<Block: BlockT> DbCache<Block> {
DbCacheTransaction {
cache: self,
tx,
cache_at_op: HashMap::new(),
cache_at_ops: HashMap::new(),
best_finalized_block: None,
}
}
/// Begin cache transaction with given ops.
pub fn transaction_with_ops<'a>(
&'a mut self,
tx: &'a mut DBTransaction,
ops: DbCacheTransactionOps<Block>,
) -> DbCacheTransaction<'a, Block> {
DbCacheTransaction {
cache: self,
tx,
cache_at_ops: ops.cache_at_ops,
best_finalized_block: ops.best_finalized_block,
}
}
/// Run post-commit cache operations.
pub fn commit(&mut self, ops: DbCacheTransactionOps<Block>) {
for (name, op) in ops.cache_at_op.into_iter() {
self.get_cache(name).on_transaction_commit(op);
pub fn commit(&mut self, ops: DbCacheTransactionOps<Block>) -> ClientResult<()> {
for (name, ops) in ops.cache_at_ops.into_iter() {
self.get_cache(name)?.on_transaction_commit(ops);
}
if let Some(best_finalized_block) = ops.best_finalized_block {
self.best_finalized_block = best_finalized_block;
}
Ok(())
}
/// Creates `ListCache` with the given name or returns a reference to the existing.
fn get_cache(&mut self, name: CacheKeyId) -> &mut ListCache<Block, Vec<u8>, self::list_storage::DbStorage> {
pub(crate) fn get_cache(
&mut self,
name: CacheKeyId,
) -> ClientResult<&mut ListCache<Block, Vec<u8>, self::list_storage::DbStorage>> {
get_cache_helper(
&mut self.cache_at,
name,
&self.db,
self.key_lookup_column,
self.header_column,
self.authorities_column,
self.cache_column,
&self.best_finalized_block
)
}
@@ -154,34 +174,49 @@ fn get_cache_helper<'a, Block: BlockT>(
header: u32,
cache: u32,
best_finalized_block: &ComplexBlockId<Block>,
) -> &'a mut ListCache<Block, Vec<u8>, self::list_storage::DbStorage> {
cache_at.entry(name).or_insert_with(|| {
ListCache::new(
self::list_storage::DbStorage::new(name.to_vec(), db.clone(),
self::list_storage::DbColumns {
meta: COLUMN_META,
key_lookup,
header,
cache,
},
),
cache_pruning_strategy(name),
best_finalized_block.clone(),
)
})
) -> ClientResult<&'a mut ListCache<Block, Vec<u8>, self::list_storage::DbStorage>> {
match cache_at.entry(name) {
Entry::Occupied(entry) => Ok(entry.into_mut()),
Entry::Vacant(entry) => {
let cache = ListCache::new(
self::list_storage::DbStorage::new(name.to_vec(), db.clone(),
self::list_storage::DbColumns {
meta: COLUMN_META,
key_lookup,
header,
cache,
},
),
cache_pruning_strategy(name),
best_finalized_block.clone(),
)?;
Ok(entry.insert(cache))
}
}
}
/// Cache operations that are to be committed after database transaction is committed.
#[derive(Default)]
pub struct DbCacheTransactionOps<Block: BlockT> {
cache_at_op: HashMap<CacheKeyId, self::list_cache::CommitOperation<Block, Vec<u8>>>,
cache_at_ops: HashMap<CacheKeyId, self::list_cache::CommitOperations<Block, Vec<u8>>>,
best_finalized_block: Option<ComplexBlockId<Block>>,
}
impl<Block: BlockT> DbCacheTransactionOps<Block> {
/// Empty transaction ops.
pub fn empty() -> DbCacheTransactionOps<Block> {
DbCacheTransactionOps {
cache_at_ops: HashMap::new(),
best_finalized_block: None,
}
}
}
/// Database-backed blockchain data cache transaction valid for single block import.
pub struct DbCacheTransaction<'a, Block: BlockT> {
cache: &'a mut DbCache<Block>,
tx: &'a mut DBTransaction,
cache_at_op: HashMap<CacheKeyId, self::list_cache::CommitOperation<Block, Vec<u8>>>,
cache_at_ops: HashMap<CacheKeyId, self::list_cache::CommitOperations<Block, Vec<u8>>>,
best_finalized_block: Option<ComplexBlockId<Block>>,
}
@@ -189,7 +224,7 @@ impl<'a, Block: BlockT> DbCacheTransaction<'a, Block> {
/// Convert transaction into post-commit operations set.
pub fn into_ops(self) -> DbCacheTransactionOps<Block> {
DbCacheTransactionOps {
cache_at_op: self.cache_at_op,
cache_at_ops: self.cache_at_ops,
best_finalized_block: self.best_finalized_block,
}
}
@@ -202,8 +237,6 @@ impl<'a, Block: BlockT> DbCacheTransaction<'a, Block> {
data_at: HashMap<CacheKeyId, Vec<u8>>,
entry_type: EntryType,
) -> ClientResult<Self> {
assert!(self.cache_at_op.is_empty());
// prepare list of caches that are not update
// (we might still need to do some cache maintenance in this case)
let missed_caches = self.cache.cache_at.keys()
@@ -212,8 +245,9 @@ impl<'a, Block: BlockT> DbCacheTransaction<'a, Block> {
.collect::<Vec<_>>();
let mut insert_op = |name: CacheKeyId, value: Option<Vec<u8>>| -> Result<(), sp_blockchain::Error> {
let cache = self.cache.get_cache(name);
let op = cache.on_block_insert(
let cache = self.cache.get_cache(name)?;
let cache_ops = self.cache_at_ops.entry(name).or_default();
cache.on_block_insert(
&mut self::list_storage::DbStorageTransaction::new(
cache.storage(),
&mut self.tx,
@@ -222,10 +256,9 @@ impl<'a, Block: BlockT> DbCacheTransaction<'a, Block> {
block.clone(),
value,
entry_type,
cache_ops,
)?;
if let Some(op) = op {
self.cache_at_op.insert(name, op);
}
Ok(())
};
@@ -245,23 +278,19 @@ impl<'a, Block: BlockT> DbCacheTransaction<'a, Block> {
pub fn on_block_finalize(
mut self,
parent: ComplexBlockId<Block>,
block: ComplexBlockId<Block>
block: ComplexBlockId<Block>,
) -> ClientResult<Self> {
assert!(self.cache_at_op.is_empty());
for (name, cache_at) in self.cache.cache_at.iter() {
let op = cache_at.on_block_finalize(
for (name, cache) in self.cache.cache_at.iter() {
let cache_ops = self.cache_at_ops.entry(*name).or_default();
cache.on_block_finalize(
&mut self::list_storage::DbStorageTransaction::new(
cache_at.storage(),
cache.storage(),
&mut self.tx
),
parent.clone(),
block.clone(),
cache_ops,
)?;
if let Some(op) = op {
self.cache_at_op.insert(name.to_owned(), op);
}
}
self.best_finalized_block = Some(block);
@@ -275,16 +304,15 @@ impl<'a, Block: BlockT> DbCacheTransaction<'a, Block> {
reverted_block: &ComplexBlockId<Block>,
) -> ClientResult<Self> {
for (name, cache) in self.cache.cache_at.iter() {
let op = cache.on_block_revert(
let cache_ops = self.cache_at_ops.entry(*name).or_default();
cache.on_block_revert(
&mut self::list_storage::DbStorageTransaction::new(
cache.storage(),
&mut self.tx
),
reverted_block,
cache_ops,
)?;
assert!(!self.cache_at_op.contains_key(name));
self.cache_at_op.insert(name.to_owned(), op);
}
Ok(self)
@@ -310,7 +338,7 @@ impl<Block: BlockT> BlockchainCache<Block> for DbCacheSync<Block> {
)?;
let tx_ops = tx.into_ops();
db.write(dbtx).map_err(db_err)?;
cache.commit(tx_ops);
cache.commit(tx_ops)?;
Ok(())
}
@@ -318,40 +346,38 @@ impl<Block: BlockT> BlockchainCache<Block> for DbCacheSync<Block> {
&self,
key: &CacheKeyId,
at: &BlockId<Block>,
) -> Option<((NumberFor<Block>, Block::Hash), Option<(NumberFor<Block>, Block::Hash)>, Vec<u8>)> {
) -> ClientResult<Option<((NumberFor<Block>, Block::Hash), Option<(NumberFor<Block>, Block::Hash)>, Vec<u8>)>> {
let mut cache = self.0.write();
let storage = cache.get_cache(*key).storage();
let cache = cache.get_cache(*key)?;
let storage = cache.storage();
let db = storage.db();
let columns = storage.columns();
let at = match *at {
BlockId::Hash(hash) => {
let header = utils::read_header::<Block>(
let header = utils::require_header::<Block>(
&**db,
columns.key_lookup,
columns.header,
BlockId::Hash(hash.clone())).ok()??;
BlockId::Hash(hash.clone()))?;
ComplexBlockId::new(hash, *header.number())
},
BlockId::Number(number) => {
let hash = utils::read_header::<Block>(
let hash = utils::require_header::<Block>(
&**db,
columns.key_lookup,
columns.header,
BlockId::Number(number.clone())).ok()??.hash();
BlockId::Number(number.clone()))?.hash();
ComplexBlockId::new(hash, number)
},
};
cache.cache_at
.get(key)?
.value_at_block(&at)
cache.value_at_block(&at)
.map(|block_and_value| block_and_value.map(|(begin_block, end_block, value)|
(
(begin_block.number, begin_block.hash),
end_block.map(|end_block| (end_block.number, end_block.hash)),
value,
)))
.ok()?
}
}
substrate/client/db/src/changes_tries_storage.rs
+1015
View File
File diff suppressed because it is too large Load Diff
substrate/client/db/src/lib.rs
+162 -491
View File
File diff suppressed because it is too large Load Diff
substrate/client/db/src/light.rs
+56 -17
View File
@@ -38,7 +38,7 @@ use codec::{Decode, Encode};
use sp_runtime::generic::{DigestItem, BlockId};
use sp_runtime::traits::{Block as BlockT, Header as HeaderT, Zero, One, NumberFor, HasherFor};
use crate::cache::{DbCacheSync, DbCache, ComplexBlockId, EntryType as CacheEntryType};
use crate::utils::{self, meta_keys, Meta, db_err, read_db, block_id_to_lookup_key, read_meta};
use crate::utils::{self, meta_keys, DatabaseType, Meta, db_err, read_db, block_id_to_lookup_key, read_meta};
use crate::{DatabaseSettings, FrozenForDuration};
use log::{trace, warn, debug};
@@ -68,13 +68,10 @@ pub struct LightStorage<Block: BlockT> {
io_stats: FrozenForDuration<kvdb::IoStats>,
}
impl<Block> LightStorage<Block>
where
Block: BlockT,
{
impl<Block: BlockT> LightStorage<Block> {
/// Create new storage with given settings.
pub fn new(config: DatabaseSettings) -> ClientResult<Self> {
let db = crate::utils::open_database(&config, columns::META, "light")?;
let db = crate::utils::open_database::<Block>(&config, DatabaseType::Light)?;
Self::from_kvdb(db as Arc<_>)
}
@@ -89,7 +86,7 @@ impl<Block> LightStorage<Block>
}
fn from_kvdb(db: Arc<dyn KeyValueDB>) -> ClientResult<Self> {
let meta = read_meta::<Block>(&*db, columns::META, columns::HEADER)?;
let meta = read_meta::<Block>(&*db, columns::HEADER)?;
let cache = DbCache::new(
db.clone(),
columns::KEY_LOOKUP,
@@ -417,7 +414,7 @@ impl<Block> LightBlockchainStorage<Block> for LightStorage<Block>
fn import_header(
&self,
header: Block::Header,
cache_at: HashMap<well_known_cache_keys::Id, Vec<u8>>,
mut cache_at: HashMap<well_known_cache_keys::Id, Vec<u8>>,
leaf_state: NewBlockState,
aux_ops: Vec<(Vec<u8>, Option<Vec<u8>>)>,
) -> ClientResult<()> {
@@ -475,6 +472,13 @@ impl<Block> LightBlockchainStorage<Block> for LightStorage<Block>
)?;
}
// update changes trie configuration cache
if !cache_at.contains_key(&well_known_cache_keys::CHANGES_TRIE_CONFIG) {
if let Some(new_configuration) = crate::changes_tries_storage::extract_new_configuration(&header) {
cache_at.insert(well_known_cache_keys::CHANGES_TRIE_CONFIG, new_configuration.encode());
}
}
{
let mut cache = self.cache.0.write();
let cache_ops = cache.transaction(&mut transaction)
@@ -487,8 +491,11 @@ impl<Block> LightBlockchainStorage<Block> for LightStorage<Block>
.into_ops();
debug!("Light DB Commit {:?} ({})", hash, number);
self.db.write(transaction).map_err(db_err)?;
cache.commit(cache_ops);
cache.commit(cache_ops)
.expect("only fails if cache with given name isn't loaded yet;\
cache is already loaded because there are cache_ops; qed");
}
self.update_meta(hash, number, leaf_state.is_best(), finalized);
@@ -543,7 +550,9 @@ impl<Block> LightBlockchainStorage<Block> for LightStorage<Block>
.into_ops();
self.db.write(transaction).map_err(db_err)?;
cache.commit(cache_ops);
cache.commit(cache_ops)
.expect("only fails if cache with given name isn't loaded yet;\
cache is already loaded because there are cache_ops; qed");
}
self.update_meta(hash, header.number().clone(), false, true);
@@ -603,7 +612,8 @@ fn cht_key<N: TryInto<u32>>(cht_type: u8, block: N) -> ClientResult<[u8; 5]> {
#[cfg(test)]
pub(crate) mod tests {
use sc_client::cht;
use sp_runtime::generic::DigestItem;
use sp_core::ChangesTrieConfiguration;
use sp_runtime::generic::{DigestItem, ChangesTrieSignal};
use sp_runtime::testing::{H256 as Hash, Header, Block as RawBlock, ExtrinsicWrapper};
use sp_blockchain::{lowest_common_ancestor, tree_route};
use super::*;
@@ -964,7 +974,7 @@ pub(crate) mod tests {
}
fn get_authorities(cache: &dyn BlockchainCache<Block>, at: BlockId<Block>) -> Option<Vec<AuthorityId>> {
cache.get_at(&well_known_cache_keys::AUTHORITIES, &at)
cache.get_at(&well_known_cache_keys::AUTHORITIES, &at).unwrap_or(None)
.and_then(|(_, _, val)| Decode::decode(&mut &val[..]).ok())
}
@@ -1148,8 +1158,8 @@ pub(crate) mod tests {
let (genesis_hash, storage) = {
let db = LightStorage::<Block>::new_test();
// before cache is initialized => None
assert_eq!(db.cache().get_at(b"test", &BlockId::Number(0)), None);
// before cache is initialized => Err
assert!(db.cache().get_at(b"test", &BlockId::Number(0)).is_err());
// insert genesis block (no value for cache is provided)
let mut genesis_hash = None;
@@ -1160,14 +1170,14 @@ pub(crate) mod tests {
});
// after genesis is inserted => None
assert_eq!(db.cache().get_at(b"test", &BlockId::Number(0)), None);
assert_eq!(db.cache().get_at(b"test", &BlockId::Number(0)).unwrap(), None);
// initialize cache
db.cache().initialize(b"test", vec![42]).unwrap();
// after genesis is inserted + cache is initialized => Some
assert_eq!(
db.cache().get_at(b"test", &BlockId::Number(0)),
db.cache().get_at(b"test", &BlockId::Number(0)).unwrap(),
Some(((0, genesis_hash.unwrap()), None, vec![42])),
);
@@ -1177,8 +1187,37 @@ pub(crate) mod tests {
// restart && check that after restart value is read from the cache
let db = LightStorage::<Block>::from_kvdb(storage as Arc<_>).expect("failed to create test-db");
assert_eq!(
db.cache().get_at(b"test", &BlockId::Number(0)),
db.cache().get_at(b"test", &BlockId::Number(0)).unwrap(),
Some(((0, genesis_hash.unwrap()), None, vec![42])),
);
}
#[test]
fn changes_trie_configuration_is_tracked_on_light_client() {
let db = LightStorage::<Block>::new_test();
let new_config = Some(ChangesTrieConfiguration::new(2, 2));
// insert block#0 && block#1 (no value for cache is provided)
let hash0 = insert_block(&db, HashMap::new(), || default_header(&Default::default(), 0));
assert_eq!(
db.cache().get_at(&well_known_cache_keys::CHANGES_TRIE_CONFIG, &BlockId::Number(0)).unwrap()
.map(|(_, _, v)| ChangesTrieConfiguration::decode(&mut &v[..]).unwrap()),
None,
);
// insert configuration at block#1 (starts from block#2)
insert_block(&db, HashMap::new(), || {
let mut header = default_header(&hash0, 1);
header.digest_mut().push(
DigestItem::ChangesTrieSignal(ChangesTrieSignal::NewConfiguration(new_config.clone()))
);
header
});
assert_eq!(
db.cache().get_at(&well_known_cache_keys::CHANGES_TRIE_CONFIG, &BlockId::Number(1)).unwrap()
.map(|(_, _, v)| Option::<ChangesTrieConfiguration>::decode(&mut &v[..]).unwrap()),
Some(new_config),
);
}
}
substrate/client/db/src/upgrade.rs
+198
View File
@@ -0,0 +1,198 @@
// Copyright 2019-2020 Parity Technologies (UK) Ltd.
// This file is part of Substrate.
// Substrate is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
// Substrate is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
// You should have received a copy of the GNU General Public License
// along with Substrate. If not, see <http://www.gnu.org/licenses/>.
//! Database upgrade logic.
use std::fs;
use std::io::{Read, Write, ErrorKind};
use std::path::{Path, PathBuf};
use std::sync::Arc;
use codec::Encode;
use kvdb_rocksdb::{Database, DatabaseConfig};
use parking_lot::RwLock;
use sp_blockchain::{well_known_cache_keys, Cache};
use sp_core::ChangesTrieConfiguration;
use sp_runtime::traits::Block as BlockT;
use crate::{
cache::{ComplexBlockId, DbCache, DbCacheSync},
utils::{DatabaseType, check_database_type, db_err, read_genesis_hash},
};
/// Version file name.
const VERSION_FILE_NAME: &'static str = "db_version";
/// Current db version.
const CURRENT_VERSION: u32 = 1;
/// Number of columns in v0.
const V0_NUM_COLUMNS: u32 = 10;
/// Upgrade database to current version.
pub fn upgrade_db<Block: BlockT>(db_path: &Path, db_type: DatabaseType) -> sp_blockchain::Result<()> {
let db_version = current_version(db_path)?;
match db_version {
0 => migrate_0_to_1::<Block>(db_path, db_type)?,
1 => (),
_ => Err(sp_blockchain::Error::Backend(format!("Future database version: {}", db_version)))?,
}
update_version(db_path)
}
/// Migration from version0 to version1:
/// 1) the number of columns has changed from 10 to 11;
/// 2) changes tries configuration are now cached.
fn migrate_0_to_1<Block: BlockT>(db_path: &Path, db_type: DatabaseType) -> sp_blockchain::Result<()> {
{
let db = open_database(db_path, db_type, V0_NUM_COLUMNS)?;
db.add_column().map_err(db_err)?;
db.flush().map_err(db_err)?;
}
let db = open_database(db_path, db_type, V0_NUM_COLUMNS + 1)?;
const V0_FULL_KEY_LOOKUP_COLUMN: u32 = 3;
const V0_FULL_HEADER_COLUMN: u32 = 4;
const V0_FULL_CACHE_COLUMN: u32 = 10; // that's the column we have just added
const V0_LIGHT_KEY_LOOKUP_COLUMN: u32 = 1;
const V0_LIGHT_HEADER_COLUMN: u32 = 2;
const V0_LIGHT_CACHE_COLUMN: u32 = 3;
let (key_lookup_column, header_column, cache_column) = match db_type {
DatabaseType::Full => (
V0_FULL_KEY_LOOKUP_COLUMN,
V0_FULL_HEADER_COLUMN,
V0_FULL_CACHE_COLUMN,
),
DatabaseType::Light => (
V0_LIGHT_KEY_LOOKUP_COLUMN,
V0_LIGHT_HEADER_COLUMN,
V0_LIGHT_CACHE_COLUMN,
),
};
let genesis_hash: Option<Block::Hash> = read_genesis_hash(&db)?;
if let Some(genesis_hash) = genesis_hash {
let cache: DbCacheSync<Block> = DbCacheSync(RwLock::new(DbCache::new(
Arc::new(db),
key_lookup_column,
header_column,
cache_column,
genesis_hash,
ComplexBlockId::new(genesis_hash, 0.into()),
)));
let changes_trie_config: Option<ChangesTrieConfiguration> = None;
cache.initialize(&well_known_cache_keys::CHANGES_TRIE_CONFIG, changes_trie_config.encode())?;
}
Ok(())
}
/// Reads current database version from the file at given path.
/// If the file does not exist returns 0.
fn current_version(path: &Path) -> sp_blockchain::Result<u32> {
let unknown_version_err = || sp_blockchain::Error::Backend("Unknown database version".into());
match fs::File::open(version_file_path(path)) {
Err(ref err) if err.kind() == ErrorKind::NotFound => Ok(0),
Err(_) => Err(unknown_version_err()),
Ok(mut file) => {
let mut s = String::new();
file.read_to_string(&mut s).map_err(|_| unknown_version_err())?;
u32::from_str_radix(&s, 10).map_err(|_| unknown_version_err())
},
}
}
/// Opens database of givent type with given number of columns.
fn open_database(db_path: &Path, db_type: DatabaseType, db_columns: u32) -> sp_blockchain::Result<Database> {
let db_path = db_path.to_str()
.ok_or_else(|| sp_blockchain::Error::Backend("Invalid database path".into()))?;
let db_cfg = DatabaseConfig::with_columns(db_columns);
let db = Database::open(&db_cfg, db_path).map_err(db_err)?;
check_database_type(&db, db_type)?;
Ok(db)
}
/// Writes current database version to the file.
/// Creates a new file if the version file does not exist yet.
fn update_version(path: &Path) -> sp_blockchain::Result<()> {
fs::create_dir_all(path).map_err(db_err)?;
let mut file = fs::File::create(version_file_path(path)).map_err(db_err)?;
file.write_all(format!("{}", CURRENT_VERSION).as_bytes()).map_err(db_err)?;
Ok(())
}
/// Returns the version file path.
fn version_file_path(path: &Path) -> PathBuf {
let mut file_path = path.to_owned();
file_path.push(VERSION_FILE_NAME);
file_path
}
#[cfg(test)]
mod tests {
use sc_state_db::PruningMode;
use crate::{DatabaseSettings, DatabaseSettingsSrc};
use crate::tests::Block;
use super::*;
fn create_db(db_path: &Path, version: Option<u32>) {
let db_cfg = DatabaseConfig::with_columns(V0_NUM_COLUMNS);
Database::open(&db_cfg, db_path.to_str().unwrap()).unwrap();
if let Some(version) = version {
fs::create_dir_all(db_path).unwrap();
let mut file = fs::File::create(version_file_path(db_path)).unwrap();
file.write_all(format!("{}", version).as_bytes()).unwrap();
}
}
fn open_database(db_path: &Path) -> sp_blockchain::Result<()> {
crate::utils::open_database::<Block>(&DatabaseSettings {
state_cache_size: 0,
state_cache_child_ratio: None,
pruning: PruningMode::ArchiveAll,
source: DatabaseSettingsSrc::Path { path: db_path.to_owned(), cache_size: None },
}, DatabaseType::Full).map(|_| ())
}
#[test]
fn downgrade_never_happens() {
let db_dir = tempdir::TempDir::new("").unwrap();
create_db(db_dir.path(), Some(CURRENT_VERSION + 1));
assert!(open_database(db_dir.path()).is_err());
}
#[test]
fn open_empty_database_works() {
let db_dir = tempdir::TempDir::new("").unwrap();
open_database(db_dir.path()).unwrap();
open_database(db_dir.path()).unwrap();
assert_eq!(current_version(db_dir.path()).unwrap(), CURRENT_VERSION);
}
#[test]
fn upgrade_from_0_to_1_works() {
for version_from_file in &[None, Some(0)] {
let db_dir = tempdir::TempDir::new("").unwrap();
let db_path = db_dir.path();
create_db(db_path, *version_from_file);
open_database(db_path).unwrap();
assert_eq!(current_version(db_path).unwrap(), CURRENT_VERSION);
}
}
}
substrate/client/db/src/utils.rs
+66 -22
View File
@@ -21,7 +21,7 @@ use std::sync::Arc;
use std::{io, convert::TryInto};
use kvdb::{KeyValueDB, DBTransaction};
#[cfg(feature = "kvdb-rocksdb")]
#[cfg(any(feature = "kvdb-rocksdb", test))]
use kvdb_rocksdb::{Database, DatabaseConfig};
use log::debug;
@@ -36,7 +36,7 @@ use crate::{DatabaseSettings, DatabaseSettingsSrc};
/// Number of columns in the db. Must be the same for both full && light dbs.
/// Otherwise RocksDb will fail to open database && check its type.
pub const NUM_COLUMNS: u32 = 10;
pub const NUM_COLUMNS: u32 = 11;
/// Meta column. The set of keys in the column is shared by full && light storages.
pub const COLUMN_META: u32 = 0;
@@ -50,6 +50,8 @@ pub mod meta_keys {
pub const FINALIZED_BLOCK: &[u8; 5] = b"final";
/// Meta information prefix for list-based caches.
pub const CACHE_META_PREFIX: &[u8; 5] = b"cache";
/// Meta information for changes tries key.
pub const CHANGES_TRIES_META: &[u8; 5] = b"ctrie";
/// Genesis block hash.
pub const GENESIS_HASH: &[u8; 3] = b"gen";
/// Leaves prefix list key.
@@ -76,6 +78,15 @@ pub struct Meta<N, H> {
/// A block lookup key: used for canonical lookup from block number to hash
pub type NumberIndexKey = [u8; 4];
/// Database type.
#[derive(Clone, Copy, Debug, PartialEq)]
pub enum DatabaseType {
/// Full node database.
Full,
/// Light node database.
Light,
}
/// Convert block number into short lookup key (LE representation) for
/// blocks that are in the canonical chain.
///
@@ -203,14 +214,17 @@ pub fn db_err(err: io::Error) -> sp_blockchain::Error {
}
/// Open RocksDB database.
pub fn open_database(
pub fn open_database<Block: BlockT>(
config: &DatabaseSettings,
col_meta: u32,
db_type: &str
db_type: DatabaseType,
) -> sp_blockchain::Result<Arc<dyn KeyValueDB>> {
let db: Arc<dyn KeyValueDB> = match &config.source {
#[cfg(feature = "kvdb-rocksdb")]
#[cfg(any(feature = "kvdb-rocksdb", test))]
DatabaseSettingsSrc::Path { path, cache_size } => {
// first upgrade database to required version
crate::upgrade::upgrade_db::<Block>(&path, db_type)?;
// and now open database assuming that it has the latest version
let mut db_config = DatabaseConfig::with_columns(NUM_COLUMNS);
if let Some(cache_size) = cache_size {
@@ -232,7 +246,7 @@ pub fn open_database(
.ok_or_else(|| sp_blockchain::Error::Backend("Invalid database path".into()))?;
Arc::new(Database::open(&db_config, &path).map_err(db_err)?)
},
#[cfg(not(feature = "kvdb-rocksdb"))]
#[cfg(not(any(feature = "kvdb-rocksdb", test)))]
DatabaseSettingsSrc::Path { .. } => {
let msg = "Try to open RocksDB database with RocksDB disabled".into();
return Err(sp_blockchain::Error::Backend(msg));
@@ -240,22 +254,28 @@ pub fn open_database(
DatabaseSettingsSrc::Custom(db) => db.clone(),
};
// check database type
match db.get(col_meta, meta_keys::TYPE).map_err(db_err)? {
check_database_type(&*db, db_type)?;
Ok(db)
}
/// Check database type.
pub fn check_database_type(db: &dyn KeyValueDB, db_type: DatabaseType) -> sp_blockchain::Result<()> {
match db.get(COLUMN_META, meta_keys::TYPE).map_err(db_err)? {
Some(stored_type) => {
if db_type.as_bytes() != &*stored_type {
if db_type.as_str().as_bytes() != &*stored_type {
return Err(sp_blockchain::Error::Backend(
format!("Unexpected database type. Expected: {}", db_type)).into());
format!("Unexpected database type. Expected: {}", db_type.as_str())).into());
}
},
None => {
let mut transaction = DBTransaction::new();
transaction.put(col_meta, meta_keys::TYPE, db_type.as_bytes());
transaction.put(COLUMN_META, meta_keys::TYPE, db_type.as_str().as_bytes());
db.write(transaction).map_err(db_err)?;
},
}
Ok(db)
Ok(())
}
/// Read database column entry for the given block.
@@ -304,20 +324,15 @@ pub fn require_header<Block: BlockT>(
}
/// Read meta from the database.
pub fn read_meta<Block>(db: &dyn KeyValueDB, col_meta: u32, col_header: u32) -> Result<
pub fn read_meta<Block>(db: &dyn KeyValueDB, col_header: u32) -> Result<
Meta<<<Block as BlockT>::Header as HeaderT>::Number, Block::Hash>,
sp_blockchain::Error,
>
where
Block: BlockT,
{
let genesis_hash: Block::Hash = match db.get(col_meta, meta_keys::GENESIS_HASH).map_err(db_err)? {
Some(h) => match Decode::decode(&mut &h[..]) {
Ok(h) => h,
Err(err) => return Err(sp_blockchain::Error::Backend(
format!("Error decoding genesis hash: {}", err)
)),
},
let genesis_hash: Block::Hash = match read_genesis_hash(db)? {
Some(genesis_hash) => genesis_hash,
None => return Ok(Meta {
best_hash: Default::default(),
best_number: Zero::zero(),
@@ -328,7 +343,7 @@ pub fn read_meta<Block>(db: &dyn KeyValueDB, col_meta: u32, col_header: u32) ->
};
let load_meta_block = |desc, key| -> Result<_, sp_blockchain::Error> {
if let Some(Some(header)) = db.get(col_meta, key).and_then(|id|
if let Some(Some(header)) = db.get(COLUMN_META, key).and_then(|id|
match id {
Some(id) => db.get(col_header, &id).map(|h| h.map(|b| Block::Header::decode(&mut &b[..]).ok())),
None => Ok(None),
@@ -354,6 +369,29 @@ pub fn read_meta<Block>(db: &dyn KeyValueDB, col_meta: u32, col_header: u32) ->
})
}
/// Read genesis hash from database.
pub fn read_genesis_hash<Hash: Decode>(db: &dyn KeyValueDB) -> sp_blockchain::Result<Option<Hash>> {
match db.get(COLUMN_META, meta_keys::GENESIS_HASH).map_err(db_err)? {
Some(h) => match Decode::decode(&mut &h[..]) {
Ok(h) => Ok(Some(h)),
Err(err) => Err(sp_blockchain::Error::Backend(
format!("Error decoding genesis hash: {}", err)
)),
},
None => Ok(None),
}
}
impl DatabaseType {
/// Returns str representation of the type.
pub fn as_str(&self) -> &'static str {
match *self {
DatabaseType::Full => "full",
DatabaseType::Light => "light",
}
}
}
#[cfg(test)]
mod tests {
use super::*;
@@ -368,4 +406,10 @@ mod tests {
_ => unreachable!(),
};
}
#[test]
fn database_type_as_str_works() {
assert_eq!(DatabaseType::Full.as_str(), "full");
assert_eq!(DatabaseType::Light.as_str(), "light");
}
}