pezkuwichain/pezkuwi-subxt
1
0
Fork 0
mirror of https://github.com/pezkuwichain/pezkuwi-subxt.git synced 2026-06-09 20:11:09 +00:00
Code Issues Packages Projects Releases Wiki Activity

Remove discarded blocks and states from database by default (#11983)

Browse Source 
* 1.Add pruning param "canonical" in sc-cli.
2.Make PruningMode's default value to ArchiveCanonical.

* Update tests in sc-state-db.

* Update tests in sc-state-db.

* 1.Add a new value `AllWithNonFinalized` in `enum BlocksPruning` which Corresponds to `blocks_pruning 0` in CLI .
2.Change value `All` to `AllFinalized` in `enum BlocksPruning` and make it to keep full finalized block history.

* Make some corresponding adjustments based on the content in the conversation.

* Update client/db/src/lib.rs

Co-authored-by: Bastian Köcher <git@kchr.de>

* Apply suggestions from code review.

* 1.Change `blocks_pruning` to be like `state_pruning` .

* Fmt and add some doc.

* Update client/cli/src/params/pruning_params.rs

Co-authored-by: Bastian Köcher <git@kchr.de>

* Update client/cli/src/params/pruning_params.rs

Co-authored-by: Bastian Köcher <git@kchr.de>

* Update doc.

* Change `new_test_with_tx_storage` to take `BlocksPruning`.

* Fmt

Co-authored-by: Bastian Köcher <git@kchr.de>
This commit is contained in:
ZhiYong
2022-09-26 15:46:59 +08:00
committed by GitHub
parent 6fee4cb3f2
commit 11fa9af104
10 changed files with 211 additions and 58 deletions
Download Patch File Download Diff File Expand all files Collapse all files
substrate/client/db/src/lib.rs
+176 -39
View File
@@ -320,10 +320,12 @@ pub struct DatabaseSettings {
}
/// Block pruning settings.
#[derive(Debug, Clone, Copy)]
#[derive(Debug, Clone, Copy, PartialEq)]
pub enum BlocksPruning {
/// Keep full block history.
All,
/// Keep full block history, of every block that was ever imported.
KeepAll,
/// Keep full finalized block history.
KeepFinalized,
/// Keep N recent finalized blocks.
Some(u32),
}
@@ -1061,19 +1063,27 @@ impl<Block: BlockT> Backend<Block> {
/// Create new memory-backed client backend for tests.
#[cfg(any(test, feature = "test-helpers"))]
pub fn new_test(blocks_pruning: u32, canonicalization_delay: u64) -> Self {
Self::new_test_with_tx_storage(blocks_pruning, canonicalization_delay)
Self::new_test_with_tx_storage(BlocksPruning::Some(blocks_pruning), canonicalization_delay)
}
/// Create new memory-backed client backend for tests.
#[cfg(any(test, feature = "test-helpers"))]
pub fn new_test_with_tx_storage(blocks_pruning: u32, canonicalization_delay: u64) -> Self {
pub fn new_test_with_tx_storage(
blocks_pruning: BlocksPruning,
canonicalization_delay: u64,
) -> Self {
let db = kvdb_memorydb::create(crate::utils::NUM_COLUMNS);
let db = sp_database::as_database(db);
let state_pruning = match blocks_pruning {
BlocksPruning::KeepAll => PruningMode::ArchiveAll,
BlocksPruning::KeepFinalized => PruningMode::ArchiveCanonical,
BlocksPruning::Some(n) => PruningMode::blocks_pruning(n),
};
let db_setting = DatabaseSettings {
trie_cache_maximum_size: Some(16 * 1024 * 1024),
state_pruning: Some(PruningMode::blocks_pruning(blocks_pruning)),
state_pruning: Some(state_pruning),
source: DatabaseSource::Custom { db, require_create_flag: true },
blocks_pruning: BlocksPruning::Some(blocks_pruning),
blocks_pruning,
};
Self::new(db_setting, canonicalization_delay).expect("failed to create test-db")
@@ -1707,32 +1717,47 @@ impl<Block: BlockT> Backend<Block> {
finalized: NumberFor<Block>,
displaced: &FinalizationOutcome<Block::Hash, NumberFor<Block>>,
) -> ClientResult<()> {
if let BlocksPruning::Some(blocks_pruning) = self.blocks_pruning {
// Always keep the last finalized block
let keep = std::cmp::max(blocks_pruning, 1);
if finalized >= keep.into() {
let number = finalized.saturating_sub(keep.into());
self.prune_block(transaction, BlockId::<Block>::number(number))?;
}
match self.blocks_pruning {
BlocksPruning::KeepAll => {},
BlocksPruning::Some(blocks_pruning) => {
// Always keep the last finalized block
let keep = std::cmp::max(blocks_pruning, 1);
if finalized >= keep.into() {
let number = finalized.saturating_sub(keep.into());
self.prune_block(transaction, BlockId::<Block>::number(number))?;
}
self.prune_displaced_branches(transaction, finalized, displaced)?;
},
BlocksPruning::KeepFinalized => {
self.prune_displaced_branches(transaction, finalized, displaced)?;
},
}
Ok(())
}
// Also discard all blocks from displaced branches
for h in displaced.leaves() {
let mut number = finalized;
let mut hash = *h;
// Follow displaced chains back until we reach a finalized block.
// Since leaves are discarded due to finality, they can't have parents
// that are canonical, but not yet finalized. So we stop deleting as soon as
// we reach canonical chain.
while self.blockchain.hash(number)? != Some(hash) {
let id = BlockId::<Block>::hash(hash);
match self.blockchain.header(id)? {
Some(header) => {
self.prune_block(transaction, id)?;
number = header.number().saturating_sub(One::one());
hash = *header.parent_hash();
},
None => break,
}
fn prune_displaced_branches(
&self,
transaction: &mut Transaction<DbHash>,
finalized: NumberFor<Block>,
displaced: &FinalizationOutcome<Block::Hash, NumberFor<Block>>,
) -> ClientResult<()> {
// Discard all blocks from displaced branches
for h in displaced.leaves() {
let mut number = finalized;
let mut hash = *h;
// Follow displaced chains back until we reach a finalized block.
// Since leaves are discarded due to finality, they can't have parents
// that are canonical, but not yet finalized. So we stop deleting as soon as
// we reach canonical chain.
while self.blockchain.hash(number)? != Some(hash) {
let id = BlockId::<Block>::hash(hash);
match self.blockchain.header(id)? {
Some(header) => {
self.prune_block(transaction, id)?;
number = header.number().saturating_sub(One::one());
hash = *header.parent_hash();
},
None => break,
}
}
}
@@ -1752,6 +1777,13 @@ impl<Block: BlockT> Backend<Block> {
columns::BODY,
id,
)?;
utils::remove_from_db(
transaction,
&*self.storage.db,
columns::KEY_LOOKUP,
columns::JUSTIFICATIONS,
id,
)?;
if let Some(index) =
read_db(&*self.storage.db, columns::KEY_LOOKUP, columns::BODY_INDEX, id)?
{
@@ -2506,7 +2538,7 @@ pub(crate) mod tests {
trie_cache_maximum_size: Some(16 * 1024 * 1024),
state_pruning: Some(PruningMode::blocks_pruning(1)),
source: DatabaseSource::Custom { db: backing, require_create_flag: false },
blocks_pruning: BlocksPruning::All,
blocks_pruning: BlocksPruning::KeepFinalized,
},
0,
)
@@ -3176,7 +3208,7 @@ pub(crate) mod tests {
#[test]
fn prune_blocks_on_finalize() {
let backend = Backend::<Block>::new_test_with_tx_storage(2, 0);
let backend = Backend::<Block>::new_test_with_tx_storage(BlocksPruning::Some(2), 0);
let mut blocks = Vec::new();
let mut prev_hash = Default::default();
for i in 0..5 {
@@ -3210,9 +3242,114 @@ pub(crate) mod tests {
assert_eq!(Some(vec![4.into()]), bc.body(BlockId::hash(blocks[4])).unwrap());
}
#[test]
fn prune_blocks_on_finalize_in_keep_all() {
let backend = Backend::<Block>::new_test_with_tx_storage(BlocksPruning::KeepAll, 0);
let mut blocks = Vec::new();
let mut prev_hash = Default::default();
for i in 0..5 {
let hash = insert_block(
&backend,
i,
prev_hash,
None,
Default::default(),
vec![i.into()],
None,
)
.unwrap();
blocks.push(hash);
prev_hash = hash;
}
let mut op = backend.begin_operation().unwrap();
backend.begin_state_operation(&mut op, BlockId::Hash(blocks[4])).unwrap();
for i in 1..3 {
op.mark_finalized(BlockId::Hash(blocks[i]), None).unwrap();
}
backend.commit_operation(op).unwrap();
let bc = backend.blockchain();
assert_eq!(Some(vec![0.into()]), bc.body(BlockId::hash(blocks[0])).unwrap());
assert_eq!(Some(vec![1.into()]), bc.body(BlockId::hash(blocks[1])).unwrap());
assert_eq!(Some(vec![2.into()]), bc.body(BlockId::hash(blocks[2])).unwrap());
assert_eq!(Some(vec![3.into()]), bc.body(BlockId::hash(blocks[3])).unwrap());
assert_eq!(Some(vec![4.into()]), bc.body(BlockId::hash(blocks[4])).unwrap());
}
#[test]
fn prune_blocks_on_finalize_with_fork_in_keep_all() {
let backend = Backend::<Block>::new_test_with_tx_storage(BlocksPruning::KeepAll, 10);
let mut blocks = Vec::new();
let mut prev_hash = Default::default();
for i in 0..5 {
let hash = insert_block(
&backend,
i,
prev_hash,
None,
Default::default(),
vec![i.into()],
None,
)
.unwrap();
blocks.push(hash);
prev_hash = hash;
}
// insert a fork at block 2
let fork_hash_root = insert_block(
&backend,
2,
blocks[1],
None,
sp_core::H256::random(),
vec![2.into()],
None,
)
.unwrap();
insert_block(
&backend,
3,
fork_hash_root,
None,
H256::random(),
vec![3.into(), 11.into()],
None,
)
.unwrap();
let mut op = backend.begin_operation().unwrap();
backend.begin_state_operation(&mut op, BlockId::Hash(blocks[4])).unwrap();
op.mark_head(BlockId::Hash(blocks[4])).unwrap();
backend.commit_operation(op).unwrap();
let bc = backend.blockchain();
assert_eq!(Some(vec![2.into()]), bc.body(BlockId::hash(fork_hash_root)).unwrap());
for i in 1..5 {
let mut op = backend.begin_operation().unwrap();
backend.begin_state_operation(&mut op, BlockId::Hash(blocks[i])).unwrap();
op.mark_finalized(BlockId::Hash(blocks[i]), None).unwrap();
backend.commit_operation(op).unwrap();
}
assert_eq!(Some(vec![0.into()]), bc.body(BlockId::hash(blocks[0])).unwrap());
assert_eq!(Some(vec![1.into()]), bc.body(BlockId::hash(blocks[1])).unwrap());
assert_eq!(Some(vec![2.into()]), bc.body(BlockId::hash(blocks[2])).unwrap());
assert_eq!(Some(vec![3.into()]), bc.body(BlockId::hash(blocks[3])).unwrap());
assert_eq!(Some(vec![4.into()]), bc.body(BlockId::hash(blocks[4])).unwrap());
assert_eq!(Some(vec![2.into()]), bc.body(BlockId::hash(fork_hash_root)).unwrap());
assert_eq!(bc.info().best_number, 4);
for i in 0..5 {
assert!(bc.hash(i).unwrap().is_some());
}
}
#[test]
fn prune_blocks_on_finalize_with_fork() {
let backend = Backend::<Block>::new_test_with_tx_storage(2, 10);
let backend = Backend::<Block>::new_test_with_tx_storage(BlocksPruning::Some(2), 10);
let mut blocks = Vec::new();
let mut prev_hash = Default::default();
for i in 0..5 {
@@ -3273,7 +3410,7 @@ pub(crate) mod tests {
#[test]
fn indexed_data_block_body() {
let backend = Backend::<Block>::new_test_with_tx_storage(1, 10);
let backend = Backend::<Block>::new_test_with_tx_storage(BlocksPruning::Some(1), 10);
let x0 = ExtrinsicWrapper::from(0u64).encode();
let x1 = ExtrinsicWrapper::from(1u64).encode();
@@ -3315,7 +3452,7 @@ pub(crate) mod tests {
#[test]
fn index_invalid_size() {
let backend = Backend::<Block>::new_test_with_tx_storage(1, 10);
let backend = Backend::<Block>::new_test_with_tx_storage(BlocksPruning::Some(1), 10);
let x0 = ExtrinsicWrapper::from(0u64).encode();
let x1 = ExtrinsicWrapper::from(1u64).encode();
@@ -3350,7 +3487,7 @@ pub(crate) mod tests {
#[test]
fn renew_transaction_storage() {
let backend = Backend::<Block>::new_test_with_tx_storage(2, 10);
let backend = Backend::<Block>::new_test_with_tx_storage(BlocksPruning::Some(2), 10);
let mut blocks = Vec::new();
let mut prev_hash = Default::default();
let x1 = ExtrinsicWrapper::from(0u64).encode();
@@ -3397,7 +3534,7 @@ pub(crate) mod tests {
#[test]
fn remove_leaf_block_works() {
let backend = Backend::<Block>::new_test_with_tx_storage(2, 10);
let backend = Backend::<Block>::new_test_with_tx_storage(BlocksPruning::Some(2), 10);
let mut blocks = Vec::new();
let mut prev_hash = Default::default();
for i in 0..2 {