Blockchain cache pruning strategy (#3395)

* blockchain cache pruning strategy * added some internal docs to cache_pruning_strategy * Update core/client/db/src/cache/mod.rs Co-Authored-By: DemiMarie-parity <48690212+DemiMarie-parity@users.noreply.github.com>
2026-05-09 00:08:00 +00:00 · 2019-09-20 09:49:15 +03:00
parent d105d3f3a1
commit eba6dd73c6
20 changed files with 161 additions and 104 deletions
@@ -52,12 +52,21 @@ use crate::cache::{CacheItemT, ComplexBlockId, EntryType};
 use crate::cache::list_entry::{Entry, StorageEntry};
 use crate::cache::list_storage::{Storage, StorageTransaction, Metadata};

+/// Pruning strategy.
+#[derive(Debug, Clone, Copy)]
+pub enum PruningStrategy<N> {
+	/// Prune entries when they're too far behind best finalized block.
+	ByDepth(N),
+	/// Do not prune old entries at all.
+	NeverPrune,
+}
+
 /// List-based cache.
 pub struct ListCache<Block: BlockT, T: CacheItemT, S: Storage<Block, T>> {
 	/// Cache storage.
 	storage: S,
-	/// Prune depth.
-	prune_depth: NumberFor<Block>,
+	/// Pruning strategy.
+	pruning_strategy: PruningStrategy<NumberFor<Block>>,
 	/// Best finalized block.
 	best_finalized_block: ComplexBlockId<Block>,
 	/// Best finalized entry (if exists).
@@ -107,7 +116,11 @@ pub enum ForkAppendResult<Block: BlockT> {

 impl<Block: BlockT, T: CacheItemT, S: Storage<Block, T>> ListCache<Block, T, S> {
 	/// Create new db list cache entry.
-	pub fn new(storage: S, prune_depth: NumberFor<Block>, best_finalized_block: ComplexBlockId<Block>) -> Self {
+	pub fn new(
+		storage: S,
+		pruning_strategy: PruningStrategy<NumberFor<Block>>,
+		best_finalized_block: ComplexBlockId<Block>,
+	) -> Self {
 		let (best_finalized_entry, unfinalized) = storage.read_meta()
 			.and_then(|meta| read_forks(&storage, meta))
 			.unwrap_or_else(|error| {
@@ -117,7 +130,7 @@ impl<Block: BlockT, T: CacheItemT, S: Storage<Block, T>> ListCache<Block, T, S>

 		ListCache {
 			storage,
-			prune_depth,
+			pruning_strategy,
 			best_finalized_block,
 			best_finalized_entry,
 			unfinalized,
@@ -362,9 +375,14 @@ impl<Block: BlockT, T: CacheItemT, S: Storage<Block, T>> ListCache<Block, T, S>
 		tx: &mut Tx,
 		block: &ComplexBlockId<Block>
 	) {
+		let prune_depth = match self.pruning_strategy {
+			PruningStrategy::ByDepth(prune_depth) => prune_depth,
+			PruningStrategy::NeverPrune => return,
+		};
+
 		let mut do_pruning = || -> ClientResult<()> {
 			// calculate last ancient block number
-			let ancient_block = match block.number.checked_sub(&self.prune_depth) {
+			let ancient_block = match block.number.checked_sub(&prune_depth) {
 				Some(number) => match self.storage.read_id(number)? {
 					Some(hash) => ComplexBlockId::new(hash, number),
 					None => return Ok(()),
@@ -669,7 +687,7 @@ pub mod tests {
 		// when block is earlier than best finalized block AND it is not finalized
 		// --- 50 ---
 		// ----------> [100]
-		assert_eq!(ListCache::<_, u64, _>::new(DummyStorage::new(), 1024, test_id(100))
+		assert_eq!(ListCache::<_, u64, _>::new(DummyStorage::new(), PruningStrategy::ByDepth(1024), test_id(100))
 			.value_at_block(&test_id(50)).unwrap(), None);
 		// when block is earlier than best finalized block AND it is finalized AND value is some
 		// [30] ---- 50 ---> [100]
@@ -679,7 +697,7 @@ pub mod tests {
 				.with_id(50, H256::from_low_u64_be(50))
 				.with_entry(test_id(100), StorageEntry { prev_valid_from: Some(test_id(30)), value: 100 })
 				.with_entry(test_id(30), StorageEntry { prev_valid_from: None, value: 30 }),
-			1024, test_id(100)
+			PruningStrategy::ByDepth(1024), test_id(100)
 		).value_at_block(&test_id(50)).unwrap(), Some((test_id(30), Some(test_id(100)), 30)));
 		// when block is the best finalized block AND value is some
 		// ---> [100]
@@ -689,7 +707,7 @@ pub mod tests {
 				.with_id(100, H256::from_low_u64_be(100))
 				.with_entry(test_id(100), StorageEntry { prev_valid_from: Some(test_id(30)), value: 100 })
 				.with_entry(test_id(30), StorageEntry { prev_valid_from: None, value: 30 }),
-			1024, test_id(100)
+			PruningStrategy::ByDepth(1024), test_id(100)
 		).value_at_block(&test_id(100)).unwrap(), Some((test_id(100), None, 100)));
 		// when block is parallel to the best finalized block
 		// ---- 100
@@ -700,7 +718,7 @@ pub mod tests {
 				.with_id(50, H256::from_low_u64_be(50))
 				.with_entry(test_id(100), StorageEntry { prev_valid_from: Some(test_id(30)), value: 100 })
 				.with_entry(test_id(30), StorageEntry { prev_valid_from: None, value: 30 }),
-			1024, test_id(100)
+			PruningStrategy::ByDepth(1024), test_id(100)
 		).value_at_block(&ComplexBlockId::new(H256::from_low_u64_be(2), 100)).unwrap(), None);

 		// when block is later than last finalized block AND there are no forks AND finalized value is Some
@@ -710,7 +728,7 @@ pub mod tests {
 				.with_meta(Some(test_id(100)), Vec::new())
 				.with_id(50, H256::from_low_u64_be(50))
 				.with_entry(test_id(100), StorageEntry { prev_valid_from: Some(test_id(30)), value: 100 }),
-			1024, test_id(100)
+			PruningStrategy::ByDepth(1024), test_id(100)
 		).value_at_block(&test_id(200)).unwrap(), Some((test_id(100), None, 100)));

 		// when block is later than last finalized block AND there are no matching forks
@@ -726,7 +744,7 @@ pub mod tests {
 				.with_header(test_header(3))
 				.with_header(test_header(4))
 				.with_header(fork_header(0, 2, 3)),
-			1024, test_id(2)
+			PruningStrategy::ByDepth(1024), test_id(2)
 		).value_at_block(&fork_id(0, 2, 3)).unwrap(), Some((correct_id(2), None, 2)));
 		// when block is later than last finalized block AND there are no matching forks
 		// AND block is not connected to finalized block
@@ -743,7 +761,7 @@ pub mod tests {
 				.with_header(test_header(4))
 				.with_header(fork_header(0, 1, 3))
 				.with_header(fork_header(0, 1, 2)),
-			1024, test_id(2)
+			PruningStrategy::ByDepth(1024), test_id(2)
 		).value_at_block(&fork_id(0, 1, 3)).unwrap(), None);

 		// when block is later than last finalized block AND it appends to unfinalized fork from the end
@@ -756,7 +774,7 @@ pub mod tests {
 				.with_entry(correct_id(4), StorageEntry { prev_valid_from: Some(correct_id(2)), value: 4 })
 				.with_header(test_header(4))
 				.with_header(test_header(5)),
-			1024, test_id(2)
+			PruningStrategy::ByDepth(1024), test_id(2)
 		).value_at_block(&correct_id(5)).unwrap(), Some((correct_id(4), None, 4)));
 		// when block is later than last finalized block AND it does not fits unfinalized fork
 		// AND it is connected to the finalized block AND finalized value is Some
@@ -771,7 +789,7 @@ pub mod tests {
 				.with_header(test_header(3))
 				.with_header(test_header(4))
 				.with_header(fork_header(0, 2, 3)),
-			1024, test_id(2)
+			PruningStrategy::ByDepth(1024), test_id(2)
 		).value_at_block(&fork_id(0, 2, 3)).unwrap(), Some((correct_id(2), None, 2)));
 	}

@@ -781,7 +799,7 @@ pub mod tests {
 		let fin = EntryType::Final;

 		// when trying to insert block < finalized number
-		assert!(ListCache::new(DummyStorage::new(), 1024, test_id(100))
+		assert!(ListCache::new(DummyStorage::new(), PruningStrategy::ByDepth(1024), test_id(100))
 			.on_block_insert(
 				&mut DummyTransaction::new(),
 				test_id(49),
@@ -790,7 +808,7 @@ pub mod tests {
 				nfin,
 			).unwrap().is_none());
 		// when trying to insert block @ finalized number
-		assert!(ListCache::new(DummyStorage::new(), 1024, test_id(100))
+		assert!(ListCache::new(DummyStorage::new(), PruningStrategy::ByDepth(1024), test_id(100))
 			.on_block_insert(
 				&mut DummyTransaction::new(),
 				test_id(99),
@@ -805,7 +823,7 @@ pub mod tests {
 			DummyStorage::new()
 				.with_meta(None, vec![test_id(4)])
 				.with_entry(test_id(4), StorageEntry { prev_valid_from: None, value: 4 }),
-			1024, test_id(2)
+			PruningStrategy::ByDepth(1024), test_id(2)
 		);
 		cache.unfinalized[0].best_block = Some(test_id(4));
 		let mut tx = DummyTransaction::new();
@@ -830,7 +848,7 @@ pub mod tests {
 				.with_meta(None, vec![correct_id(4)])
 				.with_entry(correct_id(4), StorageEntry { prev_valid_from: None, value: 4 })
 				.with_header(test_header(4)),
-			1024, test_id(2)
+			PruningStrategy::ByDepth(1024), test_id(2)
 		);
 		let mut tx = DummyTransaction::new();
 		assert_eq!(cache.on_block_insert(&mut tx, correct_id(4), correct_id(5), Some(4), nfin).unwrap(),
@@ -856,7 +874,7 @@ pub mod tests {
 				.with_header(test_header(2))
 				.with_header(test_header(3))
 				.with_header(test_header(4)),
-			1024, correct_id(2)
+			PruningStrategy::ByDepth(1024), correct_id(2)
 		);
 		let mut tx = DummyTransaction::new();
 		assert_eq!(cache.on_block_insert(&mut tx, correct_id(3), fork_id(0, 3, 4), Some(14), nfin).unwrap(),
@@ -871,7 +889,7 @@ pub mod tests {
 			DummyStorage::new()
 				.with_meta(Some(correct_id(2)), vec![])
 				.with_entry(correct_id(2), StorageEntry { prev_valid_from: None, value: 2 }),
-			1024, correct_id(2)
+			PruningStrategy::ByDepth(1024), correct_id(2)
 		);
 		let mut tx = DummyTransaction::new();
 		assert_eq!(cache.on_block_insert(&mut tx, correct_id(2), correct_id(3), Some(2), nfin).unwrap(), None);
@@ -884,7 +902,7 @@ pub mod tests {
 			DummyStorage::new()
 				.with_meta(Some(correct_id(2)), vec![])
 				.with_entry(correct_id(2), StorageEntry { prev_valid_from: None, value: 2 }),
-			1024, correct_id(2)
+			PruningStrategy::ByDepth(1024), correct_id(2)
 		);
 		let mut tx = DummyTransaction::new();
 		assert_eq!(cache.on_block_insert(&mut tx, correct_id(2), correct_id(3), Some(3), nfin).unwrap(),
@@ -894,7 +912,7 @@ pub mod tests {
 		assert_eq!(*tx.updated_meta(), Some(Metadata { finalized: Some(correct_id(2)), unfinalized: vec![correct_id(3)] }));

 		// when inserting finalized entry AND there are no previous finalized entries
-		let cache = ListCache::new(DummyStorage::new(), 1024, correct_id(2));
+		let cache = ListCache::new(DummyStorage::new(), PruningStrategy::ByDepth(1024), correct_id(2));
 		let mut tx = DummyTransaction::new();
 		assert_eq!(
 			cache.on_block_insert(&mut tx, correct_id(2), correct_id(3), Some(3), fin).unwrap(),
@@ -912,7 +930,7 @@ pub mod tests {
 			DummyStorage::new()
 				.with_meta(Some(correct_id(2)), vec![])
 				.with_entry(correct_id(2), StorageEntry { prev_valid_from: None, value: 2 }),
-			1024, correct_id(2)
+			PruningStrategy::ByDepth(1024), correct_id(2)
 		);
 		let mut tx = DummyTransaction::new();
 		assert_eq!(cache.on_block_insert(&mut tx, correct_id(2), correct_id(3), Some(2), fin).unwrap(),
@@ -940,7 +958,7 @@ pub mod tests {
 				.with_meta(Some(correct_id(2)), vec![fork_id(0, 1, 3)])
 				.with_entry(correct_id(2), StorageEntry { prev_valid_from: None, value: 2 })
 				.with_entry(fork_id(0, 1, 3), StorageEntry { prev_valid_from: None, value: 13 }),
-			1024, correct_id(2)
+			PruningStrategy::ByDepth(1024), correct_id(2)
 		);
 		let mut tx = DummyTransaction::new();
 		assert_eq!(cache.on_block_insert(&mut tx, correct_id(2), correct_id(3), Some(2), fin).unwrap(),
@@ -955,7 +973,7 @@ pub mod tests {
 				.with_meta(Some(correct_id(2)), vec![correct_id(5)])
 				.with_entry(correct_id(2), StorageEntry { prev_valid_from: None, value: 2 })
 				.with_entry(correct_id(5), StorageEntry { prev_valid_from: Some(correct_id(2)), value: 5 }),
-			1024, correct_id(2)
+			PruningStrategy::ByDepth(1024), correct_id(2)
 		);
 		let mut tx = DummyTransaction::new();
 		assert_eq!(cache.on_block_finalize(&mut tx, correct_id(2), correct_id(3)).unwrap(),
@@ -969,7 +987,7 @@ pub mod tests {
 				.with_meta(Some(correct_id(2)), vec![correct_id(5)])
 				.with_entry(correct_id(2), StorageEntry { prev_valid_from: None, value: 2 })
 				.with_entry(correct_id(5), StorageEntry { prev_valid_from: Some(correct_id(2)), value: 5 }),
-			1024, correct_id(4)
+			PruningStrategy::ByDepth(1024), correct_id(4)
 		);
 		let mut tx = DummyTransaction::new();
 		assert_eq!(
@@ -989,7 +1007,7 @@ pub mod tests {
 				.with_meta(Some(correct_id(2)), vec![fork_id(0, 1, 3)])
 				.with_entry(correct_id(2), StorageEntry { prev_valid_from: None, value: 2 })
 				.with_entry(fork_id(0, 1, 3), StorageEntry { prev_valid_from: None, value: 13 }),
-			1024, correct_id(2)
+			PruningStrategy::ByDepth(1024), correct_id(2)
 		);
 		let mut tx = DummyTransaction::new();
 		assert_eq!(cache.on_block_finalize(&mut tx, correct_id(2), correct_id(3)).unwrap(),
@@ -1004,7 +1022,7 @@ pub mod tests {
 				.with_entry(correct_id(2), StorageEntry { prev_valid_from: None, value: 2 })
 				.with_entry(correct_id(5), StorageEntry { prev_valid_from: Some(correct_id(2)), value: 5 })
 				.with_entry(correct_id(6), StorageEntry { prev_valid_from: Some(correct_id(5)), value: 6 }),
-			1024, correct_id(2)
+			PruningStrategy::ByDepth(1024), correct_id(2)
 		);

 		// when new block is appended to unfinalized fork
@@ -1043,7 +1061,7 @@ pub mod tests {
 				.with_header(test_header(3))
 				.with_header(test_header(4))
 				.with_header(test_header(5)),
-			1024, correct_id(0)
+			PruningStrategy::ByDepth(1024), correct_id(0)
 		).find_unfinalized_fork(&correct_id(4)).unwrap().unwrap().head.valid_from, correct_id(5));
 		// --- [2] ---------------> [5]
 		// ----------> [3] ---> 4
@@ -1060,7 +1078,7 @@ pub mod tests {
 				.with_header(fork_header(0, 1, 2))
 				.with_header(fork_header(0, 1, 3))
 				.with_header(fork_header(0, 1, 4)),
-			1024, correct_id(0)
+			PruningStrategy::ByDepth(1024), correct_id(0)
 		).find_unfinalized_fork(&fork_id(0, 1, 4)).unwrap().unwrap().head.valid_from, fork_id(0, 1, 3));
 		// --- [2] ---------------> [5]
 		// ----------> [3]
@@ -1080,7 +1098,7 @@ pub mod tests {
 				.with_header(fork_header(1, 1, 2))
 				.with_header(fork_header(1, 1, 3))
 				.with_header(fork_header(1, 1, 4)),
-			1024, correct_id(0)
+			PruningStrategy::ByDepth(1024), correct_id(0)
 		).find_unfinalized_fork(&fork_id(1, 1, 4)).unwrap().is_none());
 	}

@@ -1341,32 +1359,45 @@ pub mod tests {
 	}

 	#[test]
-	fn ancient_entries_are_pruned() {
-		let cache = ListCache::new(DummyStorage::new()
-			.with_id(10, H256::from_low_u64_be(10))
-			.with_id(20, H256::from_low_u64_be(20))
-			.with_id(30, H256::from_low_u64_be(30))
-			.with_entry(test_id(10), StorageEntry { prev_valid_from: None, value: 10 })
-			.with_entry(test_id(20), StorageEntry { prev_valid_from: Some(test_id(10)), value: 20 })
-			.with_entry(test_id(30), StorageEntry { prev_valid_from: Some(test_id(20)), value: 30 }),
-		10, test_id(9));
-		let mut tx = DummyTransaction::new();
+	fn ancient_entries_are_pruned_when_pruning_enabled() {
+		fn do_test(strategy: PruningStrategy<u64>) {
+			let cache = ListCache::new(DummyStorage::new()
+				.with_id(10, H256::from_low_u64_be(10))
+				.with_id(20, H256::from_low_u64_be(20))
+				.with_id(30, H256::from_low_u64_be(30))
+				.with_entry(test_id(10), StorageEntry { prev_valid_from: None, value: 10 })
+				.with_entry(test_id(20), StorageEntry { prev_valid_from: Some(test_id(10)), value: 20 })
+				.with_entry(test_id(30), StorageEntry { prev_valid_from: Some(test_id(20)), value: 30 }),
+			strategy, test_id(9));
+			let mut tx = DummyTransaction::new();

-		// when finalizing entry #10: no entries pruned
-		cache.prune_finalized_entries(&mut tx, &test_id(10));
-		assert!(tx.removed_entries().is_empty());
-		assert!(tx.inserted_entries().is_empty());
-		// when finalizing entry #19: no entries pruned
-		cache.prune_finalized_entries(&mut tx, &test_id(19));
-		assert!(tx.removed_entries().is_empty());
-		assert!(tx.inserted_entries().is_empty());
-		// when finalizing entry #20: no entries pruned
-		cache.prune_finalized_entries(&mut tx, &test_id(20));
-		assert!(tx.removed_entries().is_empty());
-		assert!(tx.inserted_entries().is_empty());
-		// when finalizing entry #30: entry 10 pruned + entry 20 is truncated
-		cache.prune_finalized_entries(&mut tx, &test_id(30));
-		assert_eq!(*tx.removed_entries(), vec![test_id(10).hash].into_iter().collect());
-		assert_eq!(*tx.inserted_entries(), vec![test_id(20).hash].into_iter().collect());
+			// when finalizing entry #10: no entries pruned
+			cache.prune_finalized_entries(&mut tx, &test_id(10));
+			assert!(tx.removed_entries().is_empty());
+			assert!(tx.inserted_entries().is_empty());
+			// when finalizing entry #19: no entries pruned
+			cache.prune_finalized_entries(&mut tx, &test_id(19));
+			assert!(tx.removed_entries().is_empty());
+			assert!(tx.inserted_entries().is_empty());
+			// when finalizing entry #20: no entries pruned
+			cache.prune_finalized_entries(&mut tx, &test_id(20));
+			assert!(tx.removed_entries().is_empty());
+			assert!(tx.inserted_entries().is_empty());
+			// when finalizing entry #30: entry 10 pruned + entry 20 is truncated (if pruning is enabled)
+			cache.prune_finalized_entries(&mut tx, &test_id(30));
+			match strategy {
+				PruningStrategy::NeverPrune => {
+					assert!(tx.removed_entries().is_empty());
+					assert!(tx.inserted_entries().is_empty());
+				},
+				PruningStrategy::ByDepth(_) => {
+					assert_eq!(*tx.removed_entries(), vec![test_id(10).hash].into_iter().collect());
+					assert_eq!(*tx.inserted_entries(), vec![test_id(20).hash].into_iter().collect());
+				},
+			}
+		}
+
+		do_test(PruningStrategy::ByDepth(10));
+		do_test(PruningStrategy::NeverPrune)
 	}
 }
@@ -21,15 +21,14 @@ use parking_lot::RwLock;

 use kvdb::{KeyValueDB, DBTransaction};

-use client::blockchain::Cache as BlockchainCache;
+use client::blockchain::{well_known_cache_keys::{self, Id as CacheKeyId}, Cache as BlockchainCache};
 use client::error::Result as ClientResult;
 use codec::{Encode, Decode};
 use sr_primitives::generic::BlockId;
 use sr_primitives::traits::{Block as BlockT, Header as HeaderT, NumberFor, Zero};
-use consensus_common::well_known_cache_keys::Id as CacheKeyId;
 use crate::utils::{self, COLUMN_META, db_err};

-use self::list_cache::ListCache;
+use self::list_cache::{ListCache, PruningStrategy};

 mod list_cache;
 mod list_entry;
@@ -166,7 +165,7 @@ fn get_cache_helper<'a, Block: BlockT>(
 					cache,
 				},
 			),
-			PRUNE_DEPTH.into(),
+			cache_pruning_strategy(name),
 			best_finalized_block.clone(),
 		)
 	})
@@ -335,3 +334,17 @@ impl<Block: BlockT> BlockchainCache<Block> for DbCacheSync<Block> {
 	}
 }

+/// Get pruning strategy for given cache.
+fn cache_pruning_strategy<N: From<u32>>(cache: CacheKeyId) -> PruningStrategy<N> {
+	// the cache is mostly used to store data from consensus engines
+	// this kind of data is only required for non-finalized blocks
+	// => by default we prune finalized cached entries
+
+	match cache {
+		// we need to keep changes tries configurations forever (or at least until changes tries,
+		// that were built using this configuration, are pruned) to make it possible to refer
+		// to old changes tries
+		well_known_cache_keys::CHANGES_TRIE_CONFIG => PruningStrategy::NeverPrune,
+		_ => PruningStrategy::ByDepth(PRUNE_DEPTH.into()),
+	}
+}