Add storage cache for child trie and notification internals (#2639)

* child cache, and test failing notifications * fix tests and no listen child on top wildcard * remove useless method * bump impl version * Update core/client/src/notifications.rs Co-Authored-By: Tomasz Drwięga <tomusdrw@users.noreply.github.com> * Update core/client/src/notifications.rs Co-Authored-By: Tomasz Drwięga <tomusdrw@users.noreply.github.com> * Update core/client/src/notifications.rs Co-Authored-By: Tomasz Drwięga <tomusdrw@users.noreply.github.com> * Update core/client/src/notifications.rs Co-Authored-By: Tomasz Drwięga <tomusdrw@users.noreply.github.com> * factoring notification methods to remove some redundant code. * test child sub removal * HStorage implementation and some type alias. * Remove HStorage cache: does not fit * fix removal * Make cache use byte length (shared) instead of number of kv * Make use of hashes cache in rpc * applying ratio on different lru caches * Fix format * break a line * Remove per element overhead of lru cache. * typo
2026-07-21 01:35:41 +00:00 · 2019-06-14 11:25:20 +02:00
parent 5c3d1f82cd
commit e777038418
16 changed files with 679 additions and 196 deletions
@@ -38,6 +38,7 @@ use std::collections::HashMap;
 use client::backend::NewBlockState;
 use client::blockchain::HeaderBackend;
 use client::ExecutionStrategies;
+use client::backend::{StorageCollection, ChildStorageCollection};
 use parity_codec::{Decode, Encode};
 use hash_db::Hasher;
 use kvdb::{KeyValueDB, DBTransaction};
@@ -69,6 +70,9 @@ use client::in_mem::Backend as InMemoryBackend;
 const CANONICALIZATION_DELAY: u64 = 4096;
 const MIN_BLOCKS_TO_KEEP_CHANGES_TRIES_FOR: u32 = 32768;

+/// Default value for storage cache child ratio.
+const DEFAULT_CHILD_RATIO: (usize, usize) = (1, 10);
+
 /// DB-backed patricia trie state, transaction type is an overlay of changes to commit.
 pub type DbState = state_machine::TrieBackend<Arc<dyn state_machine::Storage<Blake2Hasher>>, Blake2Hasher>;

@@ -169,6 +173,8 @@ pub struct DatabaseSettings {
 	pub cache_size: Option<usize>,
 	/// State cache size.
 	pub state_cache_size: usize,
+	/// Ratio of cache size dedicated to child tries.
+	pub state_cache_child_ratio: Option<(usize, usize)>,
 	/// Path to the database.
 	pub path: PathBuf,
 	/// Pruning mode.
@@ -181,7 +187,10 @@ pub fn new_client<E, S, Block, RA>(
 	executor: E,
 	genesis_storage: S,
 	execution_strategies: ExecutionStrategies,
-) -> Result<client::Client<Backend<Block>, client::LocalCallExecutor<Backend<Block>, E>, Block, RA>, client::error::Error>
+) -> Result<
+	client::Client<Backend<Block>,
+	client::LocalCallExecutor<Backend<Block>, E>, Block, RA>, client::error::Error
+>
 	where
 		Block: BlockT<Hash=H256>,
 		E: CodeExecutor<Blake2Hasher> + RuntimeInfo,
@@ -363,7 +372,8 @@ impl<Block: BlockT> client::blockchain::ProvideCache<Block> for BlockchainDb<Blo
 pub struct BlockImportOperation<Block: BlockT, H: Hasher> {
 	old_state: CachingState<Blake2Hasher, RefTrackingState<Block>, Block>,
 	db_updates: PrefixedMemoryDB<H>,
-	storage_updates: Vec<(Vec<u8>, Option<Vec<u8>>)>,
+	storage_updates: StorageCollection,
+	child_storage_updates: ChildStorageCollection,
 	changes_trie_updates: MemoryDB<H>,
 	pending_block: Option<PendingBlock<Block>>,
 	aux_ops: Vec<(Vec<u8>, Option<Vec<u8>>)>,
@@ -455,8 +465,13 @@ where Block: BlockT<Hash=H256>,
 		Ok(())
 	}

-	fn update_storage(&mut self, update: Vec<(Vec<u8>, Option<Vec<u8>>)>) -> Result<(), client::error::Error> {
+	fn update_storage(
+		&mut self,
+		update: StorageCollection,
+		child_update: ChildStorageCollection,
+	) -> Result<(), client::error::Error> {
 		self.storage_updates = update;
+		self.child_storage_updates = child_update;
 		Ok(())
 	}

@@ -670,14 +685,14 @@ impl<Block: BlockT<Hash=H256>> Backend<Block> {
 	#[cfg(feature = "kvdb-rocksdb")]
 	fn new_inner(config: DatabaseSettings, canonicalization_delay: u64) -> Result<Self, client::error::Error> {
 		let db = crate::utils::open_database(&config, columns::META, "full")?;
-		Backend::from_kvdb(db as Arc<_>, config.pruning, canonicalization_delay, config.state_cache_size)
+		Backend::from_kvdb(db as Arc<_>, canonicalization_delay, &config)
 	}

 	#[cfg(not(feature = "kvdb-rocksdb"))]
 	fn new_inner(config: DatabaseSettings, canonicalization_delay: u64) -> Result<Self, client::error::Error> {
 		log::warn!("Running without the RocksDB feature. The database will NOT be saved.");
 		let db = Arc::new(kvdb_memorydb::create(crate::utils::NUM_COLUMNS));
-		Backend::from_kvdb(db as Arc<_>, config.pruning, canonicalization_delay, config.state_cache_size)
+		Backend::from_kvdb(db as Arc<_>, canonicalization_delay, &config)
 	}

 	#[cfg(any(test, feature = "test-helpers"))]
@@ -685,26 +700,36 @@ impl<Block: BlockT<Hash=H256>> Backend<Block> {
 		use utils::NUM_COLUMNS;

 		let db = Arc::new(::kvdb_memorydb::create(NUM_COLUMNS));
+		Self::new_test_db(keep_blocks, canonicalization_delay, db as Arc<_>)
+	}

+	#[cfg(any(test, feature = "test-helpers"))]
+	pub fn new_test_db(keep_blocks: u32, canonicalization_delay: u64, db: Arc<KeyValueDB>) -> Self {
+
+		let db_setting = DatabaseSettings {
+			cache_size: None,
+			state_cache_size: 16777216,
+			state_cache_child_ratio: Some((50, 100)),
+			path: Default::default(),
+			pruning: PruningMode::keep_blocks(keep_blocks),
+		};
 		Backend::from_kvdb(
-			db as Arc<_>,
-			PruningMode::keep_blocks(keep_blocks),
+			db,
 			canonicalization_delay,
-			16777216,
+			&db_setting,
 		).expect("failed to create test-db")
 	}

 	fn from_kvdb(
 		db: Arc<dyn KeyValueDB>,
-		pruning: PruningMode,
 		canonicalization_delay: u64,
-		state_cache_size: usize
+		config: &DatabaseSettings
 	) -> Result<Self, client::error::Error> {
-		let is_archive_pruning = pruning.is_archive();
+		let is_archive_pruning = config.pruning.is_archive();
 		let blockchain = BlockchainDb::new(db.clone())?;
 		let meta = blockchain.meta.clone();
 		let map_e = |e: state_db::Error<io::Error>| ::client::error::Error::from(format!("State database error: {:?}", e));
-		let state_db: StateDb<_, _> = StateDb::new(pruning, &StateMetaDb(&*db)).map_err(map_e)?;
+		let state_db: StateDb<_, _> = StateDb::new(config.pruning.clone(), &StateMetaDb(&*db)).map_err(map_e)?;
 		let storage_db = StorageDb {
 			db: db.clone(),
 			state_db,
@@ -722,7 +747,10 @@ impl<Block: BlockT<Hash=H256>> Backend<Block> {
 			changes_trie_config: Mutex::new(None),
 			blockchain,
 			canonicalization_delay,
-			shared_cache: new_shared_cache(state_cache_size),
+			shared_cache: new_shared_cache(
+				config.state_cache_size,
+				config.state_cache_child_ratio.unwrap_or(DEFAULT_CHILD_RATIO),
+			),
 			import_lock: Default::default(),
 		})
 	}
@@ -1094,6 +1122,7 @@ impl<Block: BlockT<Hash=H256>> Backend<Block> {
 				&enacted,
 				&retracted,
 				operation.storage_updates,
+				operation.child_storage_updates,
 				Some(hash),
 				Some(number),
 				|| is_best,
@@ -1200,6 +1229,7 @@ impl<Block> client::backend::Backend<Block, Blake2Hasher> for Backend<Block> whe
 			old_state,
 			db_updates: PrefixedMemoryDB::default(),
 			storage_updates: Default::default(),
+			child_storage_updates: Default::default(),
 			changes_trie_updates: MemoryDB::default(),
 			aux_ops: Vec::new(),
 			finalized_blocks: Vec::new(),
@@ -1282,7 +1312,7 @@ impl<Block> client::backend::Backend<Block, Blake2Hasher> for Backend<Block> whe
 						|| client::error::Error::UnknownBlock(
 							format!("Error reverting to {}. Block hash not found.", best)))?;

-					best -= One::one();  // prev block
+					best -= One::one();	// prev block
 					let hash = self.blockchain.hash(best)?.ok_or_else(
 						|| client::error::Error::UnknownBlock(
 							format!("Error reverting to {}. Block hash not found.", best)))?;
@@ -1348,7 +1378,7 @@ impl<Block> client::backend::Backend<Block, Blake2Hasher> for Backend<Block> whe
 	fn destroy_state(&self, state: Self::State) -> Result<(), client::error::Error> {
 		if let Some(hash) = state.cache.parent_hash.clone() {
 			let is_best = || self.blockchain.meta.read().best_hash == hash;
-			state.release().sync_cache(&[], &[], vec![], None, None, is_best);
+			state.release().sync_cache(&[], &[], vec![], vec![], None, None, is_best);
 		}
 		Ok(())
 	}
@@ -1473,7 +1503,7 @@ mod tests {
 			db.storage.db.clone()
 		};

-		let backend = Backend::<Block>::from_kvdb(backing, PruningMode::keep_blocks(1), 0, 16777216).unwrap();
+		let backend = Backend::<Block>::new_test_db(1, 0, backing);
 		assert_eq!(backend.blockchain().info().best_number, 9);
 		for i in 0..10 {
 			assert!(backend.blockchain().hash(i).unwrap().is_some())