Add storage cache for child trie and notification internals (#2639)

* child cache, and test failing notifications * fix tests and no listen child on top wildcard * remove useless method * bump impl version * Update core/client/src/notifications.rs Co-Authored-By: Tomasz Drwięga <tomusdrw@users.noreply.github.com> * Update core/client/src/notifications.rs Co-Authored-By: Tomasz Drwięga <tomusdrw@users.noreply.github.com> * Update core/client/src/notifications.rs Co-Authored-By: Tomasz Drwięga <tomusdrw@users.noreply.github.com> * Update core/client/src/notifications.rs Co-Authored-By: Tomasz Drwięga <tomusdrw@users.noreply.github.com> * factoring notification methods to remove some redundant code. * test child sub removal * HStorage implementation and some type alias. * Remove HStorage cache: does not fit * fix removal * Make cache use byte length (shared) instead of number of kv * Make use of hashes cache in rpc * applying ratio on different lru caches * Fix format * break a line * Remove per element overhead of lru cache. * typo
2026-06-15 18:31:05 +00:00 · 2019-06-14 11:25:20 +02:00
parent 5c3d1f82cd
commit e777038418
16 changed files with 679 additions and 196 deletions
@@ -59,7 +59,10 @@ use state_machine::{
 };
 use hash_db::Hasher;

-use crate::backend::{self, BlockImportOperation, PrunableStateChangesTrieStorage};
+use crate::backend::{
+	self, BlockImportOperation, PrunableStateChangesTrieStorage,
+	StorageCollection, ChildStorageCollection
+};
 use crate::blockchain::{
 	self, Info as ChainInfo, Backend as ChainBackend,
 	HeaderBackend as ChainHeaderBackend, ProvideCache, Cache,
@@ -77,6 +80,7 @@ use substrate_telemetry::{telemetry, SUBSTRATE_INFO};

 use log::{info, trace, warn};

+
 /// Type that implements `futures::Stream` of block import events.
 pub type ImportNotifications<Block> = mpsc::UnboundedReceiver<BlockImportNotification<Block>>;

@@ -133,7 +137,15 @@ pub struct Client<B, E, Block, RA> where Block: BlockT {
 /// Client import operation, a wrapper for the backend.
 pub struct ClientImportOperation<Block: BlockT, H: Hasher<Out=Block::Hash>, B: backend::Backend<Block, H>> {
 	op: B::BlockImportOperation,
-	notify_imported: Option<(Block::Hash, BlockOrigin, Block::Header, bool, Option<Vec<(Vec<u8>, Option<Vec<u8>>)>>)>,
+	notify_imported: Option<(
+		Block::Hash,
+		BlockOrigin,
+		Block::Header,
+		bool,
+		Option<(
+			StorageCollection,
+			ChildStorageCollection,
+		)>)>,
 	notify_finalized: Vec<Block::Hash>,
 }

@@ -150,8 +162,10 @@ pub trait BlockchainEvents<Block: BlockT> {
 	/// Get storage changes event stream.
 	///
 	/// Passing `None` as `filter_keys` subscribes to all storage changes.
-	fn storage_changes_notification_stream(&self,
-		filter_keys: Option<&[StorageKey]>
+	fn storage_changes_notification_stream(
+		&self,
+		filter_keys: Option<&[StorageKey]>,
+		child_filter_keys: Option<&[(StorageKey, Option<Vec<StorageKey>>)]>,
 	) -> error::Result<StorageEventStream<Block::Hash>>;
 }

@@ -351,6 +365,14 @@ impl<B, E, Block, RA> Client<B, E, Block, RA> where
 			.map(StorageData))
 	}

+	/// Given a `BlockId` and a key, return the value under the hash in that block.
+	pub fn storage_hash(&self, id: &BlockId<Block>, key: &StorageKey)
+		-> error::Result<Option<Block::Hash>> {
+		Ok(self.state_at(id)?
+			.storage_hash(&key.0).map_err(|e| error::Error::from_state(Box::new(e)))?
+		)
+	}
+
 	/// Given a `BlockId`, a key prefix, and a child storage key, return the matching child storage keys.
 	pub fn child_storage_keys(
 		&self,
@@ -378,6 +400,18 @@ impl<B, E, Block, RA> Client<B, E, Block, RA> where
 			.map(StorageData))
 	}

+	/// Given a `BlockId`, a key and a child storage key, return the hash under the key in that block.
+	pub fn child_storage_hash(
+		&self,
+		id: &BlockId<Block>,
+		child_storage_key: &StorageKey,
+		key: &StorageKey
+	) -> error::Result<Option<Block::Hash>> {
+		Ok(self.state_at(id)?
+			.child_storage_hash(&child_storage_key.0, &key.0).map_err(|e| error::Error::from_state(Box::new(e)))?
+		)
+	}
+
 	/// Get the code at a given block.
 	pub fn code_at(&self, id: &BlockId<Block>) -> error::Result<Vec<u8>> {
 		Ok(self.storage(id, &StorageKey(well_known_keys::CODE.to_vec()))?
@@ -913,7 +947,7 @@ impl<B, E, Block, RA> Client<B, E, Block, RA> where
 			operation.op.update_db_storage(storage_update)?;
 		}
 		if let Some(storage_changes) = storage_changes.clone() {
-			operation.op.update_storage(storage_changes)?;
+			operation.op.update_storage(storage_changes.0, storage_changes.1)?;
 		}
 		if let Some(Some(changes_update)) = changes_update {
 			operation.op.update_changes_trie(changes_update)?;
@@ -942,7 +976,10 @@ impl<B, E, Block, RA> Client<B, E, Block, RA> where
 	) -> error::Result<(
 		Option<StorageUpdate<B, Block>>,
 		Option<Option<ChangesUpdate>>,
-		Option<Vec<(Vec<u8>, Option<Vec<u8>>)>>,
+		Option<(
+			Vec<(Vec<u8>, Option<Vec<u8>>)>,
+			Vec<(Vec<u8>, Vec<(Vec<u8>, Option<Vec<u8>>)>)>
+		)>
 	)>
 		where
 			E: CallExecutor<Block, Blake2Hasher> + Send + Sync + Clone,
@@ -985,7 +1022,9 @@ impl<B, E, Block, RA> Client<B, E, Block, RA> where

 				overlay.commit_prospective();

-				Ok((Some(storage_update), Some(changes_update), Some(overlay.into_committed().collect())))
+				let (top, children) = overlay.into_committed();
+				let children = children.map(|(sk, it)| (sk, it.collect())).collect();
+				Ok((Some(storage_update), Some(changes_update), Some((top.collect(), children))))
 			},
 			None => Ok((None, None, None))
 		}
@@ -1084,14 +1123,26 @@ impl<B, E, Block, RA> Client<B, E, Block, RA> where

 	fn notify_imported(
 		&self,
-		notify_import: (Block::Hash, BlockOrigin, Block::Header, bool, Option<Vec<(Vec<u8>, Option<Vec<u8>>)>>),
+		notify_import: (
+			Block::Hash, BlockOrigin,
+			Block::Header,
+			bool,
+			Option<(
+				Vec<(Vec<u8>, Option<Vec<u8>>)>,
+				Vec<(Vec<u8>, Vec<(Vec<u8>, Option<Vec<u8>>)>)>,
+				)
+			>),
 	) -> error::Result<()> {
 		let (hash, origin, header, is_new_best, storage_changes) = notify_import;

 		if let Some(storage_changes) = storage_changes {
 			// TODO [ToDr] How to handle re-orgs? Should we re-emit all storage changes?
 			self.storage_notifications.lock()
-				.trigger(&hash, storage_changes.into_iter());
+				.trigger(
+					&hash,
+					storage_changes.0.into_iter(),
+					storage_changes.1.into_iter().map(|(sk, v)| (sk, v.into_iter())),
+				);
 		}

 		let notification = BlockImportNotification::<Block> {
@@ -1467,8 +1518,12 @@ where
 	}

 	/// Get storage changes event stream.
-	fn storage_changes_notification_stream(&self, filter_keys: Option<&[StorageKey]>) -> error::Result<StorageEventStream<Block::Hash>> {
-		Ok(self.storage_notifications.lock().listen(filter_keys))
+	fn storage_changes_notification_stream(
+		&self,
+		filter_keys: Option<&[StorageKey]>,
+		child_filter_keys: Option<&[(StorageKey, Option<Vec<StorageKey>>)]>,
+	) -> error::Result<StorageEventStream<Block::Hash>> {
+		Ok(self.storage_notifications.lock().listen(filter_keys, child_filter_keys))
 	}
 }