Storage chains: indexing, renewals and reference counting (#8265)

* Transaction indexing

* Tests and fixes

* Fixed a comment

* Style

* Build

* Style

* Apply suggestions from code review

Co-authored-by: cheme <emericchevalier.pro@gmail.com>

* Code review suggestions

* Add missing impl

* Apply suggestions from code review

Co-authored-by: Bastian Köcher <bkchr@users.noreply.github.com>

* impl JoinInput

* Don't store empty slices

* JoinInput operates on slices

Co-authored-by: cheme <emericchevalier.pro@gmail.com>
Co-authored-by: Bastian Köcher <bkchr@users.noreply.github.com>

substrate/primitives/state-machine/src/ext.rs

@@ -18,7 +18,7 @@
 //! Concrete externalities implementation.
 
 use crate::{
-	StorageKey, StorageValue, OverlayedChanges,
+	StorageKey, StorageValue, OverlayedChanges, IndexOperation,
 	backend::Backend, overlayed_changes::OverlayedExtensions,
 };
 use hash_db::Hasher;
@@ -568,6 +568,36 @@ where
 		}
 	}
 
+	fn storage_index_transaction(&mut self, index: u32, offset: u32) {
+		trace!(
+			target: "state",
+			"{:04x}: IndexTransaction ({}): [{}..]",
+			self.id,
+			index,
+			offset,
+		);
+		self.overlay.add_transaction_index(IndexOperation::Insert {
+			extrinsic: index,
+			offset,
+		});
+	}
+
+	/// Renew existing piece of data storage.
+	fn storage_renew_transaction_index(&mut self, index: u32, hash: &[u8], size: u32) {
+		trace!(
+			target: "state",
+			"{:04x}: RenewTransactionIndex ({}) {} bytes",
+			self.id,
+			HexDisplay::from(&hash),
+			size,
+		);
+		self.overlay.add_transaction_index(IndexOperation::Renew {
+			extrinsic: index,
+			hash: hash.to_vec(),
+			size
+		});
+	}
+
 	#[cfg(not(feature = "std"))]
 	fn storage_changes_root(&mut self, _parent_hash: &[u8]) -> Result<Option<Vec<u8>>, ()> {
 		Ok(None)

substrate/primitives/state-machine/src/lib.rs

@@ -121,6 +121,7 @@ pub use crate::overlayed_changes::{
 	StorageChanges, StorageTransactionCache,
 	OffchainChangesCollection,
 	OffchainOverlayedChanges,
+	IndexOperation,
 };
 pub use crate::backend::Backend;
 pub use crate::trie_backend_essence::{TrieBackendStorage, Storage};

substrate/primitives/state-machine/src/overlayed_changes/mod.rs

@@ -103,12 +103,35 @@ pub struct OverlayedChanges {
 	children: Map<StorageKey, (OverlayedChangeSet, ChildInfo)>,
 	/// Offchain related changes.
 	offchain: OffchainOverlayedChanges,
+	/// Transaction index changes,
+	transaction_index_ops: Vec<IndexOperation>,
 	/// True if extrinsics stats must be collected.
 	collect_extrinsics: bool,
 	/// Collect statistic on this execution.
 	stats: StateMachineStats,
 }
 
+/// Transcation index operation.
+#[derive(Debug, Clone)]
+pub enum IndexOperation {
+	/// Insert transaction into index.
+	Insert {
+		/// Extrinsic index in the current block.
+		extrinsic: u32,
+		/// Data offset in the extrinsic.
+		offset: u32,
+	},
+	/// Renew existing transaction storage.
+	Renew {
+		/// Extrinsic index in the current block.
+		extrinsic: u32,
+		/// Referenced index hash.
+		hash: Vec<u8>,
+		/// Expected data size.
+		size: u32,
+	}
+}
+
 /// A storage changes structure that can be generated by the data collected in [`OverlayedChanges`].
 ///
 /// This contains all the changes to the storage and transactions to apply theses changes to the
@@ -137,6 +160,10 @@ pub struct StorageChanges<Transaction, H: Hasher, N: BlockNumber> {
 	/// Phantom data for block number until change trie support no_std.
 	#[cfg(not(feature = "std"))]
 	pub _ph: sp_std::marker::PhantomData<N>,
+
+	/// Changes to the transaction index,
+	#[cfg(feature = "std")]
+	pub transaction_index_changes: Vec<IndexOperation>,
 }
 
 #[cfg(feature = "std")]
@@ -149,6 +176,7 @@ impl<Transaction, H: Hasher, N: BlockNumber> StorageChanges<Transaction, H, N> {
 		Transaction,
 		H::Out,
 		Option<ChangesTrieTransaction<H, N>>,
+		Vec<IndexOperation>,
 	) {
 		(
 			self.main_storage_changes,
@@ -157,6 +185,7 @@ impl<Transaction, H: Hasher, N: BlockNumber> StorageChanges<Transaction, H, N> {
 			self.transaction,
 			self.transaction_storage_root,
 			self.changes_trie_transaction,
+			self.transaction_index_changes,
 		)
 	}
 }
@@ -214,6 +243,8 @@ impl<Transaction: Default, H: Hasher, N: BlockNumber> Default for StorageChanges
 			changes_trie_transaction: None,
 			#[cfg(not(feature = "std"))]
 			_ph: Default::default(),
+			#[cfg(feature = "std")]
+			transaction_index_changes: Default::default(),
 		}
 	}
 }
@@ -543,6 +574,9 @@ impl OverlayedChanges {
 		let (main_storage_changes, child_storage_changes) = self.drain_committed();
 		let offchain_storage_changes = self.offchain_drain_committed().collect();
 
+		#[cfg(feature = "std")]
+		let transaction_index_changes = std::mem::take(&mut self.transaction_index_ops);
+
 		Ok(StorageChanges {
 			main_storage_changes: main_storage_changes.collect(),
 			child_storage_changes: child_storage_changes.map(|(sk, it)| (sk, it.0.collect())).collect(),
@@ -551,6 +585,8 @@ impl OverlayedChanges {
 			transaction_storage_root,
 			#[cfg(feature = "std")]
 			changes_trie_transaction,
+			#[cfg(feature = "std")]
+			transaction_index_changes,
 			#[cfg(not(feature = "std"))]
 			_ph: Default::default(),
 		})
@@ -666,6 +702,11 @@ impl OverlayedChanges {
 			None => self.offchain.remove(STORAGE_PREFIX, key),
 		}
 	}
+
+	/// Add transaction index operation.
+	pub fn add_transaction_index(&mut self, op: IndexOperation) {
+		self.transaction_index_ops.push(op)
+	}
 }
 
 #[cfg(feature = "std")]