Allow transaction for offchain indexing (#7290)

* Moving offchain change set to state machine overlay change set, preparing use of change set internally. * Make change set generic over key and value, and use it for offchain indexing. * test ui change * remaining delta * generating with standard method * Remove 'drain_committed' function, and documentation. * Default constructor for enabling offchain indexing. * Remove offchain change specific iterators. * remove pub accessor * keep previous hierarchy, just expose iterator instead. * Update primitives/state-machine/src/overlayed_changes/mod.rs Co-authored-by: Tomasz Drwięga <tomusdrw@users.noreply.github.com> * fix line break * missing renamings * fix import * fix new state-machine tests. * Don't expose InnerValue type. * Add test similar to set_storage. * Remove conditional offchain storage (hard to instantiate correctly). * fix * offchain as children cannot fail if top doesn't Co-authored-by: Addie Wagenknecht <addie@nortd.com> Co-authored-by: Tomasz Drwięga <tomusdrw@users.noreply.github.com>
2026-04-27 17:28:00 +00:00 · 2021-01-22 13:27:43 +01:00
parent 20f40fbd12
commit 878f7ccf6e
13 changed files with 382 additions and 249 deletions
@@ -18,7 +18,9 @@
 //! The overlayed changes to state.

 mod changeset;
+mod offchain;

+pub use offchain::OffchainOverlayedChanges;
 use crate::{
 	backend::Backend,
 	stats::StateMachineStats,
@@ -42,8 +44,7 @@ use sp_std::collections::btree_map::{BTreeMap as Map, Entry as MapEntry};
 use sp_std::collections::btree_set::BTreeSet;
 use codec::{Decode, Encode};
 use sp_core::storage::{well_known_keys::EXTRINSIC_INDEX, ChildInfo};
-#[cfg(feature = "std")]
-use sp_core::offchain::storage::{OffchainOverlayedChanges, OffchainOverlayedChange};
+use sp_core::offchain::OffchainOverlayedChange;
 use hash_db::Hasher;
 use crate::DefaultError;
 use sp_externalities::{Extensions, Extension};
@@ -65,6 +66,9 @@ pub type StorageCollection = Vec<(StorageKey, Option<StorageValue>)>;
 /// In memory arrays of storage values for multiple child tries.
 pub type ChildStorageCollection = Vec<(StorageKey, StorageCollection)>;

+/// In memory array of storage values.
+pub type OffchainChangesCollection = Vec<((Vec<u8>, Vec<u8>), OffchainOverlayedChange)>;
+
 /// Keep trace of extrinsics index for a modified value.
 #[derive(Debug, Default, Eq, PartialEq, Clone)]
 pub struct Extrinsics(Vec<u32>);
@@ -97,13 +101,12 @@ pub struct OverlayedChanges {
 	top: OverlayedChangeSet,
 	/// Child storage changes. The map key is the child storage key without the common prefix.
 	children: Map<StorageKey, (OverlayedChangeSet, ChildInfo)>,
+	/// Offchain related changes.
+	offchain: OffchainOverlayedChanges,
 	/// True if extrinsics stats must be collected.
 	collect_extrinsics: bool,
 	/// Collect statistic on this execution.
 	stats: StateMachineStats,
-	/// Offchain related changes.
-	#[cfg(feature = "std")]
-	offchain: OffchainOverlayedChanges,
 }

 /// A storage changes structure that can be generated by the data collected in [`OverlayedChanges`].
@@ -118,8 +121,7 @@ pub struct StorageChanges<Transaction, H: Hasher, N: BlockNumber> {
 	/// All changes to the child storages.
 	pub child_storage_changes: ChildStorageCollection,
 	/// Offchain state changes to write to the offchain database.
-	#[cfg(feature = "std")]
-	pub offchain_storage_changes: OffchainOverlayedChanges,
+	pub offchain_storage_changes: OffchainChangesCollection,
 	/// A transaction for the backend that contains all changes from
 	/// [`main_storage_changes`](StorageChanges::main_storage_changes) and from
 	/// [`child_storage_changes`](StorageChanges::child_storage_changes).
@@ -143,7 +145,7 @@ impl<Transaction, H: Hasher, N: BlockNumber> StorageChanges<Transaction, H, N> {
 	pub fn into_inner(self) -> (
 		StorageCollection,
 		ChildStorageCollection,
-		OffchainOverlayedChanges,
+		OffchainChangesCollection,
 		Transaction,
 		H::Out,
 		Option<ChangesTrieTransaction<H, N>>,
@@ -205,7 +207,6 @@ impl<Transaction: Default, H: Hasher, N: BlockNumber> Default for StorageChanges
 		Self {
 			main_storage_changes: Default::default(),
 			child_storage_changes: Default::default(),
-			#[cfg(feature = "std")]
 			offchain_storage_changes: Default::default(),
 			transaction: Default::default(),
 			transaction_storage_root: Default::default(),
@@ -375,6 +376,7 @@ impl OverlayedChanges {
 		for (_, (changeset, _)) in self.children.iter_mut() {
 			changeset.start_transaction();
 		}
+		self.offchain.overlay_mut().start_transaction();
 	}

 	/// Rollback the last transaction started by `start_transaction`.
@@ -388,6 +390,8 @@ impl OverlayedChanges {
 				.expect("Top and children changesets are started in lockstep; qed");
 			!changeset.is_empty()
 		});
+		self.offchain.overlay_mut().rollback_transaction()
+			.expect("Top and offchain changesets are started in lockstep; qed");
 		Ok(())
 	}

@@ -401,6 +405,8 @@ impl OverlayedChanges {
 			changeset.commit_transaction()
 				.expect("Top and children changesets are started in lockstep; qed");
 		}
+		self.offchain.overlay_mut().commit_transaction()
+			.expect("Top and offchain changesets are started in lockstep; qed");
 		Ok(())
 	}

@@ -414,6 +420,8 @@ impl OverlayedChanges {
 			changeset.enter_runtime()
 				.expect("Top and children changesets are entering runtime in lockstep; qed")
 		}
+		self.offchain.overlay_mut().enter_runtime()
+			.expect("Top and offchain changesets are started in lockstep; qed");
 		Ok(())
 	}

@@ -427,6 +435,8 @@ impl OverlayedChanges {
 			changeset.exit_runtime()
 				.expect("Top and children changesets are entering runtime in lockstep; qed");
 		}
+		self.offchain.overlay_mut().exit_runtime()
+			.expect("Top and offchain changesets are started in lockstep; qed");
 		Ok(())
 	}

@@ -452,6 +462,16 @@ impl OverlayedChanges {
 		)
 	}

+	/// Consume all changes (top + children) and return them.
+	///
+	/// After calling this function no more changes are contained in this changeset.
+	///
+	/// Panics:
+	/// Panics if `transaction_depth() > 0`
+	pub fn offchain_drain_committed(&mut self) -> impl Iterator<Item=((StorageKey, StorageKey), OffchainOverlayedChange)> {
+		self.offchain.drain()
+	}
+
 	/// Get an iterator over all child changes as seen by the current transaction.
 	pub fn children(&self)
 		-> impl Iterator<Item=(impl Iterator<Item=(&StorageKey, &OverlayedValue)>, &ChildInfo)> {
@@ -521,12 +541,12 @@ impl OverlayedChanges {
 			.expect("Changes trie transaction was generated by `changes_trie_root`; qed");

 		let (main_storage_changes, child_storage_changes) = self.drain_committed();
+		let offchain_storage_changes = self.offchain_drain_committed().collect();

 		Ok(StorageChanges {
 			main_storage_changes: main_storage_changes.collect(),
 			child_storage_changes: child_storage_changes.map(|(sk, it)| (sk, it.0.collect())).collect(),
-			#[cfg(feature = "std")]
-			offchain_storage_changes: std::mem::take(&mut self.offchain),
+			offchain_storage_changes,
 			transaction,
 			transaction_storage_root,
 			#[cfg(feature = "std")]
@@ -633,38 +653,18 @@ impl OverlayedChanges {
 			)
 	}

-	/// Set a value in the offchain storage.
-	#[cfg(feature = "std")]
-	pub fn offchain_set_storage(&mut self, prefix: &[u8], key: &[u8], value: &[u8]) {
-		self.offchain.set(prefix, key, value);
-	}
-
-	/// Clear a value in the offchain storage.
-	#[cfg(feature = "std")]
-	pub fn offchain_remove_storage(&mut self, prefix: &[u8], key: &[u8]) {
-		self.offchain.remove(prefix, key);
-	}
-
-	/// Get a value in the offchain storage.
-	#[cfg(feature = "std")]
-	pub fn offchain_get_storage(
-		&mut self,
-		prefix: &[u8],
-		key: &[u8],
-	) -> Option<OffchainOverlayedChange> {
-		self.offchain.get(prefix, key)
-	}
-
-	/// Returns a reference to the offchain overlay.
-	#[cfg(feature = "std")]
-	pub fn offchain_overlay(&self) -> &OffchainOverlayedChanges {
+	/// Read only access ot offchain overlay.
+	pub fn offchain(&self) -> &OffchainOverlayedChanges {
 		&self.offchain
 	}

-	/// Returns a mutable reference to the offchain overlay.
-	#[cfg(feature = "std")]
-	pub fn offchain_overlay_mut(&mut self) -> &mut OffchainOverlayedChanges {
-		&mut self.offchain
+	/// Write a key value pair to the offchain storage overlay.
+	pub fn set_offchain_storage(&mut self, key: &[u8], value: Option<&[u8]>) {
+		use sp_core::offchain::STORAGE_PREFIX;
+		match value {
+			Some(value) => self.offchain.set(STORAGE_PREFIX, key, value),
+			None => self.offchain.remove(STORAGE_PREFIX, key),
+		}
 	}
 }

@@ -804,6 +804,61 @@ mod tests {
 		assert!(overlayed.storage(&key).unwrap().is_none());
 	}

+	#[test]
+	fn offchain_overlayed_storage_transactions_works() {
+		use sp_core::offchain::STORAGE_PREFIX;
+		fn check_offchain_content(
+			state: &OverlayedChanges,
+			nb_commit: usize,
+			expected: Vec<(Vec<u8>, Option<Vec<u8>>)>,
+		) {
+			let mut state = state.clone();
+			for _ in 0..nb_commit {
+				state.commit_transaction().unwrap();
+			}
+			let offchain_data: Vec<_> = state.offchain_drain_committed().collect();
+			let expected: Vec<_> = expected.into_iter().map(|(key, value)| {
+				let change = match value {
+					Some(value) => OffchainOverlayedChange::SetValue(value),
+					None => OffchainOverlayedChange::Remove,
+				};
+				((STORAGE_PREFIX.to_vec(), key), change)
+			}).collect();
+			assert_eq!(offchain_data, expected);
+		}
+
+		let mut overlayed = OverlayedChanges::default();
+
+		let key = vec![42, 69, 169, 142];
+
+		check_offchain_content(&overlayed, 0, vec![]);
+
+		overlayed.start_transaction();
+
+		overlayed.set_offchain_storage(key.as_slice(), Some(&[1, 2, 3][..]));
+		check_offchain_content(&overlayed, 1, vec![(key.clone(), Some(vec![1, 2, 3]))]);
+
+		overlayed.commit_transaction().unwrap();
+
+		check_offchain_content(&overlayed, 0, vec![(key.clone(), Some(vec![1, 2, 3]))]);
+
+		overlayed.start_transaction();
+
+		overlayed.set_offchain_storage(key.as_slice(), Some(&[][..]));
+		check_offchain_content(&overlayed, 1, vec![(key.clone(), Some(vec![]))]);
+
+		overlayed.set_offchain_storage(key.as_slice(), None);
+		check_offchain_content(&overlayed, 1, vec![(key.clone(), None)]);
+
+		overlayed.rollback_transaction().unwrap();
+
+		check_offchain_content(&overlayed, 0, vec![(key.clone(), Some(vec![1, 2, 3]))]);
+
+		overlayed.set_offchain_storage(key.as_slice(), None);
+		check_offchain_content(&overlayed, 0, vec![(key.clone(), None)]);
+	}
+
+
 	#[test]
 	fn overlayed_storage_root_works() {
 		let initial: BTreeMap<_, _> = vec![