Make offchain indexing work (#7940)

* Make offchain indexing work

This fixes some bugs with offchain indexing to make it actually working ;)

* Fix tests

* Fix browser build

* Update client/db/src/offchain.rs

Co-authored-by: cheme <emericchevalier.pro@gmail.com>

* Remove seperation between prefix and key

Co-authored-by: cheme <emericchevalier.pro@gmail.com>

substrate/primitives/api/proc-macro/src/decl_runtime_apis.rs

@@ -409,7 +409,6 @@ fn generate_call_api_at_calls(decl: &ItemTrait) -> Result<TokenStream> {
 				at: &#crate_::BlockId<Block>,
 				args: Vec<u8>,
 				changes: &std::cell::RefCell<#crate_::OverlayedChanges>,
-				offchain_changes: &std::cell::RefCell<#crate_::OffchainOverlayedChanges>,
 				storage_transaction_cache: &std::cell::RefCell<
 					#crate_::StorageTransactionCache<Block, T::StateBackend>
 				>,
@@ -439,7 +438,6 @@ fn generate_call_api_at_calls(decl: &ItemTrait) -> Result<TokenStream> {
 							native_call: None,
 							arguments: args,
 							overlayed_changes: changes,
-							offchain_changes,
 							storage_transaction_cache,
 							initialize_block,
 							context,
@@ -460,7 +458,6 @@ fn generate_call_api_at_calls(decl: &ItemTrait) -> Result<TokenStream> {
 					native_call,
 					arguments: args,
 					overlayed_changes: changes,
-					offchain_changes,
 					storage_transaction_cache,
 					initialize_block,
 					context,

substrate/primitives/api/proc-macro/src/impl_runtime_apis.rs

@@ -208,7 +208,6 @@ fn generate_runtime_api_base_structures() -> Result<TokenStream> {
 			commit_on_success: std::cell::RefCell<bool>,
 			initialized_block: std::cell::RefCell<Option<#crate_::BlockId<Block>>>,
 			changes: std::cell::RefCell<#crate_::OverlayedChanges>,
-			offchain_changes: std::cell::RefCell<#crate_::OffchainOverlayedChanges>,
 			storage_transaction_cache: std::cell::RefCell<
 				#crate_::StorageTransactionCache<Block, C::StateBackend>
 			>,
@@ -338,7 +337,6 @@ fn generate_runtime_api_base_structures() -> Result<TokenStream> {
 					commit_on_success: true.into(),
 					initialized_block: None.into(),
 					changes: Default::default(),
-					offchain_changes: Default::default(),
 					recorder: Default::default(),
 					storage_transaction_cache: Default::default(),
 				}.into()
@@ -357,7 +355,6 @@ fn generate_runtime_api_base_structures() -> Result<TokenStream> {
 					&C,
 					&Self,
 					&std::cell::RefCell<#crate_::OverlayedChanges>,
-					&std::cell::RefCell<#crate_::OffchainOverlayedChanges>,
 					&std::cell::RefCell<#crate_::StorageTransactionCache<Block, C::StateBackend>>,
 					&std::cell::RefCell<Option<#crate_::BlockId<Block>>>,
 					&Option<#crate_::ProofRecorder<Block>>,
@@ -374,7 +371,6 @@ fn generate_runtime_api_base_structures() -> Result<TokenStream> {
 					&self.call,
 					self,
 					&self.changes,
-					&self.offchain_changes,
 					&self.storage_transaction_cache,
 					&self.initialized_block,
 					&self.recorder,
@@ -531,7 +527,6 @@ impl<'a> Fold for ApiRuntimeImplToApiRuntimeApiImpl<'a> {
 							call_runtime_at,
 							core_api,
 							changes,
-							offchain_changes,
 							storage_transaction_cache,
 							initialized_block,
 							recorder
@@ -542,7 +537,6 @@ impl<'a> Fold for ApiRuntimeImplToApiRuntimeApiImpl<'a> {
 								at,
 								params_encoded,
 								changes,
-								offchain_changes,
 								storage_transaction_cache,
 								initialized_block,
 								params.map(|p| {

substrate/primitives/api/src/lib.rs

@@ -47,8 +47,6 @@ pub use sp_core::NativeOrEncoded;
 #[doc(hidden)]
 #[cfg(feature = "std")]
 pub use hash_db::Hasher;
-#[cfg(feature = "std")]
-pub use sp_core::offchain::storage::OffchainOverlayedChanges;
 #[doc(hidden)]
 #[cfg(not(feature = "std"))]
 pub use sp_core::to_substrate_wasm_fn_return_value;
@@ -521,8 +519,6 @@ pub struct CallApiAtParams<'a, Block: BlockT, C, NC, Backend: StateBackend<HashF
 	pub arguments: Vec<u8>,
 	/// The overlayed changes that are on top of the state.
 	pub overlayed_changes: &'a RefCell<OverlayedChanges>,
-	/// The overlayed changes to be applied to the offchain worker database.
-	pub offchain_changes: &'a RefCell<OffchainOverlayedChanges>,
 	/// The cache for storage transactions.
 	pub storage_transaction_cache: &'a RefCell<StorageTransactionCache<Block, Backend>>,
 	/// Determines if the function requires that `initialize_block` should be called before calling

substrate/primitives/core/src/offchain/storage.rs

@@ -41,7 +41,7 @@ impl InMemOffchainStorage {
 	/// Remove a key and its associated value from the offchain database.
 	pub fn remove(&mut self, prefix: &[u8], key: &[u8]) {
 		let key: Vec<u8> = prefix.iter().chain(key).cloned().collect();
-		let _ = self.storage.remove(&key);
+		self.storage.remove(&key);
 	}
 }
 
@@ -84,9 +84,6 @@ impl OffchainStorage for InMemOffchainStorage {
 	}
 }
 
-
-
-
 /// Change to be applied to the offchain worker db in regards to a key.
 #[derive(Debug,Clone,Hash,Eq,PartialEq)]
 pub enum OffchainOverlayedChange {
@@ -97,161 +94,47 @@ pub enum OffchainOverlayedChange {
 }
 
 /// In-memory storage for offchain workers recoding changes for the actual offchain storage implementation.
-#[derive(Debug, Clone)]
-pub enum OffchainOverlayedChanges {
-	/// Writing overlay changes to the offchain worker database is disabled by configuration.
-	Disabled,
-	/// Overlay changes can be recorded using the inner collection of this variant,
-	/// where the identifier is the tuple of `(prefix, key)`.
-	Enabled(HashMap<(Vec<u8>, Vec<u8>), OffchainOverlayedChange>),
-}
-
-impl Default for OffchainOverlayedChanges {
-	fn default() -> Self {
-		Self::Disabled
-	}
+#[derive(Debug, Clone, Default)]
+pub struct OffchainOverlayedChanges {
+	changes: HashMap<(Vec<u8>, Vec<u8>), OffchainOverlayedChange>,
 }
 
 impl OffchainOverlayedChanges {
-	/// Create the disabled variant.
-	pub fn disabled() -> Self {
-		Self::Disabled
-	}
-
-	/// Create the enabled variant.
-	pub fn enabled() -> Self {
-		Self::Enabled(HashMap::new())
-	}
-
 	/// Consume the offchain storage and iterate over all key value pairs.
-	pub fn into_iter(self) -> OffchainOverlayedChangesIntoIter {
-		OffchainOverlayedChangesIntoIter::new(self)
+	pub fn into_iter(self) -> impl Iterator<Item = ((Vec<u8>, Vec<u8>), OffchainOverlayedChange)> {
+		self.changes.into_iter()
 	}
 
 	/// Iterate over all key value pairs by reference.
-	pub fn iter<'a>(&'a self) -> OffchainOverlayedChangesIter {
-		OffchainOverlayedChangesIter::new(&self)
+	pub fn iter(&self) -> impl Iterator<Item = (&(Vec<u8>, Vec<u8>), &OffchainOverlayedChange)> {
+		self.changes.iter()
 	}
 
 	/// Drain all elements of changeset.
-	pub fn drain<'a, 'd>(&'a mut self) -> OffchainOverlayedChangesDrain<'d> where 'a: 'd {
-		OffchainOverlayedChangesDrain::new(self)
+	pub fn drain(&mut self) -> impl Iterator<Item = ((Vec<u8>, Vec<u8>), OffchainOverlayedChange)> + '_ {
+		self.changes.drain()
 	}
 
 	/// Remove a key and its associated value from the offchain database.
 	pub fn remove(&mut self, prefix: &[u8], key: &[u8]) {
-		if let Self::Enabled(ref mut storage) = self {
-			let _ = storage.insert((prefix.to_vec(), key.to_vec()), OffchainOverlayedChange::Remove);
-		}
+		self.changes.insert((prefix.to_vec(), key.to_vec()), OffchainOverlayedChange::Remove);
 	}
 
 	/// Set the value associated with a key under a prefix to the value provided.
 	pub fn set(&mut self, prefix: &[u8], key: &[u8], value: &[u8]) {
-		if let Self::Enabled(ref mut storage) = self {
-			let _ = storage.insert((prefix.to_vec(), key.to_vec()), OffchainOverlayedChange::SetValue(value.to_vec()));
-		}
+		self.changes.insert(
+			(prefix.to_vec(), key.to_vec()),
+			OffchainOverlayedChange::SetValue(value.to_vec()),
+		);
 	}
 
 	/// Obtain a associated value to the given key in storage with prefix.
 	pub fn get(&self, prefix: &[u8], key: &[u8]) -> Option<OffchainOverlayedChange> {
-		if let Self::Enabled(ref storage) = self {
-			let key = (prefix.to_vec(), key.to_vec());
-			storage.get(&key).cloned()
-		} else {
-			None
-		}
+		let key = (prefix.to_vec(), key.to_vec());
+		self.changes.get(&key).cloned()
 	}
 }
 
-use std::collections::hash_map;
-
-/// Iterate by reference over the prepared offchain worker storage changes.
-pub struct OffchainOverlayedChangesIter<'i> {
-	inner: Option<hash_map::Iter<'i, (Vec<u8>, Vec<u8>), OffchainOverlayedChange>>,
-}
-
-impl<'i> Iterator for OffchainOverlayedChangesIter<'i> {
-	type Item = (&'i (Vec<u8>, Vec<u8>), &'i OffchainOverlayedChange);
-	fn next(&mut self) -> Option<Self::Item> {
-		if let Some(ref mut iter) = self.inner {
-			iter.next()
-		} else {
-			None
-		}
-	}
-}
-
-impl<'i> OffchainOverlayedChangesIter<'i> {
-	/// Create a new iterator based on a refernce to the parent container.
-	pub fn new(container: &'i OffchainOverlayedChanges) -> Self {
-		match container {
-			OffchainOverlayedChanges::Enabled(inner) => Self {
-				inner: Some(inner.iter())
-			},
-			OffchainOverlayedChanges::Disabled => Self { inner: None, },
-		}
-	}
-}
-
-
-/// Iterate by value over the prepared offchain worker storage changes.
-pub struct OffchainOverlayedChangesIntoIter {
-	inner: Option<hash_map::IntoIter<(Vec<u8>,Vec<u8>),OffchainOverlayedChange>>,
-}
-
-impl Iterator for OffchainOverlayedChangesIntoIter {
-	type Item = ((Vec<u8>, Vec<u8>), OffchainOverlayedChange);
-	fn next(&mut self) -> Option<Self::Item> {
-		if let Some(ref mut iter) = self.inner {
-			iter.next()
-		} else {
-			None
-		}
-	}
-}
-
-impl OffchainOverlayedChangesIntoIter {
-	/// Create a new iterator by consuming the collection.
-	pub fn new(container: OffchainOverlayedChanges) -> Self {
-		match container {
-			OffchainOverlayedChanges::Enabled(inner) => Self {
-				inner: Some(inner.into_iter())
-			},
-			OffchainOverlayedChanges::Disabled => Self { inner: None, },
-		}
-	}
-}
-
-/// Iterate over all items while draining them from the collection.
-pub struct OffchainOverlayedChangesDrain<'d> {
-	inner: Option<hash_map::Drain<'d, (Vec<u8>, Vec<u8>), OffchainOverlayedChange>>,
-}
-
-impl<'d> Iterator for OffchainOverlayedChangesDrain<'d> {
-	type Item = ((Vec<u8>, Vec<u8>), OffchainOverlayedChange);
-	fn next(&mut self) -> Option<Self::Item> {
-		if let Some(ref mut iter) = self.inner {
-			iter.next()
-		} else {
-			None
-		}
-	}
-}
-
-impl<'d> OffchainOverlayedChangesDrain<'d> {
-	/// Create a new iterator by taking a mut reference to the collection,
-	/// for the lifetime of the created drain iterator.
-	pub fn new(container: &'d mut OffchainOverlayedChanges) -> Self {
-		match container {
-			OffchainOverlayedChanges::Enabled(ref mut inner) => Self {
-				inner: Some(inner.drain())
-			},
-			OffchainOverlayedChanges::Disabled => Self { inner: None, },
-		}
-	}
-}
-
-
 #[cfg(test)]
 mod test {
 	use super::*;
@@ -259,7 +142,7 @@ mod test {
 
 	#[test]
 	fn test_drain() {
-		let mut ooc = OffchainOverlayedChanges::enabled();
+		let mut ooc = OffchainOverlayedChanges::default();
 		ooc.set(STORAGE_PREFIX,b"kkk", b"vvv");
 		let drained = ooc.drain().count();
 		assert_eq!(drained, 1);
@@ -276,7 +159,7 @@ mod test {
 
 	#[test]
 	fn test_accumulated_set_remove_set() {
-		let mut ooc = OffchainOverlayedChanges::enabled();
+		let mut ooc = OffchainOverlayedChanges::default();
 		ooc.set(STORAGE_PREFIX, b"ppp", b"qqq");
 		ooc.remove(STORAGE_PREFIX, b"ppp");
 		// keys are equiv, so it will overwrite the value and the overlay will contain

substrate/primitives/state-machine/src/ext.rs

@@ -35,8 +35,6 @@ use codec::{Decode, Encode, EncodeAppend};
 use sp_std::{fmt, any::{Any, TypeId}, vec::Vec, vec, boxed::Box};
 use crate::{warn, trace, log_error};
 #[cfg(feature = "std")]
-use sp_core::offchain::storage::OffchainOverlayedChanges;
-#[cfg(feature = "std")]
 use crate::changes_trie::State as ChangesTrieState;
 use crate::StorageTransactionCache;
 #[cfg(feature = "std")]
@@ -100,9 +98,6 @@ pub struct Ext<'a, H, N, B>
 {
 	/// The overlayed changes to write to.
 	overlay: &'a mut OverlayedChanges,
-	/// The overlayed changes destined for the Offchain DB.
-	#[cfg(feature = "std")]
-	offchain_overlay: &'a mut OffchainOverlayedChanges,
 	/// The storage backend to read from.
 	backend: &'a B,
 	/// The cache for the storage transactions.
@@ -146,7 +141,6 @@ impl<'a, H, N, B> Ext<'a, H, N, B>
 	#[cfg(feature = "std")]
 	pub fn new(
 		overlay: &'a mut OverlayedChanges,
-		offchain_overlay: &'a mut OffchainOverlayedChanges,
 		storage_transaction_cache: &'a mut StorageTransactionCache<B::Transaction, H, N>,
 		backend: &'a B,
 		changes_trie_state: Option<ChangesTrieState<'a, H, N>>,
@@ -154,7 +148,6 @@ impl<'a, H, N, B> Ext<'a, H, N, B>
 	) -> Self {
 		Self {
 			overlay,
-			offchain_overlay,
 			backend,
 			changes_trie_state,
 			storage_transaction_cache,
@@ -170,12 +163,6 @@ impl<'a, H, N, B> Ext<'a, H, N, B>
 	fn mark_dirty(&mut self) {
 		self.storage_transaction_cache.reset();
 	}
-
-	/// Read only accessor for the scheduled overlay changes.
-	#[cfg(feature = "std")]
-	pub fn get_offchain_storage_changes(&self) -> &OffchainOverlayedChanges {
-		&*self.offchain_overlay
-	}
 }
 
 #[cfg(test)]
@@ -208,10 +195,10 @@ where
 {
 	#[cfg(feature = "std")]
 	fn set_offchain_storage(&mut self, key: &[u8], value: Option<&[u8]>) {
-		use ::sp_core::offchain::STORAGE_PREFIX;
+		use sp_core::offchain::STORAGE_PREFIX;
 		match value {
-			Some(value) => self.offchain_overlay.set(STORAGE_PREFIX, key, value),
-			None => self.offchain_overlay.remove(STORAGE_PREFIX, key),
+			Some(value) => self.overlay.offchain_set_storage(STORAGE_PREFIX, key, value),
+			None => self.overlay.offchain_remove_storage(STORAGE_PREFIX, key),
 		}
 	}
 
@@ -829,11 +816,9 @@ mod tests {
 		changes
 	}
 
-	fn prepare_offchain_overlay_with_changes() -> OffchainOverlayedChanges {
-		let mut ooc = OffchainOverlayedChanges::enabled();
-		ooc.set(offchain::STORAGE_PREFIX, b"k1", b"v1");
-		ooc.set(offchain::STORAGE_PREFIX, b"k2", b"v2");
-		ooc
+	fn prepare_offchain_overlay_with_changes(overlay: &mut OverlayedChanges) {
+		overlay.offchain_set_storage(offchain::STORAGE_PREFIX, b"k1", b"v1");
+		overlay.offchain_set_storage(offchain::STORAGE_PREFIX, b"k2", b"v2");
 	}
 
 	fn changes_trie_config() -> ChangesTrieConfiguration {
@@ -846,32 +831,30 @@ mod tests {
 	#[test]
 	fn storage_changes_root_is_none_when_storage_is_not_provided() {
 		let mut overlay = prepare_overlay_with_changes();
-		let mut offchain_overlay = prepare_offchain_overlay_with_changes();
 		let mut cache = StorageTransactionCache::default();
 		let backend = TestBackend::default();
-		let mut ext = TestExt::new(&mut overlay, &mut offchain_overlay, &mut cache, &backend, None, None);
+		let mut ext = TestExt::new(&mut overlay, &mut cache, &backend, None, None);
 		assert_eq!(ext.storage_changes_root(&H256::default().encode()).unwrap(), None);
 	}
 
 	#[test]
 	fn storage_changes_root_is_none_when_state_is_not_provided() {
 		let mut overlay = prepare_overlay_with_changes();
-		let mut offchain_overlay = prepare_offchain_overlay_with_changes();
 		let mut cache = StorageTransactionCache::default();
 		let backend = TestBackend::default();
-		let mut ext = TestExt::new(&mut overlay, &mut offchain_overlay, &mut cache, &backend, None, None);
+		let mut ext = TestExt::new(&mut overlay, &mut cache, &backend, None, None);
 		assert_eq!(ext.storage_changes_root(&H256::default().encode()).unwrap(), None);
 	}
 
 	#[test]
 	fn storage_changes_root_is_some_when_extrinsic_changes_are_non_empty() {
 		let mut overlay = prepare_overlay_with_changes();
-		let mut offchain_overlay = prepare_offchain_overlay_with_changes();
+		prepare_offchain_overlay_with_changes(&mut overlay);
 		let mut cache = StorageTransactionCache::default();
 		let storage = TestChangesTrieStorage::with_blocks(vec![(99, Default::default())]);
 		let state = Some(ChangesTrieState::new(changes_trie_config(), Zero::zero(), &storage));
 		let backend = TestBackend::default();
-		let mut ext = TestExt::new(&mut overlay, &mut offchain_overlay, &mut cache, &backend, state, None);
+		let mut ext = TestExt::new(&mut overlay, &mut cache, &backend, state, None);
 		assert_eq!(
 			ext.storage_changes_root(&H256::default().encode()).unwrap(),
 			Some(hex!("bb0c2ef6e1d36d5490f9766cfcc7dfe2a6ca804504c3bb206053890d6dd02376").to_vec()),
@@ -881,14 +864,14 @@ mod tests {
 	#[test]
 	fn storage_changes_root_is_some_when_extrinsic_changes_are_empty() {
 		let mut overlay = prepare_overlay_with_changes();
-		let mut offchain_overlay = prepare_offchain_overlay_with_changes();
+		prepare_offchain_overlay_with_changes(&mut overlay);
 		let mut cache = StorageTransactionCache::default();
 		overlay.set_collect_extrinsics(false);
 		overlay.set_storage(vec![1], None);
 		let storage = TestChangesTrieStorage::with_blocks(vec![(99, Default::default())]);
 		let state = Some(ChangesTrieState::new(changes_trie_config(), Zero::zero(), &storage));
 		let backend = TestBackend::default();
-		let mut ext = TestExt::new(&mut overlay, &mut offchain_overlay, &mut cache, &backend, state, None);
+		let mut ext = TestExt::new(&mut overlay, &mut cache, &backend, state, None);
 		assert_eq!(
 			ext.storage_changes_root(&H256::default().encode()).unwrap(),
 			Some(hex!("96f5aae4690e7302737b6f9b7f8567d5bbb9eac1c315f80101235a92d9ec27f4").to_vec()),
@@ -901,7 +884,7 @@ mod tests {
 		let mut overlay = OverlayedChanges::default();
 		overlay.set_storage(vec![20], None);
 		overlay.set_storage(vec![30], Some(vec![31]));
-		let mut offchain_overlay = prepare_offchain_overlay_with_changes();
+		prepare_offchain_overlay_with_changes(&mut overlay);
 		let backend = Storage {
 			top: map![
 				vec![10] => vec![10],
@@ -911,7 +894,7 @@ mod tests {
 			children_default: map![]
 		}.into();
 
-		let ext = TestExt::new(&mut overlay, &mut offchain_overlay, &mut cache, &backend, None, None);
+		let ext = TestExt::new(&mut overlay, &mut cache, &backend, None, None);
 
 		// next_backend < next_overlay
 		assert_eq!(ext.next_storage_key(&[5]), Some(vec![10]));
@@ -927,7 +910,7 @@ mod tests {
 
 		drop(ext);
 		overlay.set_storage(vec![50], Some(vec![50]));
-		let ext = TestExt::new(&mut overlay, &mut offchain_overlay, &mut cache, &backend, None, None);
+		let ext = TestExt::new(&mut overlay, &mut cache, &backend, None, None);
 
 		// next_overlay exist but next_backend doesn't exist
 		assert_eq!(ext.next_storage_key(&[40]), Some(vec![50]));
@@ -956,10 +939,9 @@ mod tests {
 			],
 		}.into();
 
+		prepare_offchain_overlay_with_changes(&mut overlay);
 
-		let mut offchain_overlay = prepare_offchain_overlay_with_changes();
-
-		let ext = TestExt::new(&mut overlay, &mut offchain_overlay, &mut cache, &backend, None, None);
+		let ext = TestExt::new(&mut overlay, &mut cache, &backend, None, None);
 
 		// next_backend < next_overlay
 		assert_eq!(ext.next_child_storage_key(child_info, &[5]), Some(vec![10]));
@@ -975,7 +957,7 @@ mod tests {
 
 		drop(ext);
 		overlay.set_child_storage(child_info, vec![50], Some(vec![50]));
-		let ext = TestExt::new(&mut overlay, &mut offchain_overlay, &mut cache, &backend, None, None);
+		let ext = TestExt::new(&mut overlay, &mut cache, &backend, None, None);
 
 		// next_overlay exist but next_backend doesn't exist
 		assert_eq!(ext.next_child_storage_key(child_info, &[40]), Some(vec![50]));
@@ -989,7 +971,7 @@ mod tests {
 		let mut overlay = OverlayedChanges::default();
 		overlay.set_child_storage(child_info, vec![20], None);
 		overlay.set_child_storage(child_info, vec![30], Some(vec![31]));
-		let mut offchain_overlay = prepare_offchain_overlay_with_changes();
+		prepare_offchain_overlay_with_changes(&mut overlay);
 		let backend = Storage {
 			top: map![],
 			children_default: map![
@@ -1004,7 +986,7 @@ mod tests {
 			],
 		}.into();
 
-		let ext = TestExt::new(&mut overlay, &mut offchain_overlay, &mut cache, &backend, None, None);
+		let ext = TestExt::new(&mut overlay, &mut cache, &backend, None, None);
 
 		assert_eq!(ext.child_storage(child_info, &[10]), Some(vec![10]));
 		assert_eq!(
@@ -1031,7 +1013,7 @@ mod tests {
 		let child_info = &child_info;
 		let mut cache = StorageTransactionCache::default();
 		let mut overlay = OverlayedChanges::default();
-		let mut offchain_overlay = prepare_offchain_overlay_with_changes();
+		prepare_offchain_overlay_with_changes(&mut overlay);
 		let backend = Storage {
 			top: map![],
 			children_default: map![
@@ -1044,7 +1026,7 @@ mod tests {
 			],
 		}.into();
 
-		let ext = TestExt::new(&mut overlay, &mut offchain_overlay, &mut cache, &backend, None, None);
+		let ext = TestExt::new(&mut overlay, &mut cache, &backend, None, None);
 
 		use sp_core::storage::well_known_keys;
 		let mut ext = ext;

substrate/primitives/state-machine/src/lib.rs

@@ -172,7 +172,6 @@ mod execution {
 	use hash_db::Hasher;
 	use codec::{Decode, Encode, Codec};
 	use sp_core::{
-		offchain::storage::OffchainOverlayedChanges,
 		storage::ChildInfo, NativeOrEncoded, NeverNativeValue, hexdisplay::HexDisplay,
 		traits::{CodeExecutor, CallInWasmExt, RuntimeCode, SpawnNamed},
 	};
@@ -299,7 +298,6 @@ mod execution {
 		method: &'a str,
 		call_data: &'a [u8],
 		overlay: &'a mut OverlayedChanges,
-		offchain_overlay: &'a mut OffchainOverlayedChanges,
 		extensions: Extensions,
 		changes_trie_state: Option<ChangesTrieState<'a, H, N>>,
 		storage_transaction_cache: Option<&'a mut StorageTransactionCache<B::Transaction, H, N>>,
@@ -329,7 +327,6 @@ mod execution {
 			backend: &'a B,
 			changes_trie_state: Option<ChangesTrieState<'a, H, N>>,
 			overlay: &'a mut OverlayedChanges,
-			offchain_overlay: &'a mut OffchainOverlayedChanges,
 			exec: &'a Exec,
 			method: &'a str,
 			call_data: &'a [u8],
@@ -347,7 +344,6 @@ mod execution {
 				call_data,
 				extensions,
 				overlay,
-				offchain_overlay,
 				changes_trie_state,
 				storage_transaction_cache: None,
 				runtime_code,
@@ -407,7 +403,6 @@ mod execution {
 
 			let mut ext = Ext::new(
 				self.overlay,
-				self.offchain_overlay,
 				cache,
 				self.backend,
 				self.changes_trie_state.clone(),
@@ -621,13 +616,11 @@ mod execution {
 		N: crate::changes_trie::BlockNumber,
 		Spawn: SpawnNamed + Send + 'static,
 	{
-		let mut offchain_overlay = OffchainOverlayedChanges::default();
 		let proving_backend = proving_backend::ProvingBackend::new(trie_backend);
 		let mut sm = StateMachine::<_, H, N, Exec>::new(
 			&proving_backend,
 			None,
 			overlay,
-			&mut offchain_overlay,
 			exec,
 			method,
 			call_data,
@@ -691,12 +684,10 @@ mod execution {
 		N: crate::changes_trie::BlockNumber,
 		Spawn: SpawnNamed + Send + 'static,
 	{
-		let mut offchain_overlay = OffchainOverlayedChanges::default();
 		let mut sm = StateMachine::<_, H, N, Exec>::new(
 			trie_backend,
 			None,
 			overlay,
-			&mut offchain_overlay,
 			exec,
 			method,
 			call_data,
@@ -879,7 +870,6 @@ mod tests {
 	use std::{result, collections::HashMap};
 	use codec::Decode;
 	use sp_core::{
-		offchain::storage::OffchainOverlayedChanges,
 		storage::ChildInfo, NativeOrEncoded, NeverNativeValue,
 		traits::CodeExecutor,
 	};
@@ -966,14 +956,12 @@ mod tests {
 	fn execute_works() {
 		let backend = trie_backend::tests::test_trie();
 		let mut overlayed_changes = Default::default();
-		let mut offchain_overlayed_changes = Default::default();
 		let wasm_code = RuntimeCode::empty();
 
 		let mut state_machine = StateMachine::new(
 			&backend,
 			changes_trie::disabled_state::<_, u64>(),
 			&mut overlayed_changes,
-			&mut offchain_overlayed_changes,
 			&DummyCodeExecutor {
 				change_changes_trie_config: false,
 				native_available: true,
@@ -998,14 +986,12 @@ mod tests {
 	fn execute_works_with_native_else_wasm() {
 		let backend = trie_backend::tests::test_trie();
 		let mut overlayed_changes = Default::default();
-		let mut offchain_overlayed_changes = Default::default();
 		let wasm_code = RuntimeCode::empty();
 
 		let mut state_machine = StateMachine::new(
 			&backend,
 			changes_trie::disabled_state::<_, u64>(),
 			&mut overlayed_changes,
-			&mut offchain_overlayed_changes,
 			&DummyCodeExecutor {
 				change_changes_trie_config: false,
 				native_available: true,
@@ -1027,14 +1013,12 @@ mod tests {
 		let mut consensus_failed = false;
 		let backend = trie_backend::tests::test_trie();
 		let mut overlayed_changes = Default::default();
-		let mut offchain_overlayed_changes = Default::default();
 		let wasm_code = RuntimeCode::empty();
 
 		let mut state_machine = StateMachine::new(
 			&backend,
 			changes_trie::disabled_state::<_, u64>(),
 			&mut overlayed_changes,
-			&mut offchain_overlayed_changes,
 			&DummyCodeExecutor {
 				change_changes_trie_config: false,
 				native_available: true,
@@ -1118,11 +1102,9 @@ mod tests {
 		overlay.set_storage(b"bbd".to_vec(), Some(b"42".to_vec()));
 
 		{
-			let mut offchain_overlay = Default::default();
 			let mut cache = StorageTransactionCache::default();
 			let mut ext = Ext::new(
 				&mut overlay,
-				&mut offchain_overlay,
 				&mut cache,
 				backend,
 				changes_trie::disabled_state::<_, u64>(),
@@ -1167,11 +1149,9 @@ mod tests {
 		overlay.set_child_storage(&child_info, b"4".to_vec(), Some(b"1312".to_vec()));
 
 		{
-			let mut offchain_overlay = Default::default();
 			let mut cache = StorageTransactionCache::default();
 			let mut ext = Ext::new(
 				&mut overlay,
-				&mut offchain_overlay,
 				&mut cache,
 				&backend,
 				changes_trie::disabled_state::<_, u64>(),
@@ -1209,11 +1189,9 @@ mod tests {
 		];
 		let backend = InMemoryBackend::<BlakeTwo256>::from(initial);
 		let mut overlay = OverlayedChanges::default();
-		let mut offchain_overlay = Default::default();
 		let mut cache = StorageTransactionCache::default();
 		let mut ext = Ext::new(
 			&mut overlay,
-			&mut offchain_overlay,
 			&mut cache,
 			&backend,
 			changes_trie::disabled_state::<_, u64>(),
@@ -1234,11 +1212,9 @@ mod tests {
 		let mut state = new_in_mem::<BlakeTwo256>();
 		let backend = state.as_trie_backend().unwrap();
 		let mut overlay = OverlayedChanges::default();
-		let mut offchain_overlay = OffchainOverlayedChanges::default();
 		let mut cache = StorageTransactionCache::default();
 		let mut ext = Ext::new(
 			&mut overlay,
-			&mut offchain_overlay,
 			&mut cache,
 			backend,
 			changes_trie::disabled_state::<_, u64>(),
@@ -1282,12 +1258,10 @@ mod tests {
 		let mut state = new_in_mem::<BlakeTwo256>();
 		let backend = state.as_trie_backend().unwrap();
 		let mut overlay = OverlayedChanges::default();
-		let mut offchain_overlay = OffchainOverlayedChanges::default();
 		let mut cache = StorageTransactionCache::default();
 		{
 			let mut ext = Ext::new(
 				&mut overlay,
-				&mut offchain_overlay,
 				&mut cache,
 				backend,
 				changes_trie::disabled_state::<_, u64>(),
@@ -1304,7 +1278,6 @@ mod tests {
 		{
 			let mut ext = Ext::new(
 				&mut overlay,
-				&mut offchain_overlay,
 				&mut cache,
 				backend,
 				changes_trie::disabled_state::<_, u64>(),
@@ -1323,7 +1296,6 @@ mod tests {
 		{
 			let ext = Ext::new(
 				&mut overlay,
-				&mut offchain_overlay,
 				&mut cache,
 				backend,
 				changes_trie::disabled_state::<_, u64>(),
@@ -1346,14 +1318,12 @@ mod tests {
 		let mut cache = StorageTransactionCache::default();
 		let mut state = new_in_mem::<BlakeTwo256>();
 		let backend = state.as_trie_backend().unwrap();
-		let mut offchain_overlay = OffchainOverlayedChanges::default();
 		let mut overlay = OverlayedChanges::default();
 
 		// For example, block initialization with event.
 		{
 			let mut ext = Ext::new(
 				&mut overlay,
-				&mut offchain_overlay,
 				&mut cache,
 				backend,
 				changes_trie::disabled_state::<_, u64>(),
@@ -1368,7 +1338,6 @@ mod tests {
 		{
 			let mut ext = Ext::new(
 				&mut overlay,
-				&mut offchain_overlay,
 				&mut cache,
 				backend,
 				changes_trie::disabled_state::<_, u64>(),
@@ -1393,7 +1362,6 @@ mod tests {
 		{
 			let mut ext = Ext::new(
 				&mut overlay,
-				&mut offchain_overlay,
 				&mut cache,
 				backend,
 				changes_trie::disabled_state::<_, u64>(),
@@ -1419,7 +1387,6 @@ mod tests {
 		{
 			let ext = Ext::new(
 				&mut overlay,
-				&mut offchain_overlay,
 				&mut cache,
 				backend,
 				changes_trie::disabled_state::<_, u64>(),
@@ -1495,14 +1462,12 @@ mod tests {
 
 		use crate::trie_backend::tests::test_trie;
 		let mut overlay = OverlayedChanges::default();
-		let mut offchain_overlay = OffchainOverlayedChanges::default();
 
 		let mut transaction = {
 			let backend = test_trie();
 			let mut cache = StorageTransactionCache::default();
 			let mut ext = Ext::new(
 				&mut overlay,
-				&mut offchain_overlay,
 				&mut cache,
 				&backend,
 				changes_trie::disabled_state::<_, u64>(),
@@ -1543,11 +1508,9 @@ mod tests {
 		assert_eq!(overlay.storage(b"bbb"), None);
 
 		{
-			let mut offchain_overlay = Default::default();
 			let mut cache = StorageTransactionCache::default();
 			let mut ext = Ext::new(
 				&mut overlay,
-				&mut offchain_overlay,
 				&mut cache,
 				backend,
 				changes_trie::disabled_state::<_, u64>(),
@@ -1571,14 +1534,12 @@ mod tests {
 
 		let backend = trie_backend::tests::test_trie();
 		let mut overlayed_changes = Default::default();
-		let mut offchain_overlayed_changes = Default::default();
 		let wasm_code = RuntimeCode::empty();
 
 		let mut state_machine = StateMachine::new(
 			&backend,
 			changes_trie::disabled_state::<_, u64>(),
 			&mut overlayed_changes,
-			&mut offchain_overlayed_changes,
 			&DummyCodeExecutor {
 				change_changes_trie_config: false,
 				native_available: true,

substrate/primitives/state-machine/src/overlayed_changes/mod.rs

@@ -43,7 +43,7 @@ use sp_std::collections::btree_set::BTreeSet;
 use codec::{Decode, Encode};
 use sp_core::storage::{well_known_keys::EXTRINSIC_INDEX, ChildInfo};
 #[cfg(feature = "std")]
-use sp_core::offchain::storage::OffchainOverlayedChanges;
+use sp_core::offchain::storage::{OffchainOverlayedChanges, OffchainOverlayedChange};
 use hash_db::Hasher;
 use crate::DefaultError;
 use sp_externalities::{Extensions, Extension};
@@ -101,6 +101,9 @@ pub struct OverlayedChanges {
 	collect_extrinsics: bool,
 	/// Collect statistic on this execution.
 	stats: StateMachineStats,
+	/// Offchain related changes.
+	#[cfg(feature = "std")]
+	offchain: OffchainOverlayedChanges,
 }
 
 /// A storage changes structure that can be generated by the data collected in [`OverlayedChanges`].
@@ -523,7 +526,7 @@ impl OverlayedChanges {
 			main_storage_changes: main_storage_changes.collect(),
 			child_storage_changes: child_storage_changes.map(|(sk, it)| (sk, it.0.collect())).collect(),
 			#[cfg(feature = "std")]
-			offchain_storage_changes: Default::default(),
+			offchain_storage_changes: std::mem::take(&mut self.offchain),
 			transaction,
 			transaction_storage_root,
 			#[cfg(feature = "std")]
@@ -629,6 +632,40 @@ impl OverlayedChanges {
 				overlay.next_change(key)
 			)
 	}
+
+	/// Set a value in the offchain storage.
+	#[cfg(feature = "std")]
+	pub fn offchain_set_storage(&mut self, prefix: &[u8], key: &[u8], value: &[u8]) {
+		self.offchain.set(prefix, key, value);
+	}
+
+	/// Clear a value in the offchain storage.
+	#[cfg(feature = "std")]
+	pub fn offchain_remove_storage(&mut self, prefix: &[u8], key: &[u8]) {
+		self.offchain.remove(prefix, key);
+	}
+
+	/// Get a value in the offchain storage.
+	#[cfg(feature = "std")]
+	pub fn offchain_get_storage(
+		&mut self,
+		prefix: &[u8],
+		key: &[u8],
+	) -> Option<OffchainOverlayedChange> {
+		self.offchain.get(prefix, key)
+	}
+
+	/// Returns a reference to the offchain overlay.
+	#[cfg(feature = "std")]
+	pub fn offchain_overlay(&self) -> &OffchainOverlayedChanges {
+		&self.offchain
+	}
+
+	/// Returns a mutable reference to the offchain overlay.
+	#[cfg(feature = "std")]
+	pub fn offchain_overlay_mut(&mut self) -> &mut OffchainOverlayedChanges {
+		&mut self.offchain
+	}
 }
 
 #[cfg(feature = "std")]
@@ -789,11 +826,9 @@ mod tests {
 		overlay.set_storage(b"dogglesworth".to_vec(), Some(b"cat".to_vec()));
 		overlay.set_storage(b"doug".to_vec(), None);
 
-		let mut offchain_overlay = Default::default();
 		let mut cache = StorageTransactionCache::default();
 		let mut ext = Ext::new(
 			&mut overlay,
-			&mut offchain_overlay,
 			&mut cache,
 			&backend,
 			crate::changes_trie::disabled_state::<_, u64>(),

substrate/primitives/state-machine/src/testing.rs

@@ -33,10 +33,7 @@ use crate::{
 use codec::{Decode, Encode};
 use hash_db::Hasher;
 use sp_core::{
-	offchain::{
-		testing::TestPersistentOffchainDB,
-		storage::OffchainOverlayedChanges
-	},
+	offchain::testing::TestPersistentOffchainDB,
 	storage::{
 		well_known_keys::{CHANGES_TRIE_CONFIG, CODE, HEAP_PAGES, is_child_storage_key},
 		Storage,
@@ -52,7 +49,6 @@ where
 	H::Out: codec::Codec + Ord,
 {
 	overlay: OverlayedChanges,
-	offchain_overlay: OffchainOverlayedChanges,
 	offchain_db: TestPersistentOffchainDB,
 	storage_transaction_cache: StorageTransactionCache<
 		<InMemoryBackend<H> as Backend<H>>::Transaction, H, N
@@ -71,7 +67,6 @@ impl<H: Hasher, N: ChangesTrieBlockNumber> TestExternalities<H, N>
 	pub fn ext(&mut self) -> Ext<H, N, InMemoryBackend<H>> {
 		Ext::new(
 			&mut self.overlay,
-			&mut self.offchain_overlay,
 			&mut self.storage_transaction_cache,
 			&self.backend,
 			match self.changes_trie_config.clone() {
@@ -109,8 +104,6 @@ impl<H: Hasher, N: ChangesTrieBlockNumber> TestExternalities<H, N>
 		storage.top.insert(HEAP_PAGES.to_vec(), 8u64.encode());
 		storage.top.insert(CODE.to_vec(), code.to_vec());
 
-		let offchain_overlay = OffchainOverlayedChanges::enabled();
-
 		let mut extensions = Extensions::default();
 		extensions.register(TaskExecutorExt::new(TaskExecutor::new()));
 
@@ -118,7 +111,6 @@ impl<H: Hasher, N: ChangesTrieBlockNumber> TestExternalities<H, N>
 
 		TestExternalities {
 			overlay,
-			offchain_overlay,
 			offchain_db,
 			changes_trie_config,
 			extensions,
@@ -128,9 +120,14 @@ impl<H: Hasher, N: ChangesTrieBlockNumber> TestExternalities<H, N>
 		}
 	}
 
+	/// Returns the overlayed changes.
+	pub fn overlayed_changes(&self) -> &OverlayedChanges {
+		&self.overlay
+	}
+
 	/// Move offchain changes from overlay to the persistent store.
 	pub fn persist_offchain_overlay(&mut self) {
-		self.offchain_db.apply_offchain_changes(&mut self.offchain_overlay);
+		self.offchain_db.apply_offchain_changes(self.overlay.offchain_overlay_mut());
 	}
 
 	/// A shared reference type around the offchain worker storage.