Make offchain indexing work (#7940)

* Make offchain indexing work

This fixes some bugs with offchain indexing to make it actually working ;)

* Fix tests

* Fix browser build

* Update client/db/src/offchain.rs

Co-authored-by: cheme <emericchevalier.pro@gmail.com>

* Remove seperation between prefix and key

Co-authored-by: cheme <emericchevalier.pro@gmail.com>

substrate/Cargo.lock

@@ -7218,12 +7218,14 @@ dependencies = [
  "parity-scale-codec",
  "parking_lot 0.11.1",
  "rand 0.7.3",
+ "sc-block-builder",
  "sc-client-api",
  "sc-client-db",
  "sc-keystore",
  "sc-network",
  "sc-transaction-pool",
  "sp-api",
+ "sp-consensus",
  "sp-core",
  "sp-offchain",
  "sp-runtime",

substrate/client/api/src/call_executor.rs

@@ -28,7 +28,7 @@ use sp_state_machine::{
 };
 use sc_executor::{RuntimeVersion, NativeVersion};
 use sp_externalities::Extensions;
-use sp_core::{NativeOrEncoded,offchain::storage::OffchainOverlayedChanges};
+use sp_core::NativeOrEncoded;
 
 use sp_api::{ProofRecorder, InitializeBlock, StorageTransactionCache};
 use crate::execution_extensions::ExecutionExtensions;
@@ -86,7 +86,6 @@ pub trait CallExecutor<B: BlockT> {
 		method: &str,
 		call_data: &[u8],
 		changes: &RefCell<OverlayedChanges>,
-		offchain_changes: &RefCell<OffchainOverlayedChanges>,
 		storage_transaction_cache: Option<&RefCell<
 			StorageTransactionCache<B, <Self::Backend as crate::backend::Backend<B>>::State>,
 		>>,

substrate/client/db/src/lib.rs

@@ -681,14 +681,12 @@ pub struct BlockImportOperation<Block: BlockT> {
 impl<Block: BlockT> BlockImportOperation<Block> {
 	fn apply_offchain(&mut self, transaction: &mut Transaction<DbHash>) {
 		for ((prefix, key), value_operation) in self.offchain_storage_updates.drain() {
-			let key: Vec<u8> = prefix
-				.into_iter()
-				.chain(sp_core::sp_std::iter::once(b'/'))
-				.chain(key.into_iter())
-				.collect();
+			let key = crate::offchain::concatenate_prefix_and_key(&prefix, &key);
 			match value_operation {
-				OffchainOverlayedChange::SetValue(val) => transaction.set_from_vec(columns::OFFCHAIN, &key, val),
-				OffchainOverlayedChange::Remove => transaction.remove(columns::OFFCHAIN, &key),
+				OffchainOverlayedChange::SetValue(val) =>
+					transaction.set_from_vec(columns::OFFCHAIN, &key, val),
+				OffchainOverlayedChange::Remove =>
+					transaction.remove(columns::OFFCHAIN, &key),
 			}
 		}
 	}

substrate/client/db/src/offchain.rs

@@ -18,10 +18,7 @@
 
 //! RocksDB-based offchain workers local storage.
 
-use std::{
-	collections::HashMap,
-	sync::Arc,
-};
+use std::{collections::HashMap, sync::Arc};
 
 use crate::{columns, Database, DbHash, Transaction};
 use parking_lot::Mutex;
@@ -43,7 +40,7 @@ impl std::fmt::Debug for LocalStorage {
 
 impl LocalStorage {
 	/// Create new offchain storage for tests (backed by memorydb)
-	#[cfg(any(test, feature = "test-helpers"))]
+	#[cfg(any(feature = "test-helpers", test))]
 	pub fn new_test() -> Self {
 		let db = kvdb_memorydb::create(crate::utils::NUM_COLUMNS);
 		let db = sp_database::as_database(db);
@@ -61,9 +58,8 @@ impl LocalStorage {
 
 impl sp_core::offchain::OffchainStorage for LocalStorage {
 	fn set(&mut self, prefix: &[u8], key: &[u8], value: &[u8]) {
-		let key: Vec<u8> = prefix.iter().chain(key).cloned().collect();
 		let mut tx = Transaction::new();
-		tx.set(columns::OFFCHAIN, &key, value);
+		tx.set(columns::OFFCHAIN, &concatenate_prefix_and_key(prefix, key), value);
 
 		if let Err(err) = self.db.commit(tx) {
 			error!("Error setting on local storage: {}", err)
@@ -71,9 +67,8 @@ impl sp_core::offchain::OffchainStorage for LocalStorage {
 	}
 
 	fn remove(&mut self, prefix: &[u8], key: &[u8]) {
-		let key: Vec<u8> = prefix.iter().chain(key).cloned().collect();
 		let mut tx = Transaction::new();
-		tx.remove(columns::OFFCHAIN, &key);
+		tx.remove(columns::OFFCHAIN, &concatenate_prefix_and_key(prefix, key));
 
 		if let Err(err) = self.db.commit(tx) {
 			error!("Error removing on local storage: {}", err)
@@ -81,8 +76,7 @@ impl sp_core::offchain::OffchainStorage for LocalStorage {
 	}
 
 	fn get(&self, prefix: &[u8], key: &[u8]) -> Option<Vec<u8>> {
-		let key: Vec<u8> = prefix.iter().chain(key).cloned().collect();
-		self.db.get(columns::OFFCHAIN, &key)
+		self.db.get(columns::OFFCHAIN, &concatenate_prefix_and_key(prefix, key))
 	}
 
 	fn compare_and_set(
@@ -92,7 +86,7 @@ impl sp_core::offchain::OffchainStorage for LocalStorage {
 		old_value: Option<&[u8]>,
 		new_value: &[u8],
 	) -> bool {
-		let key: Vec<u8> = prefix.iter().chain(item_key).cloned().collect();
+		let key = concatenate_prefix_and_key(prefix, item_key);
 		let key_lock = {
 			let mut locks = self.locks.lock();
 			locks.entry(key.clone()).or_default().clone()
@@ -122,6 +116,15 @@ impl sp_core::offchain::OffchainStorage for LocalStorage {
 	}
 }
 
+/// Concatenate the prefix and key to create an offchain key in the db.
+pub(crate) fn concatenate_prefix_and_key(prefix: &[u8], key: &[u8]) -> Vec<u8> {
+	prefix
+		.iter()
+		.chain(key.into_iter())
+		.cloned()
+		.collect()
+}
+
 #[cfg(test)]
 mod tests {
 	use super::*;

substrate/client/executor/src/integration_tests/mod.rs

@@ -477,8 +477,8 @@ fn offchain_index(wasm_method: WasmExecutionMethod) {
 
 	use sp_core::offchain::storage::OffchainOverlayedChange;
 	assert_eq!(
-		ext.ext()
-			.get_offchain_storage_changes()
+		ext.overlayed_changes()
+			.offchain_overlay()
 			.get(sp_core::offchain::STORAGE_PREFIX, b"k"),
 		Some(OffchainOverlayedChange::SetValue(b"v".to_vec()))
 	);

substrate/client/light/src/call_executor.rs

@@ -25,7 +25,6 @@ use std::{
 use codec::{Encode, Decode};
 use sp_core::{
 	convert_hash, NativeOrEncoded, traits::{CodeExecutor, SpawnNamed},
-	offchain::storage::OffchainOverlayedChanges,
 };
 use sp_runtime::{
 	generic::BlockId, traits::{One, Block as BlockT, Header as HeaderT, HashFor},
@@ -113,7 +112,6 @@ impl<Block, B, Local> CallExecutor<Block> for
 		method: &str,
 		call_data: &[u8],
 		changes: &RefCell<OverlayedChanges>,
-		offchain_changes: &RefCell<OffchainOverlayedChanges>,
 		_: Option<&RefCell<StorageTransactionCache<Block, B::State>>>,
 		initialize_block: InitializeBlock<'a, Block>,
 		_manager: ExecutionManager<EM>,
@@ -140,7 +138,6 @@ impl<Block, B, Local> CallExecutor<Block> for
 				method,
 				call_data,
 				changes,
-				offchain_changes,
 				None,
 				initialize_block,
 				ExecutionManager::NativeWhenPossible,

substrate/client/offchain/Cargo.toml

@@ -37,10 +37,12 @@ hyper = "0.13.9"
 hyper-rustls = "0.21.0"
 
 [dev-dependencies]
-sc-client-db = { version = "0.8.0", default-features = true, path = "../db/" }
+sc-client-db = { version = "0.8.0", default-features = true, path = "../db" }
+sc-block-builder = { version = "0.8.0", path = "../block-builder" }
 sc-transaction-pool = { version = "2.0.0", path = "../transaction-pool" }
 sp-transaction-pool = { version = "2.0.0", path = "../../primitives/transaction-pool" }
 sp-tracing = { version = "2.0.0", path = "../../primitives/tracing" }
+sp-consensus = { version = "0.8.1", path = "../../primitives/consensus/common" }
 substrate-test-runtime-client = { version = "2.0.0", path = "../../test-utils/runtime/client" }
 tokio = "0.2"
 lazy_static = "1.4.0"

substrate/client/offchain/src/lib.rs

@@ -60,7 +60,7 @@ pub use sp_offchain::{OffchainWorkerApi, STORAGE_PREFIX};
 pub trait NetworkProvider: NetworkStateInfo {
 	/// Set the authorized peers.
 	fn set_authorized_peers(&self, peers: HashSet<PeerId>);
-	
+
 	/// Set the authorized only flag.
 	fn set_authorized_only(&self, reserved_only: bool);
 }
@@ -238,9 +238,15 @@ mod tests {
 	use super::*;
 	use std::sync::Arc;
 	use sc_network::{Multiaddr, PeerId};
-	use substrate_test_runtime_client::{TestClient, runtime::Block};
+	use substrate_test_runtime_client::{
+		TestClient, runtime::Block, TestClientBuilderExt,
+		DefaultTestClientBuilderExt, ClientBlockImportExt,
+	};
 	use sc_transaction_pool::{BasicPool, FullChainApi};
 	use sp_transaction_pool::{TransactionPool, InPoolTransaction};
+	use sp_consensus::BlockOrigin;
+	use sc_client_api::Backend as _;
+	use sc_block_builder::BlockBuilderProvider as _;
 
 	struct TestNetwork();
 
@@ -307,4 +313,41 @@ mod tests {
 		assert_eq!(pool.0.status().ready, 1);
 		assert_eq!(pool.0.ready().next().unwrap().is_propagable(), false);
 	}
+
+	#[test]
+	fn offchain_index_set_and_clear_works() {
+		sp_tracing::try_init_simple();
+
+		let (client, backend) =
+			substrate_test_runtime_client::TestClientBuilder::new()
+				.enable_offchain_indexing_api()
+				.build_with_backend();
+		let mut client = Arc::new(client);
+		let offchain_db = backend.offchain_storage().unwrap();
+
+		let key = &b"hello"[..];
+		let value = &b"world"[..];
+		let mut block_builder = client.new_block(Default::default()).unwrap();
+		block_builder.push(
+			substrate_test_runtime_client::runtime::Extrinsic::OffchainIndexSet(
+				key.to_vec(),
+				value.to_vec(),
+			),
+		).unwrap();
+
+		let block = block_builder.build().unwrap().block;
+		client.import(BlockOrigin::Own, block).unwrap();
+
+		assert_eq!(value, &offchain_db.get(sp_offchain::STORAGE_PREFIX, &key).unwrap());
+
+		let mut block_builder = client.new_block(Default::default()).unwrap();
+		block_builder.push(
+			substrate_test_runtime_client::runtime::Extrinsic::OffchainIndexClear(key.to_vec()),
+		).unwrap();
+
+		let block = block_builder.build().unwrap().block;
+		client.import(BlockOrigin::Own, block).unwrap();
+
+		assert!(offchain_db.get(sp_offchain::STORAGE_PREFIX, &key).is_none());
+	}
 }

substrate/client/service/src/client/call_executor.rs

@@ -29,7 +29,6 @@ use sc_executor::{RuntimeVersion, RuntimeInfo, NativeVersion};
 use sp_externalities::Extensions;
 use sp_core::{
 	NativeOrEncoded, NeverNativeValue, traits::{CodeExecutor, SpawnNamed, RuntimeCode},
-	offchain::storage::OffchainOverlayedChanges,
 };
 use sp_api::{ProofRecorder, InitializeBlock, StorageTransactionCache};
 use sc_client_api::{backend, call_executor::CallExecutor};
@@ -127,11 +126,6 @@ where
 		extensions: Option<Extensions>,
 	) -> sp_blockchain::Result<Vec<u8>> {
 		let mut changes = OverlayedChanges::default();
-		let mut offchain_changes = if self.client_config.offchain_indexing_api {
-			OffchainOverlayedChanges::enabled()
-		} else {
-			OffchainOverlayedChanges::disabled()
-		};
 		let changes_trie = backend::changes_tries_state_at_block(
 			id, self.backend.changes_trie_storage()
 		)?;
@@ -145,7 +139,6 @@ where
 			&state,
 			changes_trie,
 			&mut changes,
-			&mut offchain_changes,
 			&self.executor,
 			method,
 			call_data,
@@ -176,7 +169,6 @@ where
 		method: &str,
 		call_data: &[u8],
 		changes: &RefCell<OverlayedChanges>,
-		offchain_changes: &RefCell<OffchainOverlayedChanges>,
 		storage_transaction_cache: Option<&RefCell<
 			StorageTransactionCache<Block, B::State>
 		>>,
@@ -201,7 +193,6 @@ where
 		let mut state = self.backend.state_at(*at)?;
 
 		let changes = &mut *changes.borrow_mut();
-		let offchain_changes = &mut *offchain_changes.borrow_mut();
 
 		match recorder {
 			Some(recorder) => {
@@ -213,7 +204,6 @@ where
 				let state_runtime_code = sp_state_machine::backend::BackendRuntimeCode::new(&trie_state);
 				// It is important to extract the runtime code here before we create the proof
 				// recorder.
-				
 				let runtime_code = state_runtime_code.runtime_code()
 					.map_err(sp_blockchain::Error::RuntimeCode)?;
 				let runtime_code = self.check_override(runtime_code, at)?;
@@ -227,7 +217,6 @@ where
 					&backend,
 					changes_trie_state,
 					changes,
-					offchain_changes,
 					&self.executor,
 					method,
 					call_data,
@@ -249,7 +238,6 @@ where
 					&state,
 					changes_trie_state,
 					changes,
-					offchain_changes,
 					&self.executor,
 					method,
 					call_data,
@@ -264,7 +252,6 @@ where
 
 	fn runtime_version(&self, id: &BlockId<Block>) -> sp_blockchain::Result<RuntimeVersion> {
 		let mut overlay = OverlayedChanges::default();
-		let mut offchain_overlay = OffchainOverlayedChanges::default();
 		let changes_trie_state = backend::changes_tries_state_at_block(
 			id,
 			self.backend.changes_trie_storage(),
@@ -273,7 +260,6 @@ where
 		let mut cache = StorageTransactionCache::<Block, B::State>::default();
 		let mut ext = Ext::new(
 			&mut overlay,
-			&mut offchain_overlay,
 			&mut cache,
 			&state,
 			changes_trie_state,

substrate/client/service/src/client/client.rs

@@ -1654,7 +1654,6 @@ impl<B, E, Block, RA> CallApiAt<Block> for Client<B, E, Block, RA> where
 			params.function,
 			&params.arguments,
 			params.overlayed_changes,
-			params.offchain_changes,
 			Some(params.storage_transaction_cache),
 			params.initialize_block,
 			manager,

substrate/client/service/test/src/client/light.rs

@@ -36,7 +36,7 @@ use parking_lot::Mutex;
 use substrate_test_runtime_client::{
 	runtime::{Hash, Block, Header}, TestClient, ClientBlockImportExt,
 };
-use sp_api::{InitializeBlock, StorageTransactionCache, ProofRecorder, OffchainOverlayedChanges};
+use sp_api::{InitializeBlock, StorageTransactionCache, ProofRecorder};
 use sp_consensus::BlockOrigin;
 use sc_executor::{NativeExecutor, WasmExecutionMethod, RuntimeVersion, NativeVersion};
 use sp_core::{H256, NativeOrEncoded, testing::TaskExecutor};
@@ -223,7 +223,6 @@ impl CallExecutor<Block> for DummyCallExecutor {
 		_method: &str,
 		_call_data: &[u8],
 		_changes: &RefCell<OverlayedChanges>,
-		_offchain_changes: &RefCell<OffchainOverlayedChanges>,
 		_storage_transaction_cache: Option<&RefCell<
 			StorageTransactionCache<
 				Block,

substrate/client/service/test/src/client/mod.rs

@@ -41,7 +41,7 @@ use sp_runtime::traits::{
 };
 use substrate_test_runtime::TestAPI;
 use sp_state_machine::backend::Backend as _;
-use sp_api::{ProvideRuntimeApi, OffchainOverlayedChanges};
+use sp_api::ProvideRuntimeApi;
 use sp_core::{H256, ChangesTrieConfiguration, blake2_256, testing::TaskExecutor};
 use std::collections::{HashMap, HashSet};
 use std::sync::Arc;
@@ -163,7 +163,6 @@ fn construct_block(
 	};
 	let hash = header.hash();
 	let mut overlay = OverlayedChanges::default();
-	let mut offchain_overlay = OffchainOverlayedChanges::default();
 	let backend_runtime_code = sp_state_machine::backend::BackendRuntimeCode::new(&backend);
 	let runtime_code = backend_runtime_code.runtime_code().expect("Code is part of the backend");
 	let task_executor = Box::new(TaskExecutor::new());
@@ -172,7 +171,6 @@ fn construct_block(
 		backend,
 		sp_state_machine::disabled_changes_trie_state::<_, u64>(),
 		&mut overlay,
-		&mut offchain_overlay,
 		&executor(),
 		"Core_initialize_block",
 		&header.encode(),
@@ -188,7 +186,6 @@ fn construct_block(
 			backend,
 			sp_state_machine::disabled_changes_trie_state::<_, u64>(),
 			&mut overlay,
-			&mut offchain_overlay,
 			&executor(),
 			"BlockBuilder_apply_extrinsic",
 			&tx.encode(),
@@ -204,7 +201,6 @@ fn construct_block(
 		backend,
 		sp_state_machine::disabled_changes_trie_state::<_, u64>(),
 		&mut overlay,
-		&mut offchain_overlay,
 		&executor(),
 		"BlockBuilder_finalize_block",
 		&[],
@@ -252,13 +248,11 @@ fn construct_genesis_should_work_with_native() {
 	let runtime_code = backend_runtime_code.runtime_code().expect("Code is part of the backend");
 
 	let mut overlay = OverlayedChanges::default();
-	let mut offchain_overlay = OffchainOverlayedChanges::default();
 
 	let _ = StateMachine::new(
 		&backend,
 		sp_state_machine::disabled_changes_trie_state::<_, u64>(),
 		&mut overlay,
-		&mut offchain_overlay,
 		&executor(),
 		"Core_execute_block",
 		&b1data,
@@ -288,13 +282,11 @@ fn construct_genesis_should_work_with_wasm() {
 	let runtime_code = backend_runtime_code.runtime_code().expect("Code is part of the backend");
 
 	let mut overlay = OverlayedChanges::default();
-	let mut offchain_overlay = OffchainOverlayedChanges::default();
 
 	let _ = StateMachine::new(
 		&backend,
 		sp_state_machine::disabled_changes_trie_state::<_, u64>(),
 		&mut overlay,
-		&mut offchain_overlay,
 		&executor(),
 		"Core_execute_block",
 		&b1data,
@@ -324,13 +316,11 @@ fn construct_genesis_with_bad_transaction_should_panic() {
 	let runtime_code = backend_runtime_code.runtime_code().expect("Code is part of the backend");
 
 	let mut overlay = OverlayedChanges::default();
-	let mut offchain_overlay = OffchainOverlayedChanges::default();
 
 	let r = StateMachine::new(
 		&backend,
 		sp_state_machine::disabled_changes_trie_state::<_, u64>(),
 		&mut overlay,
-		&mut offchain_overlay,
 		&executor(),
 		"Core_execute_block",
 		&b1data,

substrate/primitives/api/proc-macro/src/decl_runtime_apis.rs

@@ -409,7 +409,6 @@ fn generate_call_api_at_calls(decl: &ItemTrait) -> Result<TokenStream> {
 				at: &#crate_::BlockId<Block>,
 				args: Vec<u8>,
 				changes: &std::cell::RefCell<#crate_::OverlayedChanges>,
-				offchain_changes: &std::cell::RefCell<#crate_::OffchainOverlayedChanges>,
 				storage_transaction_cache: &std::cell::RefCell<
 					#crate_::StorageTransactionCache<Block, T::StateBackend>
 				>,
@@ -439,7 +438,6 @@ fn generate_call_api_at_calls(decl: &ItemTrait) -> Result<TokenStream> {
 							native_call: None,
 							arguments: args,
 							overlayed_changes: changes,
-							offchain_changes,
 							storage_transaction_cache,
 							initialize_block,
 							context,
@@ -460,7 +458,6 @@ fn generate_call_api_at_calls(decl: &ItemTrait) -> Result<TokenStream> {
 					native_call,
 					arguments: args,
 					overlayed_changes: changes,
-					offchain_changes,
 					storage_transaction_cache,
 					initialize_block,
 					context,

substrate/primitives/api/proc-macro/src/impl_runtime_apis.rs

@@ -208,7 +208,6 @@ fn generate_runtime_api_base_structures() -> Result<TokenStream> {
 			commit_on_success: std::cell::RefCell<bool>,
 			initialized_block: std::cell::RefCell<Option<#crate_::BlockId<Block>>>,
 			changes: std::cell::RefCell<#crate_::OverlayedChanges>,
-			offchain_changes: std::cell::RefCell<#crate_::OffchainOverlayedChanges>,
 			storage_transaction_cache: std::cell::RefCell<
 				#crate_::StorageTransactionCache<Block, C::StateBackend>
 			>,
@@ -338,7 +337,6 @@ fn generate_runtime_api_base_structures() -> Result<TokenStream> {
 					commit_on_success: true.into(),
 					initialized_block: None.into(),
 					changes: Default::default(),
-					offchain_changes: Default::default(),
 					recorder: Default::default(),
 					storage_transaction_cache: Default::default(),
 				}.into()
@@ -357,7 +355,6 @@ fn generate_runtime_api_base_structures() -> Result<TokenStream> {
 					&C,
 					&Self,
 					&std::cell::RefCell<#crate_::OverlayedChanges>,
-					&std::cell::RefCell<#crate_::OffchainOverlayedChanges>,
 					&std::cell::RefCell<#crate_::StorageTransactionCache<Block, C::StateBackend>>,
 					&std::cell::RefCell<Option<#crate_::BlockId<Block>>>,
 					&Option<#crate_::ProofRecorder<Block>>,
@@ -374,7 +371,6 @@ fn generate_runtime_api_base_structures() -> Result<TokenStream> {
 					&self.call,
 					self,
 					&self.changes,
-					&self.offchain_changes,
 					&self.storage_transaction_cache,
 					&self.initialized_block,
 					&self.recorder,
@@ -531,7 +527,6 @@ impl<'a> Fold for ApiRuntimeImplToApiRuntimeApiImpl<'a> {
 							call_runtime_at,
 							core_api,
 							changes,
-							offchain_changes,
 							storage_transaction_cache,
 							initialized_block,
 							recorder
@@ -542,7 +537,6 @@ impl<'a> Fold for ApiRuntimeImplToApiRuntimeApiImpl<'a> {
 								at,
 								params_encoded,
 								changes,
-								offchain_changes,
 								storage_transaction_cache,
 								initialized_block,
 								params.map(|p| {

substrate/primitives/api/src/lib.rs

@@ -47,8 +47,6 @@ pub use sp_core::NativeOrEncoded;
 #[doc(hidden)]
 #[cfg(feature = "std")]
 pub use hash_db::Hasher;
-#[cfg(feature = "std")]
-pub use sp_core::offchain::storage::OffchainOverlayedChanges;
 #[doc(hidden)]
 #[cfg(not(feature = "std"))]
 pub use sp_core::to_substrate_wasm_fn_return_value;
@@ -521,8 +519,6 @@ pub struct CallApiAtParams<'a, Block: BlockT, C, NC, Backend: StateBackend<HashF
 	pub arguments: Vec<u8>,
 	/// The overlayed changes that are on top of the state.
 	pub overlayed_changes: &'a RefCell<OverlayedChanges>,
-	/// The overlayed changes to be applied to the offchain worker database.
-	pub offchain_changes: &'a RefCell<OffchainOverlayedChanges>,
 	/// The cache for storage transactions.
 	pub storage_transaction_cache: &'a RefCell<StorageTransactionCache<Block, Backend>>,
 	/// Determines if the function requires that `initialize_block` should be called before calling

substrate/primitives/core/src/offchain/storage.rs

@@ -41,7 +41,7 @@ impl InMemOffchainStorage {
 	/// Remove a key and its associated value from the offchain database.
 	pub fn remove(&mut self, prefix: &[u8], key: &[u8]) {
 		let key: Vec<u8> = prefix.iter().chain(key).cloned().collect();
-		let _ = self.storage.remove(&key);
+		self.storage.remove(&key);
 	}
 }
 
@@ -84,9 +84,6 @@ impl OffchainStorage for InMemOffchainStorage {
 	}
 }
 
-
-
-
 /// Change to be applied to the offchain worker db in regards to a key.
 #[derive(Debug,Clone,Hash,Eq,PartialEq)]
 pub enum OffchainOverlayedChange {
@@ -97,161 +94,47 @@ pub enum OffchainOverlayedChange {
 }
 
 /// In-memory storage for offchain workers recoding changes for the actual offchain storage implementation.
-#[derive(Debug, Clone)]
-pub enum OffchainOverlayedChanges {
-	/// Writing overlay changes to the offchain worker database is disabled by configuration.
-	Disabled,
-	/// Overlay changes can be recorded using the inner collection of this variant,
-	/// where the identifier is the tuple of `(prefix, key)`.
-	Enabled(HashMap<(Vec<u8>, Vec<u8>), OffchainOverlayedChange>),
-}
-
-impl Default for OffchainOverlayedChanges {
-	fn default() -> Self {
-		Self::Disabled
-	}
+#[derive(Debug, Clone, Default)]
+pub struct OffchainOverlayedChanges {
+	changes: HashMap<(Vec<u8>, Vec<u8>), OffchainOverlayedChange>,
 }
 
 impl OffchainOverlayedChanges {
-	/// Create the disabled variant.
-	pub fn disabled() -> Self {
-		Self::Disabled
-	}
-
-	/// Create the enabled variant.
-	pub fn enabled() -> Self {
-		Self::Enabled(HashMap::new())
-	}
-
 	/// Consume the offchain storage and iterate over all key value pairs.
-	pub fn into_iter(self) -> OffchainOverlayedChangesIntoIter {
-		OffchainOverlayedChangesIntoIter::new(self)
+	pub fn into_iter(self) -> impl Iterator<Item = ((Vec<u8>, Vec<u8>), OffchainOverlayedChange)> {
+		self.changes.into_iter()
 	}
 
 	/// Iterate over all key value pairs by reference.
-	pub fn iter<'a>(&'a self) -> OffchainOverlayedChangesIter {
-		OffchainOverlayedChangesIter::new(&self)
+	pub fn iter(&self) -> impl Iterator<Item = (&(Vec<u8>, Vec<u8>), &OffchainOverlayedChange)> {
+		self.changes.iter()
 	}
 
 	/// Drain all elements of changeset.
-	pub fn drain<'a, 'd>(&'a mut self) -> OffchainOverlayedChangesDrain<'d> where 'a: 'd {
-		OffchainOverlayedChangesDrain::new(self)
+	pub fn drain(&mut self) -> impl Iterator<Item = ((Vec<u8>, Vec<u8>), OffchainOverlayedChange)> + '_ {
+		self.changes.drain()
 	}
 
 	/// Remove a key and its associated value from the offchain database.
 	pub fn remove(&mut self, prefix: &[u8], key: &[u8]) {
-		if let Self::Enabled(ref mut storage) = self {
-			let _ = storage.insert((prefix.to_vec(), key.to_vec()), OffchainOverlayedChange::Remove);
-		}
+		self.changes.insert((prefix.to_vec(), key.to_vec()), OffchainOverlayedChange::Remove);
 	}
 
 	/// Set the value associated with a key under a prefix to the value provided.
 	pub fn set(&mut self, prefix: &[u8], key: &[u8], value: &[u8]) {
-		if let Self::Enabled(ref mut storage) = self {
-			let _ = storage.insert((prefix.to_vec(), key.to_vec()), OffchainOverlayedChange::SetValue(value.to_vec()));
-		}
+		self.changes.insert(
+			(prefix.to_vec(), key.to_vec()),
+			OffchainOverlayedChange::SetValue(value.to_vec()),
+		);
 	}
 
 	/// Obtain a associated value to the given key in storage with prefix.
 	pub fn get(&self, prefix: &[u8], key: &[u8]) -> Option<OffchainOverlayedChange> {
-		if let Self::Enabled(ref storage) = self {
-			let key = (prefix.to_vec(), key.to_vec());
-			storage.get(&key).cloned()
-		} else {
-			None
-		}
+		let key = (prefix.to_vec(), key.to_vec());
+		self.changes.get(&key).cloned()
 	}
 }
 
-use std::collections::hash_map;
-
-/// Iterate by reference over the prepared offchain worker storage changes.
-pub struct OffchainOverlayedChangesIter<'i> {
-	inner: Option<hash_map::Iter<'i, (Vec<u8>, Vec<u8>), OffchainOverlayedChange>>,
-}
-
-impl<'i> Iterator for OffchainOverlayedChangesIter<'i> {
-	type Item = (&'i (Vec<u8>, Vec<u8>), &'i OffchainOverlayedChange);
-	fn next(&mut self) -> Option<Self::Item> {
-		if let Some(ref mut iter) = self.inner {
-			iter.next()
-		} else {
-			None
-		}
-	}
-}
-
-impl<'i> OffchainOverlayedChangesIter<'i> {
-	/// Create a new iterator based on a refernce to the parent container.
-	pub fn new(container: &'i OffchainOverlayedChanges) -> Self {
-		match container {
-			OffchainOverlayedChanges::Enabled(inner) => Self {
-				inner: Some(inner.iter())
-			},
-			OffchainOverlayedChanges::Disabled => Self { inner: None, },
-		}
-	}
-}
-
-
-/// Iterate by value over the prepared offchain worker storage changes.
-pub struct OffchainOverlayedChangesIntoIter {
-	inner: Option<hash_map::IntoIter<(Vec<u8>,Vec<u8>),OffchainOverlayedChange>>,
-}
-
-impl Iterator for OffchainOverlayedChangesIntoIter {
-	type Item = ((Vec<u8>, Vec<u8>), OffchainOverlayedChange);
-	fn next(&mut self) -> Option<Self::Item> {
-		if let Some(ref mut iter) = self.inner {
-			iter.next()
-		} else {
-			None
-		}
-	}
-}
-
-impl OffchainOverlayedChangesIntoIter {
-	/// Create a new iterator by consuming the collection.
-	pub fn new(container: OffchainOverlayedChanges) -> Self {
-		match container {
-			OffchainOverlayedChanges::Enabled(inner) => Self {
-				inner: Some(inner.into_iter())
-			},
-			OffchainOverlayedChanges::Disabled => Self { inner: None, },
-		}
-	}
-}
-
-/// Iterate over all items while draining them from the collection.
-pub struct OffchainOverlayedChangesDrain<'d> {
-	inner: Option<hash_map::Drain<'d, (Vec<u8>, Vec<u8>), OffchainOverlayedChange>>,
-}
-
-impl<'d> Iterator for OffchainOverlayedChangesDrain<'d> {
-	type Item = ((Vec<u8>, Vec<u8>), OffchainOverlayedChange);
-	fn next(&mut self) -> Option<Self::Item> {
-		if let Some(ref mut iter) = self.inner {
-			iter.next()
-		} else {
-			None
-		}
-	}
-}
-
-impl<'d> OffchainOverlayedChangesDrain<'d> {
-	/// Create a new iterator by taking a mut reference to the collection,
-	/// for the lifetime of the created drain iterator.
-	pub fn new(container: &'d mut OffchainOverlayedChanges) -> Self {
-		match container {
-			OffchainOverlayedChanges::Enabled(ref mut inner) => Self {
-				inner: Some(inner.drain())
-			},
-			OffchainOverlayedChanges::Disabled => Self { inner: None, },
-		}
-	}
-}
-
-
 #[cfg(test)]
 mod test {
 	use super::*;
@@ -259,7 +142,7 @@ mod test {
 
 	#[test]
 	fn test_drain() {
-		let mut ooc = OffchainOverlayedChanges::enabled();
+		let mut ooc = OffchainOverlayedChanges::default();
 		ooc.set(STORAGE_PREFIX,b"kkk", b"vvv");
 		let drained = ooc.drain().count();
 		assert_eq!(drained, 1);
@@ -276,7 +159,7 @@ mod test {
 
 	#[test]
 	fn test_accumulated_set_remove_set() {
-		let mut ooc = OffchainOverlayedChanges::enabled();
+		let mut ooc = OffchainOverlayedChanges::default();
 		ooc.set(STORAGE_PREFIX, b"ppp", b"qqq");
 		ooc.remove(STORAGE_PREFIX, b"ppp");
 		// keys are equiv, so it will overwrite the value and the overlay will contain

substrate/primitives/state-machine/src/ext.rs

@@ -35,8 +35,6 @@ use codec::{Decode, Encode, EncodeAppend};
 use sp_std::{fmt, any::{Any, TypeId}, vec::Vec, vec, boxed::Box};
 use crate::{warn, trace, log_error};
 #[cfg(feature = "std")]
-use sp_core::offchain::storage::OffchainOverlayedChanges;
-#[cfg(feature = "std")]
 use crate::changes_trie::State as ChangesTrieState;
 use crate::StorageTransactionCache;
 #[cfg(feature = "std")]
@@ -100,9 +98,6 @@ pub struct Ext<'a, H, N, B>
 {
 	/// The overlayed changes to write to.
 	overlay: &'a mut OverlayedChanges,
-	/// The overlayed changes destined for the Offchain DB.
-	#[cfg(feature = "std")]
-	offchain_overlay: &'a mut OffchainOverlayedChanges,
 	/// The storage backend to read from.
 	backend: &'a B,
 	/// The cache for the storage transactions.
@@ -146,7 +141,6 @@ impl<'a, H, N, B> Ext<'a, H, N, B>
 	#[cfg(feature = "std")]
 	pub fn new(
 		overlay: &'a mut OverlayedChanges,
-		offchain_overlay: &'a mut OffchainOverlayedChanges,
 		storage_transaction_cache: &'a mut StorageTransactionCache<B::Transaction, H, N>,
 		backend: &'a B,
 		changes_trie_state: Option<ChangesTrieState<'a, H, N>>,
@@ -154,7 +148,6 @@ impl<'a, H, N, B> Ext<'a, H, N, B>
 	) -> Self {
 		Self {
 			overlay,
-			offchain_overlay,
 			backend,
 			changes_trie_state,
 			storage_transaction_cache,
@@ -170,12 +163,6 @@ impl<'a, H, N, B> Ext<'a, H, N, B>
 	fn mark_dirty(&mut self) {
 		self.storage_transaction_cache.reset();
 	}
-
-	/// Read only accessor for the scheduled overlay changes.
-	#[cfg(feature = "std")]
-	pub fn get_offchain_storage_changes(&self) -> &OffchainOverlayedChanges {
-		&*self.offchain_overlay
-	}
 }
 
 #[cfg(test)]
@@ -208,10 +195,10 @@ where
 {
 	#[cfg(feature = "std")]
 	fn set_offchain_storage(&mut self, key: &[u8], value: Option<&[u8]>) {
-		use ::sp_core::offchain::STORAGE_PREFIX;
+		use sp_core::offchain::STORAGE_PREFIX;
 		match value {
-			Some(value) => self.offchain_overlay.set(STORAGE_PREFIX, key, value),
-			None => self.offchain_overlay.remove(STORAGE_PREFIX, key),
+			Some(value) => self.overlay.offchain_set_storage(STORAGE_PREFIX, key, value),
+			None => self.overlay.offchain_remove_storage(STORAGE_PREFIX, key),
 		}
 	}
 
@@ -829,11 +816,9 @@ mod tests {
 		changes
 	}
 
-	fn prepare_offchain_overlay_with_changes() -> OffchainOverlayedChanges {
-		let mut ooc = OffchainOverlayedChanges::enabled();
-		ooc.set(offchain::STORAGE_PREFIX, b"k1", b"v1");
-		ooc.set(offchain::STORAGE_PREFIX, b"k2", b"v2");
-		ooc
+	fn prepare_offchain_overlay_with_changes(overlay: &mut OverlayedChanges) {
+		overlay.offchain_set_storage(offchain::STORAGE_PREFIX, b"k1", b"v1");
+		overlay.offchain_set_storage(offchain::STORAGE_PREFIX, b"k2", b"v2");
 	}
 
 	fn changes_trie_config() -> ChangesTrieConfiguration {
@@ -846,32 +831,30 @@ mod tests {
 	#[test]
 	fn storage_changes_root_is_none_when_storage_is_not_provided() {
 		let mut overlay = prepare_overlay_with_changes();
-		let mut offchain_overlay = prepare_offchain_overlay_with_changes();
 		let mut cache = StorageTransactionCache::default();
 		let backend = TestBackend::default();
-		let mut ext = TestExt::new(&mut overlay, &mut offchain_overlay, &mut cache, &backend, None, None);
+		let mut ext = TestExt::new(&mut overlay, &mut cache, &backend, None, None);
 		assert_eq!(ext.storage_changes_root(&H256::default().encode()).unwrap(), None);
 	}
 
 	#[test]
 	fn storage_changes_root_is_none_when_state_is_not_provided() {
 		let mut overlay = prepare_overlay_with_changes();
-		let mut offchain_overlay = prepare_offchain_overlay_with_changes();
 		let mut cache = StorageTransactionCache::default();
 		let backend = TestBackend::default();
-		let mut ext = TestExt::new(&mut overlay, &mut offchain_overlay, &mut cache, &backend, None, None);
+		let mut ext = TestExt::new(&mut overlay, &mut cache, &backend, None, None);
 		assert_eq!(ext.storage_changes_root(&H256::default().encode()).unwrap(), None);
 	}
 
 	#[test]
 	fn storage_changes_root_is_some_when_extrinsic_changes_are_non_empty() {
 		let mut overlay = prepare_overlay_with_changes();
-		let mut offchain_overlay = prepare_offchain_overlay_with_changes();
+		prepare_offchain_overlay_with_changes(&mut overlay);
 		let mut cache = StorageTransactionCache::default();
 		let storage = TestChangesTrieStorage::with_blocks(vec![(99, Default::default())]);
 		let state = Some(ChangesTrieState::new(changes_trie_config(), Zero::zero(), &storage));
 		let backend = TestBackend::default();
-		let mut ext = TestExt::new(&mut overlay, &mut offchain_overlay, &mut cache, &backend, state, None);
+		let mut ext = TestExt::new(&mut overlay, &mut cache, &backend, state, None);
 		assert_eq!(
 			ext.storage_changes_root(&H256::default().encode()).unwrap(),
 			Some(hex!("bb0c2ef6e1d36d5490f9766cfcc7dfe2a6ca804504c3bb206053890d6dd02376").to_vec()),
@@ -881,14 +864,14 @@ mod tests {
 	#[test]
 	fn storage_changes_root_is_some_when_extrinsic_changes_are_empty() {
 		let mut overlay = prepare_overlay_with_changes();
-		let mut offchain_overlay = prepare_offchain_overlay_with_changes();
+		prepare_offchain_overlay_with_changes(&mut overlay);
 		let mut cache = StorageTransactionCache::default();
 		overlay.set_collect_extrinsics(false);
 		overlay.set_storage(vec![1], None);
 		let storage = TestChangesTrieStorage::with_blocks(vec![(99, Default::default())]);
 		let state = Some(ChangesTrieState::new(changes_trie_config(), Zero::zero(), &storage));
 		let backend = TestBackend::default();
-		let mut ext = TestExt::new(&mut overlay, &mut offchain_overlay, &mut cache, &backend, state, None);
+		let mut ext = TestExt::new(&mut overlay, &mut cache, &backend, state, None);
 		assert_eq!(
 			ext.storage_changes_root(&H256::default().encode()).unwrap(),
 			Some(hex!("96f5aae4690e7302737b6f9b7f8567d5bbb9eac1c315f80101235a92d9ec27f4").to_vec()),
@@ -901,7 +884,7 @@ mod tests {
 		let mut overlay = OverlayedChanges::default();
 		overlay.set_storage(vec![20], None);
 		overlay.set_storage(vec![30], Some(vec![31]));
-		let mut offchain_overlay = prepare_offchain_overlay_with_changes();
+		prepare_offchain_overlay_with_changes(&mut overlay);
 		let backend = Storage {
 			top: map![
 				vec![10] => vec![10],
@@ -911,7 +894,7 @@ mod tests {
 			children_default: map![]
 		}.into();
 
-		let ext = TestExt::new(&mut overlay, &mut offchain_overlay, &mut cache, &backend, None, None);
+		let ext = TestExt::new(&mut overlay, &mut cache, &backend, None, None);
 
 		// next_backend < next_overlay
 		assert_eq!(ext.next_storage_key(&[5]), Some(vec![10]));
@@ -927,7 +910,7 @@ mod tests {
 
 		drop(ext);
 		overlay.set_storage(vec![50], Some(vec![50]));
-		let ext = TestExt::new(&mut overlay, &mut offchain_overlay, &mut cache, &backend, None, None);
+		let ext = TestExt::new(&mut overlay, &mut cache, &backend, None, None);
 
 		// next_overlay exist but next_backend doesn't exist
 		assert_eq!(ext.next_storage_key(&[40]), Some(vec![50]));
@@ -956,10 +939,9 @@ mod tests {
 			],
 		}.into();
 
+		prepare_offchain_overlay_with_changes(&mut overlay);
 
-		let mut offchain_overlay = prepare_offchain_overlay_with_changes();
-
-		let ext = TestExt::new(&mut overlay, &mut offchain_overlay, &mut cache, &backend, None, None);
+		let ext = TestExt::new(&mut overlay, &mut cache, &backend, None, None);
 
 		// next_backend < next_overlay
 		assert_eq!(ext.next_child_storage_key(child_info, &[5]), Some(vec![10]));
@@ -975,7 +957,7 @@ mod tests {
 
 		drop(ext);
 		overlay.set_child_storage(child_info, vec![50], Some(vec![50]));
-		let ext = TestExt::new(&mut overlay, &mut offchain_overlay, &mut cache, &backend, None, None);
+		let ext = TestExt::new(&mut overlay, &mut cache, &backend, None, None);
 
 		// next_overlay exist but next_backend doesn't exist
 		assert_eq!(ext.next_child_storage_key(child_info, &[40]), Some(vec![50]));
@@ -989,7 +971,7 @@ mod tests {
 		let mut overlay = OverlayedChanges::default();
 		overlay.set_child_storage(child_info, vec![20], None);
 		overlay.set_child_storage(child_info, vec![30], Some(vec![31]));
-		let mut offchain_overlay = prepare_offchain_overlay_with_changes();
+		prepare_offchain_overlay_with_changes(&mut overlay);
 		let backend = Storage {
 			top: map![],
 			children_default: map![
@@ -1004,7 +986,7 @@ mod tests {
 			],
 		}.into();
 
-		let ext = TestExt::new(&mut overlay, &mut offchain_overlay, &mut cache, &backend, None, None);
+		let ext = TestExt::new(&mut overlay, &mut cache, &backend, None, None);
 
 		assert_eq!(ext.child_storage(child_info, &[10]), Some(vec![10]));
 		assert_eq!(
@@ -1031,7 +1013,7 @@ mod tests {
 		let child_info = &child_info;
 		let mut cache = StorageTransactionCache::default();
 		let mut overlay = OverlayedChanges::default();
-		let mut offchain_overlay = prepare_offchain_overlay_with_changes();
+		prepare_offchain_overlay_with_changes(&mut overlay);
 		let backend = Storage {
 			top: map![],
 			children_default: map![
@@ -1044,7 +1026,7 @@ mod tests {
 			],
 		}.into();
 
-		let ext = TestExt::new(&mut overlay, &mut offchain_overlay, &mut cache, &backend, None, None);
+		let ext = TestExt::new(&mut overlay, &mut cache, &backend, None, None);
 
 		use sp_core::storage::well_known_keys;
 		let mut ext = ext;

substrate/primitives/state-machine/src/lib.rs

@@ -172,7 +172,6 @@ mod execution {
 	use hash_db::Hasher;
 	use codec::{Decode, Encode, Codec};
 	use sp_core::{
-		offchain::storage::OffchainOverlayedChanges,
 		storage::ChildInfo, NativeOrEncoded, NeverNativeValue, hexdisplay::HexDisplay,
 		traits::{CodeExecutor, CallInWasmExt, RuntimeCode, SpawnNamed},
 	};
@@ -299,7 +298,6 @@ mod execution {
 		method: &'a str,
 		call_data: &'a [u8],
 		overlay: &'a mut OverlayedChanges,
-		offchain_overlay: &'a mut OffchainOverlayedChanges,
 		extensions: Extensions,
 		changes_trie_state: Option<ChangesTrieState<'a, H, N>>,
 		storage_transaction_cache: Option<&'a mut StorageTransactionCache<B::Transaction, H, N>>,
@@ -329,7 +327,6 @@ mod execution {
 			backend: &'a B,
 			changes_trie_state: Option<ChangesTrieState<'a, H, N>>,
 			overlay: &'a mut OverlayedChanges,
-			offchain_overlay: &'a mut OffchainOverlayedChanges,
 			exec: &'a Exec,
 			method: &'a str,
 			call_data: &'a [u8],
@@ -347,7 +344,6 @@ mod execution {
 				call_data,
 				extensions,
 				overlay,
-				offchain_overlay,
 				changes_trie_state,
 				storage_transaction_cache: None,
 				runtime_code,
@@ -407,7 +403,6 @@ mod execution {
 
 			let mut ext = Ext::new(
 				self.overlay,
-				self.offchain_overlay,
 				cache,
 				self.backend,
 				self.changes_trie_state.clone(),
@@ -621,13 +616,11 @@ mod execution {
 		N: crate::changes_trie::BlockNumber,
 		Spawn: SpawnNamed + Send + 'static,
 	{
-		let mut offchain_overlay = OffchainOverlayedChanges::default();
 		let proving_backend = proving_backend::ProvingBackend::new(trie_backend);
 		let mut sm = StateMachine::<_, H, N, Exec>::new(
 			&proving_backend,
 			None,
 			overlay,
-			&mut offchain_overlay,
 			exec,
 			method,
 			call_data,
@@ -691,12 +684,10 @@ mod execution {
 		N: crate::changes_trie::BlockNumber,
 		Spawn: SpawnNamed + Send + 'static,
 	{
-		let mut offchain_overlay = OffchainOverlayedChanges::default();
 		let mut sm = StateMachine::<_, H, N, Exec>::new(
 			trie_backend,
 			None,
 			overlay,
-			&mut offchain_overlay,
 			exec,
 			method,
 			call_data,
@@ -879,7 +870,6 @@ mod tests {
 	use std::{result, collections::HashMap};
 	use codec::Decode;
 	use sp_core::{
-		offchain::storage::OffchainOverlayedChanges,
 		storage::ChildInfo, NativeOrEncoded, NeverNativeValue,
 		traits::CodeExecutor,
 	};
@@ -966,14 +956,12 @@ mod tests {
 	fn execute_works() {
 		let backend = trie_backend::tests::test_trie();
 		let mut overlayed_changes = Default::default();
-		let mut offchain_overlayed_changes = Default::default();
 		let wasm_code = RuntimeCode::empty();
 
 		let mut state_machine = StateMachine::new(
 			&backend,
 			changes_trie::disabled_state::<_, u64>(),
 			&mut overlayed_changes,
-			&mut offchain_overlayed_changes,
 			&DummyCodeExecutor {
 				change_changes_trie_config: false,
 				native_available: true,
@@ -998,14 +986,12 @@ mod tests {
 	fn execute_works_with_native_else_wasm() {
 		let backend = trie_backend::tests::test_trie();
 		let mut overlayed_changes = Default::default();
-		let mut offchain_overlayed_changes = Default::default();
 		let wasm_code = RuntimeCode::empty();
 
 		let mut state_machine = StateMachine::new(
 			&backend,
 			changes_trie::disabled_state::<_, u64>(),
 			&mut overlayed_changes,
-			&mut offchain_overlayed_changes,
 			&DummyCodeExecutor {
 				change_changes_trie_config: false,
 				native_available: true,
@@ -1027,14 +1013,12 @@ mod tests {
 		let mut consensus_failed = false;
 		let backend = trie_backend::tests::test_trie();
 		let mut overlayed_changes = Default::default();
-		let mut offchain_overlayed_changes = Default::default();
 		let wasm_code = RuntimeCode::empty();
 
 		let mut state_machine = StateMachine::new(
 			&backend,
 			changes_trie::disabled_state::<_, u64>(),
 			&mut overlayed_changes,
-			&mut offchain_overlayed_changes,
 			&DummyCodeExecutor {
 				change_changes_trie_config: false,
 				native_available: true,
@@ -1118,11 +1102,9 @@ mod tests {
 		overlay.set_storage(b"bbd".to_vec(), Some(b"42".to_vec()));
 
 		{
-			let mut offchain_overlay = Default::default();
 			let mut cache = StorageTransactionCache::default();
 			let mut ext = Ext::new(
 				&mut overlay,
-				&mut offchain_overlay,
 				&mut cache,
 				backend,
 				changes_trie::disabled_state::<_, u64>(),
@@ -1167,11 +1149,9 @@ mod tests {
 		overlay.set_child_storage(&child_info, b"4".to_vec(), Some(b"1312".to_vec()));
 
 		{
-			let mut offchain_overlay = Default::default();
 			let mut cache = StorageTransactionCache::default();
 			let mut ext = Ext::new(
 				&mut overlay,
-				&mut offchain_overlay,
 				&mut cache,
 				&backend,
 				changes_trie::disabled_state::<_, u64>(),
@@ -1209,11 +1189,9 @@ mod tests {
 		];
 		let backend = InMemoryBackend::<BlakeTwo256>::from(initial);
 		let mut overlay = OverlayedChanges::default();
-		let mut offchain_overlay = Default::default();
 		let mut cache = StorageTransactionCache::default();
 		let mut ext = Ext::new(
 			&mut overlay,
-			&mut offchain_overlay,
 			&mut cache,
 			&backend,
 			changes_trie::disabled_state::<_, u64>(),
@@ -1234,11 +1212,9 @@ mod tests {
 		let mut state = new_in_mem::<BlakeTwo256>();
 		let backend = state.as_trie_backend().unwrap();
 		let mut overlay = OverlayedChanges::default();
-		let mut offchain_overlay = OffchainOverlayedChanges::default();
 		let mut cache = StorageTransactionCache::default();
 		let mut ext = Ext::new(
 			&mut overlay,
-			&mut offchain_overlay,
 			&mut cache,
 			backend,
 			changes_trie::disabled_state::<_, u64>(),
@@ -1282,12 +1258,10 @@ mod tests {
 		let mut state = new_in_mem::<BlakeTwo256>();
 		let backend = state.as_trie_backend().unwrap();
 		let mut overlay = OverlayedChanges::default();
-		let mut offchain_overlay = OffchainOverlayedChanges::default();
 		let mut cache = StorageTransactionCache::default();
 		{
 			let mut ext = Ext::new(
 				&mut overlay,
-				&mut offchain_overlay,
 				&mut cache,
 				backend,
 				changes_trie::disabled_state::<_, u64>(),
@@ -1304,7 +1278,6 @@ mod tests {
 		{
 			let mut ext = Ext::new(
 				&mut overlay,
-				&mut offchain_overlay,
 				&mut cache,
 				backend,
 				changes_trie::disabled_state::<_, u64>(),
@@ -1323,7 +1296,6 @@ mod tests {
 		{
 			let ext = Ext::new(
 				&mut overlay,
-				&mut offchain_overlay,
 				&mut cache,
 				backend,
 				changes_trie::disabled_state::<_, u64>(),
@@ -1346,14 +1318,12 @@ mod tests {
 		let mut cache = StorageTransactionCache::default();
 		let mut state = new_in_mem::<BlakeTwo256>();
 		let backend = state.as_trie_backend().unwrap();
-		let mut offchain_overlay = OffchainOverlayedChanges::default();
 		let mut overlay = OverlayedChanges::default();
 
 		// For example, block initialization with event.
 		{
 			let mut ext = Ext::new(
 				&mut overlay,
-				&mut offchain_overlay,
 				&mut cache,
 				backend,
 				changes_trie::disabled_state::<_, u64>(),
@@ -1368,7 +1338,6 @@ mod tests {
 		{
 			let mut ext = Ext::new(
 				&mut overlay,
-				&mut offchain_overlay,
 				&mut cache,
 				backend,
 				changes_trie::disabled_state::<_, u64>(),
@@ -1393,7 +1362,6 @@ mod tests {
 		{
 			let mut ext = Ext::new(
 				&mut overlay,
-				&mut offchain_overlay,
 				&mut cache,
 				backend,
 				changes_trie::disabled_state::<_, u64>(),
@@ -1419,7 +1387,6 @@ mod tests {
 		{
 			let ext = Ext::new(
 				&mut overlay,
-				&mut offchain_overlay,
 				&mut cache,
 				backend,
 				changes_trie::disabled_state::<_, u64>(),
@@ -1495,14 +1462,12 @@ mod tests {
 
 		use crate::trie_backend::tests::test_trie;
 		let mut overlay = OverlayedChanges::default();
-		let mut offchain_overlay = OffchainOverlayedChanges::default();
 
 		let mut transaction = {
 			let backend = test_trie();
 			let mut cache = StorageTransactionCache::default();
 			let mut ext = Ext::new(
 				&mut overlay,
-				&mut offchain_overlay,
 				&mut cache,
 				&backend,
 				changes_trie::disabled_state::<_, u64>(),
@@ -1543,11 +1508,9 @@ mod tests {
 		assert_eq!(overlay.storage(b"bbb"), None);
 
 		{
-			let mut offchain_overlay = Default::default();
 			let mut cache = StorageTransactionCache::default();
 			let mut ext = Ext::new(
 				&mut overlay,
-				&mut offchain_overlay,
 				&mut cache,
 				backend,
 				changes_trie::disabled_state::<_, u64>(),
@@ -1571,14 +1534,12 @@ mod tests {
 
 		let backend = trie_backend::tests::test_trie();
 		let mut overlayed_changes = Default::default();
-		let mut offchain_overlayed_changes = Default::default();
 		let wasm_code = RuntimeCode::empty();
 
 		let mut state_machine = StateMachine::new(
 			&backend,
 			changes_trie::disabled_state::<_, u64>(),
 			&mut overlayed_changes,
-			&mut offchain_overlayed_changes,
 			&DummyCodeExecutor {
 				change_changes_trie_config: false,
 				native_available: true,

substrate/primitives/state-machine/src/overlayed_changes/mod.rs

@@ -43,7 +43,7 @@ use sp_std::collections::btree_set::BTreeSet;
 use codec::{Decode, Encode};
 use sp_core::storage::{well_known_keys::EXTRINSIC_INDEX, ChildInfo};
 #[cfg(feature = "std")]
-use sp_core::offchain::storage::OffchainOverlayedChanges;
+use sp_core::offchain::storage::{OffchainOverlayedChanges, OffchainOverlayedChange};
 use hash_db::Hasher;
 use crate::DefaultError;
 use sp_externalities::{Extensions, Extension};
@@ -101,6 +101,9 @@ pub struct OverlayedChanges {
 	collect_extrinsics: bool,
 	/// Collect statistic on this execution.
 	stats: StateMachineStats,
+	/// Offchain related changes.
+	#[cfg(feature = "std")]
+	offchain: OffchainOverlayedChanges,
 }
 
 /// A storage changes structure that can be generated by the data collected in [`OverlayedChanges`].
@@ -523,7 +526,7 @@ impl OverlayedChanges {
 			main_storage_changes: main_storage_changes.collect(),
 			child_storage_changes: child_storage_changes.map(|(sk, it)| (sk, it.0.collect())).collect(),
 			#[cfg(feature = "std")]
-			offchain_storage_changes: Default::default(),
+			offchain_storage_changes: std::mem::take(&mut self.offchain),
 			transaction,
 			transaction_storage_root,
 			#[cfg(feature = "std")]
@@ -629,6 +632,40 @@ impl OverlayedChanges {
 				overlay.next_change(key)
 			)
 	}
+
+	/// Set a value in the offchain storage.
+	#[cfg(feature = "std")]
+	pub fn offchain_set_storage(&mut self, prefix: &[u8], key: &[u8], value: &[u8]) {
+		self.offchain.set(prefix, key, value);
+	}
+
+	/// Clear a value in the offchain storage.
+	#[cfg(feature = "std")]
+	pub fn offchain_remove_storage(&mut self, prefix: &[u8], key: &[u8]) {
+		self.offchain.remove(prefix, key);
+	}
+
+	/// Get a value in the offchain storage.
+	#[cfg(feature = "std")]
+	pub fn offchain_get_storage(
+		&mut self,
+		prefix: &[u8],
+		key: &[u8],
+	) -> Option<OffchainOverlayedChange> {
+		self.offchain.get(prefix, key)
+	}
+
+	/// Returns a reference to the offchain overlay.
+	#[cfg(feature = "std")]
+	pub fn offchain_overlay(&self) -> &OffchainOverlayedChanges {
+		&self.offchain
+	}
+
+	/// Returns a mutable reference to the offchain overlay.
+	#[cfg(feature = "std")]
+	pub fn offchain_overlay_mut(&mut self) -> &mut OffchainOverlayedChanges {
+		&mut self.offchain
+	}
 }
 
 #[cfg(feature = "std")]
@@ -789,11 +826,9 @@ mod tests {
 		overlay.set_storage(b"dogglesworth".to_vec(), Some(b"cat".to_vec()));
 		overlay.set_storage(b"doug".to_vec(), None);
 
-		let mut offchain_overlay = Default::default();
 		let mut cache = StorageTransactionCache::default();
 		let mut ext = Ext::new(
 			&mut overlay,
-			&mut offchain_overlay,
 			&mut cache,
 			&backend,
 			crate::changes_trie::disabled_state::<_, u64>(),

substrate/primitives/state-machine/src/testing.rs

@@ -33,10 +33,7 @@ use crate::{
 use codec::{Decode, Encode};
 use hash_db::Hasher;
 use sp_core::{
-	offchain::{
-		testing::TestPersistentOffchainDB,
-		storage::OffchainOverlayedChanges
-	},
+	offchain::testing::TestPersistentOffchainDB,
 	storage::{
 		well_known_keys::{CHANGES_TRIE_CONFIG, CODE, HEAP_PAGES, is_child_storage_key},
 		Storage,
@@ -52,7 +49,6 @@ where
 	H::Out: codec::Codec + Ord,
 {
 	overlay: OverlayedChanges,
-	offchain_overlay: OffchainOverlayedChanges,
 	offchain_db: TestPersistentOffchainDB,
 	storage_transaction_cache: StorageTransactionCache<
 		<InMemoryBackend<H> as Backend<H>>::Transaction, H, N
@@ -71,7 +67,6 @@ impl<H: Hasher, N: ChangesTrieBlockNumber> TestExternalities<H, N>
 	pub fn ext(&mut self) -> Ext<H, N, InMemoryBackend<H>> {
 		Ext::new(
 			&mut self.overlay,
-			&mut self.offchain_overlay,
 			&mut self.storage_transaction_cache,
 			&self.backend,
 			match self.changes_trie_config.clone() {
@@ -109,8 +104,6 @@ impl<H: Hasher, N: ChangesTrieBlockNumber> TestExternalities<H, N>
 		storage.top.insert(HEAP_PAGES.to_vec(), 8u64.encode());
 		storage.top.insert(CODE.to_vec(), code.to_vec());
 
-		let offchain_overlay = OffchainOverlayedChanges::enabled();
-
 		let mut extensions = Extensions::default();
 		extensions.register(TaskExecutorExt::new(TaskExecutor::new()));
 
@@ -118,7 +111,6 @@ impl<H: Hasher, N: ChangesTrieBlockNumber> TestExternalities<H, N>
 
 		TestExternalities {
 			overlay,
-			offchain_overlay,
 			offchain_db,
 			changes_trie_config,
 			extensions,
@@ -128,9 +120,14 @@ impl<H: Hasher, N: ChangesTrieBlockNumber> TestExternalities<H, N>
 		}
 	}
 
+	/// Returns the overlayed changes.
+	pub fn overlayed_changes(&self) -> &OverlayedChanges {
+		&self.overlay
+	}
+
 	/// Move offchain changes from overlay to the persistent store.
 	pub fn persist_offchain_overlay(&mut self) {
-		self.offchain_db.apply_offchain_changes(&mut self.offchain_overlay);
+		self.offchain_db.apply_offchain_changes(self.overlay.offchain_overlay_mut());
 	}
 
 	/// A shared reference type around the offchain worker storage.

substrate/test-utils/client/src/lib.rs

@@ -79,6 +79,7 @@ pub struct TestClientBuilder<Block: BlockT, Executor, Backend, G: GenesisInit> {
 	keystore: Option<SyncCryptoStorePtr>,
 	fork_blocks: ForkBlocks<Block>,
 	bad_blocks: BadBlocks<Block>,
+	enable_offchain_indexing_api: bool,
 }
 
 impl<Block: BlockT, Executor, G: GenesisInit> Default
@@ -114,6 +115,7 @@ impl<Block: BlockT, Executor, Backend, G: GenesisInit> TestClientBuilder<Block,
 			keystore: None,
 			fork_blocks: None,
 			bad_blocks: None,
+			enable_offchain_indexing_api: false,
 		}
 	}
 
@@ -175,6 +177,12 @@ impl<Block: BlockT, Executor, Backend, G: GenesisInit> TestClientBuilder<Block,
 		self
 	}
 
+	/// Enable the offchain indexing api.
+	pub fn enable_offchain_indexing_api(mut self) -> Self {
+		self.enable_offchain_indexing_api = true;
+		self
+	}
+
 	/// Build the test client with the given native executor.
 	pub fn build_with_executor<RuntimeApi>(
 		self,
@@ -219,7 +227,10 @@ impl<Block: BlockT, Executor, Backend, G: GenesisInit> TestClientBuilder<Block,
 				self.keystore,
 			),
 			None,
-			ClientConfig::default(),
+			ClientConfig {
+				offchain_indexing_api: self.enable_offchain_indexing_api,
+				..Default::default()
+			},
 		).expect("Creates new client");
 
 		let longest_chain = sc_consensus::LongestChain::new(self.backend);

substrate/test-utils/runtime/src/lib.rs

@@ -149,6 +149,8 @@ pub enum Extrinsic {
 	IncludeData(Vec<u8>),
 	StorageChange(Vec<u8>, Option<Vec<u8>>),
 	ChangesTrieConfigUpdate(Option<ChangesTrieConfiguration>),
+	OffchainIndexSet(Vec<u8>, Vec<u8>),
+	OffchainIndexClear(Vec<u8>),
 }
 
 parity_util_mem::malloc_size_of_is_0!(Extrinsic); // non-opaque extrinsic does not need this
@@ -177,6 +179,10 @@ impl BlindCheckable for Extrinsic {
 			Extrinsic::StorageChange(key, value) => Ok(Extrinsic::StorageChange(key, value)),
 			Extrinsic::ChangesTrieConfigUpdate(new_config) =>
 				Ok(Extrinsic::ChangesTrieConfigUpdate(new_config)),
+			Extrinsic::OffchainIndexSet(key, value) =>
+				Ok(Extrinsic::OffchainIndexSet(key, value)),
+			Extrinsic::OffchainIndexClear(key) =>
+				Ok(Extrinsic::OffchainIndexClear(key)),
 		}
 	}
 }
@@ -1148,13 +1154,9 @@ fn test_witness(proof: StorageProof, root: crate::Hash) {
 		root,
 	);
 	let mut overlay = sp_state_machine::OverlayedChanges::default();
-	#[cfg(feature = "std")]
-	let mut offchain_overlay = Default::default();
 	let mut cache = sp_state_machine::StorageTransactionCache::<_, _, BlockNumber>::default();
 	let mut ext = sp_state_machine::Ext::new(
 		&mut overlay,
-		#[cfg(feature = "std")]
-		&mut offchain_overlay,
 		&mut cache,
 		&backend,
 		#[cfg(feature = "std")]

substrate/test-utils/runtime/src/system.rs

@@ -261,6 +261,14 @@ fn execute_transaction_backend(utx: &Extrinsic, extrinsic_index: u32) -> ApplyEx
 			execute_storage_change(key, value.as_ref().map(|v| &**v)),
 		Extrinsic::ChangesTrieConfigUpdate(ref new_config) =>
 			execute_changes_trie_config_update(new_config.clone()),
+		Extrinsic::OffchainIndexSet(key, value) => {
+			sp_io::offchain_index::set(&key, &value);
+			Ok(Ok(()))
+		},
+		Extrinsic::OffchainIndexClear(key) => {
+			sp_io::offchain_index::clear(&key);
+			Ok(Ok(()))
+		}
 	}
 }
 

substrate/utils/frame/benchmarking-cli/src/command.rs

@@ -62,7 +62,6 @@ impl BenchmarkCmd {
 
 		let genesis_storage = spec.build_storage()?;
 		let mut changes = Default::default();
-		let mut offchain_changes = Default::default();
 		let cache_size = Some(self.database_cache_size as usize);
 		let state = BenchmarkingState::<BB>::new(genesis_storage, cache_size)?;
 		let executor = NativeExecutor::<ExecDispatch>::new(
@@ -80,7 +79,6 @@ impl BenchmarkCmd {
 			&state,
 			None,
 			&mut changes,
-			&mut offchain_changes,
 			&executor,
 			"Benchmark_dispatch_benchmark",
 			&(