diff --git a/substrate/Cargo.lock b/substrate/Cargo.lock index 1930628146..2bec30fdd8 100644 --- a/substrate/Cargo.lock +++ b/substrate/Cargo.lock @@ -7218,12 +7218,14 @@ dependencies = [ "parity-scale-codec", "parking_lot 0.11.1", "rand 0.7.3", + "sc-block-builder", "sc-client-api", "sc-client-db", "sc-keystore", "sc-network", "sc-transaction-pool", "sp-api", + "sp-consensus", "sp-core", "sp-offchain", "sp-runtime", diff --git a/substrate/client/api/src/call_executor.rs b/substrate/client/api/src/call_executor.rs index 9c0ea87ea7..5f1e0134a5 100644 --- a/substrate/client/api/src/call_executor.rs +++ b/substrate/client/api/src/call_executor.rs @@ -28,7 +28,7 @@ use sp_state_machine::{ }; use sc_executor::{RuntimeVersion, NativeVersion}; use sp_externalities::Extensions; -use sp_core::{NativeOrEncoded,offchain::storage::OffchainOverlayedChanges}; +use sp_core::NativeOrEncoded; use sp_api::{ProofRecorder, InitializeBlock, StorageTransactionCache}; use crate::execution_extensions::ExecutionExtensions; @@ -86,7 +86,6 @@ pub trait CallExecutor { method: &str, call_data: &[u8], changes: &RefCell, - offchain_changes: &RefCell, storage_transaction_cache: Option<&RefCell< StorageTransactionCache>::State>, >>, diff --git a/substrate/client/db/src/lib.rs b/substrate/client/db/src/lib.rs index 3fc95d5cdf..a2c8b56125 100644 --- a/substrate/client/db/src/lib.rs +++ b/substrate/client/db/src/lib.rs @@ -681,14 +681,12 @@ pub struct BlockImportOperation { impl BlockImportOperation { fn apply_offchain(&mut self, transaction: &mut Transaction) { for ((prefix, key), value_operation) in self.offchain_storage_updates.drain() { - let key: Vec = prefix - .into_iter() - .chain(sp_core::sp_std::iter::once(b'/')) - .chain(key.into_iter()) - .collect(); + let key = crate::offchain::concatenate_prefix_and_key(&prefix, &key); match value_operation { - OffchainOverlayedChange::SetValue(val) => transaction.set_from_vec(columns::OFFCHAIN, &key, val), - OffchainOverlayedChange::Remove => transaction.remove(columns::OFFCHAIN, &key), + OffchainOverlayedChange::SetValue(val) => + transaction.set_from_vec(columns::OFFCHAIN, &key, val), + OffchainOverlayedChange::Remove => + transaction.remove(columns::OFFCHAIN, &key), } } } diff --git a/substrate/client/db/src/offchain.rs b/substrate/client/db/src/offchain.rs index aead439734..df45c4946e 100644 --- a/substrate/client/db/src/offchain.rs +++ b/substrate/client/db/src/offchain.rs @@ -18,10 +18,7 @@ //! RocksDB-based offchain workers local storage. -use std::{ - collections::HashMap, - sync::Arc, -}; +use std::{collections::HashMap, sync::Arc}; use crate::{columns, Database, DbHash, Transaction}; use parking_lot::Mutex; @@ -43,7 +40,7 @@ impl std::fmt::Debug for LocalStorage { impl LocalStorage { /// Create new offchain storage for tests (backed by memorydb) - #[cfg(any(test, feature = "test-helpers"))] + #[cfg(any(feature = "test-helpers", test))] pub fn new_test() -> Self { let db = kvdb_memorydb::create(crate::utils::NUM_COLUMNS); let db = sp_database::as_database(db); @@ -61,9 +58,8 @@ impl LocalStorage { impl sp_core::offchain::OffchainStorage for LocalStorage { fn set(&mut self, prefix: &[u8], key: &[u8], value: &[u8]) { - let key: Vec = prefix.iter().chain(key).cloned().collect(); let mut tx = Transaction::new(); - tx.set(columns::OFFCHAIN, &key, value); + tx.set(columns::OFFCHAIN, &concatenate_prefix_and_key(prefix, key), value); if let Err(err) = self.db.commit(tx) { error!("Error setting on local storage: {}", err) @@ -71,9 +67,8 @@ impl sp_core::offchain::OffchainStorage for LocalStorage { } fn remove(&mut self, prefix: &[u8], key: &[u8]) { - let key: Vec = prefix.iter().chain(key).cloned().collect(); let mut tx = Transaction::new(); - tx.remove(columns::OFFCHAIN, &key); + tx.remove(columns::OFFCHAIN, &concatenate_prefix_and_key(prefix, key)); if let Err(err) = self.db.commit(tx) { error!("Error removing on local storage: {}", err) @@ -81,8 +76,7 @@ impl sp_core::offchain::OffchainStorage for LocalStorage { } fn get(&self, prefix: &[u8], key: &[u8]) -> Option> { - let key: Vec = prefix.iter().chain(key).cloned().collect(); - self.db.get(columns::OFFCHAIN, &key) + self.db.get(columns::OFFCHAIN, &concatenate_prefix_and_key(prefix, key)) } fn compare_and_set( @@ -92,7 +86,7 @@ impl sp_core::offchain::OffchainStorage for LocalStorage { old_value: Option<&[u8]>, new_value: &[u8], ) -> bool { - let key: Vec = prefix.iter().chain(item_key).cloned().collect(); + let key = concatenate_prefix_and_key(prefix, item_key); let key_lock = { let mut locks = self.locks.lock(); locks.entry(key.clone()).or_default().clone() @@ -122,6 +116,15 @@ impl sp_core::offchain::OffchainStorage for LocalStorage { } } +/// Concatenate the prefix and key to create an offchain key in the db. +pub(crate) fn concatenate_prefix_and_key(prefix: &[u8], key: &[u8]) -> Vec { + prefix + .iter() + .chain(key.into_iter()) + .cloned() + .collect() +} + #[cfg(test)] mod tests { use super::*; diff --git a/substrate/client/executor/src/integration_tests/mod.rs b/substrate/client/executor/src/integration_tests/mod.rs index 661d2c5d3d..e4339a9ff8 100644 --- a/substrate/client/executor/src/integration_tests/mod.rs +++ b/substrate/client/executor/src/integration_tests/mod.rs @@ -477,8 +477,8 @@ fn offchain_index(wasm_method: WasmExecutionMethod) { use sp_core::offchain::storage::OffchainOverlayedChange; assert_eq!( - ext.ext() - .get_offchain_storage_changes() + ext.overlayed_changes() + .offchain_overlay() .get(sp_core::offchain::STORAGE_PREFIX, b"k"), Some(OffchainOverlayedChange::SetValue(b"v".to_vec())) ); diff --git a/substrate/client/light/src/call_executor.rs b/substrate/client/light/src/call_executor.rs index 7115f24a77..8b403823b0 100644 --- a/substrate/client/light/src/call_executor.rs +++ b/substrate/client/light/src/call_executor.rs @@ -25,7 +25,6 @@ use std::{ use codec::{Encode, Decode}; use sp_core::{ convert_hash, NativeOrEncoded, traits::{CodeExecutor, SpawnNamed}, - offchain::storage::OffchainOverlayedChanges, }; use sp_runtime::{ generic::BlockId, traits::{One, Block as BlockT, Header as HeaderT, HashFor}, @@ -113,7 +112,6 @@ impl CallExecutor for method: &str, call_data: &[u8], changes: &RefCell, - offchain_changes: &RefCell, _: Option<&RefCell>>, initialize_block: InitializeBlock<'a, Block>, _manager: ExecutionManager, @@ -140,7 +138,6 @@ impl CallExecutor for method, call_data, changes, - offchain_changes, None, initialize_block, ExecutionManager::NativeWhenPossible, diff --git a/substrate/client/offchain/Cargo.toml b/substrate/client/offchain/Cargo.toml index 0f19caf739..7d0f01a0c7 100644 --- a/substrate/client/offchain/Cargo.toml +++ b/substrate/client/offchain/Cargo.toml @@ -37,10 +37,12 @@ hyper = "0.13.9" hyper-rustls = "0.21.0" [dev-dependencies] -sc-client-db = { version = "0.8.0", default-features = true, path = "../db/" } +sc-client-db = { version = "0.8.0", default-features = true, path = "../db" } +sc-block-builder = { version = "0.8.0", path = "../block-builder" } sc-transaction-pool = { version = "2.0.0", path = "../transaction-pool" } sp-transaction-pool = { version = "2.0.0", path = "../../primitives/transaction-pool" } sp-tracing = { version = "2.0.0", path = "../../primitives/tracing" } +sp-consensus = { version = "0.8.1", path = "../../primitives/consensus/common" } substrate-test-runtime-client = { version = "2.0.0", path = "../../test-utils/runtime/client" } tokio = "0.2" lazy_static = "1.4.0" diff --git a/substrate/client/offchain/src/lib.rs b/substrate/client/offchain/src/lib.rs index 767d2ac5a1..12b5093d47 100644 --- a/substrate/client/offchain/src/lib.rs +++ b/substrate/client/offchain/src/lib.rs @@ -60,7 +60,7 @@ pub use sp_offchain::{OffchainWorkerApi, STORAGE_PREFIX}; pub trait NetworkProvider: NetworkStateInfo { /// Set the authorized peers. fn set_authorized_peers(&self, peers: HashSet); - + /// Set the authorized only flag. fn set_authorized_only(&self, reserved_only: bool); } @@ -238,9 +238,15 @@ mod tests { use super::*; use std::sync::Arc; use sc_network::{Multiaddr, PeerId}; - use substrate_test_runtime_client::{TestClient, runtime::Block}; + use substrate_test_runtime_client::{ + TestClient, runtime::Block, TestClientBuilderExt, + DefaultTestClientBuilderExt, ClientBlockImportExt, + }; use sc_transaction_pool::{BasicPool, FullChainApi}; use sp_transaction_pool::{TransactionPool, InPoolTransaction}; + use sp_consensus::BlockOrigin; + use sc_client_api::Backend as _; + use sc_block_builder::BlockBuilderProvider as _; struct TestNetwork(); @@ -307,4 +313,41 @@ mod tests { assert_eq!(pool.0.status().ready, 1); assert_eq!(pool.0.ready().next().unwrap().is_propagable(), false); } + + #[test] + fn offchain_index_set_and_clear_works() { + sp_tracing::try_init_simple(); + + let (client, backend) = + substrate_test_runtime_client::TestClientBuilder::new() + .enable_offchain_indexing_api() + .build_with_backend(); + let mut client = Arc::new(client); + let offchain_db = backend.offchain_storage().unwrap(); + + let key = &b"hello"[..]; + let value = &b"world"[..]; + let mut block_builder = client.new_block(Default::default()).unwrap(); + block_builder.push( + substrate_test_runtime_client::runtime::Extrinsic::OffchainIndexSet( + key.to_vec(), + value.to_vec(), + ), + ).unwrap(); + + let block = block_builder.build().unwrap().block; + client.import(BlockOrigin::Own, block).unwrap(); + + assert_eq!(value, &offchain_db.get(sp_offchain::STORAGE_PREFIX, &key).unwrap()); + + let mut block_builder = client.new_block(Default::default()).unwrap(); + block_builder.push( + substrate_test_runtime_client::runtime::Extrinsic::OffchainIndexClear(key.to_vec()), + ).unwrap(); + + let block = block_builder.build().unwrap().block; + client.import(BlockOrigin::Own, block).unwrap(); + + assert!(offchain_db.get(sp_offchain::STORAGE_PREFIX, &key).is_none()); + } } diff --git a/substrate/client/service/src/client/call_executor.rs b/substrate/client/service/src/client/call_executor.rs index d6f04d7027..cc196f67a3 100644 --- a/substrate/client/service/src/client/call_executor.rs +++ b/substrate/client/service/src/client/call_executor.rs @@ -29,7 +29,6 @@ use sc_executor::{RuntimeVersion, RuntimeInfo, NativeVersion}; use sp_externalities::Extensions; use sp_core::{ NativeOrEncoded, NeverNativeValue, traits::{CodeExecutor, SpawnNamed, RuntimeCode}, - offchain::storage::OffchainOverlayedChanges, }; use sp_api::{ProofRecorder, InitializeBlock, StorageTransactionCache}; use sc_client_api::{backend, call_executor::CallExecutor}; @@ -127,11 +126,6 @@ where extensions: Option, ) -> sp_blockchain::Result> { let mut changes = OverlayedChanges::default(); - let mut offchain_changes = if self.client_config.offchain_indexing_api { - OffchainOverlayedChanges::enabled() - } else { - OffchainOverlayedChanges::disabled() - }; let changes_trie = backend::changes_tries_state_at_block( id, self.backend.changes_trie_storage() )?; @@ -145,7 +139,6 @@ where &state, changes_trie, &mut changes, - &mut offchain_changes, &self.executor, method, call_data, @@ -176,7 +169,6 @@ where method: &str, call_data: &[u8], changes: &RefCell, - offchain_changes: &RefCell, storage_transaction_cache: Option<&RefCell< StorageTransactionCache >>, @@ -201,7 +193,6 @@ where let mut state = self.backend.state_at(*at)?; let changes = &mut *changes.borrow_mut(); - let offchain_changes = &mut *offchain_changes.borrow_mut(); match recorder { Some(recorder) => { @@ -213,7 +204,6 @@ where let state_runtime_code = sp_state_machine::backend::BackendRuntimeCode::new(&trie_state); // It is important to extract the runtime code here before we create the proof // recorder. - let runtime_code = state_runtime_code.runtime_code() .map_err(sp_blockchain::Error::RuntimeCode)?; let runtime_code = self.check_override(runtime_code, at)?; @@ -227,7 +217,6 @@ where &backend, changes_trie_state, changes, - offchain_changes, &self.executor, method, call_data, @@ -249,7 +238,6 @@ where &state, changes_trie_state, changes, - offchain_changes, &self.executor, method, call_data, @@ -264,7 +252,6 @@ where fn runtime_version(&self, id: &BlockId) -> sp_blockchain::Result { let mut overlay = OverlayedChanges::default(); - let mut offchain_overlay = OffchainOverlayedChanges::default(); let changes_trie_state = backend::changes_tries_state_at_block( id, self.backend.changes_trie_storage(), @@ -273,7 +260,6 @@ where let mut cache = StorageTransactionCache::::default(); let mut ext = Ext::new( &mut overlay, - &mut offchain_overlay, &mut cache, &state, changes_trie_state, diff --git a/substrate/client/service/src/client/client.rs b/substrate/client/service/src/client/client.rs index d52a3666db..f337452e9d 100644 --- a/substrate/client/service/src/client/client.rs +++ b/substrate/client/service/src/client/client.rs @@ -1654,7 +1654,6 @@ impl CallApiAt for Client where params.function, ¶ms.arguments, params.overlayed_changes, - params.offchain_changes, Some(params.storage_transaction_cache), params.initialize_block, manager, diff --git a/substrate/client/service/test/src/client/light.rs b/substrate/client/service/test/src/client/light.rs index 201b24a6ef..b6287741fd 100644 --- a/substrate/client/service/test/src/client/light.rs +++ b/substrate/client/service/test/src/client/light.rs @@ -36,7 +36,7 @@ use parking_lot::Mutex; use substrate_test_runtime_client::{ runtime::{Hash, Block, Header}, TestClient, ClientBlockImportExt, }; -use sp_api::{InitializeBlock, StorageTransactionCache, ProofRecorder, OffchainOverlayedChanges}; +use sp_api::{InitializeBlock, StorageTransactionCache, ProofRecorder}; use sp_consensus::BlockOrigin; use sc_executor::{NativeExecutor, WasmExecutionMethod, RuntimeVersion, NativeVersion}; use sp_core::{H256, NativeOrEncoded, testing::TaskExecutor}; @@ -223,7 +223,6 @@ impl CallExecutor for DummyCallExecutor { _method: &str, _call_data: &[u8], _changes: &RefCell, - _offchain_changes: &RefCell, _storage_transaction_cache: Option<&RefCell< StorageTransactionCache< Block, diff --git a/substrate/client/service/test/src/client/mod.rs b/substrate/client/service/test/src/client/mod.rs index 9712e84e60..7498289c7b 100644 --- a/substrate/client/service/test/src/client/mod.rs +++ b/substrate/client/service/test/src/client/mod.rs @@ -41,7 +41,7 @@ use sp_runtime::traits::{ }; use substrate_test_runtime::TestAPI; use sp_state_machine::backend::Backend as _; -use sp_api::{ProvideRuntimeApi, OffchainOverlayedChanges}; +use sp_api::ProvideRuntimeApi; use sp_core::{H256, ChangesTrieConfiguration, blake2_256, testing::TaskExecutor}; use std::collections::{HashMap, HashSet}; use std::sync::Arc; @@ -163,7 +163,6 @@ fn construct_block( }; let hash = header.hash(); let mut overlay = OverlayedChanges::default(); - let mut offchain_overlay = OffchainOverlayedChanges::default(); let backend_runtime_code = sp_state_machine::backend::BackendRuntimeCode::new(&backend); let runtime_code = backend_runtime_code.runtime_code().expect("Code is part of the backend"); let task_executor = Box::new(TaskExecutor::new()); @@ -172,7 +171,6 @@ fn construct_block( backend, sp_state_machine::disabled_changes_trie_state::<_, u64>(), &mut overlay, - &mut offchain_overlay, &executor(), "Core_initialize_block", &header.encode(), @@ -188,7 +186,6 @@ fn construct_block( backend, sp_state_machine::disabled_changes_trie_state::<_, u64>(), &mut overlay, - &mut offchain_overlay, &executor(), "BlockBuilder_apply_extrinsic", &tx.encode(), @@ -204,7 +201,6 @@ fn construct_block( backend, sp_state_machine::disabled_changes_trie_state::<_, u64>(), &mut overlay, - &mut offchain_overlay, &executor(), "BlockBuilder_finalize_block", &[], @@ -252,13 +248,11 @@ fn construct_genesis_should_work_with_native() { let runtime_code = backend_runtime_code.runtime_code().expect("Code is part of the backend"); let mut overlay = OverlayedChanges::default(); - let mut offchain_overlay = OffchainOverlayedChanges::default(); let _ = StateMachine::new( &backend, sp_state_machine::disabled_changes_trie_state::<_, u64>(), &mut overlay, - &mut offchain_overlay, &executor(), "Core_execute_block", &b1data, @@ -288,13 +282,11 @@ fn construct_genesis_should_work_with_wasm() { let runtime_code = backend_runtime_code.runtime_code().expect("Code is part of the backend"); let mut overlay = OverlayedChanges::default(); - let mut offchain_overlay = OffchainOverlayedChanges::default(); let _ = StateMachine::new( &backend, sp_state_machine::disabled_changes_trie_state::<_, u64>(), &mut overlay, - &mut offchain_overlay, &executor(), "Core_execute_block", &b1data, @@ -324,13 +316,11 @@ fn construct_genesis_with_bad_transaction_should_panic() { let runtime_code = backend_runtime_code.runtime_code().expect("Code is part of the backend"); let mut overlay = OverlayedChanges::default(); - let mut offchain_overlay = OffchainOverlayedChanges::default(); let r = StateMachine::new( &backend, sp_state_machine::disabled_changes_trie_state::<_, u64>(), &mut overlay, - &mut offchain_overlay, &executor(), "Core_execute_block", &b1data, diff --git a/substrate/primitives/api/proc-macro/src/decl_runtime_apis.rs b/substrate/primitives/api/proc-macro/src/decl_runtime_apis.rs index 7c6f95c926..62ea5ed32b 100644 --- a/substrate/primitives/api/proc-macro/src/decl_runtime_apis.rs +++ b/substrate/primitives/api/proc-macro/src/decl_runtime_apis.rs @@ -409,7 +409,6 @@ fn generate_call_api_at_calls(decl: &ItemTrait) -> Result { at: &#crate_::BlockId, args: Vec, changes: &std::cell::RefCell<#crate_::OverlayedChanges>, - offchain_changes: &std::cell::RefCell<#crate_::OffchainOverlayedChanges>, storage_transaction_cache: &std::cell::RefCell< #crate_::StorageTransactionCache >, @@ -439,7 +438,6 @@ fn generate_call_api_at_calls(decl: &ItemTrait) -> Result { native_call: None, arguments: args, overlayed_changes: changes, - offchain_changes, storage_transaction_cache, initialize_block, context, @@ -460,7 +458,6 @@ fn generate_call_api_at_calls(decl: &ItemTrait) -> Result { native_call, arguments: args, overlayed_changes: changes, - offchain_changes, storage_transaction_cache, initialize_block, context, diff --git a/substrate/primitives/api/proc-macro/src/impl_runtime_apis.rs b/substrate/primitives/api/proc-macro/src/impl_runtime_apis.rs index d44792ef77..8a057383ef 100644 --- a/substrate/primitives/api/proc-macro/src/impl_runtime_apis.rs +++ b/substrate/primitives/api/proc-macro/src/impl_runtime_apis.rs @@ -208,7 +208,6 @@ fn generate_runtime_api_base_structures() -> Result { commit_on_success: std::cell::RefCell, initialized_block: std::cell::RefCell>>, changes: std::cell::RefCell<#crate_::OverlayedChanges>, - offchain_changes: std::cell::RefCell<#crate_::OffchainOverlayedChanges>, storage_transaction_cache: std::cell::RefCell< #crate_::StorageTransactionCache >, @@ -338,7 +337,6 @@ fn generate_runtime_api_base_structures() -> Result { commit_on_success: true.into(), initialized_block: None.into(), changes: Default::default(), - offchain_changes: Default::default(), recorder: Default::default(), storage_transaction_cache: Default::default(), }.into() @@ -357,7 +355,6 @@ fn generate_runtime_api_base_structures() -> Result { &C, &Self, &std::cell::RefCell<#crate_::OverlayedChanges>, - &std::cell::RefCell<#crate_::OffchainOverlayedChanges>, &std::cell::RefCell<#crate_::StorageTransactionCache>, &std::cell::RefCell>>, &Option<#crate_::ProofRecorder>, @@ -374,7 +371,6 @@ fn generate_runtime_api_base_structures() -> Result { &self.call, self, &self.changes, - &self.offchain_changes, &self.storage_transaction_cache, &self.initialized_block, &self.recorder, @@ -531,7 +527,6 @@ impl<'a> Fold for ApiRuntimeImplToApiRuntimeApiImpl<'a> { call_runtime_at, core_api, changes, - offchain_changes, storage_transaction_cache, initialized_block, recorder @@ -542,7 +537,6 @@ impl<'a> Fold for ApiRuntimeImplToApiRuntimeApiImpl<'a> { at, params_encoded, changes, - offchain_changes, storage_transaction_cache, initialized_block, params.map(|p| { diff --git a/substrate/primitives/api/src/lib.rs b/substrate/primitives/api/src/lib.rs index 265439bf37..8ce447c0d3 100644 --- a/substrate/primitives/api/src/lib.rs +++ b/substrate/primitives/api/src/lib.rs @@ -47,8 +47,6 @@ pub use sp_core::NativeOrEncoded; #[doc(hidden)] #[cfg(feature = "std")] pub use hash_db::Hasher; -#[cfg(feature = "std")] -pub use sp_core::offchain::storage::OffchainOverlayedChanges; #[doc(hidden)] #[cfg(not(feature = "std"))] pub use sp_core::to_substrate_wasm_fn_return_value; @@ -521,8 +519,6 @@ pub struct CallApiAtParams<'a, Block: BlockT, C, NC, Backend: StateBackend, /// The overlayed changes that are on top of the state. pub overlayed_changes: &'a RefCell, - /// The overlayed changes to be applied to the offchain worker database. - pub offchain_changes: &'a RefCell, /// The cache for storage transactions. pub storage_transaction_cache: &'a RefCell>, /// Determines if the function requires that `initialize_block` should be called before calling diff --git a/substrate/primitives/core/src/offchain/storage.rs b/substrate/primitives/core/src/offchain/storage.rs index ec6f91e6a5..a47361d88e 100644 --- a/substrate/primitives/core/src/offchain/storage.rs +++ b/substrate/primitives/core/src/offchain/storage.rs @@ -41,7 +41,7 @@ impl InMemOffchainStorage { /// Remove a key and its associated value from the offchain database. pub fn remove(&mut self, prefix: &[u8], key: &[u8]) { let key: Vec = prefix.iter().chain(key).cloned().collect(); - let _ = self.storage.remove(&key); + self.storage.remove(&key); } } @@ -84,9 +84,6 @@ impl OffchainStorage for InMemOffchainStorage { } } - - - /// Change to be applied to the offchain worker db in regards to a key. #[derive(Debug,Clone,Hash,Eq,PartialEq)] pub enum OffchainOverlayedChange { @@ -97,161 +94,47 @@ pub enum OffchainOverlayedChange { } /// In-memory storage for offchain workers recoding changes for the actual offchain storage implementation. -#[derive(Debug, Clone)] -pub enum OffchainOverlayedChanges { - /// Writing overlay changes to the offchain worker database is disabled by configuration. - Disabled, - /// Overlay changes can be recorded using the inner collection of this variant, - /// where the identifier is the tuple of `(prefix, key)`. - Enabled(HashMap<(Vec, Vec), OffchainOverlayedChange>), -} - -impl Default for OffchainOverlayedChanges { - fn default() -> Self { - Self::Disabled - } +#[derive(Debug, Clone, Default)] +pub struct OffchainOverlayedChanges { + changes: HashMap<(Vec, Vec), OffchainOverlayedChange>, } impl OffchainOverlayedChanges { - /// Create the disabled variant. - pub fn disabled() -> Self { - Self::Disabled - } - - /// Create the enabled variant. - pub fn enabled() -> Self { - Self::Enabled(HashMap::new()) - } - /// Consume the offchain storage and iterate over all key value pairs. - pub fn into_iter(self) -> OffchainOverlayedChangesIntoIter { - OffchainOverlayedChangesIntoIter::new(self) + pub fn into_iter(self) -> impl Iterator, Vec), OffchainOverlayedChange)> { + self.changes.into_iter() } /// Iterate over all key value pairs by reference. - pub fn iter<'a>(&'a self) -> OffchainOverlayedChangesIter { - OffchainOverlayedChangesIter::new(&self) + pub fn iter(&self) -> impl Iterator, Vec), &OffchainOverlayedChange)> { + self.changes.iter() } /// Drain all elements of changeset. - pub fn drain<'a, 'd>(&'a mut self) -> OffchainOverlayedChangesDrain<'d> where 'a: 'd { - OffchainOverlayedChangesDrain::new(self) + pub fn drain(&mut self) -> impl Iterator, Vec), OffchainOverlayedChange)> + '_ { + self.changes.drain() } /// Remove a key and its associated value from the offchain database. pub fn remove(&mut self, prefix: &[u8], key: &[u8]) { - if let Self::Enabled(ref mut storage) = self { - let _ = storage.insert((prefix.to_vec(), key.to_vec()), OffchainOverlayedChange::Remove); - } + self.changes.insert((prefix.to_vec(), key.to_vec()), OffchainOverlayedChange::Remove); } /// Set the value associated with a key under a prefix to the value provided. pub fn set(&mut self, prefix: &[u8], key: &[u8], value: &[u8]) { - if let Self::Enabled(ref mut storage) = self { - let _ = storage.insert((prefix.to_vec(), key.to_vec()), OffchainOverlayedChange::SetValue(value.to_vec())); - } + self.changes.insert( + (prefix.to_vec(), key.to_vec()), + OffchainOverlayedChange::SetValue(value.to_vec()), + ); } /// Obtain a associated value to the given key in storage with prefix. pub fn get(&self, prefix: &[u8], key: &[u8]) -> Option { - if let Self::Enabled(ref storage) = self { - let key = (prefix.to_vec(), key.to_vec()); - storage.get(&key).cloned() - } else { - None - } + let key = (prefix.to_vec(), key.to_vec()); + self.changes.get(&key).cloned() } } -use std::collections::hash_map; - -/// Iterate by reference over the prepared offchain worker storage changes. -pub struct OffchainOverlayedChangesIter<'i> { - inner: Option, Vec), OffchainOverlayedChange>>, -} - -impl<'i> Iterator for OffchainOverlayedChangesIter<'i> { - type Item = (&'i (Vec, Vec), &'i OffchainOverlayedChange); - fn next(&mut self) -> Option { - if let Some(ref mut iter) = self.inner { - iter.next() - } else { - None - } - } -} - -impl<'i> OffchainOverlayedChangesIter<'i> { - /// Create a new iterator based on a refernce to the parent container. - pub fn new(container: &'i OffchainOverlayedChanges) -> Self { - match container { - OffchainOverlayedChanges::Enabled(inner) => Self { - inner: Some(inner.iter()) - }, - OffchainOverlayedChanges::Disabled => Self { inner: None, }, - } - } -} - - -/// Iterate by value over the prepared offchain worker storage changes. -pub struct OffchainOverlayedChangesIntoIter { - inner: Option,Vec),OffchainOverlayedChange>>, -} - -impl Iterator for OffchainOverlayedChangesIntoIter { - type Item = ((Vec, Vec), OffchainOverlayedChange); - fn next(&mut self) -> Option { - if let Some(ref mut iter) = self.inner { - iter.next() - } else { - None - } - } -} - -impl OffchainOverlayedChangesIntoIter { - /// Create a new iterator by consuming the collection. - pub fn new(container: OffchainOverlayedChanges) -> Self { - match container { - OffchainOverlayedChanges::Enabled(inner) => Self { - inner: Some(inner.into_iter()) - }, - OffchainOverlayedChanges::Disabled => Self { inner: None, }, - } - } -} - -/// Iterate over all items while draining them from the collection. -pub struct OffchainOverlayedChangesDrain<'d> { - inner: Option, Vec), OffchainOverlayedChange>>, -} - -impl<'d> Iterator for OffchainOverlayedChangesDrain<'d> { - type Item = ((Vec, Vec), OffchainOverlayedChange); - fn next(&mut self) -> Option { - if let Some(ref mut iter) = self.inner { - iter.next() - } else { - None - } - } -} - -impl<'d> OffchainOverlayedChangesDrain<'d> { - /// Create a new iterator by taking a mut reference to the collection, - /// for the lifetime of the created drain iterator. - pub fn new(container: &'d mut OffchainOverlayedChanges) -> Self { - match container { - OffchainOverlayedChanges::Enabled(ref mut inner) => Self { - inner: Some(inner.drain()) - }, - OffchainOverlayedChanges::Disabled => Self { inner: None, }, - } - } -} - - #[cfg(test)] mod test { use super::*; @@ -259,7 +142,7 @@ mod test { #[test] fn test_drain() { - let mut ooc = OffchainOverlayedChanges::enabled(); + let mut ooc = OffchainOverlayedChanges::default(); ooc.set(STORAGE_PREFIX,b"kkk", b"vvv"); let drained = ooc.drain().count(); assert_eq!(drained, 1); @@ -276,7 +159,7 @@ mod test { #[test] fn test_accumulated_set_remove_set() { - let mut ooc = OffchainOverlayedChanges::enabled(); + let mut ooc = OffchainOverlayedChanges::default(); ooc.set(STORAGE_PREFIX, b"ppp", b"qqq"); ooc.remove(STORAGE_PREFIX, b"ppp"); // keys are equiv, so it will overwrite the value and the overlay will contain diff --git a/substrate/primitives/state-machine/src/ext.rs b/substrate/primitives/state-machine/src/ext.rs index 3321f0561f..c872b4eaf7 100644 --- a/substrate/primitives/state-machine/src/ext.rs +++ b/substrate/primitives/state-machine/src/ext.rs @@ -35,8 +35,6 @@ use codec::{Decode, Encode, EncodeAppend}; use sp_std::{fmt, any::{Any, TypeId}, vec::Vec, vec, boxed::Box}; use crate::{warn, trace, log_error}; #[cfg(feature = "std")] -use sp_core::offchain::storage::OffchainOverlayedChanges; -#[cfg(feature = "std")] use crate::changes_trie::State as ChangesTrieState; use crate::StorageTransactionCache; #[cfg(feature = "std")] @@ -100,9 +98,6 @@ pub struct Ext<'a, H, N, B> { /// The overlayed changes to write to. overlay: &'a mut OverlayedChanges, - /// The overlayed changes destined for the Offchain DB. - #[cfg(feature = "std")] - offchain_overlay: &'a mut OffchainOverlayedChanges, /// The storage backend to read from. backend: &'a B, /// The cache for the storage transactions. @@ -146,7 +141,6 @@ impl<'a, H, N, B> Ext<'a, H, N, B> #[cfg(feature = "std")] pub fn new( overlay: &'a mut OverlayedChanges, - offchain_overlay: &'a mut OffchainOverlayedChanges, storage_transaction_cache: &'a mut StorageTransactionCache, backend: &'a B, changes_trie_state: Option>, @@ -154,7 +148,6 @@ impl<'a, H, N, B> Ext<'a, H, N, B> ) -> Self { Self { overlay, - offchain_overlay, backend, changes_trie_state, storage_transaction_cache, @@ -170,12 +163,6 @@ impl<'a, H, N, B> Ext<'a, H, N, B> fn mark_dirty(&mut self) { self.storage_transaction_cache.reset(); } - - /// Read only accessor for the scheduled overlay changes. - #[cfg(feature = "std")] - pub fn get_offchain_storage_changes(&self) -> &OffchainOverlayedChanges { - &*self.offchain_overlay - } } #[cfg(test)] @@ -208,10 +195,10 @@ where { #[cfg(feature = "std")] fn set_offchain_storage(&mut self, key: &[u8], value: Option<&[u8]>) { - use ::sp_core::offchain::STORAGE_PREFIX; + use sp_core::offchain::STORAGE_PREFIX; match value { - Some(value) => self.offchain_overlay.set(STORAGE_PREFIX, key, value), - None => self.offchain_overlay.remove(STORAGE_PREFIX, key), + Some(value) => self.overlay.offchain_set_storage(STORAGE_PREFIX, key, value), + None => self.overlay.offchain_remove_storage(STORAGE_PREFIX, key), } } @@ -829,11 +816,9 @@ mod tests { changes } - fn prepare_offchain_overlay_with_changes() -> OffchainOverlayedChanges { - let mut ooc = OffchainOverlayedChanges::enabled(); - ooc.set(offchain::STORAGE_PREFIX, b"k1", b"v1"); - ooc.set(offchain::STORAGE_PREFIX, b"k2", b"v2"); - ooc + fn prepare_offchain_overlay_with_changes(overlay: &mut OverlayedChanges) { + overlay.offchain_set_storage(offchain::STORAGE_PREFIX, b"k1", b"v1"); + overlay.offchain_set_storage(offchain::STORAGE_PREFIX, b"k2", b"v2"); } fn changes_trie_config() -> ChangesTrieConfiguration { @@ -846,32 +831,30 @@ mod tests { #[test] fn storage_changes_root_is_none_when_storage_is_not_provided() { let mut overlay = prepare_overlay_with_changes(); - let mut offchain_overlay = prepare_offchain_overlay_with_changes(); let mut cache = StorageTransactionCache::default(); let backend = TestBackend::default(); - let mut ext = TestExt::new(&mut overlay, &mut offchain_overlay, &mut cache, &backend, None, None); + let mut ext = TestExt::new(&mut overlay, &mut cache, &backend, None, None); assert_eq!(ext.storage_changes_root(&H256::default().encode()).unwrap(), None); } #[test] fn storage_changes_root_is_none_when_state_is_not_provided() { let mut overlay = prepare_overlay_with_changes(); - let mut offchain_overlay = prepare_offchain_overlay_with_changes(); let mut cache = StorageTransactionCache::default(); let backend = TestBackend::default(); - let mut ext = TestExt::new(&mut overlay, &mut offchain_overlay, &mut cache, &backend, None, None); + let mut ext = TestExt::new(&mut overlay, &mut cache, &backend, None, None); assert_eq!(ext.storage_changes_root(&H256::default().encode()).unwrap(), None); } #[test] fn storage_changes_root_is_some_when_extrinsic_changes_are_non_empty() { let mut overlay = prepare_overlay_with_changes(); - let mut offchain_overlay = prepare_offchain_overlay_with_changes(); + prepare_offchain_overlay_with_changes(&mut overlay); let mut cache = StorageTransactionCache::default(); let storage = TestChangesTrieStorage::with_blocks(vec![(99, Default::default())]); let state = Some(ChangesTrieState::new(changes_trie_config(), Zero::zero(), &storage)); let backend = TestBackend::default(); - let mut ext = TestExt::new(&mut overlay, &mut offchain_overlay, &mut cache, &backend, state, None); + let mut ext = TestExt::new(&mut overlay, &mut cache, &backend, state, None); assert_eq!( ext.storage_changes_root(&H256::default().encode()).unwrap(), Some(hex!("bb0c2ef6e1d36d5490f9766cfcc7dfe2a6ca804504c3bb206053890d6dd02376").to_vec()), @@ -881,14 +864,14 @@ mod tests { #[test] fn storage_changes_root_is_some_when_extrinsic_changes_are_empty() { let mut overlay = prepare_overlay_with_changes(); - let mut offchain_overlay = prepare_offchain_overlay_with_changes(); + prepare_offchain_overlay_with_changes(&mut overlay); let mut cache = StorageTransactionCache::default(); overlay.set_collect_extrinsics(false); overlay.set_storage(vec![1], None); let storage = TestChangesTrieStorage::with_blocks(vec![(99, Default::default())]); let state = Some(ChangesTrieState::new(changes_trie_config(), Zero::zero(), &storage)); let backend = TestBackend::default(); - let mut ext = TestExt::new(&mut overlay, &mut offchain_overlay, &mut cache, &backend, state, None); + let mut ext = TestExt::new(&mut overlay, &mut cache, &backend, state, None); assert_eq!( ext.storage_changes_root(&H256::default().encode()).unwrap(), Some(hex!("96f5aae4690e7302737b6f9b7f8567d5bbb9eac1c315f80101235a92d9ec27f4").to_vec()), @@ -901,7 +884,7 @@ mod tests { let mut overlay = OverlayedChanges::default(); overlay.set_storage(vec![20], None); overlay.set_storage(vec![30], Some(vec![31])); - let mut offchain_overlay = prepare_offchain_overlay_with_changes(); + prepare_offchain_overlay_with_changes(&mut overlay); let backend = Storage { top: map![ vec![10] => vec![10], @@ -911,7 +894,7 @@ mod tests { children_default: map![] }.into(); - let ext = TestExt::new(&mut overlay, &mut offchain_overlay, &mut cache, &backend, None, None); + let ext = TestExt::new(&mut overlay, &mut cache, &backend, None, None); // next_backend < next_overlay assert_eq!(ext.next_storage_key(&[5]), Some(vec![10])); @@ -927,7 +910,7 @@ mod tests { drop(ext); overlay.set_storage(vec![50], Some(vec![50])); - let ext = TestExt::new(&mut overlay, &mut offchain_overlay, &mut cache, &backend, None, None); + let ext = TestExt::new(&mut overlay, &mut cache, &backend, None, None); // next_overlay exist but next_backend doesn't exist assert_eq!(ext.next_storage_key(&[40]), Some(vec![50])); @@ -956,10 +939,9 @@ mod tests { ], }.into(); + prepare_offchain_overlay_with_changes(&mut overlay); - let mut offchain_overlay = prepare_offchain_overlay_with_changes(); - - let ext = TestExt::new(&mut overlay, &mut offchain_overlay, &mut cache, &backend, None, None); + let ext = TestExt::new(&mut overlay, &mut cache, &backend, None, None); // next_backend < next_overlay assert_eq!(ext.next_child_storage_key(child_info, &[5]), Some(vec![10])); @@ -975,7 +957,7 @@ mod tests { drop(ext); overlay.set_child_storage(child_info, vec![50], Some(vec![50])); - let ext = TestExt::new(&mut overlay, &mut offchain_overlay, &mut cache, &backend, None, None); + let ext = TestExt::new(&mut overlay, &mut cache, &backend, None, None); // next_overlay exist but next_backend doesn't exist assert_eq!(ext.next_child_storage_key(child_info, &[40]), Some(vec![50])); @@ -989,7 +971,7 @@ mod tests { let mut overlay = OverlayedChanges::default(); overlay.set_child_storage(child_info, vec![20], None); overlay.set_child_storage(child_info, vec![30], Some(vec![31])); - let mut offchain_overlay = prepare_offchain_overlay_with_changes(); + prepare_offchain_overlay_with_changes(&mut overlay); let backend = Storage { top: map![], children_default: map![ @@ -1004,7 +986,7 @@ mod tests { ], }.into(); - let ext = TestExt::new(&mut overlay, &mut offchain_overlay, &mut cache, &backend, None, None); + let ext = TestExt::new(&mut overlay, &mut cache, &backend, None, None); assert_eq!(ext.child_storage(child_info, &[10]), Some(vec![10])); assert_eq!( @@ -1031,7 +1013,7 @@ mod tests { let child_info = &child_info; let mut cache = StorageTransactionCache::default(); let mut overlay = OverlayedChanges::default(); - let mut offchain_overlay = prepare_offchain_overlay_with_changes(); + prepare_offchain_overlay_with_changes(&mut overlay); let backend = Storage { top: map![], children_default: map![ @@ -1044,7 +1026,7 @@ mod tests { ], }.into(); - let ext = TestExt::new(&mut overlay, &mut offchain_overlay, &mut cache, &backend, None, None); + let ext = TestExt::new(&mut overlay, &mut cache, &backend, None, None); use sp_core::storage::well_known_keys; let mut ext = ext; diff --git a/substrate/primitives/state-machine/src/lib.rs b/substrate/primitives/state-machine/src/lib.rs index 6d85b56f8a..c8b4703755 100644 --- a/substrate/primitives/state-machine/src/lib.rs +++ b/substrate/primitives/state-machine/src/lib.rs @@ -172,7 +172,6 @@ mod execution { use hash_db::Hasher; use codec::{Decode, Encode, Codec}; use sp_core::{ - offchain::storage::OffchainOverlayedChanges, storage::ChildInfo, NativeOrEncoded, NeverNativeValue, hexdisplay::HexDisplay, traits::{CodeExecutor, CallInWasmExt, RuntimeCode, SpawnNamed}, }; @@ -299,7 +298,6 @@ mod execution { method: &'a str, call_data: &'a [u8], overlay: &'a mut OverlayedChanges, - offchain_overlay: &'a mut OffchainOverlayedChanges, extensions: Extensions, changes_trie_state: Option>, storage_transaction_cache: Option<&'a mut StorageTransactionCache>, @@ -329,7 +327,6 @@ mod execution { backend: &'a B, changes_trie_state: Option>, overlay: &'a mut OverlayedChanges, - offchain_overlay: &'a mut OffchainOverlayedChanges, exec: &'a Exec, method: &'a str, call_data: &'a [u8], @@ -347,7 +344,6 @@ mod execution { call_data, extensions, overlay, - offchain_overlay, changes_trie_state, storage_transaction_cache: None, runtime_code, @@ -407,7 +403,6 @@ mod execution { let mut ext = Ext::new( self.overlay, - self.offchain_overlay, cache, self.backend, self.changes_trie_state.clone(), @@ -621,13 +616,11 @@ mod execution { N: crate::changes_trie::BlockNumber, Spawn: SpawnNamed + Send + 'static, { - let mut offchain_overlay = OffchainOverlayedChanges::default(); let proving_backend = proving_backend::ProvingBackend::new(trie_backend); let mut sm = StateMachine::<_, H, N, Exec>::new( &proving_backend, None, overlay, - &mut offchain_overlay, exec, method, call_data, @@ -691,12 +684,10 @@ mod execution { N: crate::changes_trie::BlockNumber, Spawn: SpawnNamed + Send + 'static, { - let mut offchain_overlay = OffchainOverlayedChanges::default(); let mut sm = StateMachine::<_, H, N, Exec>::new( trie_backend, None, overlay, - &mut offchain_overlay, exec, method, call_data, @@ -879,7 +870,6 @@ mod tests { use std::{result, collections::HashMap}; use codec::Decode; use sp_core::{ - offchain::storage::OffchainOverlayedChanges, storage::ChildInfo, NativeOrEncoded, NeverNativeValue, traits::CodeExecutor, }; @@ -966,14 +956,12 @@ mod tests { fn execute_works() { let backend = trie_backend::tests::test_trie(); let mut overlayed_changes = Default::default(); - let mut offchain_overlayed_changes = Default::default(); let wasm_code = RuntimeCode::empty(); let mut state_machine = StateMachine::new( &backend, changes_trie::disabled_state::<_, u64>(), &mut overlayed_changes, - &mut offchain_overlayed_changes, &DummyCodeExecutor { change_changes_trie_config: false, native_available: true, @@ -998,14 +986,12 @@ mod tests { fn execute_works_with_native_else_wasm() { let backend = trie_backend::tests::test_trie(); let mut overlayed_changes = Default::default(); - let mut offchain_overlayed_changes = Default::default(); let wasm_code = RuntimeCode::empty(); let mut state_machine = StateMachine::new( &backend, changes_trie::disabled_state::<_, u64>(), &mut overlayed_changes, - &mut offchain_overlayed_changes, &DummyCodeExecutor { change_changes_trie_config: false, native_available: true, @@ -1027,14 +1013,12 @@ mod tests { let mut consensus_failed = false; let backend = trie_backend::tests::test_trie(); let mut overlayed_changes = Default::default(); - let mut offchain_overlayed_changes = Default::default(); let wasm_code = RuntimeCode::empty(); let mut state_machine = StateMachine::new( &backend, changes_trie::disabled_state::<_, u64>(), &mut overlayed_changes, - &mut offchain_overlayed_changes, &DummyCodeExecutor { change_changes_trie_config: false, native_available: true, @@ -1118,11 +1102,9 @@ mod tests { overlay.set_storage(b"bbd".to_vec(), Some(b"42".to_vec())); { - let mut offchain_overlay = Default::default(); let mut cache = StorageTransactionCache::default(); let mut ext = Ext::new( &mut overlay, - &mut offchain_overlay, &mut cache, backend, changes_trie::disabled_state::<_, u64>(), @@ -1167,11 +1149,9 @@ mod tests { overlay.set_child_storage(&child_info, b"4".to_vec(), Some(b"1312".to_vec())); { - let mut offchain_overlay = Default::default(); let mut cache = StorageTransactionCache::default(); let mut ext = Ext::new( &mut overlay, - &mut offchain_overlay, &mut cache, &backend, changes_trie::disabled_state::<_, u64>(), @@ -1209,11 +1189,9 @@ mod tests { ]; let backend = InMemoryBackend::::from(initial); let mut overlay = OverlayedChanges::default(); - let mut offchain_overlay = Default::default(); let mut cache = StorageTransactionCache::default(); let mut ext = Ext::new( &mut overlay, - &mut offchain_overlay, &mut cache, &backend, changes_trie::disabled_state::<_, u64>(), @@ -1234,11 +1212,9 @@ mod tests { let mut state = new_in_mem::(); let backend = state.as_trie_backend().unwrap(); let mut overlay = OverlayedChanges::default(); - let mut offchain_overlay = OffchainOverlayedChanges::default(); let mut cache = StorageTransactionCache::default(); let mut ext = Ext::new( &mut overlay, - &mut offchain_overlay, &mut cache, backend, changes_trie::disabled_state::<_, u64>(), @@ -1282,12 +1258,10 @@ mod tests { let mut state = new_in_mem::(); let backend = state.as_trie_backend().unwrap(); let mut overlay = OverlayedChanges::default(); - let mut offchain_overlay = OffchainOverlayedChanges::default(); let mut cache = StorageTransactionCache::default(); { let mut ext = Ext::new( &mut overlay, - &mut offchain_overlay, &mut cache, backend, changes_trie::disabled_state::<_, u64>(), @@ -1304,7 +1278,6 @@ mod tests { { let mut ext = Ext::new( &mut overlay, - &mut offchain_overlay, &mut cache, backend, changes_trie::disabled_state::<_, u64>(), @@ -1323,7 +1296,6 @@ mod tests { { let ext = Ext::new( &mut overlay, - &mut offchain_overlay, &mut cache, backend, changes_trie::disabled_state::<_, u64>(), @@ -1346,14 +1318,12 @@ mod tests { let mut cache = StorageTransactionCache::default(); let mut state = new_in_mem::(); let backend = state.as_trie_backend().unwrap(); - let mut offchain_overlay = OffchainOverlayedChanges::default(); let mut overlay = OverlayedChanges::default(); // For example, block initialization with event. { let mut ext = Ext::new( &mut overlay, - &mut offchain_overlay, &mut cache, backend, changes_trie::disabled_state::<_, u64>(), @@ -1368,7 +1338,6 @@ mod tests { { let mut ext = Ext::new( &mut overlay, - &mut offchain_overlay, &mut cache, backend, changes_trie::disabled_state::<_, u64>(), @@ -1393,7 +1362,6 @@ mod tests { { let mut ext = Ext::new( &mut overlay, - &mut offchain_overlay, &mut cache, backend, changes_trie::disabled_state::<_, u64>(), @@ -1419,7 +1387,6 @@ mod tests { { let ext = Ext::new( &mut overlay, - &mut offchain_overlay, &mut cache, backend, changes_trie::disabled_state::<_, u64>(), @@ -1495,14 +1462,12 @@ mod tests { use crate::trie_backend::tests::test_trie; let mut overlay = OverlayedChanges::default(); - let mut offchain_overlay = OffchainOverlayedChanges::default(); let mut transaction = { let backend = test_trie(); let mut cache = StorageTransactionCache::default(); let mut ext = Ext::new( &mut overlay, - &mut offchain_overlay, &mut cache, &backend, changes_trie::disabled_state::<_, u64>(), @@ -1543,11 +1508,9 @@ mod tests { assert_eq!(overlay.storage(b"bbb"), None); { - let mut offchain_overlay = Default::default(); let mut cache = StorageTransactionCache::default(); let mut ext = Ext::new( &mut overlay, - &mut offchain_overlay, &mut cache, backend, changes_trie::disabled_state::<_, u64>(), @@ -1571,14 +1534,12 @@ mod tests { let backend = trie_backend::tests::test_trie(); let mut overlayed_changes = Default::default(); - let mut offchain_overlayed_changes = Default::default(); let wasm_code = RuntimeCode::empty(); let mut state_machine = StateMachine::new( &backend, changes_trie::disabled_state::<_, u64>(), &mut overlayed_changes, - &mut offchain_overlayed_changes, &DummyCodeExecutor { change_changes_trie_config: false, native_available: true, diff --git a/substrate/primitives/state-machine/src/overlayed_changes/mod.rs b/substrate/primitives/state-machine/src/overlayed_changes/mod.rs index edf4c2e88e..97d7a4f057 100644 --- a/substrate/primitives/state-machine/src/overlayed_changes/mod.rs +++ b/substrate/primitives/state-machine/src/overlayed_changes/mod.rs @@ -43,7 +43,7 @@ use sp_std::collections::btree_set::BTreeSet; use codec::{Decode, Encode}; use sp_core::storage::{well_known_keys::EXTRINSIC_INDEX, ChildInfo}; #[cfg(feature = "std")] -use sp_core::offchain::storage::OffchainOverlayedChanges; +use sp_core::offchain::storage::{OffchainOverlayedChanges, OffchainOverlayedChange}; use hash_db::Hasher; use crate::DefaultError; use sp_externalities::{Extensions, Extension}; @@ -101,6 +101,9 @@ pub struct OverlayedChanges { collect_extrinsics: bool, /// Collect statistic on this execution. stats: StateMachineStats, + /// Offchain related changes. + #[cfg(feature = "std")] + offchain: OffchainOverlayedChanges, } /// A storage changes structure that can be generated by the data collected in [`OverlayedChanges`]. @@ -523,7 +526,7 @@ impl OverlayedChanges { main_storage_changes: main_storage_changes.collect(), child_storage_changes: child_storage_changes.map(|(sk, it)| (sk, it.0.collect())).collect(), #[cfg(feature = "std")] - offchain_storage_changes: Default::default(), + offchain_storage_changes: std::mem::take(&mut self.offchain), transaction, transaction_storage_root, #[cfg(feature = "std")] @@ -629,6 +632,40 @@ impl OverlayedChanges { overlay.next_change(key) ) } + + /// Set a value in the offchain storage. + #[cfg(feature = "std")] + pub fn offchain_set_storage(&mut self, prefix: &[u8], key: &[u8], value: &[u8]) { + self.offchain.set(prefix, key, value); + } + + /// Clear a value in the offchain storage. + #[cfg(feature = "std")] + pub fn offchain_remove_storage(&mut self, prefix: &[u8], key: &[u8]) { + self.offchain.remove(prefix, key); + } + + /// Get a value in the offchain storage. + #[cfg(feature = "std")] + pub fn offchain_get_storage( + &mut self, + prefix: &[u8], + key: &[u8], + ) -> Option { + self.offchain.get(prefix, key) + } + + /// Returns a reference to the offchain overlay. + #[cfg(feature = "std")] + pub fn offchain_overlay(&self) -> &OffchainOverlayedChanges { + &self.offchain + } + + /// Returns a mutable reference to the offchain overlay. + #[cfg(feature = "std")] + pub fn offchain_overlay_mut(&mut self) -> &mut OffchainOverlayedChanges { + &mut self.offchain + } } #[cfg(feature = "std")] @@ -789,11 +826,9 @@ mod tests { overlay.set_storage(b"dogglesworth".to_vec(), Some(b"cat".to_vec())); overlay.set_storage(b"doug".to_vec(), None); - let mut offchain_overlay = Default::default(); let mut cache = StorageTransactionCache::default(); let mut ext = Ext::new( &mut overlay, - &mut offchain_overlay, &mut cache, &backend, crate::changes_trie::disabled_state::<_, u64>(), diff --git a/substrate/primitives/state-machine/src/testing.rs b/substrate/primitives/state-machine/src/testing.rs index 40e37f2116..3ef1ff09b1 100644 --- a/substrate/primitives/state-machine/src/testing.rs +++ b/substrate/primitives/state-machine/src/testing.rs @@ -33,10 +33,7 @@ use crate::{ use codec::{Decode, Encode}; use hash_db::Hasher; use sp_core::{ - offchain::{ - testing::TestPersistentOffchainDB, - storage::OffchainOverlayedChanges - }, + offchain::testing::TestPersistentOffchainDB, storage::{ well_known_keys::{CHANGES_TRIE_CONFIG, CODE, HEAP_PAGES, is_child_storage_key}, Storage, @@ -52,7 +49,6 @@ where H::Out: codec::Codec + Ord, { overlay: OverlayedChanges, - offchain_overlay: OffchainOverlayedChanges, offchain_db: TestPersistentOffchainDB, storage_transaction_cache: StorageTransactionCache< as Backend>::Transaction, H, N @@ -71,7 +67,6 @@ impl TestExternalities pub fn ext(&mut self) -> Ext> { Ext::new( &mut self.overlay, - &mut self.offchain_overlay, &mut self.storage_transaction_cache, &self.backend, match self.changes_trie_config.clone() { @@ -109,8 +104,6 @@ impl TestExternalities storage.top.insert(HEAP_PAGES.to_vec(), 8u64.encode()); storage.top.insert(CODE.to_vec(), code.to_vec()); - let offchain_overlay = OffchainOverlayedChanges::enabled(); - let mut extensions = Extensions::default(); extensions.register(TaskExecutorExt::new(TaskExecutor::new())); @@ -118,7 +111,6 @@ impl TestExternalities TestExternalities { overlay, - offchain_overlay, offchain_db, changes_trie_config, extensions, @@ -128,9 +120,14 @@ impl TestExternalities } } + /// Returns the overlayed changes. + pub fn overlayed_changes(&self) -> &OverlayedChanges { + &self.overlay + } + /// Move offchain changes from overlay to the persistent store. pub fn persist_offchain_overlay(&mut self) { - self.offchain_db.apply_offchain_changes(&mut self.offchain_overlay); + self.offchain_db.apply_offchain_changes(self.overlay.offchain_overlay_mut()); } /// A shared reference type around the offchain worker storage. diff --git a/substrate/test-utils/client/src/lib.rs b/substrate/test-utils/client/src/lib.rs index 487be14a78..bf5b2b6a04 100644 --- a/substrate/test-utils/client/src/lib.rs +++ b/substrate/test-utils/client/src/lib.rs @@ -79,6 +79,7 @@ pub struct TestClientBuilder { keystore: Option, fork_blocks: ForkBlocks, bad_blocks: BadBlocks, + enable_offchain_indexing_api: bool, } impl Default @@ -114,6 +115,7 @@ impl TestClientBuilder TestClientBuilder Self { + self.enable_offchain_indexing_api = true; + self + } + /// Build the test client with the given native executor. pub fn build_with_executor( self, @@ -219,7 +227,10 @@ impl TestClientBuilder), StorageChange(Vec, Option>), ChangesTrieConfigUpdate(Option), + OffchainIndexSet(Vec, Vec), + OffchainIndexClear(Vec), } parity_util_mem::malloc_size_of_is_0!(Extrinsic); // non-opaque extrinsic does not need this @@ -177,6 +179,10 @@ impl BlindCheckable for Extrinsic { Extrinsic::StorageChange(key, value) => Ok(Extrinsic::StorageChange(key, value)), Extrinsic::ChangesTrieConfigUpdate(new_config) => Ok(Extrinsic::ChangesTrieConfigUpdate(new_config)), + Extrinsic::OffchainIndexSet(key, value) => + Ok(Extrinsic::OffchainIndexSet(key, value)), + Extrinsic::OffchainIndexClear(key) => + Ok(Extrinsic::OffchainIndexClear(key)), } } } @@ -1148,13 +1154,9 @@ fn test_witness(proof: StorageProof, root: crate::Hash) { root, ); let mut overlay = sp_state_machine::OverlayedChanges::default(); - #[cfg(feature = "std")] - let mut offchain_overlay = Default::default(); let mut cache = sp_state_machine::StorageTransactionCache::<_, _, BlockNumber>::default(); let mut ext = sp_state_machine::Ext::new( &mut overlay, - #[cfg(feature = "std")] - &mut offchain_overlay, &mut cache, &backend, #[cfg(feature = "std")] diff --git a/substrate/test-utils/runtime/src/system.rs b/substrate/test-utils/runtime/src/system.rs index 9fcb81b7b0..c379ec5de5 100644 --- a/substrate/test-utils/runtime/src/system.rs +++ b/substrate/test-utils/runtime/src/system.rs @@ -261,6 +261,14 @@ fn execute_transaction_backend(utx: &Extrinsic, extrinsic_index: u32) -> ApplyEx execute_storage_change(key, value.as_ref().map(|v| &**v)), Extrinsic::ChangesTrieConfigUpdate(ref new_config) => execute_changes_trie_config_update(new_config.clone()), + Extrinsic::OffchainIndexSet(key, value) => { + sp_io::offchain_index::set(&key, &value); + Ok(Ok(())) + }, + Extrinsic::OffchainIndexClear(key) => { + sp_io::offchain_index::clear(&key); + Ok(Ok(())) + } } } diff --git a/substrate/utils/frame/benchmarking-cli/src/command.rs b/substrate/utils/frame/benchmarking-cli/src/command.rs index 57b9a592f0..da8a1d98f0 100644 --- a/substrate/utils/frame/benchmarking-cli/src/command.rs +++ b/substrate/utils/frame/benchmarking-cli/src/command.rs @@ -62,7 +62,6 @@ impl BenchmarkCmd { let genesis_storage = spec.build_storage()?; let mut changes = Default::default(); - let mut offchain_changes = Default::default(); let cache_size = Some(self.database_cache_size as usize); let state = BenchmarkingState::::new(genesis_storage, cache_size)?; let executor = NativeExecutor::::new( @@ -80,7 +79,6 @@ impl BenchmarkCmd { &state, None, &mut changes, - &mut offchain_changes, &executor, "Benchmark_dispatch_benchmark", &(