diff --git a/substrate/Cargo.lock b/substrate/Cargo.lock index 742dd0ed85..5dddea40cb 100644 --- a/substrate/Cargo.lock +++ b/substrate/Cargo.lock @@ -4112,6 +4112,7 @@ dependencies = [ "node-primitives", "node-runtime", "pallet-contracts-rpc", + "pallet-mmr-rpc", "pallet-transaction-payment-rpc", "sc-chain-spec", "sc-client-api", @@ -5041,6 +5042,24 @@ dependencies = [ "sp-std", ] +[[package]] +name = "pallet-mmr-rpc" +version = "3.0.0" +dependencies = [ + "jsonrpc-core", + "jsonrpc-core-client", + "jsonrpc-derive", + "pallet-mmr-primitives", + "parity-scale-codec", + "serde", + "serde_json", + "sp-api", + "sp-blockchain", + "sp-core", + "sp-rpc", + "sp-runtime", +] + [[package]] name = "pallet-multisig" version = "3.0.0" @@ -7548,6 +7567,7 @@ dependencies = [ "fnv", "futures 0.3.12", "futures-timer 3.0.2", + "hex", "hyper 0.13.9", "hyper-rustls", "lazy_static", @@ -9336,6 +9356,7 @@ dependencies = [ "sc-consensus", "sc-executor", "sc-light", + "sc-offchain", "sc-service", "serde", "serde_json", diff --git a/substrate/Cargo.toml b/substrate/Cargo.toml index 9a494d6aff..4d8cfc3e97 100644 --- a/substrate/Cargo.toml +++ b/substrate/Cargo.toml @@ -89,6 +89,7 @@ members = [ "frame/membership", "frame/merkle-mountain-range", "frame/merkle-mountain-range/primitives", + "frame/merkle-mountain-range/rpc", "frame/metadata", "frame/multisig", "frame/nicks", diff --git a/substrate/bin/node-template/node/src/service.rs b/substrate/bin/node-template/node/src/service.rs index 368767fbdd..4ea54dc817 100644 --- a/substrate/bin/node-template/node/src/service.rs +++ b/substrate/bin/node-template/node/src/service.rs @@ -141,7 +141,7 @@ pub fn new_full(mut config: Configuration) -> Result if config.offchain_worker.enabled { sc_service::build_offchain_workers( - &config, backend.clone(), task_manager.spawn_handle(), client.clone(), network.clone(), + &config, task_manager.spawn_handle(), client.clone(), network.clone(), ); } @@ -323,7 +323,7 @@ pub fn new_light(mut config: Configuration) -> Result if config.offchain_worker.enabled { sc_service::build_offchain_workers( - &config, backend.clone(), task_manager.spawn_handle(), client.clone(), network.clone(), + &config, task_manager.spawn_handle(), client.clone(), network.clone(), ); } diff --git a/substrate/bin/node/cli/src/service.rs b/substrate/bin/node/cli/src/service.rs index 6fed5bf5c6..92f30a7257 100644 --- a/substrate/bin/node/cli/src/service.rs +++ b/substrate/bin/node/cli/src/service.rs @@ -219,7 +219,7 @@ pub fn new_full_base( if config.offchain_worker.enabled { sc_service::build_offchain_workers( - &config, backend.clone(), task_manager.spawn_handle(), client.clone(), network.clone(), + &config, task_manager.spawn_handle(), client.clone(), network.clone(), ); } @@ -430,7 +430,7 @@ pub fn new_light_base(mut config: Configuration) -> Result<( if config.offchain_worker.enabled { sc_service::build_offchain_workers( - &config, backend.clone(), task_manager.spawn_handle(), client.clone(), network.clone(), + &config, task_manager.spawn_handle(), client.clone(), network.clone(), ); } diff --git a/substrate/bin/node/rpc/Cargo.toml b/substrate/bin/node/rpc/Cargo.toml index 1689d0e824..7a25b6d8b0 100644 --- a/substrate/bin/node/rpc/Cargo.toml +++ b/substrate/bin/node/rpc/Cargo.toml @@ -15,6 +15,7 @@ jsonrpc-core = "15.1.0" node-primitives = { version = "2.0.0", path = "../primitives" } node-runtime = { version = "2.0.0", path = "../runtime" } pallet-contracts-rpc = { version = "3.0.0", path = "../../../frame/contracts/rpc/" } +pallet-mmr-rpc = { version = "3.0.0", path = "../../../frame/merkle-mountain-range/rpc/" } pallet-transaction-payment-rpc = { version = "3.0.0", path = "../../../frame/transaction-payment/rpc/" } sc-client-api = { version = "3.0.0", path = "../../../client/api" } sc-consensus-babe = { version = "0.9.0", path = "../../../client/consensus/babe" } diff --git a/substrate/bin/node/rpc/src/lib.rs b/substrate/bin/node/rpc/src/lib.rs index e68ca6843b..1d9f88c8c9 100644 --- a/substrate/bin/node/rpc/src/lib.rs +++ b/substrate/bin/node/rpc/src/lib.rs @@ -116,6 +116,7 @@ pub fn create_full( HeaderMetadata + Sync + Send + 'static, C::Api: substrate_frame_rpc_system::AccountNonceApi, C::Api: pallet_contracts_rpc::ContractsRuntimeApi, + C::Api: pallet_mmr_rpc::MmrRuntimeApi::Hash>, C::Api: pallet_transaction_payment_rpc::TransactionPaymentRuntimeApi, C::Api: BabeApi, C::Api: BlockBuilder, @@ -126,6 +127,7 @@ pub fn create_full( { use substrate_frame_rpc_system::{FullSystem, SystemApi}; use pallet_contracts_rpc::{Contracts, ContractsApi}; + use pallet_mmr_rpc::{MmrApi, Mmr}; use pallet_transaction_payment_rpc::{TransactionPayment, TransactionPaymentApi}; let mut io = jsonrpc_core::IoHandler::default(); @@ -161,6 +163,9 @@ pub fn create_full( io.extend_with( ContractsApi::to_delegate(Contracts::new(client.clone())) ); + io.extend_with( + MmrApi::to_delegate(Mmr::new(client.clone())) + ); io.extend_with( TransactionPaymentApi::to_delegate(TransactionPayment::new(client.clone())) ); diff --git a/substrate/bin/node/runtime/src/lib.rs b/substrate/bin/node/runtime/src/lib.rs index 55f42b5723..8bb5cf0858 100644 --- a/substrate/bin/node/runtime/src/lib.rs +++ b/substrate/bin/node/runtime/src/lib.rs @@ -1377,23 +1377,31 @@ impl_runtime_apis! { impl pallet_mmr::primitives::MmrApi< Block, - mmr::Leaf, mmr::Hash, > for Runtime { - fn generate_proof(leaf_index: u64) -> Result<(mmr::Leaf, mmr::Proof), mmr::Error> { + fn generate_proof(leaf_index: u64) + -> Result<(mmr::EncodableOpaqueLeaf, mmr::Proof), mmr::Error> + { Mmr::generate_proof(leaf_index) + .map(|(leaf, proof)| (mmr::EncodableOpaqueLeaf::from_leaf(&leaf), proof)) } - fn verify_proof(leaf: mmr::Leaf, proof: mmr::Proof) -> Result<(), mmr::Error> { + fn verify_proof(leaf: mmr::EncodableOpaqueLeaf, proof: mmr::Proof) + -> Result<(), mmr::Error> + { + let leaf: mmr::Leaf = leaf + .into_opaque_leaf() + .try_decode() + .ok_or(mmr::Error::Verify)?; Mmr::verify_leaf(leaf, proof) } fn verify_proof_stateless( root: mmr::Hash, - leaf: Vec, + leaf: mmr::EncodableOpaqueLeaf, proof: mmr::Proof ) -> Result<(), mmr::Error> { - let node = mmr::DataOrHash::Data(mmr::OpaqueLeaf(leaf)); + let node = mmr::DataOrHash::Data(leaf.into_opaque_leaf()); pallet_mmr::verify_leaf_proof::(root, node, proof) } } diff --git a/substrate/bin/node/testing/src/bench.rs b/substrate/bin/node/testing/src/bench.rs index a6f65b86a0..668284101b 100644 --- a/substrate/bin/node/testing/src/bench.rs +++ b/substrate/bin/node/testing/src/bench.rs @@ -417,13 +417,14 @@ impl BenchDb { }; let task_executor = TaskExecutor::new(); - let (client, backend) = sc_service::new_client( - db_config, + let backend = sc_service::new_db_backend(db_config).expect("Should not fail"); + let client = sc_service::new_client( + backend.clone(), NativeExecutor::new(WasmExecutionMethod::Compiled, None, 8), &keyring.generate_genesis(), None, None, - ExecutionExtensions::new(profile.into_execution_strategies(), None), + ExecutionExtensions::new(profile.into_execution_strategies(), None, None), Box::new(task_executor.clone()), None, Default::default(), diff --git a/substrate/client/api/src/execution_extensions.rs b/substrate/client/api/src/execution_extensions.rs index 1b13f2c6cf..2f17408b7d 100644 --- a/substrate/client/api/src/execution_extensions.rs +++ b/substrate/client/api/src/execution_extensions.rs @@ -26,7 +26,7 @@ use std::sync::{Weak, Arc}; use codec::Decode; use sp_core::{ ExecutionContext, - offchain::{self, OffchainExt, TransactionPoolExt}, + offchain::{self, OffchainWorkerExt, TransactionPoolExt, OffchainDbExt}, }; use sp_keystore::{KeystoreExt, SyncCryptoStorePtr}; use sp_runtime::{ @@ -76,6 +76,18 @@ impl ExtensionsFactory for () { } } +/// Create a Offchain DB accessor object. +pub trait DbExternalitiesFactory: Send + Sync { + /// Create [`offchain::DbExternalities`] instance. + fn create(&self) -> Box; +} + +impl DbExternalitiesFactory for T { + fn create(&self) -> Box { + Box::new(self.clone()) + } +} + /// A producer of execution extensions for offchain calls. /// /// This crate aggregates extensions available for the offchain calls @@ -84,6 +96,7 @@ impl ExtensionsFactory for () { pub struct ExecutionExtensions { strategies: ExecutionStrategies, keystore: Option, + offchain_db: Option>, // FIXME: these two are only RwLock because of https://github.com/paritytech/substrate/issues/4587 // remove when fixed. // To break retain cycle between `Client` and `TransactionPool` we require this @@ -99,6 +112,7 @@ impl Default for ExecutionExtensions { Self { strategies: Default::default(), keystore: None, + offchain_db: None, transaction_pool: RwLock::new(None), extensions_factory: RwLock::new(Box::new(())), } @@ -110,12 +124,14 @@ impl ExecutionExtensions { pub fn new( strategies: ExecutionStrategies, keystore: Option, + offchain_db: Option>, ) -> Self { let transaction_pool = RwLock::new(None); let extensions_factory = Box::new(()); Self { strategies, keystore, + offchain_db, extensions_factory: RwLock::new(extensions_factory), transaction_pool, } @@ -164,9 +180,22 @@ impl ExecutionExtensions { } } + if capabilities.has(offchain::Capability::OffchainDbRead) || + capabilities.has(offchain::Capability::OffchainDbWrite) + { + if let Some(offchain_db) = self.offchain_db.as_ref() { + extensions.register( + OffchainDbExt::new(offchain::LimitedExternalities::new( + capabilities, + offchain_db.create(), + )) + ); + } + } + if let ExecutionContext::OffchainCall(Some(ext)) = context { extensions.register( - OffchainExt::new(offchain::LimitedExternalities::new(capabilities, ext.0)), + OffchainWorkerExt::new(offchain::LimitedExternalities::new(capabilities, ext.0)), ); } diff --git a/substrate/client/executor/src/integration_tests/mod.rs b/substrate/client/executor/src/integration_tests/mod.rs index b28e3ca243..d08f830f40 100644 --- a/substrate/client/executor/src/integration_tests/mod.rs +++ b/substrate/client/executor/src/integration_tests/mod.rs @@ -21,7 +21,7 @@ use codec::{Encode, Decode}; use hex_literal::hex; use sp_core::{ blake2_128, blake2_256, ed25519, sr25519, map, Pair, - offchain::{OffchainExt, testing}, + offchain::{OffchainWorkerExt, OffchainDbExt, testing}, traits::{Externalities, CallInWasm}, }; use sc_runtime_test::wasm_binary_unwrap; @@ -468,7 +468,7 @@ test_wasm_execution!(offchain_index); fn offchain_index(wasm_method: WasmExecutionMethod) { let mut ext = TestExternalities::default(); let (offchain, _state) = testing::TestOffchainExt::new(); - ext.register_extension(OffchainExt::new(offchain)); + ext.register_extension(OffchainWorkerExt::new(offchain)); call_in_wasm( "test_offchain_index_set", &[0], @@ -487,7 +487,8 @@ test_wasm_execution!(offchain_local_storage_should_work); fn offchain_local_storage_should_work(wasm_method: WasmExecutionMethod) { let mut ext = TestExternalities::default(); let (offchain, state) = testing::TestOffchainExt::new(); - ext.register_extension(OffchainExt::new(offchain)); + ext.register_extension(OffchainDbExt::new(offchain.clone())); + ext.register_extension(OffchainWorkerExt::new(offchain)); assert_eq!( call_in_wasm( "test_offchain_local_storage", @@ -504,7 +505,7 @@ test_wasm_execution!(offchain_http_should_work); fn offchain_http_should_work(wasm_method: WasmExecutionMethod) { let mut ext = TestExternalities::default(); let (offchain, state) = testing::TestOffchainExt::new(); - ext.register_extension(OffchainExt::new(offchain)); + ext.register_extension(OffchainWorkerExt::new(offchain)); state.write().expect_request(testing::PendingRequest { method: "POST".into(), uri: "http://localhost:12345".into(), diff --git a/substrate/client/offchain/Cargo.toml b/substrate/client/offchain/Cargo.toml index 5671affb6f..9aca829c70 100644 --- a/substrate/client/offchain/Cargo.toml +++ b/substrate/client/offchain/Cargo.toml @@ -14,23 +14,24 @@ targets = ["x86_64-unknown-linux-gnu"] [dependencies] bytes = "0.5" -sc-client-api = { version = "3.0.0", path = "../api" } -sp-api = { version = "3.0.0", path = "../../primitives/api" } +codec = { package = "parity-scale-codec", version = "2.0.0", features = ["derive"] } +hex = "0.4" fnv = "1.0.6" futures = "0.3.9" futures-timer = "3.0.1" log = "0.4.8" -threadpool = "1.7" num_cpus = "1.10" -sp-offchain = { version = "3.0.0", path = "../../primitives/offchain" } -codec = { package = "parity-scale-codec", version = "2.0.0", features = ["derive"] } parking_lot = "0.11.1" -sp-core = { version = "3.0.0", path = "../../primitives/core" } rand = "0.7.2" +sc-client-api = { version = "3.0.0", path = "../api" } +sc-keystore = { version = "3.0.0", path = "../keystore" } +sc-network = { version = "0.9.0", path = "../network" } +sp-api = { version = "3.0.0", path = "../../primitives/api" } +sp-core = { version = "3.0.0", path = "../../primitives/core" } +sp-offchain = { version = "3.0.0", path = "../../primitives/offchain" } sp-runtime = { version = "3.0.0", path = "../../primitives/runtime" } sp-utils = { version = "3.0.0", path = "../../primitives/utils" } -sc-network = { version = "0.9.0", path = "../network" } -sc-keystore = { version = "3.0.0", path = "../keystore" } +threadpool = "1.7" [target.'cfg(not(target_os = "unknown"))'.dependencies] hyper = "0.13.9" diff --git a/substrate/client/offchain/src/api.rs b/substrate/client/offchain/src/api.rs index 64c5060fb0..9b5ff69b72 100644 --- a/substrate/client/offchain/src/api.rs +++ b/substrate/client/offchain/src/api.rs @@ -26,12 +26,11 @@ use std::{ use crate::NetworkProvider; use futures::Future; -use log::error; use sc_network::{PeerId, Multiaddr}; use codec::{Encode, Decode}; use sp_core::OpaquePeerId; use sp_core::offchain::{ - Externalities as OffchainExt, HttpRequestId, Timestamp, HttpRequestStatus, HttpError, + self, HttpRequestId, Timestamp, HttpRequestStatus, HttpError, OffchainStorage, OpaqueNetworkState, OpaqueMultiaddr, StorageKind, }; pub use sp_offchain::STORAGE_PREFIX; @@ -47,22 +46,9 @@ mod http_dummy; mod timestamp; -/// Asynchronous offchain API. -/// -/// NOTE this is done to prevent recursive calls into the runtime (which are not supported currently). -pub(crate) struct Api { - /// Offchain Workers database. - db: Storage, - /// A provider for substrate networking. - network_provider: Arc, - /// Is this node a potential validator? - is_validator: bool, - /// Everything HTTP-related is handled by a different struct. - http: http::HttpApi, -} - fn unavailable_yet(name: &str) -> R { - error!( + log::error!( + target: "sc_offchain", "The {:?} API is not available for offchain workers yet. Follow \ https://github.com/paritytech/substrate/issues/1458 for details", name ); @@ -71,7 +57,109 @@ fn unavailable_yet(name: &str) -> R { const LOCAL_DB: &str = "LOCAL (fork-aware) DB"; -impl OffchainExt for Api { +/// Offchain DB reference. +#[derive(Debug, Clone)] +pub struct Db { + /// Persistent storage database. + persistent: Storage, +} + +impl Db { + /// Create new instance of Offchain DB. + pub fn new(persistent: Storage) -> Self { + Self { persistent } + } + + /// Create new instance of Offchain DB, backed by given backend. + pub fn factory_from_backend(backend: &Backend) -> Option< + Box + > where + Backend: sc_client_api::Backend, + Block: sp_runtime::traits::Block, + Storage: 'static, + { + sc_client_api::Backend::offchain_storage(backend).map(|db| + Box::new(Self::new(db)) as _ + ) + } +} + +impl offchain::DbExternalities for Db { + fn local_storage_set(&mut self, kind: StorageKind, key: &[u8], value: &[u8]) { + log::debug!( + target: "sc_offchain", + "{:?}: Write: {:?} <= {:?}", kind, hex::encode(key), hex::encode(value) + ); + match kind { + StorageKind::PERSISTENT => self.persistent.set(STORAGE_PREFIX, key, value), + StorageKind::LOCAL => unavailable_yet(LOCAL_DB), + } + } + + fn local_storage_clear(&mut self, kind: StorageKind, key: &[u8]) { + log::debug!( + target: "sc_offchain", + "{:?}: Clear: {:?}", kind, hex::encode(key) + ); + match kind { + StorageKind::PERSISTENT => self.persistent.remove(STORAGE_PREFIX, key), + StorageKind::LOCAL => unavailable_yet(LOCAL_DB), + } + } + + fn local_storage_compare_and_set( + &mut self, + kind: StorageKind, + key: &[u8], + old_value: Option<&[u8]>, + new_value: &[u8], + ) -> bool { + log::debug!( + target: "sc_offchain", + "{:?}: CAS: {:?} <= {:?} vs {:?}", + kind, + hex::encode(key), + hex::encode(new_value), + old_value.as_ref().map(hex::encode), + ); + match kind { + StorageKind::PERSISTENT => { + self.persistent.compare_and_set(STORAGE_PREFIX, key, old_value, new_value) + }, + StorageKind::LOCAL => unavailable_yet(LOCAL_DB), + } + } + + fn local_storage_get(&mut self, kind: StorageKind, key: &[u8]) -> Option> { + let result = match kind { + StorageKind::PERSISTENT => self.persistent.get(STORAGE_PREFIX, key), + StorageKind::LOCAL => unavailable_yet(LOCAL_DB), + }; + log::debug!( + target: "sc_offchain", + "{:?}: Read: {:?} => {:?}", + kind, + hex::encode(key), + result.as_ref().map(hex::encode) + ); + result + } +} + +/// Asynchronous offchain API. +/// +/// NOTE this is done to prevent recursive calls into the runtime +/// (which are not supported currently). +pub(crate) struct Api { + /// A provider for substrate networking. + network_provider: Arc, + /// Is this node a potential validator? + is_validator: bool, + /// Everything HTTP-related is handled by a different struct. + http: http::HttpApi, +} + +impl offchain::Externalities for Api { fn is_validator(&self) -> bool { self.is_validator } @@ -98,42 +186,6 @@ impl OffchainExt for Api { rand::random() } - fn local_storage_set(&mut self, kind: StorageKind, key: &[u8], value: &[u8]) { - match kind { - StorageKind::PERSISTENT => self.db.set(STORAGE_PREFIX, key, value), - StorageKind::LOCAL => unavailable_yet(LOCAL_DB), - } - } - - fn local_storage_clear(&mut self, kind: StorageKind, key: &[u8]) { - match kind { - StorageKind::PERSISTENT => self.db.remove(STORAGE_PREFIX, key), - StorageKind::LOCAL => unavailable_yet(LOCAL_DB), - } - } - - fn local_storage_compare_and_set( - &mut self, - kind: StorageKind, - key: &[u8], - old_value: Option<&[u8]>, - new_value: &[u8], - ) -> bool { - match kind { - StorageKind::PERSISTENT => { - self.db.compare_and_set(STORAGE_PREFIX, key, old_value, new_value) - }, - StorageKind::LOCAL => unavailable_yet(LOCAL_DB), - } - } - - fn local_storage_get(&mut self, kind: StorageKind, key: &[u8]) -> Option> { - match kind { - StorageKind::PERSISTENT => self.db.get(STORAGE_PREFIX, key), - StorageKind::LOCAL => unavailable_yet(LOCAL_DB), - } - } - fn http_request_start( &mut self, method: &str, @@ -270,16 +322,14 @@ pub(crate) struct AsyncApi { impl AsyncApi { /// Creates new Offchain extensions API implementation an the asynchronous processing part. - pub fn new( - db: S, + pub fn new( network_provider: Arc, is_validator: bool, shared_client: SharedClient, - ) -> (Api, Self) { + ) -> (Api, Self) { let (http_api, http_worker) = http::http(shared_client); let api = Api { - db, network_provider, is_validator, http: http_api, @@ -303,9 +353,10 @@ impl AsyncApi { #[cfg(test)] mod tests { use super::*; - use std::{convert::{TryFrom, TryInto}, time::SystemTime}; use sc_client_db::offchain::LocalStorage; use sc_network::{NetworkStateInfo, PeerId}; + use sp_core::offchain::{Externalities, DbExternalities}; + use std::{convert::{TryFrom, TryInto}, time::SystemTime}; struct TestNetwork(); @@ -329,20 +380,22 @@ mod tests { } } - fn offchain_api() -> (Api, AsyncApi) { + fn offchain_api() -> (Api, AsyncApi) { sp_tracing::try_init_simple(); - let db = LocalStorage::new_test(); let mock = Arc::new(TestNetwork()); let shared_client = SharedClient::new(); AsyncApi::new( - db, mock, false, shared_client, ) } + fn offchain_db() -> Db { + Db::new(LocalStorage::new_test()) + } + #[test] fn should_get_timestamp() { let mut api = offchain_api().0; @@ -381,7 +434,7 @@ mod tests { fn should_set_and_get_local_storage() { // given let kind = StorageKind::PERSISTENT; - let mut api = offchain_api().0; + let mut api = offchain_db(); let key = b"test"; // when @@ -396,7 +449,7 @@ mod tests { fn should_compare_and_set_local_storage() { // given let kind = StorageKind::PERSISTENT; - let mut api = offchain_api().0; + let mut api = offchain_db(); let key = b"test"; api.local_storage_set(kind, key, b"value"); @@ -413,7 +466,7 @@ mod tests { fn should_compare_and_set_local_storage_with_none() { // given let kind = StorageKind::PERSISTENT; - let mut api = offchain_api().0; + let mut api = offchain_db(); let key = b"test"; // when diff --git a/substrate/client/offchain/src/lib.rs b/substrate/client/offchain/src/lib.rs index f456efb755..717f02eccd 100644 --- a/substrate/client/offchain/src/lib.rs +++ b/substrate/client/offchain/src/lib.rs @@ -46,13 +46,13 @@ use sp_api::{ApiExt, ProvideRuntimeApi}; use futures::future::Future; use log::{debug, warn}; use sc_network::{ExHashT, NetworkService, NetworkStateInfo, PeerId}; -use sp_core::{offchain::{self, OffchainStorage}, ExecutionContext, traits::SpawnNamed}; +use sp_core::{offchain, ExecutionContext, traits::SpawnNamed}; use sp_runtime::{generic::BlockId, traits::{self, Header}}; use futures::{prelude::*, future::ready}; mod api; -use api::SharedClient; +pub use api::Db as OffchainDb; pub use sp_offchain::{OffchainWorkerApi, STORAGE_PREFIX}; /// NetworkProvider provides [`OffchainWorkers`] with all necessary hooks into the @@ -80,21 +80,19 @@ where } /// An offchain workers manager. -pub struct OffchainWorkers { +pub struct OffchainWorkers { client: Arc, - db: Storage, _block: PhantomData, thread_pool: Mutex, - shared_client: SharedClient, + shared_client: api::SharedClient, } -impl OffchainWorkers { +impl OffchainWorkers { /// Creates new `OffchainWorkers`. - pub fn new(client: Arc, db: Storage) -> Self { - let shared_client = SharedClient::new(); + pub fn new(client: Arc) -> Self { + let shared_client = api::SharedClient::new(); Self { client, - db, _block: PhantomData, thread_pool: Mutex::new(ThreadPool::new(num_cpus::get())), shared_client, @@ -102,9 +100,8 @@ impl OffchainWorkers fmt::Debug for OffchainWorkers< +impl fmt::Debug for OffchainWorkers< Client, - Storage, Block, > { fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { @@ -112,15 +109,13 @@ impl fmt::Debug for OffchainWorkers< } } -impl OffchainWorkers< +impl OffchainWorkers< Client, - Storage, Block, > where Block: traits::Block, Client: ProvideRuntimeApi + Send + Sync + 'static, Client::Api: OffchainWorkerApi, - Storage: OffchainStorage + 'static, { /// Start the offchain workers after given block. #[must_use] @@ -150,7 +145,6 @@ impl OffchainWorkers< debug!("Checking offchain workers at {:?}: version:{}", at, version); if version > 0 { let (api, runner) = api::AsyncApi::new( - self.db.clone(), network_provider, is_validator, self.shared_client.clone(), @@ -197,10 +191,10 @@ impl OffchainWorkers< } /// Inform the offchain worker about new imported blocks -pub async fn notification_future( +pub async fn notification_future( is_validator: bool, client: Arc, - offchain: Arc>, + offchain: Arc>, spawner: Spawner, network_provider: Arc, ) @@ -208,7 +202,6 @@ pub async fn notification_future( Block: traits::Block, Client: ProvideRuntimeApi + sc_client_api::BlockchainEvents + Send + Sync + 'static, Client::Api: OffchainWorkerApi, - Storage: OffchainStorage + 'static, Spawner: SpawnNamed { client.import_notification_stream().for_each(move |n| { @@ -300,12 +293,11 @@ mod tests { spawner, client.clone(), )); - let db = sc_client_db::offchain::LocalStorage::new_test(); let network = Arc::new(TestNetwork()); let header = client.header(&BlockId::number(0)).unwrap().unwrap(); // when - let offchain = OffchainWorkers::new(client, db); + let offchain = OffchainWorkers::new(client); futures::executor::block_on( offchain.on_block_imported(&header, network, false) ); @@ -317,6 +309,8 @@ mod tests { #[test] fn offchain_index_set_and_clear_works() { + use sp_core::offchain::OffchainStorage; + sp_tracing::try_init_simple(); let (client, backend) = diff --git a/substrate/client/service/src/builder.rs b/substrate/client/service/src/builder.rs index d0fa10a44d..8a5f63ab7b 100644 --- a/substrate/client/service/src/builder.rs +++ b/substrate/client/service/src/builder.rs @@ -39,7 +39,7 @@ use futures::{ channel::oneshot, }; use sc_keystore::LocalKeystore; -use log::{info, warn}; +use log::info; use sc_network::config::{Role, OnDemand}; use sc_network::NetworkService; use sc_network::block_request_handler::{self, BlockRequestHandler}; @@ -338,13 +338,17 @@ pub fn new_full_parts( transaction_storage: config.transaction_storage.clone(), }; + + let backend = new_db_backend(db_config)?; + let extensions = sc_client_api::execution_extensions::ExecutionExtensions::new( config.execution_strategies.clone(), Some(keystore_container.sync_keystore()), + sc_offchain::OffchainDb::factory_from_backend(&*backend), ); - new_client( - db_config, + let client = new_client( + backend.clone(), executor, chain_spec.as_storage_builder(), fork_blocks, @@ -357,7 +361,9 @@ pub fn new_full_parts( offchain_indexing_api: config.offchain_worker.indexing_enabled, wasm_runtime_overrides: config.wasm_runtime_overrides.clone(), }, - )? + )?; + + (client, backend) }; Ok(( @@ -420,9 +426,20 @@ pub fn new_light_parts( Ok((client, backend, keystore_container, task_manager, on_demand)) } -/// Create an instance of db-backed client. -pub fn new_client( +/// Create an instance of default DB-backend backend. +pub fn new_db_backend( settings: DatabaseSettings, +) -> Result>, sp_blockchain::Error> where + Block: BlockT, +{ + const CANONICALIZATION_DELAY: u64 = 4096; + + Ok(Arc::new(Backend::new(settings, CANONICALIZATION_DELAY)?)) +} + +/// Create an instance of client backed by given backend. +pub fn new_client( + backend: Arc>, executor: E, genesis_storage: &dyn BuildStorage, fork_blocks: ForkBlocks, @@ -431,38 +448,30 @@ pub fn new_client( spawn_handle: Box, prometheus_registry: Option, config: ClientConfig, -) -> Result<( +) -> Result< crate::client::Client< Backend, crate::client::LocalCallExecutor, E>, Block, RA, >, - Arc>, -), sp_blockchain::Error, > where Block: BlockT, E: CodeExecutor + RuntimeInfo, { - const CANONICALIZATION_DELAY: u64 = 4096; - - let backend = Arc::new(Backend::new(settings, CANONICALIZATION_DELAY)?); let executor = crate::client::LocalCallExecutor::new(backend.clone(), executor, spawn_handle, config.clone())?; - Ok(( - crate::client::Client::new( - backend.clone(), - executor, - genesis_storage, - fork_blocks, - bad_blocks, - execution_extensions, - prometheus_registry, - config, - )?, + Ok(crate::client::Client::new( backend, - )) + executor, + genesis_storage, + fork_blocks, + bad_blocks, + execution_extensions, + prometheus_registry, + config, + )?) } /// Parameters to pass into `build`. @@ -499,28 +508,18 @@ pub struct SpawnTasksParams<'a, TBl: BlockT, TCl, TExPool, TRpc, Backend> { } /// Build a shared offchain workers instance. -pub fn build_offchain_workers( +pub fn build_offchain_workers( config: &Configuration, - backend: Arc, spawn_handle: SpawnTaskHandle, client: Arc, network: Arc::Hash>>, -) -> Option>> +) -> Option>> where - TBl: BlockT, TBackend: sc_client_api::Backend, - >::OffchainStorage: 'static, + TBl: BlockT, TCl: Send + Sync + ProvideRuntimeApi + BlockchainEvents + 'static, >::Api: sc_offchain::OffchainWorkerApi, { - let offchain_workers = match backend.offchain_storage() { - Some(db) => { - Some(Arc::new(sc_offchain::OffchainWorkers::new(client.clone(), db))) - }, - None => { - warn!("Offchain workers disabled, due to lack of offchain storage support in backend."); - None - }, - }; + let offchain_workers = Some(Arc::new(sc_offchain::OffchainWorkers::new(client.clone()))); // Inform the offchain worker about new imported blocks if let Some(offchain) = offchain_workers.clone() { diff --git a/substrate/client/service/src/client/client.rs b/substrate/client/service/src/client/client.rs index 263ff7b9c5..6e9fdea092 100644 --- a/substrate/client/service/src/client/client.rs +++ b/substrate/client/service/src/client/client.rs @@ -205,7 +205,11 @@ pub fn new_with_backend( B: backend::LocalBackend + 'static, { let call_executor = LocalCallExecutor::new(backend.clone(), executor, spawn_handle, config.clone())?; - let extensions = ExecutionExtensions::new(Default::default(), keystore); + let extensions = ExecutionExtensions::new( + Default::default(), + keystore, + sc_offchain::OffchainDb::factory_from_backend(&*backend), + ); Client::new( backend, call_executor, diff --git a/substrate/client/service/src/lib.rs b/substrate/client/service/src/lib.rs index 39bad8f2f3..4ca784558d 100644 --- a/substrate/client/service/src/lib.rs +++ b/substrate/client/service/src/lib.rs @@ -51,7 +51,7 @@ use sp_utils::{status_sinks, mpsc::{tracing_unbounded, TracingUnboundedReceiver} pub use self::error::Error; pub use self::builder::{ - new_full_client, new_client, new_full_parts, new_light_parts, + new_full_client, new_db_backend, new_client, new_full_parts, new_light_parts, spawn_tasks, build_network, build_offchain_workers, BuildNetworkParams, KeystoreContainer, NetworkStarter, SpawnTasksParams, TFullClient, TLightClient, TFullBackend, TLightBackend, TLightBackendWithHash, TLightClientWithBackend, diff --git a/substrate/frame/election-provider-multi-phase/src/mock.rs b/substrate/frame/election-provider-multi-phase/src/mock.rs index eb38a4cd52..e7a2924fd2 100644 --- a/substrate/frame/election-provider-multi-phase/src/mock.rs +++ b/substrate/frame/election-provider-multi-phase/src/mock.rs @@ -27,7 +27,7 @@ use parking_lot::RwLock; use sp_core::{ offchain::{ testing::{PoolState, TestOffchainExt, TestTransactionPoolExt}, - OffchainExt, TransactionPoolExt, + OffchainDbExt, OffchainWorkerExt, TransactionPoolExt, }, H256, }; @@ -369,7 +369,8 @@ impl ExtBuilder { seed[0..4].copy_from_slice(&iters.to_le_bytes()); offchain_state.write().seed = seed; - ext.register_extension(OffchainExt::new(offchain)); + ext.register_extension(OffchainDbExt::new(offchain.clone())); + ext.register_extension(OffchainWorkerExt::new(offchain)); ext.register_extension(TransactionPoolExt::new(pool)); (ext, pool_state) diff --git a/substrate/frame/example-offchain-worker/src/tests.rs b/substrate/frame/example-offchain-worker/src/tests.rs index 6f73ffcb9e..7707e7d61e 100644 --- a/substrate/frame/example-offchain-worker/src/tests.rs +++ b/substrate/frame/example-offchain-worker/src/tests.rs @@ -22,7 +22,7 @@ use codec::Decode; use frame_support::{assert_ok, parameter_types}; use sp_core::{ H256, - offchain::{OffchainExt, TransactionPoolExt, testing}, + offchain::{OffchainWorkerExt, TransactionPoolExt, testing}, sr25519::Signature, }; @@ -144,7 +144,7 @@ fn it_aggregates_the_price() { fn should_make_http_call_and_parse_result() { let (offchain, state) = testing::TestOffchainExt::new(); let mut t = sp_io::TestExternalities::default(); - t.register_extension(OffchainExt::new(offchain)); + t.register_extension(OffchainWorkerExt::new(offchain)); price_oracle_response(&mut state.write()); @@ -160,7 +160,7 @@ fn should_make_http_call_and_parse_result() { fn knows_how_to_mock_several_http_calls() { let (offchain, state) = testing::TestOffchainExt::new(); let mut t = sp_io::TestExternalities::default(); - t.register_extension(OffchainExt::new(offchain)); + t.register_extension(OffchainWorkerExt::new(offchain)); { let mut state = state.write(); @@ -217,7 +217,7 @@ fn should_submit_signed_transaction_on_chain() { let mut t = sp_io::TestExternalities::default(); - t.register_extension(OffchainExt::new(offchain)); + t.register_extension(OffchainWorkerExt::new(offchain)); t.register_extension(TransactionPoolExt::new(pool)); t.register_extension(KeystoreExt(Arc::new(keystore))); @@ -255,7 +255,7 @@ fn should_submit_unsigned_transaction_on_chain_for_any_account() { .clone(); let mut t = sp_io::TestExternalities::default(); - t.register_extension(OffchainExt::new(offchain)); + t.register_extension(OffchainWorkerExt::new(offchain)); t.register_extension(TransactionPoolExt::new(pool)); t.register_extension(KeystoreExt(Arc::new(keystore))); @@ -308,7 +308,7 @@ fn should_submit_unsigned_transaction_on_chain_for_all_accounts() { .clone(); let mut t = sp_io::TestExternalities::default(); - t.register_extension(OffchainExt::new(offchain)); + t.register_extension(OffchainWorkerExt::new(offchain)); t.register_extension(TransactionPoolExt::new(pool)); t.register_extension(KeystoreExt(Arc::new(keystore))); @@ -349,7 +349,7 @@ fn should_submit_raw_unsigned_transaction_on_chain() { let keystore = KeyStore::new(); let mut t = sp_io::TestExternalities::default(); - t.register_extension(OffchainExt::new(offchain)); + t.register_extension(OffchainWorkerExt::new(offchain)); t.register_extension(TransactionPoolExt::new(pool)); t.register_extension(KeystoreExt(Arc::new(keystore))); diff --git a/substrate/frame/im-online/src/tests.rs b/substrate/frame/im-online/src/tests.rs index dc6fc4f373..919b639dd6 100644 --- a/substrate/frame/im-online/src/tests.rs +++ b/substrate/frame/im-online/src/tests.rs @@ -23,7 +23,8 @@ use super::*; use crate::mock::*; use sp_core::OpaquePeerId; use sp_core::offchain::{ - OffchainExt, + OffchainDbExt, + OffchainWorkerExt, TransactionPoolExt, testing::{TestOffchainExt, TestTransactionPoolExt}, }; @@ -205,7 +206,8 @@ fn should_generate_heartbeats() { let mut ext = new_test_ext(); let (offchain, _state) = TestOffchainExt::new(); let (pool, state) = TestTransactionPoolExt::new(); - ext.register_extension(OffchainExt::new(offchain)); + ext.register_extension(OffchainDbExt::new(offchain.clone())); + ext.register_extension(OffchainWorkerExt::new(offchain)); ext.register_extension(TransactionPoolExt::new(pool)); ext.execute_with(|| { @@ -310,7 +312,8 @@ fn should_not_send_a_report_if_already_online() { let mut ext = new_test_ext(); let (offchain, _state) = TestOffchainExt::new(); let (pool, pool_state) = TestTransactionPoolExt::new(); - ext.register_extension(OffchainExt::new(offchain)); + ext.register_extension(OffchainDbExt::new(offchain.clone())); + ext.register_extension(OffchainWorkerExt::new(offchain)); ext.register_extension(TransactionPoolExt::new(pool)); ext.execute_with(|| { diff --git a/substrate/frame/merkle-mountain-range/primitives/src/lib.rs b/substrate/frame/merkle-mountain-range/primitives/src/lib.rs index f1ee15b48b..0887535dca 100644 --- a/substrate/frame/merkle-mountain-range/primitives/src/lib.rs +++ b/substrate/frame/merkle-mountain-range/primitives/src/lib.rs @@ -360,6 +360,11 @@ impl OpaqueLeaf { pub fn from_encoded_leaf(encoded_leaf: Vec) -> Self { OpaqueLeaf(encoded_leaf) } + + /// Attempt to decode the leaf into expected concrete type. + pub fn try_decode(&self) -> Option { + codec::Decode::decode(&mut &*self.0).ok() + } } impl FullLeaf for OpaqueLeaf { @@ -368,18 +373,49 @@ impl FullLeaf for OpaqueLeaf { } } +/// A type-safe wrapper for the concrete leaf type. +/// +/// This structure serves merely to avoid passing raw `Vec` around. +/// It must be `Vec`-encoding compatible. +/// +/// It is different from [`OpaqueLeaf`], because it does implement `Codec` +/// and the encoding has to match raw `Vec` encoding. +#[derive(codec::Encode, codec::Decode, RuntimeDebug, PartialEq, Eq)] +pub struct EncodableOpaqueLeaf(pub Vec); + +impl EncodableOpaqueLeaf { + /// Convert a concrete leaf into encodable opaque version. + pub fn from_leaf(leaf: &T) -> Self { + let opaque = OpaqueLeaf::from_leaf(leaf); + Self::from_opaque_leaf(opaque) + } + + /// Given an opaque leaf, make it encodable. + pub fn from_opaque_leaf(opaque: OpaqueLeaf) -> Self { + Self(opaque.0) + } + + /// Try to convert into a [OpaqueLeaf]. + pub fn into_opaque_leaf(self) -> OpaqueLeaf { + // wrap into `OpaqueLeaf` type + OpaqueLeaf::from_encoded_leaf(self.0) + } +} + sp_api::decl_runtime_apis! { /// API to interact with MMR pallet. - pub trait MmrApi { + pub trait MmrApi { /// Generate MMR proof for a leaf under given index. - fn generate_proof(leaf_index: u64) -> Result<(Leaf, Proof), Error>; + #[skip_initialize_block] + fn generate_proof(leaf_index: u64) -> Result<(EncodableOpaqueLeaf, Proof), Error>; /// Verify MMR proof against on-chain MMR. /// /// Note this function will use on-chain MMR root hash and check if the proof /// matches the hash. /// See [Self::verify_proof_stateless] for a stateless verifier. - fn verify_proof(leaf: Leaf, proof: Proof) -> Result<(), Error>; + #[skip_initialize_block] + fn verify_proof(leaf: EncodableOpaqueLeaf, proof: Proof) -> Result<(), Error>; /// Verify MMR proof against given root hash. /// @@ -387,7 +423,8 @@ sp_api::decl_runtime_apis! { /// proof is verified against given MMR root hash. /// /// The leaf data is expected to be encoded in it's compact form. - fn verify_proof_stateless(root: Hash, leaf: Vec, proof: Proof) + #[skip_initialize_block] + fn verify_proof_stateless(root: Hash, leaf: EncodableOpaqueLeaf, proof: Proof) -> Result<(), Error>; } } @@ -535,7 +572,7 @@ mod tests { } #[test] - fn opaque_leaves_should_be_scale_compatible_with_concrete_ones() { + fn opaque_leaves_should_be_full_leaf_compatible() { // given let a = Test::Data("Hello World!".into()); let b = Test::Data("".into()); @@ -564,4 +601,36 @@ mod tests { opaque, ); } + + #[test] + fn encode_opaque_leaf_should_be_scale_compatible() { + use codec::Encode; + + // given + let a = Test::Data("Hello World!".into()); + let case1 = EncodableOpaqueLeaf::from_leaf(&a); + let case2 = EncodableOpaqueLeaf::from_opaque_leaf(OpaqueLeaf(a.encode())); + let case3 = a.encode().encode(); + + // when + let encoded = vec![&case1, &case2] + .into_iter() + .map(|x| x.encode()) + .collect::>(); + let decoded = vec![&*encoded[0], &*encoded[1], &*case3] + .into_iter() + .map(|x| EncodableOpaqueLeaf::decode(&mut &*x)) + .collect::>(); + + // then + assert_eq!(case1, case2); + assert_eq!(encoded[0], encoded[1]); + // then encoding should also match double-encoded leaf. + assert_eq!(encoded[0], case3); + + assert_eq!(decoded[0], decoded[1]); + assert_eq!(decoded[1], decoded[2]); + assert_eq!(decoded[0], Ok(case2)); + assert_eq!(decoded[1], Ok(case1)); + } } diff --git a/substrate/frame/merkle-mountain-range/rpc/Cargo.toml b/substrate/frame/merkle-mountain-range/rpc/Cargo.toml new file mode 100644 index 0000000000..4730dbc7ea --- /dev/null +++ b/substrate/frame/merkle-mountain-range/rpc/Cargo.toml @@ -0,0 +1,29 @@ +[package] +name = "pallet-mmr-rpc" +version = "3.0.0" +authors = ["Parity Technologies "] +edition = "2018" +license = "Apache-2.0" +homepage = "https://substrate.dev" +repository = "https://github.com/paritytech/substrate/" +description = "Node-specific RPC methods for interaction with Merkle Mountain Range pallet." +publish = false + +[package.metadata.docs.rs] +targets = ["x86_64-unknown-linux-gnu"] + +[dependencies] +codec = { package = "parity-scale-codec", version = "2.0.0" } +jsonrpc-core = "15.1.0" +jsonrpc-core-client = "15.1.0" +jsonrpc-derive = "15.1.0" +pallet-mmr-primitives = { version = "3.0.0", path = "../primitives" } +serde = { version = "1.0.101", features = ["derive"] } +sp-api = { version = "3.0.0", path = "../../../primitives/api" } +sp-blockchain = { version = "3.0.0", path = "../../../primitives/blockchain" } +sp-core = { version = "3.0.0", path = "../../../primitives/core" } +sp-rpc = { version = "3.0.0", path = "../../../primitives/rpc" } +sp-runtime = { version = "3.0.0", path = "../../../primitives/runtime" } + +[dev-dependencies] +serde_json = "1.0.41" diff --git a/substrate/frame/merkle-mountain-range/rpc/src/lib.rs b/substrate/frame/merkle-mountain-range/rpc/src/lib.rs new file mode 100644 index 0000000000..5277f4fa47 --- /dev/null +++ b/substrate/frame/merkle-mountain-range/rpc/src/lib.rs @@ -0,0 +1,222 @@ +// This file is part of Substrate. + +// Copyright (C) 2021 Parity Technologies (UK) Ltd. +// SPDX-License-Identifier: Apache-2.0 + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#![warn(missing_docs)] + +//! Node-specific RPC methods for interaction with Merkle Mountain Range pallet. + +use std::sync::Arc; + +use codec::{Codec, Encode}; +use jsonrpc_core::{Error, ErrorCode, Result}; +use jsonrpc_derive::rpc; +use serde::{Deserialize, Serialize}; +use sp_api::ProvideRuntimeApi; +use sp_blockchain::HeaderBackend; +use sp_core::Bytes; +use sp_runtime::{ + generic::BlockId, + traits::{Block as BlockT}, +}; +use pallet_mmr_primitives::{Error as MmrError, Proof}; + +pub use pallet_mmr_primitives::MmrApi as MmrRuntimeApi; + +/// Retrieved MMR leaf and its proof. +#[derive(Serialize, Deserialize, Debug, Clone, PartialEq, Eq)] +#[serde(rename_all = "camelCase")] +pub struct LeafProof { + /// Block hash the proof was generated for. + pub block_hash: BlockHash, + /// SCALE-encoded leaf data. + pub leaf: Bytes, + /// SCALE-encoded proof data. See [pallet_mmr_primitives::Proof]. + pub proof: Bytes, +} + +impl LeafProof { + /// Create new `LeafProof` from given concrete `leaf` and `proof`. + pub fn new( + block_hash: BlockHash, + leaf: Leaf, + proof: Proof, + ) -> Self where + Leaf: Encode, + MmrHash: Encode, + { + Self { + block_hash, + leaf: Bytes(leaf.encode()), + proof: Bytes(proof.encode()), + } + } +} + +/// MMR RPC methods. +#[rpc] +pub trait MmrApi { + /// Generate MMR proof for given leaf index. + /// + /// This method calls into a runtime with MMR pallet included and attempts to generate + /// MMR proof for leaf at given `leaf_index`. + /// Optionally, a block hash at which the runtime should be queried can be specified. + /// + /// Returns the (full) leaf itself and a proof for this leaf (compact encoding, i.e. hash of + /// the leaf). Both parameters are SCALE-encoded. + #[rpc(name = "mmr_generateProof")] + fn generate_proof( + &self, + leaf_index: u64, + at: Option, + ) -> Result>; +} + +/// An implementation of MMR specific RPC methods. +pub struct Mmr { + client: Arc, + _marker: std::marker::PhantomData, +} + +impl Mmr { + /// Create new `Mmr` with the given reference to the client. + pub fn new(client: Arc) -> Self { + Self { + client, + _marker: Default::default(), + } + } +} + +impl MmrApi<::Hash,> for Mmr +where + Block: BlockT, + C: Send + Sync + 'static + ProvideRuntimeApi + HeaderBackend, + C::Api: MmrRuntimeApi< + Block, + MmrHash, + >, + MmrHash: Codec + Send + Sync + 'static, +{ + fn generate_proof( + &self, + leaf_index: u64, + at: Option<::Hash>, + ) -> Result::Hash>> { + let api = self.client.runtime_api(); + let block_hash = at.unwrap_or_else(|| + // If the block hash is not supplied assume the best block. + self.client.info().best_hash + ); + + let (leaf, proof) = api + .generate_proof_with_context( + &BlockId::hash(block_hash), + sp_core::ExecutionContext::OffchainCall(None), + leaf_index, + ) + .map_err(runtime_error_into_rpc_error)? + .map_err(mmr_error_into_rpc_error)?; + + Ok(LeafProof::new(block_hash, leaf, proof)) + } +} + +const RUNTIME_ERROR: i64 = 8000; +const MMR_ERROR: i64 = 8010; + +/// Converts a mmr-specific error into an RPC error. +fn mmr_error_into_rpc_error(err: MmrError) -> Error { + match err { + MmrError::LeafNotFound => Error { + code: ErrorCode::ServerError(MMR_ERROR + 1), + message: "Leaf was not found".into(), + data: Some(format!("{:?}", err).into()), + }, + MmrError::GenerateProof => Error { + code: ErrorCode::ServerError(MMR_ERROR + 2), + message: "Error while generating the proof".into(), + data: Some(format!("{:?}", err).into()), + }, + _ => Error { + code: ErrorCode::ServerError(MMR_ERROR), + message: "Unexpected MMR error".into(), + data: Some(format!("{:?}", err).into()), + }, + } +} + +/// Converts a runtime trap into an RPC error. +fn runtime_error_into_rpc_error(err: impl std::fmt::Debug) -> Error { + Error { + code: ErrorCode::ServerError(RUNTIME_ERROR), + message: "Runtime trapped".into(), + data: Some(format!("{:?}", err).into()), + } +} + +#[cfg(test)] +mod tests { + use super::*; + use sp_core::H256; + + #[test] + fn should_serialize_leaf_proof() { + // given + let leaf = vec![1_u8, 2, 3, 4]; + let proof = Proof { + leaf_index: 1, + leaf_count: 9, + items: vec![H256::repeat_byte(1), H256::repeat_byte(2)], + }; + + let leaf_proof = LeafProof::new(H256::repeat_byte(0), leaf, proof); + + // when + let actual = serde_json::to_string(&leaf_proof).unwrap(); + + // then + assert_eq!( + actual, + r#"{"blockHash":"0x0000000000000000000000000000000000000000000000000000000000000000","leaf":"0x1001020304","proof":"0x010000000000000009000000000000000801010101010101010101010101010101010101010101010101010101010101010202020202020202020202020202020202020202020202020202020202020202"}"# + ); + } + + #[test] + fn should_deserialize_leaf_proof() { + // given + let expected = LeafProof { + block_hash: H256::repeat_byte(0), + leaf: Bytes(vec![1_u8, 2, 3, 4].encode()), + proof: Bytes(Proof { + leaf_index: 1, + leaf_count: 9, + items: vec![H256::repeat_byte(1), H256::repeat_byte(2)], + }.encode()), + }; + + // when + let actual: LeafProof = serde_json::from_str(r#"{ + "blockHash":"0x0000000000000000000000000000000000000000000000000000000000000000", + "leaf":"0x1001020304", + "proof":"0x010000000000000009000000000000000801010101010101010101010101010101010101010101010101010101010101010202020202020202020202020202020202020202020202020202020202020202" + }"#).unwrap(); + + // then + assert_eq!(actual, expected); + + } +} diff --git a/substrate/frame/merkle-mountain-range/src/mmr/storage.rs b/substrate/frame/merkle-mountain-range/src/mmr/storage.rs index 0bff53f2fb..021c0716b1 100644 --- a/substrate/frame/merkle-mountain-range/src/mmr/storage.rs +++ b/substrate/frame/merkle-mountain-range/src/mmr/storage.rs @@ -62,10 +62,9 @@ impl mmr_lib::MMRStore> for Storage mmr_lib::Result>> { let key = Module::::offchain_key(pos); // Retrieve the element from Off-chain DB. - Ok( - sp_io::offchain ::local_storage_get(sp_core::offchain::StorageKind::PERSISTENT, &key) - .and_then(|v| codec::Decode::decode(&mut &*v).ok()) - ) + Ok(sp_io::offchain + ::local_storage_get(sp_core::offchain::StorageKind::PERSISTENT, &key) + .and_then(|v| codec::Decode::decode(&mut &*v).ok())) } fn append(&mut self, _: u64, _: Vec>) -> mmr_lib::Result<()> { @@ -95,9 +94,8 @@ impl mmr_lib::MMRStore> for Storage>::insert(size, elem.hash()); // Indexing API is used to store the full leaf content. - elem.using_encoded(|elem| { - sp_io::offchain_index::set(&Module::::offchain_key(size), elem) - }); + let key = Module::::offchain_key(size); + elem.using_encoded(|elem| sp_io::offchain_index::set(&key, elem)); size += 1; if let Node::Data(..) = elem { diff --git a/substrate/frame/merkle-mountain-range/src/tests.rs b/substrate/frame/merkle-mountain-range/src/tests.rs index 63e4ec2257..ea522dc51c 100644 --- a/substrate/frame/merkle-mountain-range/src/tests.rs +++ b/substrate/frame/merkle-mountain-range/src/tests.rs @@ -23,7 +23,7 @@ use sp_core::{ H256, offchain::{ testing::TestOffchainExt, - OffchainExt, + OffchainWorkerExt, OffchainDbExt, }, }; use pallet_mmr_primitives::{Proof, Compact}; @@ -34,7 +34,8 @@ pub(crate) fn new_test_ext() -> sp_io::TestExternalities { fn register_offchain_ext(ext: &mut sp_io::TestExternalities) { let (offchain, _offchain_state) = TestOffchainExt::with_offchain_db(ext.offchain_db()); - ext.register_extension(OffchainExt::new(offchain)); + ext.register_extension(OffchainDbExt::new(offchain.clone())); + ext.register_extension(OffchainWorkerExt::new(offchain)); } fn new_block() -> u64 { diff --git a/substrate/frame/session/src/historical/offchain.rs b/substrate/frame/session/src/historical/offchain.rs index 38cf09124c..f095be9e44 100644 --- a/substrate/frame/session/src/historical/offchain.rs +++ b/substrate/frame/session/src/historical/offchain.rs @@ -147,7 +147,8 @@ mod tests { use sp_core::crypto::key_types::DUMMY; use sp_core::offchain::{ testing::TestOffchainExt, - OffchainExt, + OffchainDbExt, + OffchainWorkerExt, StorageKind, }; @@ -181,7 +182,8 @@ mod tests { seed[0..4].copy_from_slice(&ITERATIONS.to_le_bytes()); offchain_state.write().seed = seed; - ext.register_extension(OffchainExt::new(offchain)); + ext.register_extension(OffchainDbExt::new(offchain.clone())); + ext.register_extension(OffchainWorkerExt::new(offchain)); ext } diff --git a/substrate/frame/staking/fuzzer/src/submit_solution.rs b/substrate/frame/staking/fuzzer/src/submit_solution.rs index b661a83a1b..63ec189d44 100644 --- a/substrate/frame/staking/fuzzer/src/submit_solution.rs +++ b/substrate/frame/staking/fuzzer/src/submit_solution.rs @@ -26,7 +26,7 @@ use pallet_staking::testing_utils::*; use frame_support::{assert_ok, storage::StorageValue, traits::UnfilteredDispatchable}; use frame_system::RawOrigin; use sp_runtime::DispatchError; -use sp_core::offchain::{testing::TestOffchainExt, OffchainExt}; +use sp_core::offchain::{testing::TestOffchainExt, OffchainWorkerExt, OffchainDbExt}; use pallet_staking::{EraElectionStatus, ElectionStatus, Module as Staking, Call as StakingCall}; mod mock; @@ -55,7 +55,8 @@ pub fn new_test_ext(iterations: u32) -> sp_io::TestExternalities { seed[0..4].copy_from_slice(&iterations.to_le_bytes()); offchain_state.write().seed = seed; - ext.register_extension(OffchainExt::new(offchain)); + ext.register_extension(OffchainDbExt::new(offchain.clone())); + ext.register_extension(OffchainWorkerExt::new(offchain)); ext } diff --git a/substrate/frame/staking/src/tests.rs b/substrate/frame/staking/src/tests.rs index 529cd7b87c..92e1862e39 100644 --- a/substrate/frame/staking/src/tests.rs +++ b/substrate/frame/staking/src/tests.rs @@ -2917,7 +2917,7 @@ mod offchain_election { use parking_lot::RwLock; use sp_core::offchain::{ testing::{PoolState, TestOffchainExt, TestTransactionPoolExt}, - OffchainExt, TransactionPoolExt, + OffchainWorkerExt, TransactionPoolExt, OffchainDbExt, }; use sp_io::TestExternalities; use sp_npos_elections::StakedAssignment; @@ -2960,7 +2960,8 @@ mod offchain_election { seed[0..4].copy_from_slice(&iterations.to_le_bytes()); offchain_state.write().seed = seed; - ext.register_extension(OffchainExt::new(offchain)); + ext.register_extension(OffchainDbExt::new(offchain.clone())); + ext.register_extension(OffchainWorkerExt::new(offchain)); ext.register_extension(TransactionPoolExt::new(pool)); pool_state diff --git a/substrate/primitives/core/src/lib.rs b/substrate/primitives/core/src/lib.rs index 7fc9fa0919..c72f38ea08 100644 --- a/substrate/primitives/core/src/lib.rs +++ b/substrate/primitives/core/src/lib.rs @@ -119,9 +119,10 @@ impl ExecutionContext { match self { Importing | Syncing | BlockConstruction => offchain::Capabilities::none(), - // Enable keystore and transaction pool by default for offchain calls. + // Enable keystore, transaction pool and Offchain DB reads by default for offchain calls. OffchainCall(None) => [ offchain::Capability::Keystore, + offchain::Capability::OffchainDbRead, offchain::Capability::TransactionPool, ][..].into(), OffchainCall(Some((_, capabilities))) => *capabilities, diff --git a/substrate/primitives/core/src/offchain/mod.rs b/substrate/primitives/core/src/offchain/mod.rs index 6a08df1d7f..8b587b887e 100644 --- a/substrate/primitives/core/src/offchain/mod.rs +++ b/substrate/primitives/core/src/offchain/mod.rs @@ -29,10 +29,10 @@ pub mod storage; #[cfg(feature = "std")] pub mod testing; -/// Local storage prefix used by the Offchain Worker API to +/// Persistent storage prefix used by the Offchain Worker API when creating a DB key. pub const STORAGE_PREFIX : &[u8] = b"storage"; -/// Offchain workers local storage. +/// Offchain DB persistent (non-fork-aware) storage. pub trait OffchainStorage: Clone + Send + Sync { /// Persist a value in storage under given key and prefix. fn set(&mut self, prefix: &[u8], key: &[u8], value: &[u8]); @@ -263,9 +263,9 @@ pub enum Capability { /// Access to opaque network state. NetworkState = 16, /// Access to offchain worker DB (read only). - OffchainWorkerDbRead = 32, + OffchainDbRead = 32, /// Access to offchain worker DB (writes). - OffchainWorkerDbWrite = 64, + OffchainDbWrite = 64, /// Manage the authorized nodes NodeAuthorization = 128, } @@ -293,7 +293,7 @@ impl Capabilities { [ Capability::TransactionPool, Capability::Keystore, - Capability::OffchainWorkerDbRead, + Capability::OffchainDbRead, ][..].into() } @@ -337,42 +337,6 @@ pub trait Externalities: Send { /// Obviously fine in the off-chain worker context. fn random_seed(&mut self) -> [u8; 32]; - /// Sets a value in the local storage. - /// - /// Note this storage is not part of the consensus, it's only accessible by - /// offchain worker tasks running on the same machine. It _is_ persisted between runs. - fn local_storage_set(&mut self, kind: StorageKind, key: &[u8], value: &[u8]); - - /// Removes a value in the local storage. - /// - /// Note this storage is not part of the consensus, it's only accessible by - /// offchain worker tasks running on the same machine. It _is_ persisted between runs. - fn local_storage_clear(&mut self, kind: StorageKind, key: &[u8]); - - /// Sets a value in the local storage if it matches current value. - /// - /// Since multiple offchain workers may be running concurrently, to prevent - /// data races use CAS to coordinate between them. - /// - /// Returns `true` if the value has been set, `false` otherwise. - /// - /// Note this storage is not part of the consensus, it's only accessible by - /// offchain worker tasks running on the same machine. It _is_ persisted between runs. - fn local_storage_compare_and_set( - &mut self, - kind: StorageKind, - key: &[u8], - old_value: Option<&[u8]>, - new_value: &[u8], - ) -> bool; - - /// Gets a value from the local storage. - /// - /// If the value does not exist in the storage `None` will be returned. - /// Note this storage is not part of the consensus, it's only accessible by - /// offchain worker tasks running on the same machine. It _is_ persisted between runs. - fn local_storage_get(&mut self, kind: StorageKind, key: &[u8]) -> Option>; - /// Initiates a http request given HTTP verb and the URL. /// /// Meta is a future-reserved field containing additional, parity-scale-codec encoded parameters. @@ -521,28 +485,6 @@ impl Externalities for Box { (&mut **self).random_seed() } - fn local_storage_set(&mut self, kind: StorageKind, key: &[u8], value: &[u8]) { - (&mut **self).local_storage_set(kind, key, value) - } - - fn local_storage_clear(&mut self, kind: StorageKind, key: &[u8]) { - (&mut **self).local_storage_clear(kind, key) - } - - fn local_storage_compare_and_set( - &mut self, - kind: StorageKind, - key: &[u8], - old_value: Option<&[u8]>, - new_value: &[u8], - ) -> bool { - (&mut **self).local_storage_compare_and_set(kind, key, old_value, new_value) - } - - fn local_storage_get(&mut self, kind: StorageKind, key: &[u8]) -> Option> { - (&mut **self).local_storage_get(kind, key) - } - fn http_request_start(&mut self, method: &str, uri: &str, meta: &[u8]) -> Result { (&mut **self).http_request_start(method, uri, meta) } @@ -582,7 +524,7 @@ impl Externalities for Box { } } -/// An `OffchainExternalities` implementation with limited capabilities. +/// An `*Externalities` implementation with limited capabilities. pub struct LimitedExternalities { capabilities: Capabilities, externalities: T, @@ -633,32 +575,6 @@ impl Externalities for LimitedExternalities { self.externalities.random_seed() } - fn local_storage_set(&mut self, kind: StorageKind, key: &[u8], value: &[u8]) { - self.check(Capability::OffchainWorkerDbWrite, "local_storage_set"); - self.externalities.local_storage_set(kind, key, value) - } - - fn local_storage_clear(&mut self, kind: StorageKind, key: &[u8]) { - self.check(Capability::OffchainWorkerDbWrite, "local_storage_clear"); - self.externalities.local_storage_clear(kind, key) - } - - fn local_storage_compare_and_set( - &mut self, - kind: StorageKind, - key: &[u8], - old_value: Option<&[u8]>, - new_value: &[u8], - ) -> bool { - self.check(Capability::OffchainWorkerDbWrite, "local_storage_compare_and_set"); - self.externalities.local_storage_compare_and_set(kind, key, old_value, new_value) - } - - fn local_storage_get(&mut self, kind: StorageKind, key: &[u8]) -> Option> { - self.check(Capability::OffchainWorkerDbRead, "local_storage_get"); - self.externalities.local_storage_get(kind, key) - } - fn http_request_start(&mut self, method: &str, uri: &str, meta: &[u8]) -> Result { self.check(Capability::Http, "http_request_start"); self.externalities.http_request_start(method, uri, meta) @@ -707,18 +623,123 @@ impl Externalities for LimitedExternalities { #[cfg(feature = "std")] sp_externalities::decl_extension! { - /// The offchain extension that will be registered at the Substrate externalities. - pub struct OffchainExt(Box); + /// The offchain worker extension that will be registered at the Substrate externalities. + pub struct OffchainWorkerExt(Box); } #[cfg(feature = "std")] -impl OffchainExt { +impl OffchainWorkerExt { /// Create a new instance of `Self`. pub fn new(offchain: O) -> Self { Self(Box::new(offchain)) } } +/// A externalities extension for accessing the Offchain DB. +pub trait DbExternalities: Send { + /// Sets a value in the local storage. + /// + /// Note this storage is not part of the consensus, it's only accessible by + /// offchain worker tasks running on the same machine. It _is_ persisted between runs. + fn local_storage_set(&mut self, kind: StorageKind, key: &[u8], value: &[u8]); + + /// Removes a value in the local storage. + /// + /// Note this storage is not part of the consensus, it's only accessible by + /// offchain worker tasks running on the same machine. It _is_ persisted between runs. + fn local_storage_clear(&mut self, kind: StorageKind, key: &[u8]); + + /// Sets a value in the local storage if it matches current value. + /// + /// Since multiple offchain workers may be running concurrently, to prevent + /// data races use CAS to coordinate between them. + /// + /// Returns `true` if the value has been set, `false` otherwise. + /// + /// Note this storage is not part of the consensus, it's only accessible by + /// offchain worker tasks running on the same machine. It _is_ persisted between runs. + fn local_storage_compare_and_set( + &mut self, + kind: StorageKind, + key: &[u8], + old_value: Option<&[u8]>, + new_value: &[u8], + ) -> bool; + + /// Gets a value from the local storage. + /// + /// If the value does not exist in the storage `None` will be returned. + /// Note this storage is not part of the consensus, it's only accessible by + /// offchain worker tasks running on the same machine. It _is_ persisted between runs. + fn local_storage_get(&mut self, kind: StorageKind, key: &[u8]) -> Option>; +} + +impl DbExternalities for Box { + fn local_storage_set(&mut self, kind: StorageKind, key: &[u8], value: &[u8]) { + (&mut **self).local_storage_set(kind, key, value) + } + + fn local_storage_clear(&mut self, kind: StorageKind, key: &[u8]) { + (&mut **self).local_storage_clear(kind, key) + } + + fn local_storage_compare_and_set( + &mut self, + kind: StorageKind, + key: &[u8], + old_value: Option<&[u8]>, + new_value: &[u8], + ) -> bool { + (&mut **self).local_storage_compare_and_set(kind, key, old_value, new_value) + } + + fn local_storage_get(&mut self, kind: StorageKind, key: &[u8]) -> Option> { + (&mut **self).local_storage_get(kind, key) + } +} + +impl DbExternalities for LimitedExternalities { + fn local_storage_set(&mut self, kind: StorageKind, key: &[u8], value: &[u8]) { + self.check(Capability::OffchainDbWrite, "local_storage_set"); + self.externalities.local_storage_set(kind, key, value) + } + + fn local_storage_clear(&mut self, kind: StorageKind, key: &[u8]) { + self.check(Capability::OffchainDbWrite, "local_storage_clear"); + self.externalities.local_storage_clear(kind, key) + } + + fn local_storage_compare_and_set( + &mut self, + kind: StorageKind, + key: &[u8], + old_value: Option<&[u8]>, + new_value: &[u8], + ) -> bool { + self.check(Capability::OffchainDbWrite, "local_storage_compare_and_set"); + self.externalities.local_storage_compare_and_set(kind, key, old_value, new_value) + } + + fn local_storage_get(&mut self, kind: StorageKind, key: &[u8]) -> Option> { + self.check(Capability::OffchainDbRead, "local_storage_get"); + self.externalities.local_storage_get(kind, key) + } +} + +#[cfg(feature = "std")] +sp_externalities::decl_extension! { + /// The offchain database extension that will be registered at the Substrate externalities. + pub struct OffchainDbExt(Box); +} + +#[cfg(feature = "std")] +impl OffchainDbExt { + /// Create a new instance of `OffchainDbExt`. + pub fn new(offchain: O) -> Self { + Self(Box::new(offchain)) + } +} + /// Abstraction over transaction pool. /// /// This trait is currently used within the `ExternalitiesExtension` diff --git a/substrate/primitives/core/src/offchain/testing.rs b/substrate/primitives/core/src/offchain/testing.rs index da486a3d03..bdec7bf4ef 100644 --- a/substrate/primitives/core/src/offchain/testing.rs +++ b/substrate/primitives/core/src/offchain/testing.rs @@ -244,44 +244,6 @@ impl offchain::Externalities for TestOffchainExt { self.0.read().seed } - fn local_storage_set(&mut self, kind: StorageKind, key: &[u8], value: &[u8]) { - let mut state = self.0.write(); - match kind { - StorageKind::LOCAL => state.local_storage.set(b"", key, value), - StorageKind::PERSISTENT => state.persistent_storage.set(b"", key, value), - }; - } - - fn local_storage_clear(&mut self, kind: StorageKind, key: &[u8]) { - let mut state = self.0.write(); - match kind { - StorageKind::LOCAL => state.local_storage.remove(b"", key), - StorageKind::PERSISTENT => state.persistent_storage.remove(b"", key), - }; - } - - fn local_storage_compare_and_set( - &mut self, - kind: StorageKind, - key: &[u8], - old_value: Option<&[u8]>, - new_value: &[u8] - ) -> bool { - let mut state = self.0.write(); - match kind { - StorageKind::LOCAL => state.local_storage.compare_and_set(b"", key, old_value, new_value), - StorageKind::PERSISTENT => state.persistent_storage.compare_and_set(b"", key, old_value, new_value), - } - } - - fn local_storage_get(&mut self, kind: StorageKind, key: &[u8]) -> Option> { - let state = self.0.read(); - match kind { - StorageKind::LOCAL => state.local_storage.get(TestPersistentOffchainDB::PREFIX, key), - StorageKind::PERSISTENT => state.persistent_storage.get(key), - } - } - fn http_request_start(&mut self, method: &str, uri: &str, meta: &[u8]) -> Result { let mut state = self.0.write(); let id = RequestId(state.requests.len() as u16); @@ -393,6 +355,48 @@ impl offchain::Externalities for TestOffchainExt { } } +impl offchain::DbExternalities for TestOffchainExt { + fn local_storage_set(&mut self, kind: StorageKind, key: &[u8], value: &[u8]) { + let mut state = self.0.write(); + match kind { + StorageKind::LOCAL => state.local_storage.set(b"", key, value), + StorageKind::PERSISTENT => state.persistent_storage.set(b"", key, value), + }; + } + + fn local_storage_clear(&mut self, kind: StorageKind, key: &[u8]) { + let mut state = self.0.write(); + match kind { + StorageKind::LOCAL => state.local_storage.remove(b"", key), + StorageKind::PERSISTENT => state.persistent_storage.remove(b"", key), + }; + } + + fn local_storage_compare_and_set( + &mut self, + kind: StorageKind, + key: &[u8], + old_value: Option<&[u8]>, + new_value: &[u8] + ) -> bool { + let mut state = self.0.write(); + match kind { + StorageKind::LOCAL => state.local_storage + .compare_and_set(b"", key, old_value, new_value), + StorageKind::PERSISTENT => state.persistent_storage + .compare_and_set(b"", key, old_value, new_value), + } + } + + fn local_storage_get(&mut self, kind: StorageKind, key: &[u8]) -> Option> { + let state = self.0.read(); + match kind { + StorageKind::LOCAL => state.local_storage.get(TestPersistentOffchainDB::PREFIX, key), + StorageKind::PERSISTENT => state.persistent_storage.get(key), + } + } +} + /// The internal state of the fake transaction pool. #[derive(Default)] pub struct PoolState { diff --git a/substrate/primitives/io/src/lib.rs b/substrate/primitives/io/src/lib.rs index 521d831dfc..e123008e5a 100644 --- a/substrate/primitives/io/src/lib.rs +++ b/substrate/primitives/io/src/lib.rs @@ -39,7 +39,7 @@ use tracing; use sp_core::{ crypto::Pair, traits::{CallInWasmExt, TaskExecutorExt, RuntimeSpawnExt}, - offchain::{OffchainExt, TransactionPoolExt}, + offchain::{OffchainDbExt, OffchainWorkerExt, TransactionPoolExt}, hexdisplay::HexDisplay, storage::ChildInfo, }; @@ -843,7 +843,7 @@ pub trait Offchain { /// Even if this function returns `true`, it does not mean that any keys are configured /// and that the validator is registered in the chain. fn is_validator(&mut self) -> bool { - self.extension::() + self.extension::() .expect("is_validator can be called only in the offchain worker context") .is_validator() } @@ -860,21 +860,21 @@ pub trait Offchain { /// Returns information about the local node's network state. fn network_state(&mut self) -> Result { - self.extension::() + self.extension::() .expect("network_state can be called only in the offchain worker context") .network_state() } /// Returns current UNIX timestamp (in millis) fn timestamp(&mut self) -> Timestamp { - self.extension::() + self.extension::() .expect("timestamp can be called only in the offchain worker context") .timestamp() } /// Pause the execution until `deadline` is reached. fn sleep_until(&mut self, deadline: Timestamp) { - self.extension::() + self.extension::() .expect("sleep_until can be called only in the offchain worker context") .sleep_until(deadline) } @@ -884,7 +884,7 @@ pub trait Offchain { /// This is a truly random, non-deterministic seed generated by host environment. /// Obviously fine in the off-chain worker context. fn random_seed(&mut self) -> [u8; 32] { - self.extension::() + self.extension::() .expect("random_seed can be called only in the offchain worker context") .random_seed() } @@ -894,8 +894,9 @@ pub trait Offchain { /// Note this storage is not part of the consensus, it's only accessible by /// offchain worker tasks running on the same machine. It IS persisted between runs. fn local_storage_set(&mut self, kind: StorageKind, key: &[u8], value: &[u8]) { - self.extension::() - .expect("local_storage_set can be called only in the offchain worker context") + self.extension::() + .expect("local_storage_set can be called only in the offchain call context with + OffchainDb extension") .local_storage_set(kind, key, value) } @@ -904,8 +905,9 @@ pub trait Offchain { /// Note this storage is not part of the consensus, it's only accessible by /// offchain worker tasks running on the same machine. It IS persisted between runs. fn local_storage_clear(&mut self, kind: StorageKind, key: &[u8]) { - self.extension::() - .expect("local_storage_clear can be called only in the offchain worker context") + self.extension::() + .expect("local_storage_clear can be called only in the offchain call context with + OffchainDb extension") .local_storage_clear(kind, key) } @@ -925,9 +927,15 @@ pub trait Offchain { old_value: Option>, new_value: &[u8], ) -> bool { - self.extension::() - .expect("local_storage_compare_and_set can be called only in the offchain worker context") - .local_storage_compare_and_set(kind, key, old_value.as_ref().map(|v| v.deref()), new_value) + self.extension::() + .expect("local_storage_compare_and_set can be called only in the offchain call context + with OffchainDb extension") + .local_storage_compare_and_set( + kind, + key, + old_value.as_ref().map(|v| v.deref()), + new_value, + ) } /// Gets a value from the local storage. @@ -936,8 +944,9 @@ pub trait Offchain { /// Note this storage is not part of the consensus, it's only accessible by /// offchain worker tasks running on the same machine. It IS persisted between runs. fn local_storage_get(&mut self, kind: StorageKind, key: &[u8]) -> Option> { - self.extension::() - .expect("local_storage_get can be called only in the offchain worker context") + self.extension::() + .expect("local_storage_get can be called only in the offchain call context with + OffchainDb extension") .local_storage_get(kind, key) } @@ -951,7 +960,7 @@ pub trait Offchain { uri: &str, meta: &[u8], ) -> Result { - self.extension::() + self.extension::() .expect("http_request_start can be called only in the offchain worker context") .http_request_start(method, uri, meta) } @@ -963,7 +972,7 @@ pub trait Offchain { name: &str, value: &str, ) -> Result<(), ()> { - self.extension::() + self.extension::() .expect("http_request_add_header can be called only in the offchain worker context") .http_request_add_header(request_id, name, value) } @@ -980,7 +989,7 @@ pub trait Offchain { chunk: &[u8], deadline: Option, ) -> Result<(), HttpError> { - self.extension::() + self.extension::() .expect("http_request_write_body can be called only in the offchain worker context") .http_request_write_body(request_id, chunk, deadline) } @@ -997,7 +1006,7 @@ pub trait Offchain { ids: &[HttpRequestId], deadline: Option, ) -> Vec { - self.extension::() + self.extension::() .expect("http_response_wait can be called only in the offchain worker context") .http_response_wait(ids, deadline) } @@ -1007,7 +1016,7 @@ pub trait Offchain { /// Returns a vector of pairs `(HeaderKey, HeaderValue)`. /// NOTE response headers have to be read before response body. fn http_response_headers(&mut self, request_id: HttpRequestId) -> Vec<(Vec, Vec)> { - self.extension::() + self.extension::() .expect("http_response_headers can be called only in the offchain worker context") .http_response_headers(request_id) } @@ -1026,7 +1035,7 @@ pub trait Offchain { buffer: &mut [u8], deadline: Option, ) -> Result { - self.extension::() + self.extension::() .expect("http_response_read_body can be called only in the offchain worker context") .http_response_read_body(request_id, buffer, deadline) .map(|r| r as u32) @@ -1034,7 +1043,7 @@ pub trait Offchain { /// Set the authorized nodes and authorized_only flag. fn set_authorized_nodes(&mut self, nodes: Vec, authorized_only: bool) { - self.extension::() + self.extension::() .expect("set_authorized_nodes can be called only in the offchain worker context") .set_authorized_nodes(nodes, authorized_only) } diff --git a/substrate/primitives/runtime/src/offchain/http.rs b/substrate/primitives/runtime/src/offchain/http.rs index 31eec32f6a..a346460897 100644 --- a/substrate/primitives/runtime/src/offchain/http.rs +++ b/substrate/primitives/runtime/src/offchain/http.rs @@ -518,7 +518,7 @@ mod tests { use super::*; use sp_io::TestExternalities; use sp_core::offchain::{ - OffchainExt, + OffchainWorkerExt, testing, }; @@ -526,7 +526,7 @@ mod tests { fn should_send_a_basic_request_and_get_response() { let (offchain, state) = testing::TestOffchainExt::new(); let mut t = TestExternalities::default(); - t.register_extension(OffchainExt::new(offchain)); + t.register_extension(OffchainWorkerExt::new(offchain)); t.execute_with(|| { let request: Request = Request::get("http://localhost:1234"); @@ -567,7 +567,7 @@ mod tests { fn should_send_a_post_request() { let (offchain, state) = testing::TestOffchainExt::new(); let mut t = TestExternalities::default(); - t.register_extension(OffchainExt::new(offchain)); + t.register_extension(OffchainWorkerExt::new(offchain)); t.execute_with(|| { let pending = Request::default() diff --git a/substrate/primitives/runtime/src/offchain/storage.rs b/substrate/primitives/runtime/src/offchain/storage.rs index 56bebf956c..794ae4255a 100644 --- a/substrate/primitives/runtime/src/offchain/storage.rs +++ b/substrate/primitives/runtime/src/offchain/storage.rs @@ -104,7 +104,7 @@ mod tests { use super::*; use sp_io::TestExternalities; use sp_core::offchain::{ - OffchainExt, + OffchainDbExt, testing, }; @@ -112,7 +112,7 @@ mod tests { fn should_set_and_get() { let (offchain, state) = testing::TestOffchainExt::new(); let mut t = TestExternalities::default(); - t.register_extension(OffchainExt::new(offchain)); + t.register_extension(OffchainDbExt::new(offchain)); t.execute_with(|| { let val = StorageValue::persistent(b"testval"); @@ -134,7 +134,7 @@ mod tests { fn should_mutate() { let (offchain, state) = testing::TestOffchainExt::new(); let mut t = TestExternalities::default(); - t.register_extension(OffchainExt::new(offchain)); + t.register_extension(OffchainDbExt::new(offchain)); t.execute_with(|| { let val = StorageValue::persistent(b"testval"); diff --git a/substrate/primitives/runtime/src/offchain/storage_lock.rs b/substrate/primitives/runtime/src/offchain/storage_lock.rs index 416689cadf..4c66db6c38 100644 --- a/substrate/primitives/runtime/src/offchain/storage_lock.rs +++ b/substrate/primitives/runtime/src/offchain/storage_lock.rs @@ -453,7 +453,7 @@ pub trait BlockNumberProvider { #[cfg(test)] mod tests { use super::*; - use sp_core::offchain::{testing, OffchainExt}; + use sp_core::offchain::{testing, OffchainWorkerExt, OffchainDbExt}; use sp_io::TestExternalities; const VAL_1: u32 = 0u32; @@ -463,7 +463,8 @@ mod tests { fn storage_lock_write_unlock_lock_read_unlock() { let (offchain, state) = testing::TestOffchainExt::new(); let mut t = TestExternalities::default(); - t.register_extension(OffchainExt::new(offchain)); + t.register_extension(OffchainDbExt::new(offchain.clone())); + t.register_extension(OffchainWorkerExt::new(offchain)); t.execute_with(|| { let mut lock = StorageLock::<'_, Time>::new(b"lock_1"); @@ -493,7 +494,8 @@ mod tests { fn storage_lock_and_forget() { let (offchain, state) = testing::TestOffchainExt::new(); let mut t = TestExternalities::default(); - t.register_extension(OffchainExt::new(offchain)); + t.register_extension(OffchainDbExt::new(offchain.clone())); + t.register_extension(OffchainWorkerExt::new(offchain)); t.execute_with(|| { let mut lock = StorageLock::<'_, Time>::new(b"lock_2"); @@ -517,7 +519,8 @@ mod tests { fn storage_lock_and_let_expire_and_lock_again() { let (offchain, state) = testing::TestOffchainExt::new(); let mut t = TestExternalities::default(); - t.register_extension(OffchainExt::new(offchain)); + t.register_extension(OffchainDbExt::new(offchain.clone())); + t.register_extension(OffchainWorkerExt::new(offchain)); t.execute_with(|| { let sleep_until = offchain::timestamp().add(Duration::from_millis(500)); @@ -549,7 +552,8 @@ mod tests { fn extend_active_lock() { let (offchain, state) = testing::TestOffchainExt::new(); let mut t = TestExternalities::default(); - t.register_extension(OffchainExt::new(offchain)); + t.register_extension(OffchainDbExt::new(offchain.clone())); + t.register_extension(OffchainWorkerExt::new(offchain)); t.execute_with(|| { let lock_expiration = Duration::from_millis(300); diff --git a/substrate/test-utils/client/Cargo.toml b/substrate/test-utils/client/Cargo.toml index 1f62e32ddf..df1cca2101 100644 --- a/substrate/test-utils/client/Cargo.toml +++ b/substrate/test-utils/client/Cargo.toml @@ -24,6 +24,7 @@ sc-client-db = { version = "0.9.0", features = ["test-helpers"], path = "../../c sc-consensus = { version = "0.9.0", path = "../../client/consensus/common" } sc-executor = { version = "0.9.0", path = "../../client/executor" } sc-light = { version = "3.0.0", path = "../../client/light" } +sc-offchain = { version = "3.0.0", path = "../../client/offchain" } sc-service = { version = "0.9.0", default-features = false, features = ["test-helpers"], path = "../../client/service" } sp-blockchain = { version = "3.0.0", path = "../../primitives/blockchain" } sp-consensus = { version = "0.9.0", path = "../../primitives/consensus/common" } diff --git a/substrate/test-utils/client/src/lib.rs b/substrate/test-utils/client/src/lib.rs index bf5b2b6a04..cdeefccc40 100644 --- a/substrate/test-utils/client/src/lib.rs +++ b/substrate/test-utils/client/src/lib.rs @@ -198,6 +198,7 @@ impl TestClientBuilder + 'static, Backend: sc_client_api::backend::Backend, + >::OffchainStorage: 'static, { let storage = { let mut storage = self.genesis_init.genesis_storage(); @@ -225,6 +226,7 @@ impl TestClientBuilder, _>::new( &state,