feat/ocw/bookkeeping (#5200)

Co-Authored-By: Kian Paimani <5588131+kianenigma@users.noreply.github.com>
Co-Authored-By: Tomasz Drwięga <tomusdrw@users.noreply.github.com>

substrate/client/src/call_executor.rs

@@ -25,9 +25,10 @@ use sp_state_machine::{
 };
 use sc_executor::{RuntimeVersion, RuntimeInfo, NativeVersion};
 use sp_externalities::Extensions;
-use sp_core::{NativeOrEncoded, NeverNativeValue, traits::CodeExecutor};
+use sp_core::{NativeOrEncoded, NeverNativeValue, traits::CodeExecutor, offchain::storage::OffchainOverlayedChanges};
 use sp_api::{ProofRecorder, InitializeBlock, StorageTransactionCache};
 use sc_client_api::{backend, call_executor::CallExecutor, CloneableSpawn};
+use crate::client::ClientConfig;
 
 /// Call executor that executes methods locally, querying all required
 /// data from local backend.
@@ -35,6 +36,7 @@ pub struct LocalCallExecutor<B, E> {
 	backend: Arc<B>,
 	executor: E,
 	spawn_handle: Box<dyn CloneableSpawn>,
+	client_config: ClientConfig,
 }
 
 impl<B, E> LocalCallExecutor<B, E> {
@@ -43,11 +45,13 @@ impl<B, E> LocalCallExecutor<B, E> {
 		backend: Arc<B>,
 		executor: E,
 		spawn_handle: Box<dyn CloneableSpawn>,
+		client_config: ClientConfig,
 	) -> Self {
 		LocalCallExecutor {
 			backend,
 			executor,
 			spawn_handle,
+			client_config,
 		}
 	}
 }
@@ -58,6 +62,7 @@ impl<B, E> Clone for LocalCallExecutor<B, E> where E: Clone {
 			backend: self.backend.clone(),
 			executor: self.executor.clone(),
 			spawn_handle: self.spawn_handle.clone(),
+			client_config: self.client_config.clone(),
 		}
 	}
 }
@@ -81,6 +86,11 @@ where
 		extensions: Option<Extensions>,
 	) -> sp_blockchain::Result<Vec<u8>> {
 		let mut changes = OverlayedChanges::default();
+		let mut offchain_changes = if self.client_config.offchain_indexing_api {
+			OffchainOverlayedChanges::enabled()
+		} else {
+			OffchainOverlayedChanges::disabled()
+		};
 		let changes_trie = backend::changes_tries_state_at_block(
 			id, self.backend.changes_trie_storage()
 		)?;
@@ -90,6 +100,7 @@ where
 			&state,
 			changes_trie,
 			&mut changes,
+			&mut offchain_changes,
 			&self.executor,
 			method,
 			call_data,
@@ -120,6 +131,7 @@ where
 		method: &str,
 		call_data: &[u8],
 		changes: &RefCell<OverlayedChanges>,
+		offchain_changes: &RefCell<OffchainOverlayedChanges>,
 		storage_transaction_cache: Option<&RefCell<
 			StorageTransactionCache<Block, B::State>
 		>>,
@@ -143,6 +155,9 @@ where
 
 		let mut state = self.backend.state_at(*at)?;
 
+		let changes = &mut *changes.borrow_mut();
+		let offchain_changes = &mut *offchain_changes.borrow_mut();
+
 		match recorder {
 			Some(recorder) => {
 				let trie_state = state.as_trie_backend()
@@ -160,11 +175,11 @@ where
 					recorder.clone(),
 				);
 
-				let changes = &mut *changes.borrow_mut();
 				let mut state_machine = StateMachine::new(
 					&backend,
 					changes_trie_state,
 					changes,
+					offchain_changes,
 					&self.executor,
 					method,
 					call_data,
@@ -179,11 +194,11 @@ where
 			None => {
 				let state_runtime_code = sp_state_machine::backend::BackendRuntimeCode::new(&state);
 				let runtime_code = state_runtime_code.runtime_code()?;
-				let changes = &mut *changes.borrow_mut();
 				let mut state_machine = StateMachine::new(
 					&state,
 					changes_trie_state,
 					changes,
+					offchain_changes,
 					&self.executor,
 					method,
 					call_data,
@@ -198,6 +213,7 @@ where
 
 	fn runtime_version(&self, id: &BlockId<Block>) -> sp_blockchain::Result<RuntimeVersion> {
 		let mut overlay = OverlayedChanges::default();
+		let mut offchain_overlay = OffchainOverlayedChanges::default();
 		let changes_trie_state = backend::changes_tries_state_at_block(
 			id,
 			self.backend.changes_trie_storage(),
@@ -206,6 +222,7 @@ where
 		let mut cache = StorageTransactionCache::<Block, B::State>::default();
 		let mut ext = Ext::new(
 			&mut overlay,
+			&mut offchain_overlay,
 			&mut cache,
 			&state,
 			changes_trie_state,

substrate/client/src/client.rs

@@ -99,6 +99,7 @@ pub struct Client<B, E, Block, RA> where Block: BlockT {
 	importing_block: RwLock<Option<Block::Hash>>,
 	block_rules: BlockRules<Block>,
 	execution_extensions: ExecutionExtensions<Block>,
+	config: ClientConfig,
 	_phantom: PhantomData<RA>,
 }
 
@@ -136,6 +137,7 @@ pub fn new_in_mem<E, Block, S, RA>(
 	keystore: Option<sp_core::traits::BareCryptoStorePtr>,
 	prometheus_registry: Option<Registry>,
 	spawn_handle: Box<dyn CloneableSpawn>,
+	config: ClientConfig,
 ) -> sp_blockchain::Result<Client<
 	in_mem::Backend<Block>,
 	LocalCallExecutor<in_mem::Backend<Block>, E>,
@@ -146,7 +148,24 @@ pub fn new_in_mem<E, Block, S, RA>(
 	S: BuildStorage,
 	Block: BlockT,
 {
-	new_with_backend(Arc::new(in_mem::Backend::new()), executor, genesis_storage, keystore, spawn_handle, prometheus_registry)
+	new_with_backend(
+		Arc::new(in_mem::Backend::new()),
+		executor,
+		genesis_storage,
+		keystore,
+		spawn_handle,
+		prometheus_registry,
+		config,
+	)
+}
+
+/// Relevant client configuration items relevant for the client.
+#[derive(Debug,Clone,Default)]
+pub struct ClientConfig {
+	/// Enable the offchain worker db.
+	pub offchain_worker_enabled: bool,
+	/// If true, allows access from the runtime to write into offchain worker db.
+	pub offchain_indexing_api: bool,
 }
 
 /// Create a client with the explicitly provided backend.
@@ -158,6 +177,7 @@ pub fn new_with_backend<B, E, Block, S, RA>(
 	keystore: Option<sp_core::traits::BareCryptoStorePtr>,
 	spawn_handle: Box<dyn CloneableSpawn>,
 	prometheus_registry: Option<Registry>,
+	config: ClientConfig,
 ) -> sp_blockchain::Result<Client<B, LocalCallExecutor<B, E>, Block, RA>>
 	where
 		E: CodeExecutor + RuntimeInfo,
@@ -165,7 +185,7 @@ pub fn new_with_backend<B, E, Block, S, RA>(
 		Block: BlockT,
 		B: backend::LocalBackend<Block> + 'static,
 {
-	let call_executor = LocalCallExecutor::new(backend.clone(), executor, spawn_handle);
+	let call_executor = LocalCallExecutor::new(backend.clone(), executor, spawn_handle, config.clone());
 	let extensions = ExecutionExtensions::new(Default::default(), keystore);
 	Client::new(
 		backend,
@@ -175,6 +195,7 @@ pub fn new_with_backend<B, E, Block, S, RA>(
 		Default::default(),
 		extensions,
 		prometheus_registry,
+		config,
 	)
 }
 
@@ -243,6 +264,7 @@ impl<B, E, Block, RA> Client<B, E, Block, RA> where
 	B: backend::Backend<Block>,
 	E: CallExecutor<Block>,
 	Block: BlockT,
+	Block::Header: Clone,
 {
 	/// Creates new Substrate Client with given blockchain and code executor.
 	pub fn new(
@@ -253,6 +275,7 @@ impl<B, E, Block, RA> Client<B, E, Block, RA> where
 		bad_blocks: BadBlocks<Block>,
 		execution_extensions: ExecutionExtensions<Block>,
 		prometheus_registry: Option<Registry>,
+		config: ClientConfig,
 	) -> sp_blockchain::Result<Self> {
 		if backend.blockchain().header(BlockId::Number(Zero::zero()))?.is_none() {
 			let genesis_storage = build_genesis_storage.build_storage()?;
@@ -282,6 +305,7 @@ impl<B, E, Block, RA> Client<B, E, Block, RA> where
 			importing_block: Default::default(),
 			block_rules: BlockRules::new(fork_blocks, bad_blocks),
 			execution_extensions,
+			config,
 			_phantom: Default::default(),
 		})
 	}
@@ -614,7 +638,7 @@ impl<B, E, Block, RA> Client<B, E, Block, RA> where
 		if let Ok(ImportResult::Imported(ref aux)) = result {
 			if aux.is_new_best {
 				use rand::Rng;
-				
+
 				// don't send telemetry block import events during initial sync for every
 				// block to avoid spamming the telemetry server, these events will be randomly
 				// sent at a rate of 1/10.
@@ -696,7 +720,22 @@ impl<B, E, Block, RA> Client<B, E, Block, RA> where
 
 				operation.op.update_cache(new_cache);
 
-				let (main_sc, child_sc, tx, _, changes_trie_tx) = storage_changes.into_inner();
+				let (
+					main_sc,
+					child_sc,
+					offchain_sc,
+					tx, _,
+					changes_trie_tx,
+				) = storage_changes.into_inner();
+
+				if self.config.offchain_indexing_api {
+					// if let Some(mut offchain_storage) = self.backend.offchain_storage() {
+					// 	offchain_sc.iter().for_each(|(k,v)| {
+					// 		offchain_storage.set(b"block-import-info", k,v)
+					// 	});
+					// }
+					operation.op.update_offchain_storage(offchain_sc)?;
+				}
 
 				operation.op.update_db_storage(tx)?;
 				operation.op.update_storage(main_sc.clone(), child_sc.clone())?;
@@ -1554,6 +1593,7 @@ impl<B, E, Block, RA> CallApiAt<Block> for Client<B, E, Block, RA> where
 			params.function,
 			&params.arguments,
 			params.overlayed_changes,
+			params.offchain_changes,
 			Some(params.storage_transaction_cache),
 			params.initialize_block,
 			manager,
@@ -3493,6 +3533,7 @@ pub(crate) mod tests {
 				None,
 				None,
 				sp_core::tasks::executor(),
+				Default::default(),
 			)
 			.unwrap();
 

substrate/client/src/genesis.rs

@@ -55,6 +55,7 @@ mod tests {
 	};
 	use sp_runtime::traits::BlakeTwo256;
 	use sp_core::tasks::executor as tasks_executor;
+	use sp_core::offchain::storage::OffchainOverlayedChanges;
 	use hex_literal::*;
 
 	native_executor_instance!(
@@ -90,6 +91,7 @@ mod tests {
 		};
 		let hash = header.hash();
 		let mut overlay = OverlayedChanges::default();
+		let mut offchain_overlay = OffchainOverlayedChanges::default();
 		let backend_runtime_code = sp_state_machine::backend::BackendRuntimeCode::new(&backend);
 		let runtime_code = backend_runtime_code.runtime_code().expect("Code is part of the backend");
 
@@ -97,6 +99,7 @@ mod tests {
 			backend,
 			sp_state_machine::disabled_changes_trie_state::<_, u64>(),
 			&mut overlay,
+			&mut offchain_overlay,
 			&executor(),
 			"Core_initialize_block",
 			&header.encode(),
@@ -112,6 +115,7 @@ mod tests {
 				backend,
 				sp_state_machine::disabled_changes_trie_state::<_, u64>(),
 				&mut overlay,
+				&mut offchain_overlay,
 				&executor(),
 				"BlockBuilder_apply_extrinsic",
 				&tx.encode(),
@@ -127,6 +131,7 @@ mod tests {
 			backend,
 			sp_state_machine::disabled_changes_trie_state::<_, u64>(),
 			&mut overlay,
+			&mut offchain_overlay,
 			&executor(),
 			"BlockBuilder_finalize_block",
 			&[],
@@ -174,10 +179,13 @@ mod tests {
 		let runtime_code = backend_runtime_code.runtime_code().expect("Code is part of the backend");
 
 		let mut overlay = OverlayedChanges::default();
+		let mut offchain_overlay = OffchainOverlayedChanges::default();
+
 		let _ = StateMachine::new(
 			&backend,
 			sp_state_machine::disabled_changes_trie_state::<_, u64>(),
 			&mut overlay,
+			&mut offchain_overlay,
 			&executor(),
 			"Core_execute_block",
 			&b1data,
@@ -206,10 +214,13 @@ mod tests {
 		let runtime_code = backend_runtime_code.runtime_code().expect("Code is part of the backend");
 
 		let mut overlay = OverlayedChanges::default();
+		let mut offchain_overlay = OffchainOverlayedChanges::default();
+
 		let _ = StateMachine::new(
 			&backend,
 			sp_state_machine::disabled_changes_trie_state::<_, u64>(),
 			&mut overlay,
+			&mut offchain_overlay,
 			&executor(),
 			"Core_execute_block",
 			&b1data,
@@ -238,10 +249,13 @@ mod tests {
 		let runtime_code = backend_runtime_code.runtime_code().expect("Code is part of the backend");
 
 		let mut overlay = OverlayedChanges::default();
+		let mut offchain_overlay = OffchainOverlayedChanges::default();
+
 		let r = StateMachine::new(
 			&backend,
 			sp_state_machine::disabled_changes_trie_state::<_, u64>(),
 			&mut overlay,
+			&mut offchain_overlay,
 			&executor(),
 			"Core_execute_block",
 			&b1data,

substrate/client/src/lib.rs

@@ -63,6 +63,7 @@
 //! 		backend.clone(),
 //! 		NativeExecutor::<LocalExecutor>::new(WasmExecutionMethod::Interpreted, None, 8),
 //!         sp_core::tasks::executor(),
+//! 		Default::default(),
 //!		),
 //! 	// This parameter provides the storage for the chain genesis.
 //! 	&<Storage>::default(),
@@ -70,6 +71,7 @@
 //! 	Default::default(),
 //! 	Default::default(),
 //!		None,
+//! 	Default::default(),
 //! );
 //! ```
 //!
@@ -99,8 +101,8 @@ pub use crate::{
 	client::{
 		new_with_backend,
 		new_in_mem,
-		BlockBackend, ImportNotifications, FinalityNotifications, BlockchainEvents, LockImportRun,
-		BlockImportNotification, Client, ClientInfo, ExecutionStrategies, FinalityNotification,
+		ImportNotifications, FinalityNotifications, BlockchainEvents, LockImportRun,
+		BlockImportNotification, Client, ClientConfig, ClientInfo, ExecutionStrategies, FinalityNotification,
 		LongestChain, BlockOf, ProvideUncles, BadBlocks, ForkBlocks, apply_aux,
 	},
 	leaves::LeafSet,

substrate/client/src/light/call_executor.rs

@@ -21,7 +21,7 @@ use std::{
 };
 
 use codec::{Encode, Decode};
-use sp_core::{convert_hash, NativeOrEncoded, traits::CodeExecutor};
+use sp_core::{convert_hash, NativeOrEncoded, traits::CodeExecutor, offchain::storage::OffchainOverlayedChanges};
 use sp_runtime::{
 	generic::BlockId, traits::{One, Block as BlockT, Header as HeaderT, HashFor},
 };
@@ -108,6 +108,7 @@ impl<Block, B, Local> CallExecutor<Block> for
 		method: &str,
 		call_data: &[u8],
 		changes: &RefCell<OverlayedChanges>,
+		offchain_changes: &RefCell<OffchainOverlayedChanges>,
 		_: Option<&RefCell<StorageTransactionCache<Block, B::State>>>,
 		initialize_block: InitializeBlock<'a, Block>,
 		_manager: ExecutionManager<EM>,
@@ -134,6 +135,7 @@ impl<Block, B, Local> CallExecutor<Block> for
 				method,
 				call_data,
 				changes,
+				offchain_changes,
 				None,
 				initialize_block,
 				ExecutionManager::NativeWhenPossible,
@@ -338,6 +340,7 @@ mod tests {
 			_method: &str,
 			_call_data: &[u8],
 			_changes: &RefCell<OverlayedChanges>,
+			_offchain_changes: &RefCell<OffchainOverlayedChanges>,
 			_storage_transaction_cache: Option<&RefCell<
 				StorageTransactionCache<
 					Block,

substrate/client/src/light/mod.rs

@@ -31,7 +31,7 @@ use sp_blockchain::Result as ClientResult;
 use prometheus_endpoint::Registry;
 
 use crate::call_executor::LocalCallExecutor;
-use crate::client::Client;
+use crate::client::{Client,ClientConfig};
 use sc_client_api::{
 	light::Storage as BlockchainStorage, CloneableSpawn,
 };
@@ -77,7 +77,7 @@ pub fn new_light<B, S, RA, E>(
 		S: BlockchainStorage<B> + 'static,
 		E: CodeExecutor + RuntimeInfo + Clone + 'static,
 {
-	let local_executor = LocalCallExecutor::new(backend.clone(), code_executor, spawn_handle.clone());
+	let local_executor = LocalCallExecutor::new(backend.clone(), code_executor, spawn_handle.clone(), ClientConfig::default());
 	let executor = GenesisCallExecutor::new(backend.clone(), local_executor);
 	Client::new(
 		backend,
@@ -87,6 +87,7 @@ pub fn new_light<B, S, RA, E>(
 		Default::default(),
 		Default::default(),
 		prometheus_registry,
+		ClientConfig::default(),
 	)
 }