create parallel tasks extension (#5249)

substrate/client/src/call_executor.rs

@@ -27,13 +27,14 @@ use sc_executor::{RuntimeVersion, RuntimeInfo, NativeVersion};
 use sp_externalities::Extensions;
 use sp_core::{NativeOrEncoded, NeverNativeValue, traits::CodeExecutor};
 use sp_api::{ProofRecorder, InitializeBlock, StorageTransactionCache};
-use sc_client_api::{backend, call_executor::CallExecutor};
+use sc_client_api::{backend, call_executor::CallExecutor, CloneableSpawn};
 
 /// Call executor that executes methods locally, querying all required
 /// data from local backend.
 pub struct LocalCallExecutor<B, E> {
 	backend: Arc<B>,
 	executor: E,
+	spawn_handle: Box<dyn CloneableSpawn>,
 }
 
 impl<B, E> LocalCallExecutor<B, E> {
@@ -41,10 +42,12 @@ impl<B, E> LocalCallExecutor<B, E> {
 	pub fn new(
 		backend: Arc<B>,
 		executor: E,
+		spawn_handle: Box<dyn CloneableSpawn>,
 	) -> Self {
 		LocalCallExecutor {
 			backend,
 			executor,
+			spawn_handle,
 		}
 	}
 }
@@ -54,6 +57,7 @@ impl<B, E> Clone for LocalCallExecutor<B, E> where E: Clone {
 		LocalCallExecutor {
 			backend: self.backend.clone(),
 			executor: self.executor.clone(),
+			spawn_handle: self.spawn_handle.clone(),
 		}
 	}
 }
@@ -91,6 +95,7 @@ where
 			call_data,
 			extensions.unwrap_or_default(),
 			&state_runtime_code.runtime_code()?,
+			self.spawn_handle.clone(),
 		).execute_using_consensus_failure_handler::<_, NeverNativeValue, fn() -> _>(
 			strategy.get_manager(),
 			None,
@@ -164,6 +169,7 @@ where
 					call_data,
 					extensions.unwrap_or_default(),
 					&runtime_code,
+					self.spawn_handle.clone(),
 				)
 				// TODO: https://github.com/paritytech/substrate/issues/4455
 				// .with_storage_transaction_cache(storage_transaction_cache.as_mut().map(|c| &mut **c))
@@ -180,6 +186,7 @@ where
 					call_data,
 					extensions.unwrap_or_default(),
 					&state_runtime_code.runtime_code()?,
+					self.spawn_handle.clone(),
 				)
 				.with_storage_transaction_cache(storage_transaction_cache.as_mut().map(|c| &mut **c))
 				.execute_using_consensus_failure_handler(execution_manager, native_call)
@@ -218,6 +225,7 @@ where
 			trie_state,
 			overlay,
 			&self.executor,
+			self.spawn_handle.clone(),
 			method,
 			call_data,
 			&sp_state_machine::backend::BackendRuntimeCode::new(trie_state).runtime_code()?,

substrate/client/src/client.rs

@@ -76,7 +76,7 @@ pub use sc_client_api::{
 	},
 	execution_extensions::{ExecutionExtensions, ExecutionStrategies},
 	notifications::{StorageNotifications, StorageEventStream},
-	CallExecutor, ExecutorProvider, ProofProvider,
+	CallExecutor, ExecutorProvider, ProofProvider, CloneableSpawn,
 };
 use sp_blockchain::Error;
 use prometheus_endpoint::Registry;
@@ -135,6 +135,7 @@ pub fn new_in_mem<E, Block, S, RA>(
 	genesis_storage: &S,
 	keystore: Option<sp_core::traits::BareCryptoStorePtr>,
 	prometheus_registry: Option<Registry>,
+	spawn_handle: Box<dyn CloneableSpawn>,
 ) -> sp_blockchain::Result<Client<
 	in_mem::Backend<Block>,
 	LocalCallExecutor<in_mem::Backend<Block>, E>,
@@ -145,7 +146,7 @@ pub fn new_in_mem<E, Block, S, RA>(
 	S: BuildStorage,
 	Block: BlockT,
 {
-	new_with_backend(Arc::new(in_mem::Backend::new()), executor, genesis_storage, keystore, prometheus_registry)
+	new_with_backend(Arc::new(in_mem::Backend::new()), executor, genesis_storage, keystore, spawn_handle, prometheus_registry)
 }
 
 /// Create a client with the explicitly provided backend.
@@ -155,6 +156,7 @@ pub fn new_with_backend<B, E, Block, S, RA>(
 	executor: E,
 	build_genesis_storage: &S,
 	keystore: Option<sp_core::traits::BareCryptoStorePtr>,
+	spawn_handle: Box<dyn CloneableSpawn>,
 	prometheus_registry: Option<Registry>,
 ) -> sp_blockchain::Result<Client<B, LocalCallExecutor<B, E>, Block, RA>>
 	where
@@ -163,7 +165,7 @@ pub fn new_with_backend<B, E, Block, S, RA>(
 		Block: BlockT,
 		B: backend::LocalBackend<Block> + 'static,
 {
-	let call_executor = LocalCallExecutor::new(backend.clone(), executor);
+	let call_executor = LocalCallExecutor::new(backend.clone(), executor, spawn_handle);
 	let extensions = ExecutionExtensions::new(Default::default(), keystore);
 	Client::new(
 		backend,
@@ -1124,7 +1126,13 @@ impl<B, E, Block, RA> ProofProvider<Block> for Client<B, E, Block, RA> where
 
 		let state = self.state_at(id)?;
 		let header = self.prepare_environment_block(id)?;
-		prove_execution(state, header, &self.executor, method, call_data).map(|(r, p)| {
+		prove_execution(
+			state,
+			header,
+			&self.executor,
+			method,
+			call_data,
+		).map(|(r, p)| {
 			(r, StorageProof::merge(vec![p, code_proof]))
 		})
 	}
@@ -3482,6 +3490,7 @@ pub(crate) mod tests {
 				&substrate_test_runtime_client::GenesisParameters::default().genesis_storage(),
 				None,
 				None,
+				sp_core::tasks::executor(),
 			)
 			.unwrap();
 

substrate/client/src/genesis.rs

@@ -54,6 +54,7 @@ mod tests {
 		AccountKeyring, Sr25519Keyring,
 	};
 	use sp_runtime::traits::BlakeTwo256;
+	use sp_core::tasks::executor as tasks_executor;
 	use hex_literal::*;
 
 	native_executor_instance!(
@@ -101,6 +102,7 @@ mod tests {
 			&header.encode(),
 			Default::default(),
 			&runtime_code,
+			tasks_executor(),
 		).execute(
 			ExecutionStrategy::NativeElseWasm,
 		).unwrap();
@@ -115,6 +117,7 @@ mod tests {
 				&tx.encode(),
 				Default::default(),
 				&runtime_code,
+				tasks_executor(),
 			).execute(
 				ExecutionStrategy::NativeElseWasm,
 			).unwrap();
@@ -129,6 +132,7 @@ mod tests {
 			&[],
 			Default::default(),
 			&runtime_code,
+			tasks_executor(),
 		).execute(
 			ExecutionStrategy::NativeElseWasm,
 		).unwrap();
@@ -179,6 +183,7 @@ mod tests {
 			&b1data,
 			Default::default(),
 			&runtime_code,
+			tasks_executor(),
 		).execute(
 			ExecutionStrategy::NativeElseWasm,
 		).unwrap();
@@ -210,6 +215,7 @@ mod tests {
 			&b1data,
 			Default::default(),
 			&runtime_code,
+			tasks_executor(),
 		).execute(
 			ExecutionStrategy::AlwaysWasm,
 		).unwrap();
@@ -241,6 +247,7 @@ mod tests {
 			&b1data,
 			Default::default(),
 			&runtime_code,
+			tasks_executor(),
 		).execute(
 			ExecutionStrategy::NativeElseWasm,
 		);

substrate/client/src/lib.rs

@@ -62,6 +62,7 @@
 //! 	LocalCallExecutor::new(
 //! 		backend.clone(),
 //! 		NativeExecutor::<LocalExecutor>::new(WasmExecutionMethod::Interpreted, None, 8),
+//!         sp_core::tasks::executor(),
 //!		),
 //! 	// This parameter provides the storage for the chain genesis.
 //! 	&<Storage>::default(),

substrate/client/src/light/call_executor.rs

@@ -28,7 +28,7 @@ use sp_runtime::{
 use sp_externalities::Extensions;
 use sp_state_machine::{
 	self, Backend as StateBackend, OverlayedChanges, ExecutionStrategy, create_proof_check_backend,
-	execution_proof_check_on_trie_backend, ExecutionManager, StorageProof,
+	execution_proof_check_on_trie_backend, ExecutionManager, StorageProof, CloneableSpawn,
 };
 use hash_db::Hasher;
 
@@ -216,6 +216,7 @@ pub fn prove_execution<Block, S, E>(
 /// Proof should include both environment preparation proof and method execution proof.
 pub fn check_execution_proof<Header, E, H>(
 	executor: &E,
+	spawn_handle: Box<dyn CloneableSpawn>,
 	request: &RemoteCallRequest<Header>,
 	remote_proof: StorageProof,
 ) -> ClientResult<Vec<u8>>
@@ -227,6 +228,7 @@ pub fn check_execution_proof<Header, E, H>(
 {
 	check_execution_proof_with_make_header::<Header, E, H, _>(
 		executor,
+		spawn_handle,
 		request,
 		remote_proof,
 		|header| <Header as HeaderT>::new(
@@ -241,6 +243,7 @@ pub fn check_execution_proof<Header, E, H>(
 
 fn check_execution_proof_with_make_header<Header, E, H, MakeNextHeader: Fn(&Header) -> Header>(
 	executor: &E,
+	spawn_handle: Box<dyn CloneableSpawn>,
 	request: &RemoteCallRequest<Header>,
 	remote_proof: StorageProof,
 	make_next_header: MakeNextHeader,
@@ -267,6 +270,7 @@ fn check_execution_proof_with_make_header<Header, E, H, MakeNextHeader: Fn(&Head
 		&trie_backend,
 		&mut changes,
 		executor,
+		spawn_handle.clone(),
 		"Core_initialize_block",
 		&next_header.encode(),
 		&runtime_code,
@@ -277,6 +281,7 @@ fn check_execution_proof_with_make_header<Header, E, H, MakeNextHeader: Fn(&Head
 		&trie_backend,
 		&mut changes,
 		executor,
+		spawn_handle,
 		&request.method,
 		&request.call_data,
 		&runtime_code,
@@ -292,7 +297,7 @@ mod tests {
 		runtime::{Header, Digest, Block}, TestClient, ClientBlockImportExt,
 	};
 	use sc_executor::{NativeExecutor, WasmExecutionMethod};
-	use sp_core::H256;
+	use sp_core::{H256, tasks::executor as tasks_executor};
 	use sc_client_api::backend::{Backend, NewBlockState};
 	use crate::in_mem::Backend as InMemBackend;
 	use sc_client_api::ProofProvider;
@@ -387,6 +392,7 @@ mod tests {
 			// check remote execution proof locally
 			let local_result = check_execution_proof::<_, _, BlakeTwo256>(
 				&local_executor(),
+				tasks_executor(),
 				&RemoteCallRequest {
 					block: substrate_test_runtime_client::runtime::Hash::default(),
 					header: remote_header,
@@ -414,6 +420,7 @@ mod tests {
 			// check remote execution proof locally
 			let execution_result = check_execution_proof_with_make_header::<_, _, BlakeTwo256, _>(
 				&local_executor(),
+				tasks_executor(),
 				&RemoteCallRequest {
 					block: substrate_test_runtime_client::runtime::Hash::default(),
 					header: remote_header,

substrate/client/src/light/fetcher.rs

@@ -30,7 +30,7 @@ use sp_runtime::traits::{
 use sp_state_machine::{
 	ChangesTrieRootsStorage, ChangesTrieAnchorBlockId, ChangesTrieConfigurationRange,
 	InMemoryChangesTrieStorage, TrieBackend, read_proof_check, key_changes_proof_check_with_db,
-	read_child_proof_check,
+	read_child_proof_check, CloneableSpawn,
 };
 pub use sp_state_machine::StorageProof;
 use sp_blockchain::{Error as ClientError, Result as ClientResult};
@@ -50,14 +50,15 @@ use crate::light::call_executor::check_execution_proof;
 pub struct LightDataChecker<E, H, B: BlockT, S: BlockchainStorage<B>> {
 	blockchain: Arc<Blockchain<S>>,
 	executor: E,
+	spawn_handle: Box<dyn CloneableSpawn>,
 	_hasher: PhantomData<(B, H)>,
 }
 
 impl<E, H, B: BlockT, S: BlockchainStorage<B>> LightDataChecker<E, H, B, S> {
 	/// Create new light data checker.
-	pub fn new(blockchain: Arc<Blockchain<S>>, executor: E) -> Self {
+	pub fn new(blockchain: Arc<Blockchain<S>>, executor: E, spawn_handle: Box<dyn CloneableSpawn>) -> Self {
 		Self {
-			blockchain, executor, _hasher: PhantomData
+			blockchain, executor, spawn_handle, _hasher: PhantomData
 		}
 	}
 
@@ -254,7 +255,12 @@ impl<E, Block, H, S> FetchChecker<Block> for LightDataChecker<E, H, Block, S>
 		request: &RemoteCallRequest<Block::Header>,
 		remote_proof: StorageProof,
 	) -> ClientResult<Vec<u8>> {
-		check_execution_proof::<_, _, H>(&self.executor, request, remote_proof)
+		check_execution_proof::<_, _, H>(
+			&self.executor,
+			self.spawn_handle.clone(),
+			request,
+			remote_proof,
+		)
 	}
 
 	fn check_changes_proof(
@@ -338,7 +344,8 @@ pub mod tests {
 	use sc_client_api::backend::NewBlockState;
 	use substrate_test_runtime_client::{
 		blockchain::HeaderBackend, AccountKeyring, ClientBlockImportExt,
-		runtime::{self, Hash, Block, Header, Extrinsic}
+		runtime::{self, Hash, Block, Header, Extrinsic},
+		tasks_executor,
 	};
 	use sp_consensus::BlockOrigin;
 
@@ -395,7 +402,8 @@ pub mod tests {
 		).unwrap();
 		let local_checker = LightDataChecker::new(
 			Arc::new(DummyBlockchain::new(DummyStorage::new())),
-			local_executor()
+			local_executor(),
+			tasks_executor(),
 		);
 		(local_checker, remote_block_header, remote_read_proof, heap_pages)
 	}
@@ -444,6 +452,7 @@ pub mod tests {
 		let local_checker = LightDataChecker::new(
 			Arc::new(DummyBlockchain::new(DummyStorage::new())),
 			local_executor(),
+			tasks_executor(),
 		);
 		(local_checker, remote_block_header, remote_read_proof, child_value)
 	}
@@ -474,6 +483,7 @@ pub mod tests {
 		let local_checker = LightDataChecker::new(
 			Arc::new(DummyBlockchain::new(DummyStorage::new())),
 			local_executor(),
+			tasks_executor(),
 		);
 		(local_checker, local_cht_root, remote_block_header, remote_header_proof)
 	}
@@ -559,6 +569,7 @@ pub mod tests {
 		let local_checker = TestChecker::new(
 			Arc::new(DummyBlockchain::new(DummyStorage::new())),
 			local_executor(),
+			tasks_executor(),
 		);
 		let local_checker = &local_checker as &dyn FetchChecker<Block>;
 		let max = remote_client.chain_info().best_number;
@@ -633,6 +644,7 @@ pub mod tests {
 		let local_checker = TestChecker::new(
 			Arc::new(DummyBlockchain::new(local_storage)),
 			local_executor(),
+			tasks_executor(),
 		);
 
 		// check proof on local client
@@ -667,6 +679,7 @@ pub mod tests {
 		let local_checker = TestChecker::new(
 			Arc::new(DummyBlockchain::new(DummyStorage::new())),
 			local_executor(),
+			tasks_executor(),
 		);
 		let local_checker = &local_checker as &dyn FetchChecker<Block>;
 		let max = remote_client.chain_info().best_number;
@@ -754,6 +767,7 @@ pub mod tests {
 		let local_checker = TestChecker::new(
 			Arc::new(DummyBlockchain::new(DummyStorage::new())),
 			local_executor(),
+			tasks_executor(),
 		);
 		assert!(local_checker.check_changes_tries_proof(4, &remote_proof.roots,
 			remote_proof.roots_proof.clone()).is_err());
@@ -764,6 +778,7 @@ pub mod tests {
 		let local_checker = TestChecker::new(
 			Arc::new(DummyBlockchain::new(local_storage)),
 			local_executor(),
+			tasks_executor(),
 		);
 		let result = local_checker.check_changes_tries_proof(
 			4, &remote_proof.roots, StorageProof::empty()
@@ -781,6 +796,7 @@ pub mod tests {
 		let local_checker = TestChecker::new(
 			Arc::new(DummyBlockchain::new(DummyStorage::new())),
 			local_executor(),
+			tasks_executor(),
 		);
 
 		let body_request = RemoteBodyRequest {
@@ -804,6 +820,7 @@ pub mod tests {
 		let local_checker = TestChecker::new(
 			Arc::new(DummyBlockchain::new(DummyStorage::new())),
 			local_executor(),
+			tasks_executor(),
 		);
 
 		let body_request = RemoteBodyRequest {

substrate/client/src/light/mod.rs

@@ -33,7 +33,7 @@ use prometheus_endpoint::Registry;
 use crate::call_executor::LocalCallExecutor;
 use crate::client::Client;
 use sc_client_api::{
-	light::Storage as BlockchainStorage,
+	light::Storage as BlockchainStorage, CloneableSpawn,
 };
 use crate::light::backend::Backend;
 use crate::light::blockchain::Blockchain;
@@ -59,6 +59,7 @@ pub fn new_light<B, S, RA, E>(
 	backend: Arc<Backend<S, HashFor<B>>>,
 	genesis_storage: &dyn BuildStorage,
 	code_executor: E,
+	spawn_handle: Box<dyn CloneableSpawn>,
 	prometheus_registry: Option<Registry>,
 ) -> ClientResult<
 		Client<
@@ -76,7 +77,7 @@ pub fn new_light<B, S, RA, E>(
 		S: BlockchainStorage<B> + 'static,
 		E: CodeExecutor + RuntimeInfo + Clone + 'static,
 {
-	let local_executor = LocalCallExecutor::new(backend.clone(), code_executor);
+	let local_executor = LocalCallExecutor::new(backend.clone(), code_executor, spawn_handle.clone());
 	let executor = GenesisCallExecutor::new(backend.clone(), local_executor);
 	Client::new(
 		backend,
@@ -93,9 +94,10 @@ pub fn new_light<B, S, RA, E>(
 pub fn new_fetch_checker<E, B: BlockT, S: BlockchainStorage<B>>(
 	blockchain: Arc<Blockchain<S>>,
 	executor: E,
+	spawn_handle: Box<dyn CloneableSpawn>,
 ) -> LightDataChecker<E, HashFor<B>, B, S>
 	where
 		E: CodeExecutor,
 {
-	LightDataChecker::new(blockchain, executor)
+	LightDataChecker::new(blockchain, executor, spawn_handle)
 }