Prepare for asynchronous transaction validation in tx pool (#3650)

* async txpool API

* Update core/rpc/src/author/mod.rs

Co-Authored-By: Tomasz Drwięga <tomusdrw@users.noreply.github.com>

* Update core/transaction-pool/graph/src/pool.rs

Co-Authored-By: Tomasz Drwięga <tomusdrw@users.noreply.github.com>

* Pool -> Pool + ValidatedPool

* removed lost block_on when importing xt from network

* fix grumbles

* alias for future::Executor in rpc

* removed executor from Author RPCs

* Pool + SharedValidatedPool -> Pool

* fix compilation after merge

* another fix

* another fix

substrate/core/rpc/api/src/author/error.rs

@@ -22,6 +22,9 @@ use jsonrpc_core as rpc;
 /// Author RPC Result type.
 pub type Result<T> = std::result::Result<T, Error>;
 
+/// Author RPC future Result type.
+pub type FutureResult<T> = Box<dyn rpc::futures::Future<Item = T, Error = Error> + Send>;
+
 /// Author RPC errors.
 #[derive(Debug, derive_more::Display, derive_more::From)]
 pub enum Error {

substrate/core/rpc/api/src/author/mod.rs

@@ -24,7 +24,7 @@ use jsonrpc_pubsub::{typed::Subscriber, SubscriptionId};
 use primitives::{
 	Bytes
 };
-use self::error::Result;
+use self::error::{FutureResult, Result};
 use txpool::watcher::Status;
 
 pub use self::gen_client::Client as AuthorClient;
@@ -37,7 +37,7 @@ pub trait AuthorApi<Hash, BlockHash> {
 
 	/// Submit hex-encoded extrinsic for inclusion in block.
 	#[rpc(name = "author_submitExtrinsic")]
-	fn submit_extrinsic(&self, extrinsic: Bytes) -> Result<Hash>;
+	fn submit_extrinsic(&self, extrinsic: Bytes) -> FutureResult<Hash>;
 
 	/// Insert a key into the keystore.
 	#[rpc(name = "author_insertKey")]

substrate/core/rpc/api/src/lib.rs

@@ -25,7 +25,7 @@ mod helpers;
 mod subscriptions;
 
 pub use jsonrpc_core::IoHandlerExtension as RpcExtension;
-pub use subscriptions::Subscriptions;
+pub use subscriptions::{Subscriptions, TaskExecutor};
 pub use helpers::Receiver;
 
 pub mod author;

substrate/core/rpc/api/src/subscriptions.rs

@@ -25,6 +25,9 @@ use jsonrpc_core::futures::{Future, future};
 
 type Id = u64;
 
+/// Alias for a an implementation of `futures::future::Executor`.
+pub type TaskExecutor = Arc<dyn future::Executor<Box<dyn Future<Item = (), Error = ()> + Send>> + Send + Sync>;
+
 /// Generate unique ids for subscriptions.
 #[derive(Clone, Debug)]
 pub struct IdProvider {
@@ -53,12 +56,12 @@ impl IdProvider {
 pub struct Subscriptions {
 	next_id: IdProvider,
 	active_subscriptions: Arc<Mutex<HashMap<Id, oneshot::Sender<()>>>>,
-	executor: Arc<dyn future::Executor<Box<dyn Future<Item = (), Error = ()> + Send>> + Send + Sync>,
+	executor: TaskExecutor,
 }
 
 impl Subscriptions {
 	/// Creates new `Subscriptions` object.
-	pub fn new(executor: Arc<dyn future::Executor<Box<dyn Future<Item = (), Error = ()> + Send>> + Send + Sync>) -> Self {
+	pub fn new(executor: TaskExecutor) -> Self {
 		Subscriptions {
 			next_id: Default::default(),
 			active_subscriptions: Default::default(),

substrate/core/rpc/src/author/mod.rs

@@ -20,13 +20,18 @@
 mod tests;
 
 use std::{sync::Arc, convert::TryInto};
+use futures03::future::{FutureExt, TryFutureExt};
+use log::warn;
 
 use client::{self, Client};
-use rpc::futures::{Sink, Future};
+use rpc::futures::{
+	Sink, Future,
+	stream::Stream as _,
+	future::result,
+};
 use futures03::{StreamExt as _, compat::Compat};
 use api::Subscriptions;
 use jsonrpc_pubsub::{typed::Subscriber, SubscriptionId};
-use log::warn;
 use codec::{Encode, Decode};
 use primitives::{Bytes, Blake2Hasher, H256, traits::BareCryptoStorePtr};
 use sr_primitives::{generic, traits::{self, ProvideRuntimeApi}};
@@ -44,7 +49,7 @@ use session::SessionKeys;
 
 /// Re-export the API for backward compatibility.
 pub use api::author::*;
-use self::error::{Error, Result};
+use self::error::{Error, FutureResult, Result};
 
 /// Authoring API
 pub struct Author<B, E, P, RA> where P: PoolChainApi + Sync + Send + 'static {
@@ -108,15 +113,19 @@ impl<B, E, P, RA> AuthorApi<ExHash<P>, BlockHash<P>> for Author<B, E, P, RA> whe
 		).map(Into::into).map_err(|e| Error::Client(Box::new(e)))
 	}
 
-	fn submit_extrinsic(&self, ext: Bytes) -> Result<ExHash<P>> {
-		let xt = Decode::decode(&mut &ext[..])?;
+	fn submit_extrinsic(&self, ext: Bytes) -> FutureResult<ExHash<P>> {
+		let xt = match Decode::decode(&mut &ext[..]) {
+			Ok(xt) => xt,
+			Err(err) => return Box::new(result(Err(err.into()))),
+		};
 		let best_block_hash = self.client.info().chain.best_hash;
-		self.pool
+		Box::new(self.pool
 			.submit_one(&generic::BlockId::hash(best_block_hash), xt)
+			.compat()
 			.map_err(|e| e.into_pool_error()
 				.map(Into::into)
-				.unwrap_or_else(|e| error::Error::Verification(Box::new(e)).into())
-			)
+				.unwrap_or_else(|e| error::Error::Verification(Box::new(e)).into()))
+		)
 	}
 
 	fn pending_extrinsics(&self) -> Result<Vec<Bytes>> {
@@ -151,17 +160,20 @@ impl<B, E, P, RA> AuthorApi<ExHash<P>, BlockHash<P>> for Author<B, E, P, RA> whe
 	) {
 		let submit = || -> Result<_> {
 			let best_block_hash = self.client.info().chain.best_hash;
-			let dxt = <<P as PoolChainApi>::Block as traits::Block>::Extrinsic::decode(&mut &xt[..])?;
-			self.pool
+			let dxt = <<P as PoolChainApi>::Block as traits::Block>::Extrinsic::decode(&mut &xt[..])
+				.map_err(error::Error::from)?;
+			Ok(self.pool
 				.submit_and_watch(&generic::BlockId::hash(best_block_hash), dxt)
+				.boxed()
+				.compat()
 				.map_err(|e| e.into_pool_error()
-					.map(Into::into)
+					.map(error::Error::from)
 					.unwrap_or_else(|e| error::Error::Verification(Box::new(e)).into())
-				)
+				))
 		};
 
-		let watcher = match submit() {
-			Ok(watcher) => watcher,
+		let future_watcher = match submit() {
+			Ok(future_watcher) => future_watcher,
 			Err(err) => {
 				// reject the subscriber (ignore errors - we don't care if subscriber is no longer there).
 				let _ = subscriber.reject(err.into());
@@ -169,12 +181,23 @@ impl<B, E, P, RA> AuthorApi<ExHash<P>, BlockHash<P>> for Author<B, E, P, RA> whe
 			},
 		};
 
+		// make 'future' watcher be a future with output = stream of watcher events
+		let future_watcher = future_watcher
+			.map_err(|err| { warn!("Failed to submit extrinsic: {}", err); })
+			.map(|watcher| Compat::new(watcher.into_stream().map(|v| Ok::<_, ()>(Ok(v)))));
+
+		// convert a 'future' watcher into the stream with single element = stream of watcher events
+		let watcher_stream = future_watcher.into_stream();
+
+		// and now flatten the 'watcher_stream' so that we'll have the stream with watcher events
+		let watcher_stream = watcher_stream.flatten();
+
 		self.subscriptions.add(subscriber, move |sink| {
 			sink
 				.sink_map_err(|e| warn!("Error sending notifications: {:?}", e))
-				.send_all(Compat::new(watcher.into_stream().map(|v| Ok::<_, ()>(Ok(v)))))
+				.send_all(watcher_stream)
 				.map(|_| ())
-		})
+		});
 	}
 
 	fn unwatch_extrinsic(&self, _metadata: Option<Self::Metadata>, id: SubscriptionId) -> Result<bool> {

substrate/core/rpc/src/author/tests.rs

@@ -21,9 +21,8 @@ use assert_matches::assert_matches;
 use codec::Encode;
 use transaction_pool::{
 	txpool::Pool,
-	ChainApi,
+	FullChainApi,
 };
-use futures::Stream;
 use primitives::{
 	H256, blake2_256, hexdisplay::HexDisplay, traits::BareCryptoStore,
 	testing::{ED25519, SR25519, KeyStore}, ed25519, crypto::Pair
@@ -51,7 +50,7 @@ fn submit_transaction_should_not_cause_error() {
 	let keystore = KeyStore::new();
 	let p = Author {
 		client: client.clone(),
-		pool: Arc::new(Pool::new(Default::default(), ChainApi::new(client))),
+		pool: Arc::new(Pool::new(Default::default(), FullChainApi::new(client))),
 		subscriptions: Subscriptions::new(Arc::new(runtime.executor())),
 		keystore: keystore.clone(),
 	};
@@ -59,11 +58,11 @@ fn submit_transaction_should_not_cause_error() {
 	let h: H256 = blake2_256(&xt).into();
 
 	assert_matches!(
-		AuthorApi::submit_extrinsic(&p, xt.clone().into()),
+		AuthorApi::submit_extrinsic(&p, xt.clone().into()).wait(),
 		Ok(h2) if h == h2
 	);
 	assert!(
-		AuthorApi::submit_extrinsic(&p, xt.into()).is_err()
+		AuthorApi::submit_extrinsic(&p, xt.into()).wait().is_err()
 	);
 }
 
@@ -74,7 +73,7 @@ fn submit_rich_transaction_should_not_cause_error() {
 	let keystore = KeyStore::new();
 	let p = Author {
 		client: client.clone(),
-		pool: Arc::new(Pool::new(Default::default(), ChainApi::new(client.clone()))),
+		pool: Arc::new(Pool::new(Default::default(), FullChainApi::new(client.clone()))),
 		subscriptions: Subscriptions::new(Arc::new(runtime.executor())),
 		keystore: keystore.clone(),
 	};
@@ -82,11 +81,11 @@ fn submit_rich_transaction_should_not_cause_error() {
 	let h: H256 = blake2_256(&xt).into();
 
 	assert_matches!(
-		AuthorApi::submit_extrinsic(&p, xt.clone().into()),
+		AuthorApi::submit_extrinsic(&p, xt.clone().into()).wait(),
 		Ok(h2) if h == h2
 	);
 	assert!(
-		AuthorApi::submit_extrinsic(&p, xt.into()).is_err()
+		AuthorApi::submit_extrinsic(&p, xt.into()).wait().is_err()
 	);
 }
 
@@ -95,7 +94,7 @@ fn should_watch_extrinsic() {
 	//given
 	let mut runtime = runtime::Runtime::new().unwrap();
 	let client = Arc::new(test_client::new());
-	let pool = Arc::new(Pool::new(Default::default(), ChainApi::new(client.clone())));
+	let pool = Arc::new(Pool::new(Default::default(), FullChainApi::new(client.clone())));
 	let keystore = KeyStore::new();
 	let p = Author {
 		client,
@@ -120,7 +119,7 @@ fn should_watch_extrinsic() {
 		};
 		tx.into_signed_tx()
 	};
-	AuthorApi::submit_extrinsic(&p, replacement.encode().into()).unwrap();
+	AuthorApi::submit_extrinsic(&p, replacement.encode().into()).wait().unwrap();
 	let (res, data) = runtime.block_on(data.into_future()).unwrap();
 	assert_eq!(
 		res,
@@ -137,7 +136,7 @@ fn should_watch_extrinsic() {
 fn should_return_pending_extrinsics() {
 	let runtime = runtime::Runtime::new().unwrap();
 	let client = Arc::new(test_client::new());
-	let pool = Arc::new(Pool::new(Default::default(), ChainApi::new(client.clone())));
+	let pool = Arc::new(Pool::new(Default::default(), FullChainApi::new(client.clone())));
 	let keystore = KeyStore::new();
 	let p = Author {
 		client,
@@ -146,7 +145,7 @@ fn should_return_pending_extrinsics() {
 		keystore: keystore.clone(),
 	};
 	let ex = uxt(AccountKeyring::Alice, 0);
-	AuthorApi::submit_extrinsic(&p, ex.encode().into()).unwrap();
+	AuthorApi::submit_extrinsic(&p, ex.encode().into()).wait().unwrap();
  	assert_matches!(
 		p.pending_extrinsics(),
 		Ok(ref expected) if *expected == vec![Bytes(ex.encode())]
@@ -157,7 +156,7 @@ fn should_return_pending_extrinsics() {
 fn should_remove_extrinsics() {
 	let runtime = runtime::Runtime::new().unwrap();
 	let client = Arc::new(test_client::new());
-	let pool = Arc::new(Pool::new(Default::default(), ChainApi::new(client.clone())));
+	let pool = Arc::new(Pool::new(Default::default(), FullChainApi::new(client.clone())));
 	let keystore = KeyStore::new();
 	let p = Author {
 		client,
@@ -166,11 +165,11 @@ fn should_remove_extrinsics() {
 		keystore: keystore.clone(),
 	};
 	let ex1 = uxt(AccountKeyring::Alice, 0);
-	p.submit_extrinsic(ex1.encode().into()).unwrap();
+	p.submit_extrinsic(ex1.encode().into()).wait().unwrap();
 	let ex2 = uxt(AccountKeyring::Alice, 1);
-	p.submit_extrinsic(ex2.encode().into()).unwrap();
+	p.submit_extrinsic(ex2.encode().into()).wait().unwrap();
 	let ex3 = uxt(AccountKeyring::Bob, 0);
-	let hash3 = p.submit_extrinsic(ex3.encode().into()).unwrap();
+	let hash3 = p.submit_extrinsic(ex3.encode().into()).wait().unwrap();
 	assert_eq!(pool.status().ready, 3);
 
 	// now remove all 3
@@ -190,7 +189,7 @@ fn should_insert_key() {
 	let keystore = KeyStore::new();
 	let p = Author {
 		client: client.clone(),
-		pool: Arc::new(Pool::new(Default::default(), ChainApi::new(client))),
+		pool: Arc::new(Pool::new(Default::default(), FullChainApi::new(client))),
 		subscriptions: Subscriptions::new(Arc::new(runtime.executor())),
 		keystore: keystore.clone(),
 	};
@@ -216,7 +215,7 @@ fn should_rotate_keys() {
 	let client = Arc::new(test_client::TestClientBuilder::new().set_keystore(keystore.clone()).build());
 	let p = Author {
 		client: client.clone(),
-		pool: Arc::new(Pool::new(Default::default(), ChainApi::new(client))),
+		pool: Arc::new(Pool::new(Default::default(), FullChainApi::new(client))),
 		subscriptions: Subscriptions::new(Arc::new(runtime.executor())),
 		keystore: keystore.clone(),
 	};