Fix quadratic iterations over transaction pool (#4736)

* transaction pool changes

* service & network changes

* address review

* reduce future pool

substrate/client/transaction-pool/graph/src/base_pool.rs

@@ -374,9 +374,9 @@ impl<Hash: hash::Hash + Member + Serialize, Ex: std::fmt::Debug> BasePool<Hash,
 	///
 	/// Includes both ready and future pool. For every hash in the `hashes`
 	/// iterator an `Option` is produced (so the resulting `Vec` always have the same length).
-	pub fn by_hash(&self, hashes: &[Hash]) -> Vec<Option<Arc<Transaction<Hash, Ex>>>> {
-		let ready = self.ready.by_hash(hashes);
-		let future = self.future.by_hash(hashes);
+	pub fn by_hashes(&self, hashes: &[Hash]) -> Vec<Option<Arc<Transaction<Hash, Ex>>>> {
+		let ready = self.ready.by_hashes(hashes);
+		let future = self.future.by_hashes(hashes);
 
 		ready
 			.into_iter()
@@ -385,6 +385,11 @@ impl<Hash: hash::Hash + Member + Serialize, Ex: std::fmt::Debug> BasePool<Hash,
 			.collect()
 	}
 
+	/// Returns pool transaction by hash.
+	pub fn ready_by_hash(&self, hash: &Hash) -> Option<Arc<Transaction<Hash, Ex>>> {
+		self.ready.by_hash(hash)
+	}
+
 	/// Makes sure that the transactions in the queues stay within provided limits.
 	///
 	/// Removes and returns worst transactions from the queues and all transactions that depend on them.

substrate/client/transaction-pool/graph/src/future.rs

@@ -160,7 +160,7 @@ impl<Hash: hash::Hash + Eq + Clone, Ex> FutureTransactions<Hash, Ex> {
 	}
 
 	/// Returns a list of known transactions
-	pub fn by_hash(&self, hashes: &[Hash]) -> Vec<Option<Arc<Transaction<Hash, Ex>>>> {
+	pub fn by_hashes(&self, hashes: &[Hash]) -> Vec<Option<Arc<Transaction<Hash, Ex>>>> {
 		hashes.iter().map(|h| self.waiting.get(h).map(|x| x.transaction.clone())).collect()
 	}
 

substrate/client/transaction-pool/graph/src/pool.rs

@@ -38,7 +38,7 @@ use sp_transaction_pool::{error, PoolStatus};
 use crate::validated_pool::{ValidatedPool, ValidatedTransaction};
 
 /// Modification notification event stream type;
-pub type EventStream = mpsc::UnboundedReceiver<()>;
+pub type EventStream<H> = mpsc::UnboundedReceiver<H>;
 
 /// Extrinsic hash type for a pool.
 pub type ExHash<A> = <A as ChainApi>::Hash;
@@ -105,11 +105,11 @@ impl Default for Options {
 	fn default() -> Self {
 		Options {
 			ready: base::Limit {
-				count: 512,
-				total_bytes: 10 * 1024 * 1024,
+				count: 8192,
+				total_bytes: 20 * 1024 * 1024,
 			},
 			future: base::Limit {
-				count: 128,
+				count: 512,
 				total_bytes: 1 * 1024 * 1024,
 			},
 			reject_future_transactions: false,
@@ -331,7 +331,7 @@ impl<B: ChainApi> Pool<B> {
 	///
 	/// Consumers of this stream should use the `ready` method to actually get the
 	/// pending transactions in the right order.
-	pub fn import_notification_stream(&self) -> EventStream {
+	pub fn import_notification_stream(&self) -> EventStream<ExHash<B>> {
 		self.validated_pool.import_notification_stream()
 	}
 
@@ -437,6 +437,11 @@ impl<B: ChainApi> Pool<B> {
 
 		(hash, validity)
 	}
+
+	/// Get ready transaction by hash, if it present in the pool.
+	pub fn ready_transaction(&self, hash: &ExHash<B>) -> Option<TransactionFor<B>> {
+		self.validated_pool.ready_by_hash(hash)
+	}
 }
 
 impl<B: ChainApi> Clone for Pool<B> {
@@ -638,8 +643,8 @@ mod tests {
 
 		// then
 		let mut it = futures::executor::block_on_stream(stream);
-		assert_eq!(it.next(), Some(()));
-		assert_eq!(it.next(), Some(()));
+		assert_eq!(it.next(), Some(32));
+		assert_eq!(it.next(), Some(33));
 		assert_eq!(it.next(), None);
 	}
 

substrate/client/transaction-pool/graph/src/ready.rs

@@ -230,8 +230,13 @@ impl<Hash: hash::Hash + Member + Serialize, Ex> ReadyTransactions<Hash, Ex> {
 		self.ready.read().contains_key(hash)
 	}
 
-	/// Retrieve transaction by hash
-	pub fn by_hash(&self, hashes: &[Hash]) -> Vec<Option<Arc<Transaction<Hash, Ex>>>> {
+	/// Retrive transaction by hash
+	pub fn by_hash(&self, hash: &Hash) -> Option<Arc<Transaction<Hash, Ex>>> {
+		self.by_hashes(&[hash.clone()]).into_iter().next().unwrap_or(None)
+	}
+
+	/// Retrieve transactions by hash
+	pub fn by_hashes(&self, hashes: &[Hash]) -> Vec<Option<Arc<Transaction<Hash, Ex>>>> {
 		let ready = self.ready.read();
 		hashes.iter().map(|hash| {
 			ready.get(hash).map(|x| x.transaction.transaction.clone())

substrate/client/transaction-pool/graph/src/validated_pool.rs

@@ -70,7 +70,7 @@ pub(crate) struct ValidatedPool<B: ChainApi> {
 		ExHash<B>,
 		ExtrinsicFor<B>,
 	>>,
-	import_notification_sinks: Mutex<Vec<mpsc::UnboundedSender<()>>>,
+	import_notification_sinks: Mutex<Vec<mpsc::UnboundedSender<ExHash<B>>>>,
 	rotator: PoolRotator<ExHash<B>>,
 }
 
@@ -125,8 +125,8 @@ impl<B: ChainApi> ValidatedPool<B> {
 			ValidatedTransaction::Valid(tx) => {
 				let imported = self.pool.write().import(tx)?;
 
-				if let base::Imported::Ready { .. } = imported {
-					self.import_notification_sinks.lock().retain(|sink| sink.unbounded_send(()).is_ok());
+				if let base::Imported::Ready { ref hash, .. } = imported {
+					self.import_notification_sinks.lock().retain(|sink| sink.unbounded_send(hash.clone()).is_ok());
 				}
 
 				let mut listener = self.listener.write();
@@ -320,7 +320,7 @@ impl<B: ChainApi> ValidatedPool<B> {
 
 	/// For each extrinsic, returns tags that it provides (if known), or None (if it is unknown).
 	pub fn extrinsics_tags(&self, hashes: &[ExHash<B>]) -> Vec<Option<Vec<Tag>>> {
-		self.pool.read().by_hash(&hashes)
+		self.pool.read().by_hashes(&hashes)
 			.into_iter()
 			.map(|existing_in_pool| existing_in_pool
 				.map(|transaction| transaction.provides.iter().cloned()
@@ -328,6 +328,11 @@ impl<B: ChainApi> ValidatedPool<B> {
 			.collect()
 	}
 
+	/// Get ready transaction by hash
+	pub fn ready_by_hash(&self, hash: &ExHash<B>) -> Option<TransactionFor<B>> {
+		self.pool.read().ready_by_hash(hash)
+	}
+
 	/// Prunes ready transactions that provide given list of tags.
 	pub fn prune_tags(
 		&self,
@@ -444,7 +449,7 @@ impl<B: ChainApi> ValidatedPool<B> {
 	}
 
 	/// Return an event stream of transactions imported to the pool.
-	pub fn import_notification_stream(&self) -> EventStream {
+	pub fn import_notification_stream(&self) -> EventStream<ExHash<B>> {
 		let (sink, stream) = mpsc::unbounded();
 		self.import_notification_sinks.lock().push(sink);
 		stream