Transaction pool: Ensure that we prune transactions properly (#8963)

* Transaction pool: Ensure that we prune transactions properly

There was a bug in the transaction pool that we didn't pruned
transactions properly because we called `prune_known`, instead of `prune`.

This bug was introduced by:
https://github.com/paritytech/substrate/pull/4629

This is required to have stale extrinsics being removed properly, so
that they don't fill up the tx pool.

* Fix compilation

* Fix benches

* ...

substrate/client/transaction-pool/src/api.rs

@@ -81,7 +81,7 @@ impl<Client, Block> FullChainApi<Client, Block> {
 impl<Client, Block> sc_transaction_graph::ChainApi for FullChainApi<Client, Block>
 where
 	Block: BlockT,
-	Client: ProvideRuntimeApi<Block> + BlockBackend<Block> + BlockIdTo<Block>,
+	Client: ProvideRuntimeApi<Block> + BlockBackend<Block> + BlockIdTo<Block> + HeaderBackend<Block>,
 	Client: Send + Sync + 'static,
 	Client::Api: TaggedTransactionQueue<Block>,
 {
@@ -150,6 +150,13 @@ where
 			(<traits::HashFor::<Block> as traits::Hash>::hash(x), x.len())
 		})
 	}
+
+	fn block_header(
+		&self,
+		at: &BlockId<Self::Block>,
+	) -> Result<Option<<Self::Block as BlockT>::Header>, Self::Error> {
+		self.client.header(*at).map_err(Into::into)
+	}
 }
 
 /// Helper function to validate a transaction using a full chain API.
@@ -162,7 +169,7 @@ fn validate_transaction_blocking<Client, Block>(
 ) -> error::Result<TransactionValidity>
 where
 	Block: BlockT,
-	Client: ProvideRuntimeApi<Block> + BlockBackend<Block> + BlockIdTo<Block>,
+	Client: ProvideRuntimeApi<Block> + BlockBackend<Block> + BlockIdTo<Block> + HeaderBackend<Block>,
 	Client: Send + Sync + 'static,
 	Client::Api: TaggedTransactionQueue<Block>,
 {
@@ -193,7 +200,7 @@ where
 impl<Client, Block> FullChainApi<Client, Block>
 where
 	Block: BlockT,
-	Client: ProvideRuntimeApi<Block> + BlockBackend<Block> + BlockIdTo<Block>,
+	Client: ProvideRuntimeApi<Block> + BlockBackend<Block> + BlockIdTo<Block> + HeaderBackend<Block>,
 	Client: Send + Sync + 'static,
 	Client::Api: TaggedTransactionQueue<Block>,
 {
@@ -333,4 +340,11 @@ impl<Client, F, Block> sc_transaction_graph::ChainApi for
 			Ok(Some(transactions))
 		}.boxed()
 	}
+
+	fn block_header(
+		&self,
+		at: &BlockId<Self::Block>,
+	) -> Result<Option<<Self::Block as BlockT>::Header>, Self::Error> {
+		self.client.header(*at).map_err(Into::into)
+	}
 }

substrate/client/transaction-pool/src/lib.rs

@@ -40,7 +40,7 @@ use parking_lot::Mutex;
 
 use sp_runtime::{
 	generic::BlockId,
-	traits::{Block as BlockT, NumberFor, AtLeast32Bit, Extrinsic, Zero},
+	traits::{Block as BlockT, NumberFor, AtLeast32Bit, Extrinsic, Zero, Header as HeaderT},
 };
 use sp_core::traits::SpawnNamed;
 use sp_transaction_pool::{
@@ -379,6 +379,7 @@ where
 	Block: BlockT,
 	Client: sp_api::ProvideRuntimeApi<Block>
 		+ sc_client_api::BlockBackend<Block>
+		+ sc_client_api::blockchain::HeaderBackend<Block>
 		+ sp_runtime::traits::BlockIdTo<Block>
 		+ sc_client_api::ExecutorProvider<Block>
 		+ sc_client_api::UsageProvider<Block>
@@ -419,6 +420,7 @@ where
 	Block: BlockT,
 	Client: sp_api::ProvideRuntimeApi<Block>
 		+ sc_client_api::BlockBackend<Block>
+		+ sc_client_api::blockchain::HeaderBackend<Block>
 		+ sp_runtime::traits::BlockIdTo<Block>,
 	Client: Send + Sync + 'static,
 	Client::Api: sp_transaction_pool::runtime_api::TaggedTransactionQueue<Block>,
@@ -555,19 +557,32 @@ async fn prune_known_txs_for_block<Block: BlockT, Api: ChainApi<Block = Block>>(
 	api: &Api,
 	pool: &sc_transaction_graph::Pool<Api>,
 ) -> Vec<ExtrinsicHash<Api>> {
-	let hashes = api.block_body(&block_id).await
+	let extrinsics = api.block_body(&block_id).await
 		.unwrap_or_else(|e| {
 			log::warn!("Prune known transactions: error request {:?}!", e);
 			None
 		})
-		.unwrap_or_default()
-		.into_iter()
+		.unwrap_or_default();
+
+	let hashes = extrinsics.iter()
 		.map(|tx| pool.hash_of(&tx))
 		.collect::<Vec<_>>();
 
 	log::trace!(target: "txpool", "Pruning transactions: {:?}", hashes);
 
-	if let Err(e) = pool.prune_known(&block_id, &hashes) {
+	let header = match api.block_header(&block_id) {
+		Ok(Some(h)) => h,
+		Ok(None) => {
+			log::debug!(target: "txpool", "Could not find header for {:?}.", block_id);
+			return hashes
+		},
+		Err(e) => {
+			log::debug!(target: "txpool", "Error retrieving header for {:?}: {:?}", block_id, e);
+			return hashes
+		}
+	};
+
+	if let Err(e) = pool.prune(&block_id, &BlockId::hash(*header.parent_hash()), &extrinsics).await {
 		log::error!("Cannot prune known in the pool {:?}!", e);
 	}
 

substrate/client/transaction-pool/src/testing/pool.rs

@@ -306,31 +306,6 @@ fn should_not_retain_invalid_hashes_from_retracted() {
 	assert_eq!(pool.status().ready, 0);
 }
 
-#[test]
-fn should_revalidate_transaction_multiple_times() {
-	let xt = uxt(Alice, 209);
-
-	let (pool, _guard, mut notifier) = maintained_pool();
-
-	block_on(pool.submit_one(&BlockId::number(0), SOURCE, xt.clone())).expect("1. Imported");
-	assert_eq!(pool.status().ready, 1);
-
-	let header = pool.api.push_block(1, vec![xt.clone()], true);
-
-	block_on(pool.maintain(block_event(header)));
-
-	block_on(pool.submit_one(&BlockId::number(0), SOURCE, xt.clone())).expect("1. Imported");
-	assert_eq!(pool.status().ready, 1);
-
-	let header = pool.api.push_block(2, vec![], true);
-	pool.api.add_invalid(&xt);
-
-	block_on(pool.maintain(block_event(header)));
-	block_on(notifier.next());
-
-	assert_eq!(pool.status().ready, 0);
-}
-
 #[test]
 fn should_revalidate_across_many_blocks() {
 	let xt1 = uxt(Alice, 209);
@@ -1002,21 +977,13 @@ fn pruning_a_transaction_should_remove_it_from_best_transaction() {
 	let xt1 = Extrinsic::IncludeData(Vec::new());
 
 	block_on(pool.submit_one(&BlockId::number(0), SOURCE, xt1.clone())).expect("1. Imported");
+	assert_eq!(pool.status().ready, 1);
 	let header = pool.api.push_block(1, vec![xt1.clone()], true);
 
 	// This will prune `xt1`.
 	block_on(pool.maintain(block_event(header)));
 
-	// Submit the tx again.
-	block_on(pool.submit_one(&BlockId::number(1), SOURCE, xt1.clone())).expect("2. Imported");
-
-	let mut iterator = block_on(pool.ready_at(1));
-
-	assert_eq!(iterator.next().unwrap().data, xt1.clone());
-
-	// If the tx was not removed from the best txs, the tx would be
-	// returned a second time by the iterator.
-	assert!(iterator.next().is_none());
+	assert_eq!(pool.status().ready, 0);
 }
 
 #[test]
@@ -1038,3 +1005,79 @@ fn only_revalidate_on_best_block() {
 
 	assert_eq!(pool.status().ready, 1);
 }
+
+#[test]
+fn stale_transactions_are_pruned() {
+	sp_tracing::try_init_simple();
+
+	// Our initial transactions
+	let xts = vec![
+		Transfer {
+			from: Alice.into(),
+			to: Bob.into(),
+			nonce: 1,
+			amount: 1,
+		},
+		Transfer {
+			from: Alice.into(),
+			to: Bob.into(),
+			nonce: 2,
+			amount: 1,
+		},
+		Transfer {
+			from: Alice.into(),
+			to: Bob.into(),
+			nonce: 3,
+			amount: 1,
+		},
+	];
+
+	let (pool, _guard, _notifier) = maintained_pool();
+
+	xts.into_iter().for_each(|xt| {
+		block_on(
+			pool.submit_one(&BlockId::number(0), SOURCE, xt.into_signed_tx()),
+		).expect("1. Imported");
+	});
+	assert_eq!(pool.status().ready, 0);
+	assert_eq!(pool.status().future, 3);
+
+	// Almost the same as our initial transactions, but with some different `amount`s to make them
+	// generate a different hash
+	let xts = vec![
+		Transfer {
+			from: Alice.into(),
+			to: Bob.into(),
+			nonce: 1,
+			amount: 2,
+		}.into_signed_tx(),
+		Transfer {
+			from: Alice.into(),
+			to: Bob.into(),
+			nonce: 2,
+			amount: 2,
+		}.into_signed_tx(),
+		Transfer {
+			from: Alice.into(),
+			to: Bob.into(),
+			nonce: 3,
+			amount: 2,
+		}.into_signed_tx(),
+	];
+
+	// Import block
+	let header = pool.api.push_block(1, xts, true);
+	block_on(pool.maintain(block_event(header)));
+	// The imported transactions have a different hash and should not evict our initial
+	// transactions.
+	assert_eq!(pool.status().future, 3);
+
+	// Import enough blocks to make our transactions stale
+	for n in 1..66 {
+		let header = pool.api.push_block(n, vec![], true);
+		block_on(pool.maintain(block_event(header)));
+	}
+
+	assert_eq!(pool.status().future, 0);
+	assert_eq!(pool.status().ready, 0);
+}