Recover transaction pool on light client (#3833)

* recover tx pool on light client * revert local tests fix * removed import renamings * futures03::Future -> std::future::Future * Update core/transaction-pool/graph/src/error.rs Co-Authored-By: Tomasz Drwięga <tomusdrw@users.noreply.github.com> * replace remove_from_ready with remove_invalid * avoid excess hashing * debug -> warn * TransactionPool + BasicTransactionPool * pause future tx reject when resubmitting * bump impl_version to make CI happy * and revert back local test fixes * alter doc to restart CI * Transaction::clone() -> Transaction::duplicate() * transactions -> updated_tranasctions * remove explicit consensus-common ref * ::std:: -> std:: * manual set/unset flag -> calling clusore with given flag value * removed comments * removed force argument * BestIterator -> Box<Iterator> * separate crate for TxPool + Maintainer trait * long line fix * pos-merge fix * fix benches compilation * Rename txpoolapi to txpool_api * Clean up. * Finalize merge. * post-merge fix * Move transaction pool api to primitives directly. * Consistent naming for txpool-runtime-api * Warn about missing docs. * Move abstraction for offchain calls to tx-pool-api. * Merge RPC instantiation. * Update cargo.lock * Post merge fixes. * Avoid depending on client. * Fix build
2026-06-09 20:11:09 +00:00 · 2019-11-28 03:00:54 +03:00
parent 3e26fceda4
commit a782021ee8
64 changed files with 2370 additions and 667 deletions
@@ -21,7 +21,6 @@ use std::{
 };

 use crate::base_pool as base;
-use crate::error;
 use crate::watcher::Watcher;
 use serde::Serialize;

@@ -35,6 +34,8 @@ use sr_primitives::{
 	traits::{self, SaturatedConversion},
 	transaction_validity::{TransactionValidity, TransactionTag as Tag, TransactionValidityError},
 };
+use txpool_api::{error, PoolStatus};
+
 use crate::validated_pool::{ValidatedPool, ValidatedTransaction};

 /// Modification notification event stream type;
@@ -92,6 +93,8 @@ pub struct Options {
 	pub ready: base::Limit,
 	/// Future queue limits.
 	pub future: base::Limit,
+	/// Reject future transactions.
+	pub reject_future_transactions: bool,
 }

 impl Default for Options {
@@ -105,6 +108,7 @@ impl Default for Options {
 				count: 128,
 				total_bytes: 1 * 1024 * 1024,
 			},
+			reject_future_transactions: false,
 		}
 	}
 }
@@ -131,7 +135,9 @@ impl<B: ChainApi> Pool<B> {
 		let validated_pool = self.validated_pool.clone();
 		self.verify(at, xts, force)
 			.map(move |validated_transactions| validated_transactions
-				.map(|validated_transactions| validated_pool.submit(validated_transactions)))
+				.map(|validated_transactions| validated_pool.submit(validated_transactions
+					.into_iter()
+					.map(|(_, tx)| tx))))
 	}

 	/// Imports one unverified extrinsic to the pool
@@ -161,10 +167,40 @@ impl<B: ChainApi> Pool<B> {
 		let validated_pool = self.validated_pool.clone();
 		Either::Right(
 			self.verify_one(at, block_number, xt, false)
-				.map(move |validated_transactions| validated_pool.submit_and_watch(validated_transactions))
+				.map(move |validated_transactions| validated_pool.submit_and_watch(validated_transactions.1))
 		)
 	}

+	/// Revalidate all ready transactions.
+	///
+	/// Returns future that performs validation of all ready transactions and
+	/// then resubmits all transactions back to the pool.
+	pub fn revalidate_ready(&self, at: &BlockId<B::Block>) -> impl Future<Output=Result<(), B::Error>> {
+		let validated_pool = self.validated_pool.clone();
+		let ready = self.validated_pool.ready().map(|tx| tx.data.clone());
+		self.verify(at, ready, false)
+			.map(move |revalidated_transactions| revalidated_transactions.map(
+				move |revalidated_transactions| validated_pool.resubmit(revalidated_transactions)
+			))
+	}
+
+	/// Prunes known ready transactions.
+	///
+	/// Used to clear the pool from transactions that were part of recently imported block.
+	/// The main difference from the `prune` is that we do not revalidate any transactions
+	/// and ignore unknown passed hashes.
+	pub fn prune_known(&self, at: &BlockId<B::Block>, hashes: &[ExHash<B>]) -> Result<(), B::Error> {
+		// Get details of all extrinsics that are already in the pool
+		let in_pool_tags = self.validated_pool.extrinsics_tags(hashes)
+			.into_iter().filter_map(|x| x).flat_map(|x| x);
+
+		// Prune all transactions that provide given tags
+		let prune_status = self.validated_pool.prune_tags(in_pool_tags)?;
+		let pruned_transactions = hashes.into_iter().cloned()
+			.chain(prune_status.pruned.iter().map(|tx| tx.hash.clone()));
+		self.validated_pool.fire_pruned(at, pruned_transactions)
+	}
+
 	/// Prunes ready transactions.
 	///
 	/// Used to clear the pool from transactions that were part of recently imported block.
@@ -184,7 +220,8 @@ impl<B: ChainApi> Pool<B> {
 			extrinsics.len()
 		);
 		// Get details of all extrinsics that are already in the pool
-		let (in_pool_hashes, in_pool_tags) = self.validated_pool.extrinsics_tags(extrinsics);
+		let in_pool_hashes = extrinsics.iter().map(|extrinsic| self.hash_of(extrinsic)).collect::<Vec<_>>();
+		let in_pool_tags = self.validated_pool.extrinsics_tags(&in_pool_hashes);

 		// Zip the ones from the pool with the full list (we get pairs `(Extrinsic, Option<Vec<Tag>>)`)
 		let all = extrinsics.iter().zip(in_pool_tags.into_iter());
@@ -274,7 +311,7 @@ impl<B: ChainApi> Pool<B> {
 					&at,
 					known_imported_hashes,
 					pruned_hashes,
-					reverified_transactions,
+					reverified_transactions.into_iter().map(|(_, xt)| xt).collect(),
 				))
 			)))
 	}
@@ -303,7 +340,7 @@ impl<B: ChainApi> Pool<B> {
 	}

 	/// Returns pool status.
-	pub fn status(&self) -> base::Status {
+	pub fn status(&self) -> PoolStatus {
 		self.validated_pool.status()
 	}

@@ -325,7 +362,7 @@ impl<B: ChainApi> Pool<B> {
 		at: &BlockId<B::Block>,
 		xts: impl IntoIterator<Item=ExtrinsicFor<B>>,
 		force: bool,
-	) -> impl Future<Output=Result<Vec<ValidatedTransactionFor<B>>, B::Error>> {
+	) -> impl Future<Output=Result<HashMap<ExHash<B>, ValidatedTransactionFor<B>>, B::Error>> {
 		// we need a block number to compute tx validity
 		let block_number = match self.resolve_block_number(at) {
 			Ok(block_number) => block_number,
@@ -338,7 +375,7 @@ impl<B: ChainApi> Pool<B> {
 		);

 		// make single validation future that waits all until all extrinsics are validated
-		Either::Right(join_all(validation_futures).then(|x| ready(Ok(x))))
+		Either::Right(join_all(validation_futures).then(|x| ready(Ok(x.into_iter().collect()))))
 	}

 	/// Returns future that validates single transaction at given block.
@@ -348,14 +385,17 @@ impl<B: ChainApi> Pool<B> {
 		block_number: NumberFor<B>,
 		xt: ExtrinsicFor<B>,
 		force: bool,
-	) -> impl Future<Output=ValidatedTransactionFor<B>> {
+	) -> impl Future<Output=(ExHash<B>, ValidatedTransactionFor<B>)> {
 		let (hash, bytes) = self.validated_pool.api().hash_and_length(&xt);
 		if !force && self.validated_pool.is_banned(&hash) {
-			return Either::Left(ready(ValidatedTransaction::Invalid(hash, error::Error::TemporarilyBanned.into())))
+			return Either::Left(ready((
+				hash.clone(),
+				ValidatedTransaction::Invalid(hash, error::Error::TemporarilyBanned.into()),
+			)))
 		}

 		Either::Right(self.validated_pool.api().validate_transaction(block_id, xt.clone())
-			.then(move |validation_result| ready(match validation_result {
+			.then(move |validation_result| ready((hash.clone(), match validation_result {
 				Ok(validity) => match validity {
 					Ok(validity) => if validity.provides.is_empty() {
 						ValidatedTransaction::Invalid(hash, error::Error::NoTagsProvided.into())
@@ -379,7 +419,7 @@ impl<B: ChainApi> Pool<B> {
 						ValidatedTransaction::Unknown(hash, error::Error::UnknownTransaction(e).into()),
 				},
 				Err(e) => ValidatedTransaction::Invalid(hash, e),
-			})))
+			}))))
 	}
 }

@@ -391,50 +431,30 @@ impl<B: ChainApi> Clone for Pool<B> {
 	}
 }

-impl<A: ChainApi> sr_primitives::offchain::TransactionPool<A::Block> for Pool<A> {
-	fn submit_at(
-		&self,
-		at: &BlockId<A::Block>,
-		extrinsic: <A::Block as sr_primitives::traits::Block>::Extrinsic,
-	) -> Result<(), ()> {
-		log::debug!(
-			target: "txpool",
-			"(offchain call) Submitting a transaction to the pool: {:?}",
-			extrinsic
-		);
-
-		let result = futures::executor::block_on(self.submit_one(&at, extrinsic));
-
-		result.map(|_| ())
-			.map_err(|e| log::warn!(
-				target: "txpool",
-				"(offchain call) Error submitting a transaction to the pool: {:?}",
-				e
-			))
-	}
-}
-
 #[cfg(test)]
 mod tests {
 	use std::{
-		collections::HashMap,
+		collections::{HashMap, HashSet},
 		time::Instant,
 	};
 	use parking_lot::Mutex;
 	use futures::executor::block_on;
 	use super::*;
+	use txpool_api::TransactionStatus;
 	use sr_primitives::transaction_validity::{ValidTransaction, InvalidTransaction};
 	use codec::Encode;
 	use test_runtime::{Block, Extrinsic, Transfer, H256, AccountId};
 	use assert_matches::assert_matches;
 	use crate::base_pool::Limit;
-	use crate::watcher;

 	const INVALID_NONCE: u64 = 254;

 	#[derive(Clone, Debug, Default)]
 	struct TestApi {
 		delay: Arc<Mutex<Option<std::sync::mpsc::Receiver<()>>>>,
+		invalidate: Arc<Mutex<HashSet<u64>>>,
+		clear_requirements: Arc<Mutex<HashSet<u64>>>,
+		add_requirements: Arc<Mutex<HashSet<u64>>>,
 	}

 	impl ChainApi for TestApi {
@@ -449,6 +469,7 @@ mod tests {
 			at: &BlockId<Self::Block>,
 			uxt: ExtrinsicFor<Self>,
 		) -> Self::ValidationFuture {
+			let hash = self.hash_and_length(&uxt).0;
 			let block_number = self.block_id_to_number(at).unwrap().unwrap();
 			let nonce = uxt.transfer().nonce;

@@ -462,16 +483,30 @@ mod tests {
 				}
 			}

+			if self.invalidate.lock().contains(&hash) {
+				return futures::future::ready(Ok(InvalidTransaction::Custom(0).into()));
+			}
+
 			futures::future::ready(if nonce < block_number {
 				Ok(InvalidTransaction::Stale.into())
 			} else {
-				Ok(Ok(ValidTransaction {
+				let mut transaction = ValidTransaction {
 					priority: 4,
 					requires: if nonce > block_number { vec![vec![nonce as u8 - 1]] } else { vec![] },
 					provides: if nonce == INVALID_NONCE { vec![] } else { vec![vec![nonce as u8]] },
 					longevity: 3,
 					propagate: true,
-				}))
+				};
+
+				if self.clear_requirements.lock().contains(&hash) {
+					transaction.requires.clear();
+				}
+
+				if self.add_requirements.lock().contains(&hash) {
+					transaction.requires.push(vec![128]);
+				}
+
+				Ok(Ok(transaction))
 			})
 		}

@@ -651,6 +686,7 @@ mod tests {
 		let pool = Pool::new(Options {
 			ready: limit.clone(),
 			future: limit.clone(),
+			..Default::default()
 		}, TestApi::default());

 		let hash1 = block_on(pool.submit_one(&BlockId::Number(0), uxt(Transfer {
@@ -685,6 +721,7 @@ mod tests {
 		let pool = Pool::new(Options {
 			ready: limit.clone(),
 			future: limit.clone(),
+			..Default::default()
 		}, TestApi::default());

 		// when
@@ -742,8 +779,8 @@ mod tests {

 			// then
 			let mut stream = futures::executor::block_on_stream(watcher.into_stream());
-			assert_eq!(stream.next(), Some(watcher::Status::Ready));
-			assert_eq!(stream.next(), Some(watcher::Status::Finalized(H256::from_low_u64_be(2).into())));
+			assert_eq!(stream.next(), Some(TransactionStatus::Ready));
+			assert_eq!(stream.next(), Some(TransactionStatus::Finalized(H256::from_low_u64_be(2).into())));
 			assert_eq!(stream.next(), None);
 		}

@@ -767,8 +804,8 @@ mod tests {

 			// then
 			let mut stream = futures::executor::block_on_stream(watcher.into_stream());
-			assert_eq!(stream.next(), Some(watcher::Status::Ready));
-			assert_eq!(stream.next(), Some(watcher::Status::Finalized(H256::from_low_u64_be(2).into())));
+			assert_eq!(stream.next(), Some(TransactionStatus::Ready));
+			assert_eq!(stream.next(), Some(TransactionStatus::Finalized(H256::from_low_u64_be(2).into())));
 			assert_eq!(stream.next(), None);
 		}

@@ -796,8 +833,8 @@ mod tests {

 			// then
 			let mut stream = futures::executor::block_on_stream(watcher.into_stream());
-			assert_eq!(stream.next(), Some(watcher::Status::Future));
-			assert_eq!(stream.next(), Some(watcher::Status::Ready));
+			assert_eq!(stream.next(), Some(TransactionStatus::Future));
+			assert_eq!(stream.next(), Some(TransactionStatus::Ready));
 		}

 		#[test]
@@ -819,8 +856,8 @@ mod tests {

 			// then
 			let mut stream = futures::executor::block_on_stream(watcher.into_stream());
-			assert_eq!(stream.next(), Some(watcher::Status::Ready));
-			assert_eq!(stream.next(), Some(watcher::Status::Invalid));
+			assert_eq!(stream.next(), Some(TransactionStatus::Ready));
+			assert_eq!(stream.next(), Some(TransactionStatus::Invalid));
 			assert_eq!(stream.next(), None);
 		}

@@ -846,8 +883,8 @@ mod tests {

 			// then
 			let mut stream = futures::executor::block_on_stream(watcher.into_stream());
-			assert_eq!(stream.next(), Some(watcher::Status::Ready));
-			assert_eq!(stream.next(), Some(watcher::Status::Broadcast(peers)));
+			assert_eq!(stream.next(), Some(TransactionStatus::Ready));
+			assert_eq!(stream.next(), Some(TransactionStatus::Broadcast(peers)));
 		}

 		#[test]
@@ -860,6 +897,7 @@ mod tests {
 			let pool = Pool::new(Options {
 				ready: limit.clone(),
 				future: limit.clone(),
+				..Default::default()
 			}, TestApi::default());

 			let xt = uxt(Transfer {
@@ -883,8 +921,8 @@ mod tests {

 			// then
 			let mut stream = futures::executor::block_on_stream(watcher.into_stream());
-			assert_eq!(stream.next(), Some(watcher::Status::Ready));
-			assert_eq!(stream.next(), Some(watcher::Status::Dropped));
+			assert_eq!(stream.next(), Some(TransactionStatus::Ready));
+			assert_eq!(stream.next(), Some(TransactionStatus::Dropped));
 		}

 		#[test]
@@ -941,5 +979,81 @@ mod tests {
 			assert_eq!(pool.status().future, 0);
 		}
 	}
+
+	#[test]
+	fn should_revalidate_ready_transactions() {
+		fn transfer(nonce: u64) -> Extrinsic {
+			uxt(Transfer {
+				from: AccountId::from_h256(H256::from_low_u64_be(1)),
+				to: AccountId::from_h256(H256::from_low_u64_be(2)),
+				amount: 5,
+				nonce,
+			})
+		}
+
+		// given
+		let pool = pool();
+		let tx0 = transfer(0);
+		let hash0 = pool.validated_pool.api().hash_and_length(&tx0).0;
+		let watcher0 = block_on(pool.submit_and_watch(&BlockId::Number(0), tx0)).unwrap();
+		let tx1 = transfer(1);
+		let hash1 = pool.validated_pool.api().hash_and_length(&tx1).0;
+		let watcher1 = block_on(pool.submit_and_watch(&BlockId::Number(0), tx1)).unwrap();
+		let tx2 = transfer(2);
+		let hash2 = pool.validated_pool.api().hash_and_length(&tx2).0;
+		let watcher2 = block_on(pool.submit_and_watch(&BlockId::Number(0), tx2)).unwrap();
+		let tx3 = transfer(3);
+		let hash3 = pool.validated_pool.api().hash_and_length(&tx3).0;
+		let watcher3 = block_on(pool.submit_and_watch(&BlockId::Number(0), tx3)).unwrap();
+		let tx4 = transfer(4);
+		let hash4 = pool.validated_pool.api().hash_and_length(&tx4).0;
+		let watcher4 = block_on(pool.submit_and_watch(&BlockId::Number(0), tx4)).unwrap();
+		assert_eq!(pool.status().ready, 5);
+
+		// when
+		pool.validated_pool.api().invalidate.lock().insert(hash3);
+		pool.validated_pool.api().clear_requirements.lock().insert(hash1);
+		pool.validated_pool.api().add_requirements.lock().insert(hash0);
+		block_on(pool.revalidate_ready(&BlockId::Number(0))).unwrap();
+
+		// then
+		// hash0 now has unsatisfied requirements => it is moved to the future queue
+		// hash1 is now independent of hash0 => it is in ready queue
+		// hash2 still depends on hash1 => it is in ready queue
+		// hash3 is now invalid => it is removed from the pool
+		// hash4 now depends on invalidated hash3 => it is moved to the future queue
+		//
+		// events for hash3 are: Ready, Invalid
+		// events for hash4 are: Ready, Invalid
+		assert_eq!(pool.status().ready, 2);
+		assert_eq!(
+			futures::executor::block_on_stream(watcher3.into_stream()).collect::<Vec<_>>(),
+			vec![TransactionStatus::Ready, TransactionStatus::Invalid],
+		);
+
+		// when
+		pool.validated_pool.remove_invalid(&[hash0, hash1, hash2, hash4]);
+
+		// then
+		// events for hash0 are: Ready, Future, Invalid
+		// events for hash1 are: Ready, Invalid
+		// events for hash2 are: Ready, Invalid
+		assert_eq!(
+			futures::executor::block_on_stream(watcher0.into_stream()).collect::<Vec<_>>(),
+			vec![TransactionStatus::Ready, TransactionStatus::Future, TransactionStatus::Invalid],
+		);
+		assert_eq!(
+			futures::executor::block_on_stream(watcher1.into_stream()).collect::<Vec<_>>(),
+			vec![TransactionStatus::Ready, TransactionStatus::Invalid],
+		);
+		assert_eq!(
+			futures::executor::block_on_stream(watcher2.into_stream()).collect::<Vec<_>>(),
+			vec![TransactionStatus::Ready, TransactionStatus::Invalid],
+		);
+		assert_eq!(
+			futures::executor::block_on_stream(watcher4.into_stream()).collect::<Vec<_>>(),
+			vec![TransactionStatus::Ready, TransactionStatus::Future, TransactionStatus::Invalid],
+		);
+	}
 }