Reorganising the repository - external renames and moves (#4074)

* Adding first rough ouline of the repository structure

* Remove old CI stuff

* add title

* formatting fixes

* move node-exits job's script to scripts dir

* Move docs into subdir

* move to bin

* move maintainence scripts, configs and helpers into its own dir

* add .local to ignore

* move core->client

* start up 'test' area

* move test client

* move test runtime

* make test move compile

* Add dependencies rule enforcement.

* Fix indexing.

* Update docs to reflect latest changes

* Moving /srml->/paint

* update docs

* move client/sr-* -> primitives/

* clean old readme

* remove old broken code in rhd

* update lock

* Step 1.

* starting to untangle client

* Fix after merge.

* start splitting out client interfaces

* move children and blockchain interfaces

* Move trie and state-machine to primitives.

* Fix WASM builds.

* fixing broken imports

* more interface moves

* move backend and light to interfaces

* move CallExecutor

* move cli off client

* moving around more interfaces

* re-add consensus crates into the mix

* fix subkey path

* relieve client from executor

* starting to pull out client from grandpa

* move is_decendent_of out of client

* grandpa still depends on client directly

* lemme tests pass

* rename srml->paint

* Make it compile.

* rename interfaces->client-api

* Move keyring to primitives.

* fixup libp2p dep

* fix broken use

* allow dependency enforcement to fail

* move fork-tree

* Moving wasm-builder

* make env

* move build-script-utils

* fixup broken crate depdencies and names

* fix imports for authority discovery

* fix typo

* update cargo.lock

* fixing imports

* Fix paths and add missing crates

* re-add missing crates

substrate/client/transaction-pool/graph/src/base_pool.rs

@@ -0,0 +1,975 @@
+// Copyright 2018-2019 Parity Technologies (UK) Ltd.
+// This file is part of Substrate.
+
+// Substrate is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// Substrate is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with Substrate.  If not, see <http://www.gnu.org/licenses/>.
+
+//! A basic version of the dependency graph.
+//!
+//! For a more full-featured pool, have a look at the `pool` module.
+
+use std::{
+	collections::HashSet,
+	fmt,
+	hash,
+	sync::Arc,
+};
+
+use log::{trace, debug, warn};
+use serde::Serialize;
+use primitives::hexdisplay::HexDisplay;
+use sr_primitives::traits::Member;
+use sr_primitives::transaction_validity::{
+	TransactionTag as Tag,
+	TransactionLongevity as Longevity,
+	TransactionPriority as Priority,
+};
+
+use crate::error;
+use crate::future::{FutureTransactions, WaitingTransaction};
+use crate::ready::ReadyTransactions;
+
+/// Successful import result.
+#[derive(Debug, PartialEq, Eq)]
+pub enum Imported<Hash, Ex> {
+	/// Transaction was successfully imported to Ready queue.
+	Ready {
+		/// Hash of transaction that was successfully imported.
+		hash: Hash,
+		/// Transactions that got promoted from the Future queue.
+		promoted: Vec<Hash>,
+		/// Transactions that failed to be promoted from the Future queue and are now discarded.
+		failed: Vec<Hash>,
+		/// Transactions removed from the Ready pool (replaced).
+		removed: Vec<Arc<Transaction<Hash, Ex>>>,
+	},
+	/// Transaction was successfully imported to Future queue.
+	Future {
+		/// Hash of transaction that was successfully imported.
+		hash: Hash,
+	}
+}
+
+impl<Hash, Ex> Imported<Hash, Ex> {
+	/// Returns the hash of imported transaction.
+	pub fn hash(&self) -> &Hash {
+		use self::Imported::*;
+		match *self {
+			Ready { ref hash, .. } => hash,
+			Future { ref hash, .. } => hash,
+		}
+	}
+}
+
+/// Status of pruning the queue.
+#[derive(Debug)]
+pub struct PruneStatus<Hash, Ex> {
+	/// A list of imports that satisfying the tag triggered.
+	pub promoted: Vec<Imported<Hash, Ex>>,
+	/// A list of transactions that failed to be promoted and now are discarded.
+	pub failed: Vec<Hash>,
+	/// A list of transactions that got pruned from the ready queue.
+	pub pruned: Vec<Arc<Transaction<Hash, Ex>>>,
+}
+
+/// Immutable transaction
+#[cfg_attr(test, derive(Clone))]
+#[derive(PartialEq, Eq)]
+pub struct Transaction<Hash, Extrinsic> {
+	/// Raw extrinsic representing that transaction.
+	pub data: Extrinsic,
+	/// Number of bytes encoding of the transaction requires.
+	pub bytes: usize,
+	/// Transaction hash (unique)
+	pub hash: Hash,
+	/// Transaction priority (higher = better)
+	pub priority: Priority,
+	/// At which block the transaction becomes invalid?
+	pub valid_till: Longevity,
+	/// Tags required by the transaction.
+	pub requires: Vec<Tag>,
+	/// Tags that this transaction provides.
+	pub provides: Vec<Tag>,
+	/// Should that transaction be propagated.
+	pub propagate: bool,
+}
+
+impl<Hash, Extrinsic> Transaction<Hash, Extrinsic> {
+	/// Returns `true` if the transaction should be propagated to other peers.
+	pub fn is_propagateable(&self) -> bool {
+		self.propagate
+	}
+}
+
+impl<Hash, Extrinsic> fmt::Debug for Transaction<Hash, Extrinsic> where
+	Hash: fmt::Debug,
+	Extrinsic: fmt::Debug,
+{
+	fn fmt(&self, fmt: &mut fmt::Formatter) -> fmt::Result {
+		fn print_tags(fmt: &mut fmt::Formatter, tags: &[Tag]) -> fmt::Result {
+			let mut it = tags.iter();
+			if let Some(t) = it.next() {
+				write!(fmt, "{}", HexDisplay::from(t))?;
+			}
+			for t in it {
+				write!(fmt, ",{}", HexDisplay::from(t))?;
+			}
+			Ok(())
+		}
+
+		write!(fmt, "Transaction {{ ")?;
+		write!(fmt, "hash: {:?}, ", &self.hash)?;
+		write!(fmt, "priority: {:?}, ", &self.priority)?;
+		write!(fmt, "valid_till: {:?}, ", &self.valid_till)?;
+		write!(fmt, "bytes: {:?}, ", &self.bytes)?;
+		write!(fmt, "propagate: {:?}, ", &self.propagate)?;
+		write!(fmt, "requires: [")?;
+		print_tags(fmt, &self.requires)?;
+		write!(fmt, "], provides: [")?;
+		print_tags(fmt, &self.provides)?;
+		write!(fmt, "], ")?;
+		write!(fmt, "data: {:?}", &self.data)?;
+		write!(fmt, "}}")?;
+		Ok(())
+	}
+}
+
+/// Store last pruned tags for given number of invocations.
+const RECENTLY_PRUNED_TAGS: usize = 2;
+
+/// Transaction pool.
+///
+/// Builds a dependency graph for all transactions in the pool and returns
+/// the ones that are currently ready to be executed.
+///
+/// General note:
+/// If function returns some transactions it usually means that importing them
+/// as-is for the second time will fail or produce unwanted results.
+/// Most likely it is required to revalidate them and recompute set of
+/// required tags.
+#[derive(Debug)]
+pub struct BasePool<Hash: hash::Hash + Eq, Ex> {
+	future: FutureTransactions<Hash, Ex>,
+	ready: ReadyTransactions<Hash, Ex>,
+	/// Store recently pruned tags (for last two invocations).
+	///
+	/// This is used to make sure we don't accidentally put
+	/// transactions to future in case they were just stuck in verification.
+	recently_pruned: [HashSet<Tag>; RECENTLY_PRUNED_TAGS],
+	recently_pruned_index: usize,
+}
+
+impl<Hash: hash::Hash + Eq, Ex> Default for BasePool<Hash, Ex> {
+	fn default() -> Self {
+		BasePool {
+			future: Default::default(),
+			ready: Default::default(),
+			recently_pruned: Default::default(),
+			recently_pruned_index: 0,
+		}
+	}
+}
+
+impl<Hash: hash::Hash + Member + Serialize, Ex: ::std::fmt::Debug> BasePool<Hash, Ex> {
+	/// Imports transaction to the pool.
+	///
+	/// The pool consists of two parts: Future and Ready.
+	/// The former contains transactions that require some tags that are not yet provided by
+	/// other transactions in the pool.
+	/// The latter contains transactions that have all the requirements satisfied and are
+	/// ready to be included in the block.
+	pub fn import(
+		&mut self,
+		tx: Transaction<Hash, Ex>,
+	) -> error::Result<Imported<Hash, Ex>> {
+		if self.future.contains(&tx.hash) || self.ready.contains(&tx.hash) {
+			return Err(error::Error::AlreadyImported(Box::new(tx.hash.clone())))
+		}
+
+		let tx = WaitingTransaction::new(
+			tx,
+			self.ready.provided_tags(),
+			&self.recently_pruned,
+		);
+		trace!(target: "txpool", "[{:?}] {:?}", tx.transaction.hash, tx);
+		debug!(target: "txpool", "[{:?}] Importing to {}", tx.transaction.hash, if tx.is_ready() { "ready" } else { "future" });
+
+		// If all tags are not satisfied import to future.
+		if !tx.is_ready() {
+			let hash = tx.transaction.hash.clone();
+			self.future.import(tx);
+			return Ok(Imported::Future { hash });
+		}
+
+		self.import_to_ready(tx)
+	}
+
+	/// Imports transaction to ready queue.
+	///
+	/// NOTE the transaction has to have all requirements satisfied.
+	fn import_to_ready(&mut self, tx: WaitingTransaction<Hash, Ex>) -> error::Result<Imported<Hash, Ex>> {
+		let hash = tx.transaction.hash.clone();
+		let mut promoted = vec![];
+		let mut failed = vec![];
+		let mut removed = vec![];
+
+		let mut first = true;
+		let mut to_import = vec![tx];
+
+		loop {
+			// take first transaction from the list
+			let tx = match to_import.pop() {
+				Some(tx) => tx,
+				None => break,
+			};
+
+			// find transactions in Future that it unlocks
+			to_import.append(&mut self.future.satisfy_tags(&tx.transaction.provides));
+
+			// import this transaction
+			let current_hash = tx.transaction.hash.clone();
+			match self.ready.import(tx) {
+				Ok(mut replaced) => {
+					if !first {
+						promoted.push(current_hash);
+					}
+					// The transactions were removed from the ready pool. We might attempt to re-import them.
+					removed.append(&mut replaced);
+				},
+				// transaction failed to be imported.
+				Err(e) => if first {
+					debug!(target: "txpool", "[{:?}] Error importing: {:?}", current_hash, e);
+					return Err(e)
+				} else {
+					failed.push(current_hash);
+				},
+			}
+			first = false;
+		}
+
+		// An edge case when importing transaction caused
+		// some future transactions to be imported and that
+		// future transactions pushed out current transaction.
+		// This means that there is a cycle and the transactions should
+		// be moved back to future, since we can't resolve it.
+		if removed.iter().any(|tx| tx.hash == hash) {
+			// We still need to remove all transactions that we promoted
+			// since they depend on each other and will never get to the best iterator.
+			self.ready.remove_invalid(&promoted);
+
+			debug!(target: "txpool", "[{:?}] Cycle detected, bailing.", hash);
+			return Err(error::Error::CycleDetected)
+		}
+
+		Ok(Imported::Ready {
+			hash,
+			promoted,
+			failed,
+			removed,
+		})
+	}
+
+	/// Returns an iterator over ready transactions in the pool.
+	pub fn ready(&self) -> impl Iterator<Item=Arc<Transaction<Hash, Ex>>> {
+		self.ready.get()
+	}
+
+	/// Returns an iterator over future transactions in the pool.
+	pub fn futures(&self) -> impl Iterator<Item=&Transaction<Hash, Ex>> {
+		self.future.all()
+	}
+
+	/// Returns pool transactions given list of hashes.
+	///
+	/// Includes both ready and future pool. For every hash in the `hashes`
+	/// iterator an `Option` is produced (so the resulting `Vec` always have the same length).
+	pub fn by_hash(&self, hashes: &[Hash]) -> Vec<Option<Arc<Transaction<Hash, Ex>>>> {
+		let ready = self.ready.by_hash(hashes);
+		let future = self.future.by_hash(hashes);
+
+		ready
+			.into_iter()
+			.zip(future)
+			.map(|(a, b)| a.or(b))
+			.collect()
+	}
+
+	/// Makes sure that the transactions in the queues stay within provided limits.
+	///
+	/// Removes and returns worst transactions from the queues and all transactions that depend on them.
+	/// Technically the worst transaction should be evaluated by computing the entire pending set.
+	/// We use a simplified approach to remove the transaction that occupies the pool for the longest time.
+	pub fn enforce_limits(&mut self, ready: &Limit, future: &Limit) -> Vec<Arc<Transaction<Hash, Ex>>> {
+		let mut removed = vec![];
+
+		while ready.is_exceeded(self.ready.len(), self.ready.bytes()) {
+			// find the worst transaction
+			let minimal = self.ready
+				.fold(|minimal, current| {
+					let transaction = &current.transaction;
+					match minimal {
+						None => Some(transaction.clone()),
+						Some(ref tx) if tx.insertion_id > transaction.insertion_id => {
+							Some(transaction.clone())
+						},
+						other => other,
+					}
+				});
+
+			if let Some(minimal) = minimal {
+				removed.append(&mut self.remove_invalid(&[minimal.transaction.hash.clone()]))
+			} else {
+				break;
+			}
+		}
+
+		while future.is_exceeded(self.future.len(), self.future.bytes()) {
+			// find the worst transaction
+			let minimal = self.future
+				.fold(|minimal, current| {
+					match minimal {
+						None => Some(current.clone()),
+						Some(ref tx) if tx.imported_at > current.imported_at => {
+							Some(current.clone())
+						},
+						other => other,
+					}
+				});
+
+			if let Some(minimal) = minimal {
+				removed.append(&mut self.remove_invalid(&[minimal.transaction.hash.clone()]))
+			} else {
+				break;
+			}
+		}
+
+		removed
+	}
+
+	/// Removes all transactions represented by the hashes and all other transactions
+	/// that depend on them.
+	///
+	/// Returns a list of actually removed transactions.
+	/// NOTE some transactions might still be valid, but were just removed because
+	/// they were part of a chain, you may attempt to re-import them later.
+	/// NOTE If you want to remove ready transactions that were already used
+	/// and you don't want them to be stored in the pool use `prune_tags` method.
+	pub fn remove_invalid(&mut self, hashes: &[Hash]) -> Vec<Arc<Transaction<Hash, Ex>>> {
+		let mut removed = self.ready.remove_invalid(hashes);
+		removed.extend(self.future.remove(hashes));
+		removed
+	}
+
+	/// Prunes transactions that provide given list of tags.
+	///
+	/// This will cause all transactions that provide these tags to be removed from the pool,
+	/// but unlike `remove_invalid`, dependent transactions are not touched.
+	/// Additional transactions from future queue might be promoted to ready if you satisfy tags
+	/// that the pool didn't previously know about.
+	pub fn prune_tags(&mut self, tags: impl IntoIterator<Item=Tag>) -> PruneStatus<Hash, Ex> {
+		let mut to_import = vec![];
+		let mut pruned = vec![];
+		let recently_pruned = &mut self.recently_pruned[self.recently_pruned_index];
+		self.recently_pruned_index = (self.recently_pruned_index + 1) % RECENTLY_PRUNED_TAGS;
+		recently_pruned.clear();
+
+		for tag in tags {
+			// make sure to promote any future transactions that could be unlocked
+			to_import.append(&mut self.future.satisfy_tags(::std::iter::once(&tag)));
+			// and actually prune transactions in ready queue
+			pruned.append(&mut self.ready.prune_tags(tag.clone()));
+			// store the tags for next submission
+			recently_pruned.insert(tag);
+		}
+
+		let mut promoted = vec![];
+		let mut failed = vec![];
+		for tx in to_import {
+			let hash = tx.transaction.hash.clone();
+			match self.import_to_ready(tx) {
+				Ok(res) => promoted.push(res),
+				Err(e) => {
+					warn!(target: "txpool", "[{:?}] Failed to promote during pruning: {:?}", hash, e);
+					failed.push(hash)
+				},
+			}
+		}
+
+		PruneStatus {
+			pruned,
+			failed,
+			promoted,
+		}
+	}
+
+	/// Get pool status.
+	pub fn status(&self) -> Status {
+		Status {
+			ready: self.ready.len(),
+			ready_bytes: self.ready.bytes(),
+			future: self.future.len(),
+			future_bytes: self.future.bytes(),
+		}
+	}
+}
+
+/// Pool status
+#[derive(Debug)]
+pub struct Status {
+	/// Number of transactions in the ready queue.
+	pub ready: usize,
+	/// Sum of bytes of ready transaction encodings.
+	pub ready_bytes: usize,
+	/// Number of transactions in the future queue.
+	pub future: usize,
+	/// Sum of bytes of ready transaction encodings.
+	pub future_bytes: usize,
+}
+
+impl Status {
+	/// Returns true if the are no transactions in the pool.
+	pub fn is_empty(&self) -> bool {
+		self.ready == 0 && self.future == 0
+	}
+}
+
+/// Queue limits
+#[derive(Debug, Clone)]
+pub struct Limit {
+	/// Maximal number of transactions in the queue.
+	pub count: usize,
+	/// Maximal size of encodings of all transactions in the queue.
+	pub total_bytes: usize,
+}
+
+impl Limit {
+	/// Returns true if any of the provided values exceeds the limit.
+	pub fn is_exceeded(&self, count: usize, bytes: usize) -> bool {
+		self.count < count || self.total_bytes < bytes
+	}
+}
+
+#[cfg(test)]
+mod tests {
+	use super::*;
+
+	type Hash = u64;
+
+	fn pool() -> BasePool<Hash, Vec<u8>> {
+		BasePool::default()
+	}
+
+	#[test]
+	fn should_import_transaction_to_ready() {
+		// given
+		let mut pool = pool();
+
+		// when
+		pool.import(Transaction {
+			data: vec![1u8],
+			bytes: 1,
+			hash: 1u64,
+			priority: 5u64,
+			valid_till: 64u64,
+			requires: vec![],
+			provides: vec![vec![1]],
+			propagate: true,
+		}).unwrap();
+
+		// then
+		assert_eq!(pool.ready().count(), 1);
+		assert_eq!(pool.ready.len(), 1);
+	}
+
+	#[test]
+	fn should_not_import_same_transaction_twice() {
+		// given
+		let mut pool = pool();
+
+		// when
+		pool.import(Transaction {
+			data: vec![1u8],
+			bytes: 1,
+			hash: 1,
+			priority: 5u64,
+			valid_till: 64u64,
+			requires: vec![],
+			provides: vec![vec![1]],
+			propagate: true,
+		}).unwrap();
+		pool.import(Transaction {
+			data: vec![1u8],
+			bytes: 1,
+			hash: 1,
+			priority: 5u64,
+			valid_till: 64u64,
+			requires: vec![],
+			provides: vec![vec![1]],
+			propagate: true,
+		}).unwrap_err();
+
+		// then
+		assert_eq!(pool.ready().count(), 1);
+		assert_eq!(pool.ready.len(), 1);
+	}
+
+
+	#[test]
+	fn should_import_transaction_to_future_and_promote_it_later() {
+		// given
+		let mut pool = pool();
+
+		// when
+		pool.import(Transaction {
+			data: vec![1u8],
+			bytes: 1,
+			hash: 1,
+			priority: 5u64,
+			valid_till: 64u64,
+			requires: vec![vec![0]],
+			provides: vec![vec![1]],
+			propagate: true,
+		}).unwrap();
+		assert_eq!(pool.ready().count(), 0);
+		assert_eq!(pool.ready.len(), 0);
+		pool.import(Transaction {
+			data: vec![2u8],
+			bytes: 1,
+			hash: 2,
+			priority: 5u64,
+			valid_till: 64u64,
+			requires: vec![],
+			provides: vec![vec![0]],
+			propagate: true,
+		}).unwrap();
+
+		// then
+		assert_eq!(pool.ready().count(), 2);
+		assert_eq!(pool.ready.len(), 2);
+	}
+
+	#[test]
+	fn should_promote_a_subgraph() {
+		// given
+		let mut pool = pool();
+
+		// when
+		pool.import(Transaction {
+			data: vec![1u8],
+			bytes: 1,
+			hash: 1,
+			priority: 5u64,
+			valid_till: 64u64,
+			requires: vec![vec![0]],
+			provides: vec![vec![1]],
+			propagate: true,
+		}).unwrap();
+		pool.import(Transaction {
+			data: vec![3u8],
+			bytes: 1,
+			hash: 3,
+			priority: 5u64,
+			valid_till: 64u64,
+			requires: vec![vec![2]],
+			provides: vec![],
+			propagate: true,
+		}).unwrap();
+		pool.import(Transaction {
+			data: vec![2u8],
+			bytes: 1,
+			hash: 2,
+			priority: 5u64,
+			valid_till: 64u64,
+			requires: vec![vec![1]],
+			provides: vec![vec![3], vec![2]],
+			propagate: true,
+		}).unwrap();
+		pool.import(Transaction {
+			data: vec![4u8],
+			bytes: 1,
+			hash: 4,
+			priority: 1_000u64,
+			valid_till: 64u64,
+			requires: vec![vec![3], vec![4]],
+			provides: vec![],
+			propagate: true,
+		}).unwrap();
+		assert_eq!(pool.ready().count(), 0);
+		assert_eq!(pool.ready.len(), 0);
+
+		let res = pool.import(Transaction {
+			data: vec![5u8],
+			bytes: 1,
+			hash: 5,
+			priority: 5u64,
+			valid_till: 64u64,
+			requires: vec![],
+			provides: vec![vec![0], vec![4]],
+			propagate: true,
+		}).unwrap();
+
+		// then
+		let mut it = pool.ready().into_iter().map(|tx| tx.data[0]);
+
+		assert_eq!(it.next(), Some(5));
+		assert_eq!(it.next(), Some(1));
+		assert_eq!(it.next(), Some(2));
+		assert_eq!(it.next(), Some(4));
+		assert_eq!(it.next(), Some(3));
+		assert_eq!(it.next(), None);
+		assert_eq!(res, Imported::Ready {
+			hash: 5,
+			promoted: vec![1, 2, 3, 4],
+			failed: vec![],
+			removed: vec![],
+		});
+	}
+
+	#[test]
+	fn should_handle_a_cycle() {
+		// given
+		let mut pool = pool();
+		pool.import(Transaction {
+			data: vec![1u8],
+			bytes: 1,
+			hash: 1,
+			priority: 5u64,
+			valid_till: 64u64,
+			requires: vec![vec![0]],
+			provides: vec![vec![1]],
+			propagate: true,
+		}).unwrap();
+		pool.import(Transaction {
+			data: vec![3u8],
+			bytes: 1,
+			hash: 3,
+			priority: 5u64,
+			valid_till: 64u64,
+			requires: vec![vec![1]],
+			provides: vec![vec![2]],
+			propagate: true,
+		}).unwrap();
+		assert_eq!(pool.ready().count(), 0);
+		assert_eq!(pool.ready.len(), 0);
+
+		// when
+		pool.import(Transaction {
+			data: vec![2u8],
+			bytes: 1,
+			hash: 2,
+			priority: 5u64,
+			valid_till: 64u64,
+			requires: vec![vec![2]],
+			provides: vec![vec![0]],
+			propagate: true,
+		}).unwrap();
+
+		// then
+		{
+			let mut it = pool.ready().into_iter().map(|tx| tx.data[0]);
+			assert_eq!(it.next(), None);
+		}
+		// all transactions occupy the Future queue - it's fine
+		assert_eq!(pool.future.len(), 3);
+
+		// let's close the cycle with one additional transaction
+		let res = pool.import(Transaction {
+			data: vec![4u8],
+			bytes: 1,
+			hash: 4,
+			priority: 50u64,
+			valid_till: 64u64,
+			requires: vec![],
+			provides: vec![vec![0]],
+			propagate: true,
+		}).unwrap();
+		let mut it = pool.ready().into_iter().map(|tx| tx.data[0]);
+		assert_eq!(it.next(), Some(4));
+		assert_eq!(it.next(), Some(1));
+		assert_eq!(it.next(), Some(3));
+		assert_eq!(it.next(), None);
+		assert_eq!(res, Imported::Ready {
+			hash: 4,
+			promoted: vec![1, 3],
+			failed: vec![2],
+			removed: vec![],
+		});
+		assert_eq!(pool.future.len(), 0);
+	}
+
+	#[test]
+	fn should_handle_a_cycle_with_low_priority() {
+		// given
+		let mut pool = pool();
+		pool.import(Transaction {
+			data: vec![1u8],
+			bytes: 1,
+			hash: 1,
+			priority: 5u64,
+			valid_till: 64u64,
+			requires: vec![vec![0]],
+			provides: vec![vec![1]],
+			propagate: true,
+		}).unwrap();
+		pool.import(Transaction {
+			data: vec![3u8],
+			bytes: 1,
+			hash: 3,
+			priority: 5u64,
+			valid_till: 64u64,
+			requires: vec![vec![1]],
+			provides: vec![vec![2]],
+			propagate: true,
+		}).unwrap();
+		assert_eq!(pool.ready().count(), 0);
+		assert_eq!(pool.ready.len(), 0);
+
+		// when
+		pool.import(Transaction {
+			data: vec![2u8],
+			bytes: 1,
+			hash: 2,
+			priority: 5u64,
+			valid_till: 64u64,
+			requires: vec![vec![2]],
+			provides: vec![vec![0]],
+			propagate: true,
+		}).unwrap();
+
+		// then
+		{
+			let mut it = pool.ready().into_iter().map(|tx| tx.data[0]);
+			assert_eq!(it.next(), None);
+		}
+		// all transactions occupy the Future queue - it's fine
+		assert_eq!(pool.future.len(), 3);
+
+		// let's close the cycle with one additional transaction
+		let err = pool.import(Transaction {
+			data: vec![4u8],
+			bytes: 1,
+			hash: 4,
+			priority: 1u64, // lower priority than Tx(2)
+			valid_till: 64u64,
+			requires: vec![],
+			provides: vec![vec![0]],
+			propagate: true,
+		}).unwrap_err();
+		let mut it = pool.ready().into_iter().map(|tx| tx.data[0]);
+		assert_eq!(it.next(), None);
+		assert_eq!(pool.ready.len(), 0);
+		assert_eq!(pool.future.len(), 0);
+		if let error::Error::CycleDetected = err {
+		} else {
+			assert!(false, "Invalid error kind: {:?}", err);
+		}
+	}
+
+	#[test]
+	fn should_remove_invalid_transactions() {
+		// given
+		let mut pool = pool();
+		pool.import(Transaction {
+			data: vec![5u8],
+			bytes: 1,
+			hash: 5,
+			priority: 5u64,
+			valid_till: 64u64,
+			requires: vec![],
+			provides: vec![vec![0], vec![4]],
+			propagate: true,
+		}).unwrap();
+		pool.import(Transaction {
+			data: vec![1u8],
+			bytes: 1,
+			hash: 1,
+			priority: 5u64,
+			valid_till: 64u64,
+			requires: vec![vec![0]],
+			provides: vec![vec![1]],
+			propagate: true,
+		}).unwrap();
+		pool.import(Transaction {
+			data: vec![3u8],
+			bytes: 1,
+			hash: 3,
+			priority: 5u64,
+			valid_till: 64u64,
+			requires: vec![vec![2]],
+			provides: vec![],
+			propagate: true,
+		}).unwrap();
+		pool.import(Transaction {
+			data: vec![2u8],
+			bytes: 1,
+			hash: 2,
+			priority: 5u64,
+			valid_till: 64u64,
+			requires: vec![vec![1]],
+			provides: vec![vec![3], vec![2]],
+			propagate: true,
+		}).unwrap();
+		pool.import(Transaction {
+			data: vec![4u8],
+			bytes: 1,
+			hash: 4,
+			priority: 1_000u64,
+			valid_till: 64u64,
+			requires: vec![vec![3], vec![4]],
+			provides: vec![],
+			propagate: true,
+		}).unwrap();
+		// future
+		pool.import(Transaction {
+			data: vec![6u8],
+			bytes: 1,
+			hash: 6,
+			priority: 1_000u64,
+			valid_till: 64u64,
+			requires: vec![vec![11]],
+			provides: vec![],
+			propagate: true,
+		}).unwrap();
+		assert_eq!(pool.ready().count(), 5);
+		assert_eq!(pool.future.len(), 1);
+
+		// when
+		pool.remove_invalid(&[6, 1]);
+
+		// then
+		assert_eq!(pool.ready().count(), 1);
+		assert_eq!(pool.future.len(), 0);
+	}
+
+	#[test]
+	fn should_prune_ready_transactions() {
+		// given
+		let mut pool = pool();
+		// future (waiting for 0)
+		pool.import(Transaction {
+			data: vec![5u8],
+			bytes: 1,
+			hash: 5,
+			priority: 5u64,
+			valid_till: 64u64,
+			requires: vec![vec![0]],
+			provides: vec![vec![100]],
+			propagate: true,
+		}).unwrap();
+		// ready
+		pool.import(Transaction {
+			data: vec![1u8],
+			bytes: 1,
+			hash: 1,
+			priority: 5u64,
+			valid_till: 64u64,
+			requires: vec![],
+			provides: vec![vec![1]],
+			propagate: true,
+		}).unwrap();
+		pool.import(Transaction {
+			data: vec![2u8],
+			bytes: 1,
+			hash: 2,
+			priority: 5u64,
+			valid_till: 64u64,
+			requires: vec![vec![2]],
+			provides: vec![vec![3]],
+			propagate: true,
+		}).unwrap();
+		pool.import(Transaction {
+			data: vec![3u8],
+			bytes: 1,
+			hash: 3,
+			priority: 5u64,
+			valid_till: 64u64,
+			requires: vec![vec![1]],
+			provides: vec![vec![2]],
+			propagate: true,
+		}).unwrap();
+		pool.import(Transaction {
+			data: vec![4u8],
+			bytes: 1,
+			hash: 4,
+			priority: 1_000u64,
+			valid_till: 64u64,
+			requires: vec![vec![3], vec![2]],
+			provides: vec![vec![4]],
+			propagate: true,
+		}).unwrap();
+
+		assert_eq!(pool.ready().count(), 4);
+		assert_eq!(pool.future.len(), 1);
+
+		// when
+		let result = pool.prune_tags(vec![vec![0], vec![2]]);
+
+		// then
+		assert_eq!(result.pruned.len(), 2);
+		assert_eq!(result.failed.len(), 0);
+		assert_eq!(result.promoted[0], Imported::Ready {
+			hash: 5,
+			promoted: vec![],
+			failed: vec![],
+			removed: vec![],
+		});
+		assert_eq!(result.promoted.len(), 1);
+		assert_eq!(pool.future.len(), 0);
+		assert_eq!(pool.ready.len(), 3);
+		assert_eq!(pool.ready().count(), 3);
+	}
+
+	#[test]
+	fn transaction_debug() {
+		assert_eq!(
+			format!("{:?}", Transaction {
+				data: vec![4u8],
+				bytes: 1,
+				hash: 4,
+				priority: 1_000u64,
+				valid_till: 64u64,
+				requires: vec![vec![3], vec![2]],
+				provides: vec![vec![4]],
+				propagate: true,
+			}),
+			"Transaction { \
+hash: 4, priority: 1000, valid_till: 64, bytes: 1, propagate: true, \
+requires: [03,02], provides: [04], data: [4]}".to_owned()
+		);
+	}
+
+	#[test]
+	fn transaction_propagation() {
+		assert_eq!(Transaction {
+				data: vec![4u8],
+				bytes: 1,
+				hash: 4,
+				priority: 1_000u64,
+				valid_till: 64u64,
+				requires: vec![vec![3], vec![2]],
+				provides: vec![vec![4]],
+				propagate: true,
+		}.is_propagateable(), true);
+
+		assert_eq!(Transaction {
+				data: vec![4u8],
+				bytes: 1,
+				hash: 4,
+				priority: 1_000u64,
+				valid_till: 64u64,
+				requires: vec![vec![3], vec![2]],
+				provides: vec![vec![4]],
+				propagate: false,
+		}.is_propagateable(), false);
+	}
+}

substrate/client/transaction-pool/graph/src/error.rs

@@ -0,0 +1,79 @@
+// Copyright 2018-2019 Parity Technologies (UK) Ltd.
+// This file is part of Substrate.
+
+// Substrate is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// Substrate is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with Substrate.  If not, see <http://www.gnu.org/licenses/>.
+
+//! Transaction pool errors.
+
+use sr_primitives::transaction_validity::{
+	TransactionPriority as Priority, InvalidTransaction, UnknownTransaction,
+};
+
+/// Transaction pool result.
+pub type Result<T> = std::result::Result<T, Error>;
+
+/// Transaction pool error type.
+#[derive(Debug, derive_more::Display, derive_more::From)]
+pub enum Error {
+	/// Transaction is not verifiable yet, but might be in the future.
+	#[display(fmt="Unknown transaction validity: {:?}", _0)]
+	UnknownTransaction(UnknownTransaction),
+	/// Transaction is invalid.
+	#[display(fmt="Invalid transaction validity: {:?}", _0)]
+	InvalidTransaction(InvalidTransaction),
+	/// The transaction validity returned no "provides" tag.
+	///
+	/// Such transactions are not accepted to the pool, since we use those tags
+	/// to define identity of transactions (occupance of the same "slot").
+	#[display(fmt="The transaction does not provide any tags, so the pool can't identify it.")]
+	NoTagsProvided,
+	/// The transaction is temporarily banned.
+	#[display(fmt="Temporarily Banned")]
+	TemporarilyBanned,
+	/// The transaction is already in the pool.
+	#[display(fmt="[{:?}] Already imported", _0)]
+	AlreadyImported(Box<dyn std::any::Any + Send>),
+	/// The transaction cannot be imported cause it's a replacement and has too low priority.
+	#[display(fmt="Too low priority ({} > {})", old, new)]
+	TooLowPriority {
+		/// Transaction already in the pool.
+		old: Priority,
+		/// Transaction entering the pool.
+		new: Priority
+	},
+	/// Deps cycle etected and we couldn't import transaction.
+	#[display(fmt="Cycle Detected")]
+	CycleDetected,
+	/// Transaction was dropped immediately after it got inserted.
+	#[display(fmt="Transaction couldn't enter the pool because of the limit.")]
+	ImmediatelyDropped,
+	/// Invalid block id.
+	InvalidBlockId(String),
+}
+
+impl std::error::Error for Error {}
+
+/// Transaction pool error conversion.
+pub trait IntoPoolError: ::std::error::Error + Send + Sized {
+	/// Try to extract original `Error`
+	///
+	/// This implementation is optional and used only to
+	/// provide more descriptive error messages for end users
+	/// of RPC API.
+	fn into_pool_error(self) -> ::std::result::Result<Error, Self> { Err(self) }
+}
+
+impl IntoPoolError for Error {
+	fn into_pool_error(self) -> ::std::result::Result<Error, Self> { Ok(self) }
+}

substrate/client/transaction-pool/graph/src/future.rs

@@ -0,0 +1,239 @@
+// Copyright 2018-2019 Parity Technologies (UK) Ltd.
+// This file is part of Substrate.
+
+// Substrate is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// Substrate is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with Substrate.  If not, see <http://www.gnu.org/licenses/>.
+
+use std::{
+	collections::{HashMap, HashSet},
+	fmt,
+	hash,
+	sync::Arc,
+	time,
+};
+
+use primitives::hexdisplay::HexDisplay;
+use sr_primitives::transaction_validity::{
+	TransactionTag as Tag,
+};
+
+use crate::base_pool::Transaction;
+
+/// Transaction with partially satisfied dependencies.
+pub struct WaitingTransaction<Hash, Ex> {
+	/// Transaction details.
+	pub transaction: Arc<Transaction<Hash, Ex>>,
+	/// Tags that are required and have not been satisfied yet by other transactions in the pool.
+	pub missing_tags: HashSet<Tag>,
+	/// Time of import to the Future Queue.
+	pub imported_at: time::Instant,
+}
+
+impl<Hash: fmt::Debug, Ex: fmt::Debug> fmt::Debug for WaitingTransaction<Hash, Ex> {
+	fn fmt(&self, fmt: &mut fmt::Formatter) -> fmt::Result {
+		write!(fmt, "WaitingTransaction {{ ")?;
+		write!(fmt, "imported_at: {:?}, ", self.imported_at)?;
+		write!(fmt, "transaction: {:?}, ", self.transaction)?;
+		write!(fmt, "missing_tags: {{")?;
+		let mut it = self.missing_tags.iter().map(|tag| HexDisplay::from(tag));
+		if let Some(tag) = it.next() {
+			write!(fmt, "{}", tag)?;
+		}
+		for tag in it {
+			write!(fmt, ", {}", tag)?;
+		}
+		write!(fmt, " }}}}")
+	}
+}
+
+impl<Hash, Ex> Clone for WaitingTransaction<Hash, Ex> {
+	fn clone(&self) -> Self {
+		WaitingTransaction {
+			transaction: self.transaction.clone(),
+			missing_tags: self.missing_tags.clone(),
+			imported_at: self.imported_at.clone(),
+		}
+	}
+}
+
+impl<Hash, Ex> WaitingTransaction<Hash, Ex> {
+	/// Creates a new `WaitingTransaction`.
+	///
+	/// Computes the set of missing tags based on the requirements and tags that
+	/// are provided by all transactions in the ready queue.
+	pub fn new(
+		transaction: Transaction<Hash, Ex>,
+		provided: &HashMap<Tag, Hash>,
+		recently_pruned: &[HashSet<Tag>],
+	) -> Self {
+		let missing_tags = transaction.requires
+			.iter()
+			.filter(|tag| {
+				// is true if the tag is already satisfied either via transaction in the pool
+				// or one that was recently included.
+				let is_provided = provided.contains_key(&**tag) || recently_pruned.iter().any(|x| x.contains(&**tag));
+				!is_provided
+			})
+			.cloned()
+			.collect();
+
+		WaitingTransaction {
+			transaction: Arc::new(transaction),
+			missing_tags,
+			imported_at: time::Instant::now(),
+		}
+	}
+
+	/// Marks the tag as satisfied.
+	pub fn satisfy_tag(&mut self, tag: &Tag) {
+		self.missing_tags.remove(tag);
+	}
+
+	/// Returns true if transaction has all requirements satisfied.
+	pub fn is_ready(&self) -> bool {
+		self.missing_tags.is_empty()
+	}
+}
+
+/// A pool of transactions that are not yet ready to be included in the block.
+///
+/// Contains transactions that are still awaiting for some other transactions that
+/// could provide a tag that they require.
+#[derive(Debug)]
+pub struct FutureTransactions<Hash: hash::Hash + Eq, Ex> {
+	/// tags that are not yet provided by any transaction and we await for them
+	wanted_tags: HashMap<Tag, HashSet<Hash>>,
+	/// Transactions waiting for a particular other transaction
+	waiting: HashMap<Hash, WaitingTransaction<Hash, Ex>>,
+}
+
+impl<Hash: hash::Hash + Eq, Ex> Default for FutureTransactions<Hash, Ex> {
+	fn default() -> Self {
+		FutureTransactions {
+			wanted_tags: Default::default(),
+			waiting: Default::default(),
+		}
+	}
+}
+
+const WAITING_PROOF: &str = r"#
+In import we always insert to `waiting` if we push to `wanted_tags`;
+when removing from `waiting` we always clear `wanted_tags`;
+every hash from `wanted_tags` is always present in `waiting`;
+qed
+#";
+
+impl<Hash: hash::Hash + Eq + Clone, Ex> FutureTransactions<Hash, Ex> {
+	/// Import transaction to Future queue.
+	///
+	/// Only transactions that don't have all their tags satisfied should occupy
+	/// the Future queue.
+	/// As soon as required tags are provided by some other transactions that are ready
+	/// we should remove the transactions from here and move them to the Ready queue.
+	pub fn import(&mut self, tx: WaitingTransaction<Hash, Ex>) {
+		assert!(!tx.is_ready(), "Transaction is ready.");
+		assert!(!self.waiting.contains_key(&tx.transaction.hash), "Transaction is already imported.");
+
+		// Add all tags that are missing
+		for tag in &tx.missing_tags {
+			let entry = self.wanted_tags.entry(tag.clone()).or_insert_with(HashSet::new);
+			entry.insert(tx.transaction.hash.clone());
+		}
+
+		// Add the transaction to a by-hash waiting map
+		self.waiting.insert(tx.transaction.hash.clone(), tx);
+	}
+
+	/// Returns true if given hash is part of the queue.
+	pub fn contains(&self, hash: &Hash) -> bool {
+		self.waiting.contains_key(hash)
+	}
+
+	/// Returns a list of known transactions
+	pub fn by_hash(&self, hashes: &[Hash]) -> Vec<Option<Arc<Transaction<Hash, Ex>>>> {
+		hashes.iter().map(|h| self.waiting.get(h).map(|x| x.transaction.clone())).collect()
+	}
+
+	/// Satisfies provided tags in transactions that are waiting for them.
+	///
+	/// Returns (and removes) transactions that became ready after their last tag got
+	/// satisfied and now we can remove them from Future and move to Ready queue.
+	pub fn satisfy_tags<T: AsRef<Tag>>(&mut self, tags: impl IntoIterator<Item=T>) -> Vec<WaitingTransaction<Hash, Ex>> {
+		let mut became_ready = vec![];
+
+		for tag in tags {
+			if let Some(hashes) = self.wanted_tags.remove(tag.as_ref()) {
+				for hash in hashes {
+					let is_ready = {
+						let tx = self.waiting.get_mut(&hash).expect(WAITING_PROOF);
+						tx.satisfy_tag(tag.as_ref());
+						tx.is_ready()
+					};
+
+					if is_ready {
+						let tx = self.waiting.remove(&hash).expect(WAITING_PROOF);
+						became_ready.push(tx);
+					}
+				}
+			}
+		}
+
+		became_ready
+	}
+
+	/// Removes transactions for given list of hashes.
+	///
+	/// Returns a list of actually removed transactions.
+	pub fn remove(&mut self, hashes: &[Hash]) -> Vec<Arc<Transaction<Hash, Ex>>> {
+		let mut removed = vec![];
+		for hash in hashes {
+			if let Some(waiting_tx) = self.waiting.remove(hash) {
+				// remove from wanted_tags as well
+				for tag in waiting_tx.missing_tags {
+					let remove = if let Some(wanted) = self.wanted_tags.get_mut(&tag) {
+						wanted.remove(hash);
+						wanted.is_empty()
+					} else { false };
+					if remove {
+						self.wanted_tags.remove(&tag);
+					}
+				}
+				// add to result
+				removed.push(waiting_tx.transaction)
+			}
+		}
+		removed
+	}
+
+	/// Fold a list of future transactions to compute a single value.
+	pub fn fold<R, F: FnMut(Option<R>, &WaitingTransaction<Hash, Ex>) -> Option<R>>(&mut self, f: F) -> Option<R> {
+		self.waiting
+			.values()
+			.fold(None, f)
+	}
+
+	/// Returns iterator over all future transactions
+	pub fn all(&self) -> impl Iterator<Item=&Transaction<Hash, Ex>> {
+		self.waiting.values().map(|waiting| &*waiting.transaction)
+	}
+
+	/// Returns number of transactions in the Future queue.
+	pub fn len(&self) -> usize {
+		self.waiting.len()
+	}
+
+	/// Returns sum of encoding lengths of all transactions in this queue.
+	pub fn bytes(&self) -> usize {
+		self.waiting.values().fold(0, |acc, tx| acc + tx.transaction.bytes)
+	}
+}

substrate/client/transaction-pool/graph/src/lib.rs

@@ -0,0 +1,44 @@
+// Copyright 2018-2019 Parity Technologies (UK) Ltd.
+// This file is part of Substrate.
+
+// Substrate is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// Substrate is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with Substrate.  If not, see <http://www.gnu.org/licenses/>.
+
+//! Generic Transaction Pool
+//!
+//! The pool is based on dependency graph between transactions
+//! and their priority.
+//! The pool is able to return an iterator that traverses transaction
+//! graph in the correct order taking into account priorities and dependencies.
+
+#![warn(missing_docs)]
+#![warn(unused_extern_crates)]
+
+mod future;
+mod listener;
+mod pool;
+mod ready;
+mod rotator;
+mod validated_pool;
+
+pub mod base_pool;
+pub mod error;
+pub mod watcher;
+
+pub use self::error::IntoPoolError;
+pub use self::base_pool::{Transaction, Status};
+pub use self::pool::{
+	Pool,
+	Options, ChainApi, EventStream, ExtrinsicFor,
+	BlockHash, ExHash, NumberFor, TransactionFor,
+};

substrate/client/transaction-pool/graph/src/listener.rs

@@ -0,0 +1,104 @@
+
+// Copyright 2018-2019 Parity Technologies (UK) Ltd.
+// This file is part of Substrate.
+
+// Substrate is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// Substrate is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with Substrate.  If not, see <http://www.gnu.org/licenses/>.
+
+use std::{
+	collections::HashMap,
+	fmt,
+	hash,
+};
+use serde::Serialize;
+use crate::watcher;
+use sr_primitives::traits;
+use log::{debug, trace, warn};
+
+/// Extrinsic pool default listener.
+pub struct Listener<H: hash::Hash + Eq, H2> {
+	watchers: HashMap<H, watcher::Sender<H, H2>>
+}
+
+impl<H: hash::Hash + Eq, H2> Default for Listener<H, H2> {
+	fn default() -> Self {
+		Listener {
+			watchers: Default::default(),
+		}
+	}
+}
+
+impl<H: hash::Hash + traits::Member + Serialize, H2: Clone + fmt::Debug> Listener<H, H2> {
+	fn fire<F>(&mut self, hash: &H, fun: F) where F: FnOnce(&mut watcher::Sender<H, H2>) {
+		let clean = if let Some(h) = self.watchers.get_mut(hash) {
+			fun(h);
+			h.is_done()
+		} else {
+			false
+		};
+
+		if clean {
+			self.watchers.remove(hash);
+		}
+	}
+
+	/// Creates a new watcher for given verified extrinsic.
+	///
+	/// The watcher can be used to subscribe to lifecycle events of that extrinsic.
+	pub fn create_watcher(&mut self, hash: H) -> watcher::Watcher<H, H2> {
+		let sender = self.watchers.entry(hash.clone()).or_insert_with(watcher::Sender::default);
+		sender.new_watcher(hash)
+	}
+
+	/// Notify the listeners about extrinsic broadcast.
+	pub fn broadcasted(&mut self, hash: &H, peers: Vec<String>) {
+		trace!(target: "txpool", "[{:?}] Broadcasted", hash);
+		self.fire(hash, |watcher| watcher.broadcast(peers));
+	}
+
+	/// New transaction was added to the ready pool or promoted from the future pool.
+	pub fn ready(&mut self, tx: &H, old: Option<&H>) {
+		trace!(target: "txpool", "[{:?}] Ready (replaced: {:?})", tx, old);
+		self.fire(tx, |watcher| watcher.ready());
+		if let Some(old) = old {
+			self.fire(old, |watcher| watcher.usurped(tx.clone()));
+		}
+	}
+
+	/// New transaction was added to the future pool.
+	pub fn future(&mut self, tx: &H) {
+		trace!(target: "txpool", "[{:?}] Future", tx);
+		self.fire(tx, |watcher| watcher.future());
+	}
+
+	/// Transaction was dropped from the pool because of the limit.
+	pub fn dropped(&mut self, tx: &H, by: Option<&H>) {
+		trace!(target: "txpool", "[{:?}] Dropped (replaced by {:?})", tx, by);
+		self.fire(tx, |watcher| match by {
+			Some(t) => watcher.usurped(t.clone()),
+			None => watcher.dropped(),
+		})
+	}
+
+	/// Transaction was removed as invalid.
+	pub fn invalid(&mut self, tx: &H) {
+		warn!(target: "transaction-pool", "Extrinsic invalid: {:?}", tx);
+		self.fire(tx, |watcher| watcher.invalid());
+	}
+
+	/// Transaction was pruned from the pool.
+	pub fn pruned(&mut self, header_hash: H2, tx: &H) {
+		debug!(target: "txpool", "[{:?}] Pruned at {:?}", tx, header_hash);
+		self.fire(tx, |watcher| watcher.finalized(header_hash))
+	}
+}

substrate/client/transaction-pool/graph/src/pool.rs

@@ -0,0 +1,919 @@
+// Copyright 2018-2019 Parity Technologies (UK) Ltd.
+// This file is part of Substrate.
+
+// Substrate is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// Substrate is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with Substrate.  If not, see <http://www.gnu.org/licenses/>.
+
+use std::{
+	hash,
+	collections::HashMap,
+	sync::Arc,
+};
+
+use crate::base_pool as base;
+use crate::error;
+use crate::watcher::Watcher;
+use serde::Serialize;
+
+use futures::{
+	Future, FutureExt,
+	channel::mpsc,
+	future::{Either, ready, join_all},
+};
+use sr_primitives::{
+	generic::BlockId,
+	traits::{self, SaturatedConversion},
+	transaction_validity::{TransactionValidity, TransactionTag as Tag, TransactionValidityError},
+};
+use crate::validated_pool::{ValidatedPool, ValidatedTransaction};
+
+/// Modification notification event stream type;
+pub type EventStream = mpsc::UnboundedReceiver<()>;
+
+/// Extrinsic hash type for a pool.
+pub type ExHash<A> = <A as ChainApi>::Hash;
+/// Block hash type for a pool.
+pub type BlockHash<A> = <<A as ChainApi>::Block as traits::Block>::Hash;
+/// Extrinsic type for a pool.
+pub type ExtrinsicFor<A> = <<A as ChainApi>::Block as traits::Block>::Extrinsic;
+/// Block number type for the ChainApi
+pub type NumberFor<A> = traits::NumberFor<<A as ChainApi>::Block>;
+/// A type of transaction stored in the pool
+pub type TransactionFor<A> = Arc<base::Transaction<ExHash<A>, ExtrinsicFor<A>>>;
+/// A type of validated transaction stored in the pool.
+pub type ValidatedTransactionFor<A> = ValidatedTransaction<
+	ExHash<A>,
+	ExtrinsicFor<A>,
+	<A as ChainApi>::Error,
+>;
+
+/// Concrete extrinsic validation and query logic.
+pub trait ChainApi: Send + Sync {
+	/// Block type.
+	type Block: traits::Block;
+	/// Transaction Hash type
+	type Hash: hash::Hash + Eq + traits::Member + Serialize;
+	/// Error type.
+	type Error: From<error::Error> + error::IntoPoolError;
+	/// Validate transaction future.
+	type ValidationFuture: Future<Output=Result<TransactionValidity, Self::Error>> + Send + Unpin;
+
+	/// Verify extrinsic at given block.
+	fn validate_transaction(
+		&self,
+		at: &BlockId<Self::Block>,
+		uxt: ExtrinsicFor<Self>,
+	) -> Self::ValidationFuture;
+
+	/// Returns a block number given the block id.
+	fn block_id_to_number(&self, at: &BlockId<Self::Block>) -> Result<Option<NumberFor<Self>>, Self::Error>;
+
+	/// Returns a block hash given the block id.
+	fn block_id_to_hash(&self, at: &BlockId<Self::Block>) -> Result<Option<BlockHash<Self>>, Self::Error>;
+
+	/// Returns hash and encoding length of the extrinsic.
+	fn hash_and_length(&self, uxt: &ExtrinsicFor<Self>) -> (Self::Hash, usize);
+}
+
+/// Pool configuration options.
+#[derive(Debug, Clone)]
+pub struct Options {
+	/// Ready queue limits.
+	pub ready: base::Limit,
+	/// Future queue limits.
+	pub future: base::Limit,
+}
+
+impl Default for Options {
+	fn default() -> Self {
+		Options {
+			ready: base::Limit {
+				count: 512,
+				total_bytes: 10 * 1024 * 1024,
+			},
+			future: base::Limit {
+				count: 128,
+				total_bytes: 1 * 1024 * 1024,
+			},
+		}
+	}
+}
+
+/// Extrinsics pool that performs validation.
+pub struct Pool<B: ChainApi> {
+	validated_pool: Arc<ValidatedPool<B>>,
+}
+
+impl<B: ChainApi> Pool<B> {
+	/// Create a new transaction pool.
+	pub fn new(options: Options, api: B) -> Self {
+		Pool {
+			validated_pool: Arc::new(ValidatedPool::new(options, api)),
+		}
+	}
+
+	/// Imports a bunch of unverified extrinsics to the pool
+	pub fn submit_at<T>(&self, at: &BlockId<B::Block>, xts: T, force: bool)
+		-> impl Future<Output=Result<Vec<Result<ExHash<B>, B::Error>>, B::Error>>
+	where
+		T: IntoIterator<Item=ExtrinsicFor<B>>
+	{
+		let validated_pool = self.validated_pool.clone();
+		self.verify(at, xts, force)
+			.map(move |validated_transactions| validated_transactions
+				.map(|validated_transactions| validated_pool.submit(validated_transactions)))
+	}
+
+	/// Imports one unverified extrinsic to the pool
+	pub fn submit_one(
+		&self,
+		at: &BlockId<B::Block>,
+		xt: ExtrinsicFor<B>,
+	) -> impl Future<Output=Result<ExHash<B>, B::Error>> {
+		self.submit_at(at, std::iter::once(xt), false)
+			.map(|import_result| import_result.and_then(|mut import_result| import_result
+				.pop()
+				.expect("One extrinsic passed; one result returned; qed")
+			))
+	}
+
+	/// Import a single extrinsic and starts to watch their progress in the pool.
+	pub fn submit_and_watch(
+		&self,
+		at: &BlockId<B::Block>,
+		xt: ExtrinsicFor<B>,
+	) -> impl Future<Output=Result<Watcher<ExHash<B>, BlockHash<B>>, B::Error>> {
+		let block_number = match self.resolve_block_number(at) {
+			Ok(block_number) => block_number,
+			Err(err) => return Either::Left(ready(Err(err)))
+		};
+
+		let validated_pool = self.validated_pool.clone();
+		Either::Right(
+			self.verify_one(at, block_number, xt, false)
+				.map(move |validated_transactions| validated_pool.submit_and_watch(validated_transactions))
+		)
+	}
+
+	/// Prunes ready transactions.
+	///
+	/// Used to clear the pool from transactions that were part of recently imported block.
+	/// To perform pruning we need the tags that each extrinsic provides and to avoid calling
+	/// into runtime too often we first lookup all extrinsics that are in the pool and get
+	/// their provided tags from there. Otherwise we query the runtime at the `parent` block.
+	pub fn prune(
+		&self,
+		at: &BlockId<B::Block>,
+		parent: &BlockId<B::Block>,
+		extrinsics: &[ExtrinsicFor<B>],
+	) -> impl Future<Output=Result<(), B::Error>> {
+		log::debug!(
+			target: "txpool",
+			"Starting pruning of block {:?} (extrinsics: {})",
+			at,
+			extrinsics.len()
+		);
+		// Get details of all extrinsics that are already in the pool
+		let (in_pool_hashes, in_pool_tags) = self.validated_pool.extrinsics_tags(extrinsics);
+
+		// Zip the ones from the pool with the full list (we get pairs `(Extrinsic, Option<Vec<Tag>>)`)
+		let all = extrinsics.iter().zip(in_pool_tags.into_iter());
+
+		// Prepare future that collect tags for all extrinsics
+		let future_tags = join_all(all
+			.map(|(extrinsic, in_pool_tags)|
+				match in_pool_tags {
+					// reuse the tags for extrinsics that were found in the pool
+					Some(tags) => Either::Left(
+						ready(tags)
+					),
+					// if it's not found in the pool query the runtime at parent block
+					// to get validity info and tags that the extrinsic provides.
+					None => Either::Right(self.validated_pool.api().validate_transaction(parent, extrinsic.clone())
+						.then(|validity| ready(match validity {
+							Ok(Ok(validity)) => validity.provides,
+							// silently ignore invalid extrinsics,
+							// cause they might just be inherent
+							_ => Vec::new(),
+						}))),
+				}
+			));
+
+		// Prune transactions by tags
+		let at = at.clone();
+		let self_clone = self.clone();
+		future_tags.then(move |tags| self_clone.prune_tags(
+			&at,
+			tags.into_iter().flat_map(|tags| tags),
+			in_pool_hashes,
+		))
+	}
+
+	/// Prunes ready transactions that provide given list of tags.
+	///
+	/// Given tags are assumed to be always provided now, so all transactions
+	/// in the Future Queue that require that particular tag (and have other
+	/// requirements satisfied) are promoted to Ready Queue.
+	///
+	/// Moreover for each provided tag we remove transactions in the pool that:
+	/// 1. Provide that tag directly
+	/// 2. Are a dependency of pruned transaction.
+	///
+	/// Returns transactions that have been removed from the pool and must be reverified
+	/// before reinserting to the pool.
+	///
+	/// By removing predecessor transactions as well we might actually end up
+	/// pruning too much, so all removed transactions are reverified against
+	/// the runtime (`validate_transaction`) to make sure they are invalid.
+	///
+	/// However we avoid revalidating transactions that are contained within
+	/// the second parameter of `known_imported_hashes`. These transactions
+	/// (if pruned) are not revalidated and become temporarily banned to
+	/// prevent importing them in the (near) future.
+	pub fn prune_tags(
+		&self,
+		at: &BlockId<B::Block>,
+		tags: impl IntoIterator<Item=Tag>,
+		known_imported_hashes: impl IntoIterator<Item=ExHash<B>> + Clone,
+	) -> impl Future<Output=Result<(), B::Error>> {
+		log::trace!(target: "txpool", "Pruning at {:?}", at);
+		// Prune all transactions that provide given tags
+		let prune_status = match self.validated_pool.prune_tags(tags) {
+			Ok(prune_status) => prune_status,
+			Err(e) => return Either::Left(ready(Err(e))),
+		};
+
+		// Make sure that we don't revalidate extrinsics that were part of the recently
+		// imported block. This is especially important for UTXO-like chains cause the
+		// inputs are pruned so such transaction would go to future again.
+		self.validated_pool.ban(&std::time::Instant::now(), known_imported_hashes.clone().into_iter());
+
+		// Try to re-validate pruned transactions since some of them might be still valid.
+		// note that `known_imported_hashes` will be rejected here due to temporary ban.
+		let pruned_hashes = prune_status.pruned.iter().map(|tx| tx.hash.clone()).collect::<Vec<_>>();
+		let pruned_transactions = prune_status.pruned.into_iter().map(|tx| tx.data.clone());
+		let reverify_future = self.verify(at, pruned_transactions, false);
+
+		log::trace!(target: "txpool", "Prunning at {:?}. Resubmitting transactions.", at);
+		// And finally - submit reverified transactions back to the pool
+		let at = at.clone();
+		let validated_pool = self.validated_pool.clone();
+		Either::Right(reverify_future.then(move |reverified_transactions|
+			ready(reverified_transactions.and_then(|reverified_transactions|
+				validated_pool.resubmit_pruned(
+					&at,
+					known_imported_hashes,
+					pruned_hashes,
+					reverified_transactions,
+				))
+			)))
+	}
+
+	/// Return an event stream of transactions imported to the pool.
+	pub fn import_notification_stream(&self) -> EventStream {
+		self.validated_pool.import_notification_stream()
+	}
+
+	/// Invoked when extrinsics are broadcasted.
+	pub fn on_broadcasted(&self, propagated: HashMap<ExHash<B>, Vec<String>>) {
+		self.validated_pool.on_broadcasted(propagated)
+	}
+
+	/// Remove from the pool.
+	pub fn remove_invalid(&self, hashes: &[ExHash<B>]) -> Vec<TransactionFor<B>> {
+		self.validated_pool.remove_invalid(hashes)
+	}
+
+	/// Get an iterator for ready transactions ordered by priority
+	pub fn ready(&self) -> impl Iterator<Item=TransactionFor<B>> {
+		self.validated_pool.ready()
+	}
+
+	/// Returns pool status.
+	pub fn status(&self) -> base::Status {
+		self.validated_pool.status()
+	}
+
+	/// Returns transaction hash
+	pub fn hash_of(&self, xt: &ExtrinsicFor<B>) -> ExHash<B> {
+		self.validated_pool.api().hash_and_length(xt).0
+	}
+
+	/// Resolves block number by id.
+	fn resolve_block_number(&self, at: &BlockId<B::Block>) -> Result<NumberFor<B>, B::Error> {
+		self.validated_pool.api().block_id_to_number(at)
+			.and_then(|number| number.ok_or_else(||
+				error::Error::InvalidBlockId(format!("{:?}", at)).into()))
+	}
+
+	/// Returns future that validates a bunch of transactions at given block.
+	fn verify(
+		&self,
+		at: &BlockId<B::Block>,
+		xts: impl IntoIterator<Item=ExtrinsicFor<B>>,
+		force: bool,
+	) -> impl Future<Output=Result<Vec<ValidatedTransactionFor<B>>, B::Error>> {
+		// we need a block number to compute tx validity
+		let block_number = match self.resolve_block_number(at) {
+			Ok(block_number) => block_number,
+			Err(err) => return Either::Left(ready(Err(err))),
+		};
+
+		// for each xt, prepare a validation future
+		let validation_futures = xts.into_iter().map(move |xt|
+			self.verify_one(at, block_number, xt, force)
+		);
+
+		// make single validation future that waits all until all extrinsics are validated
+		Either::Right(join_all(validation_futures).then(|x| ready(Ok(x))))
+	}
+
+	/// Returns future that validates single transaction at given block.
+	fn verify_one(
+		&self,
+		block_id: &BlockId<B::Block>,
+		block_number: NumberFor<B>,
+		xt: ExtrinsicFor<B>,
+		force: bool,
+	) -> impl Future<Output=ValidatedTransactionFor<B>> {
+		let (hash, bytes) = self.validated_pool.api().hash_and_length(&xt);
+		if !force && self.validated_pool.is_banned(&hash) {
+			return Either::Left(ready(ValidatedTransaction::Invalid(hash, error::Error::TemporarilyBanned.into())))
+		}
+
+		Either::Right(self.validated_pool.api().validate_transaction(block_id, xt.clone())
+			.then(move |validation_result| ready(match validation_result {
+				Ok(validity) => match validity {
+					Ok(validity) => if validity.provides.is_empty() {
+						ValidatedTransaction::Invalid(hash, error::Error::NoTagsProvided.into())
+					} else {
+						ValidatedTransaction::Valid(base::Transaction {
+							data: xt,
+							bytes,
+							hash,
+							priority: validity.priority,
+							requires: validity.requires,
+							provides: validity.provides,
+							propagate: validity.propagate,
+							valid_till: block_number
+								.saturated_into::<u64>()
+								.saturating_add(validity.longevity),
+						})
+					},
+					Err(TransactionValidityError::Invalid(e)) =>
+						ValidatedTransaction::Invalid(hash, error::Error::InvalidTransaction(e).into()),
+					Err(TransactionValidityError::Unknown(e)) =>
+						ValidatedTransaction::Unknown(hash, error::Error::UnknownTransaction(e).into()),
+				},
+				Err(e) => ValidatedTransaction::Invalid(hash, e),
+			})))
+	}
+}
+
+impl<B: ChainApi> Clone for Pool<B> {
+	fn clone(&self) -> Self {
+		Self {
+			validated_pool: self.validated_pool.clone(),
+		}
+	}
+}
+
+#[cfg(test)]
+mod tests {
+	use std::{
+		collections::HashMap,
+		time::Instant,
+	};
+	use parking_lot::Mutex;
+	use futures::executor::block_on;
+	use super::*;
+	use sr_primitives::transaction_validity::{ValidTransaction, InvalidTransaction};
+	use codec::Encode;
+	use test_runtime::{Block, Extrinsic, Transfer, H256, AccountId};
+	use assert_matches::assert_matches;
+	use crate::base_pool::Limit;
+	use crate::watcher;
+
+	const INVALID_NONCE: u64 = 254;
+
+	#[derive(Clone, Debug, Default)]
+	struct TestApi {
+		delay: Arc<Mutex<Option<std::sync::mpsc::Receiver<()>>>>,
+	}
+
+	impl ChainApi for TestApi {
+		type Block = Block;
+		type Hash = u64;
+		type Error = error::Error;
+		type ValidationFuture = futures::future::Ready<error::Result<TransactionValidity>>;
+
+		/// Verify extrinsic at given block.
+		fn validate_transaction(
+			&self,
+			at: &BlockId<Self::Block>,
+			uxt: ExtrinsicFor<Self>,
+		) -> Self::ValidationFuture {
+			let block_number = self.block_id_to_number(at).unwrap().unwrap();
+			let nonce = uxt.transfer().nonce;
+
+			// This is used to control the test flow.
+			if nonce > 0 {
+				let opt = self.delay.lock().take();
+				if let Some(delay) = opt {
+					if delay.recv().is_err() {
+						println!("Error waiting for delay!");
+					}
+				}
+			}
+
+			futures::future::ready(if nonce < block_number {
+				Ok(InvalidTransaction::Stale.into())
+			} else {
+				Ok(Ok(ValidTransaction {
+					priority: 4,
+					requires: if nonce > block_number { vec![vec![nonce as u8 - 1]] } else { vec![] },
+					provides: if nonce == INVALID_NONCE { vec![] } else { vec![vec![nonce as u8]] },
+					longevity: 3,
+					propagate: true,
+				}))
+			})
+		}
+
+		/// Returns a block number given the block id.
+		fn block_id_to_number(&self, at: &BlockId<Self::Block>) -> Result<Option<NumberFor<Self>>, Self::Error> {
+			Ok(match at {
+				BlockId::Number(num) => Some(*num),
+				BlockId::Hash(_) => None,
+			})
+		}
+
+		/// Returns a block hash given the block id.
+		fn block_id_to_hash(&self, at: &BlockId<Self::Block>) -> Result<Option<BlockHash<Self>>, Self::Error> {
+			Ok(match at {
+				BlockId::Number(num) => Some(H256::from_low_u64_be(*num)).into(),
+				BlockId::Hash(_) => None,
+			})
+		}
+
+		/// Hash the extrinsic.
+		fn hash_and_length(&self, uxt: &ExtrinsicFor<Self>) -> (Self::Hash, usize) {
+			let len = uxt.encode().len();
+			(
+				(H256::from(uxt.transfer().from.clone()).to_low_u64_be() << 5) + uxt.transfer().nonce,
+				len
+			)
+		}
+	}
+
+	fn uxt(transfer: Transfer) -> Extrinsic {
+		Extrinsic::Transfer(transfer, Default::default())
+	}
+
+	fn pool() -> Pool<TestApi> {
+		Pool::new(Default::default(), TestApi::default())
+	}
+
+	#[test]
+	fn should_validate_and_import_transaction() {
+		// given
+		let pool = pool();
+
+		// when
+		let hash = block_on(pool.submit_one(&BlockId::Number(0), uxt(Transfer {
+			from: AccountId::from_h256(H256::from_low_u64_be(1)),
+			to: AccountId::from_h256(H256::from_low_u64_be(2)),
+			amount: 5,
+			nonce: 0,
+		}))).unwrap();
+
+		// then
+		assert_eq!(pool.ready().map(|v| v.hash).collect::<Vec<_>>(), vec![hash]);
+	}
+
+	#[test]
+	fn should_reject_if_temporarily_banned() {
+		// given
+		let pool = pool();
+		let uxt = uxt(Transfer {
+			from: AccountId::from_h256(H256::from_low_u64_be(1)),
+			to: AccountId::from_h256(H256::from_low_u64_be(2)),
+			amount: 5,
+			nonce: 0,
+		});
+
+		// when
+		pool.validated_pool.rotator().ban(&Instant::now(), vec![pool.hash_of(&uxt)]);
+		let res = block_on(pool.submit_one(&BlockId::Number(0), uxt));
+		assert_eq!(pool.status().ready, 0);
+		assert_eq!(pool.status().future, 0);
+
+		// then
+		assert_matches!(res.unwrap_err(), error::Error::TemporarilyBanned);
+	}
+
+	#[test]
+	fn should_notify_about_pool_events() {
+		let stream = {
+			// given
+			let pool = pool();
+			let stream = pool.import_notification_stream();
+
+			// when
+			let _hash = block_on(pool.submit_one(&BlockId::Number(0), uxt(Transfer {
+				from: AccountId::from_h256(H256::from_low_u64_be(1)),
+				to: AccountId::from_h256(H256::from_low_u64_be(2)),
+				amount: 5,
+				nonce: 0,
+			}))).unwrap();
+			let _hash = block_on(pool.submit_one(&BlockId::Number(0), uxt(Transfer {
+				from: AccountId::from_h256(H256::from_low_u64_be(1)),
+				to: AccountId::from_h256(H256::from_low_u64_be(2)),
+				amount: 5,
+				nonce: 1,
+			}))).unwrap();
+			// future doesn't count
+			let _hash = block_on(pool.submit_one(&BlockId::Number(0), uxt(Transfer {
+				from: AccountId::from_h256(H256::from_low_u64_be(1)),
+				to: AccountId::from_h256(H256::from_low_u64_be(2)),
+				amount: 5,
+				nonce: 3,
+			}))).unwrap();
+
+			assert_eq!(pool.status().ready, 2);
+			assert_eq!(pool.status().future, 1);
+			stream
+		};
+
+		// then
+		let mut it = futures::executor::block_on_stream(stream);
+		assert_eq!(it.next(), Some(()));
+		assert_eq!(it.next(), Some(()));
+		assert_eq!(it.next(), None);
+	}
+
+	#[test]
+	fn should_clear_stale_transactions() {
+		// given
+		let pool = pool();
+		let hash1 = block_on(pool.submit_one(&BlockId::Number(0), uxt(Transfer {
+			from: AccountId::from_h256(H256::from_low_u64_be(1)),
+			to: AccountId::from_h256(H256::from_low_u64_be(2)),
+			amount: 5,
+			nonce: 0,
+		}))).unwrap();
+		let hash2 = block_on(pool.submit_one(&BlockId::Number(0), uxt(Transfer {
+			from: AccountId::from_h256(H256::from_low_u64_be(1)),
+			to: AccountId::from_h256(H256::from_low_u64_be(2)),
+			amount: 5,
+			nonce: 1,
+		}))).unwrap();
+		let hash3 = block_on(pool.submit_one(&BlockId::Number(0), uxt(Transfer {
+			from: AccountId::from_h256(H256::from_low_u64_be(1)),
+			to: AccountId::from_h256(H256::from_low_u64_be(2)),
+			amount: 5,
+			nonce: 3,
+		}))).unwrap();
+
+		// when
+		pool.validated_pool.clear_stale(&BlockId::Number(5)).unwrap();
+
+		// then
+		assert_eq!(pool.ready().count(), 0);
+		assert_eq!(pool.status().future, 0);
+		assert_eq!(pool.status().ready, 0);
+		// make sure they are temporarily banned as well
+		assert!(pool.validated_pool.rotator().is_banned(&hash1));
+		assert!(pool.validated_pool.rotator().is_banned(&hash2));
+		assert!(pool.validated_pool.rotator().is_banned(&hash3));
+	}
+
+	#[test]
+	fn should_ban_mined_transactions() {
+		// given
+		let pool = pool();
+		let hash1 = block_on(pool.submit_one(&BlockId::Number(0), uxt(Transfer {
+			from: AccountId::from_h256(H256::from_low_u64_be(1)),
+			to: AccountId::from_h256(H256::from_low_u64_be(2)),
+			amount: 5,
+			nonce: 0,
+		}))).unwrap();
+
+		// when
+		block_on(pool.prune_tags(&BlockId::Number(1), vec![vec![0]], vec![hash1.clone()])).unwrap();
+
+		// then
+		assert!(pool.validated_pool.rotator().is_banned(&hash1));
+	}
+
+	#[test]
+	fn should_limit_futures() {
+		// given
+		let limit = Limit {
+			count: 100,
+			total_bytes: 200,
+		};
+		let pool = Pool::new(Options {
+			ready: limit.clone(),
+			future: limit.clone(),
+		}, TestApi::default());
+
+		let hash1 = block_on(pool.submit_one(&BlockId::Number(0), uxt(Transfer {
+			from: AccountId::from_h256(H256::from_low_u64_be(1)),
+			to: AccountId::from_h256(H256::from_low_u64_be(2)),
+			amount: 5,
+			nonce: 1,
+		}))).unwrap();
+		assert_eq!(pool.status().future, 1);
+
+		// when
+		let hash2 = block_on(pool.submit_one(&BlockId::Number(0), uxt(Transfer {
+			from: AccountId::from_h256(H256::from_low_u64_be(2)),
+			to: AccountId::from_h256(H256::from_low_u64_be(2)),
+			amount: 5,
+			nonce: 10,
+		}))).unwrap();
+
+		// then
+		assert_eq!(pool.status().future, 1);
+		assert!(pool.validated_pool.rotator().is_banned(&hash1));
+		assert!(!pool.validated_pool.rotator().is_banned(&hash2));
+	}
+
+	#[test]
+	fn should_error_if_reject_immediately() {
+		// given
+		let limit = Limit {
+			count: 100,
+			total_bytes: 10,
+		};
+		let pool = Pool::new(Options {
+			ready: limit.clone(),
+			future: limit.clone(),
+		}, TestApi::default());
+
+		// when
+		block_on(pool.submit_one(&BlockId::Number(0), uxt(Transfer {
+			from: AccountId::from_h256(H256::from_low_u64_be(1)),
+			to: AccountId::from_h256(H256::from_low_u64_be(2)),
+			amount: 5,
+			nonce: 1,
+		}))).unwrap_err();
+
+		// then
+		assert_eq!(pool.status().ready, 0);
+		assert_eq!(pool.status().future, 0);
+	}
+
+	#[test]
+	fn should_reject_transactions_with_no_provides() {
+		// given
+		let pool = pool();
+
+		// when
+		let err = block_on(pool.submit_one(&BlockId::Number(0), uxt(Transfer {
+			from: AccountId::from_h256(H256::from_low_u64_be(1)),
+			to: AccountId::from_h256(H256::from_low_u64_be(2)),
+			amount: 5,
+			nonce: INVALID_NONCE,
+		}))).unwrap_err();
+
+		// then
+		assert_eq!(pool.status().ready, 0);
+		assert_eq!(pool.status().future, 0);
+		assert_matches!(err, error::Error::NoTagsProvided);
+	}
+
+	mod listener {
+		use super::*;
+
+		#[test]
+		fn should_trigger_ready_and_finalized() {
+			// given
+			let pool = pool();
+			let watcher = block_on(pool.submit_and_watch(&BlockId::Number(0), uxt(Transfer {
+				from: AccountId::from_h256(H256::from_low_u64_be(1)),
+				to: AccountId::from_h256(H256::from_low_u64_be(2)),
+				amount: 5,
+				nonce: 0,
+			}))).unwrap();
+			assert_eq!(pool.status().ready, 1);
+			assert_eq!(pool.status().future, 0);
+
+			// when
+			block_on(pool.prune_tags(&BlockId::Number(2), vec![vec![0u8]], vec![])).unwrap();
+			assert_eq!(pool.status().ready, 0);
+			assert_eq!(pool.status().future, 0);
+
+			// then
+			let mut stream = futures::executor::block_on_stream(watcher.into_stream());
+			assert_eq!(stream.next(), Some(watcher::Status::Ready));
+			assert_eq!(stream.next(), Some(watcher::Status::Finalized(H256::from_low_u64_be(2).into())));
+			assert_eq!(stream.next(), None);
+		}
+
+		#[test]
+		fn should_trigger_ready_and_finalized_when_pruning_via_hash() {
+			// given
+			let pool = pool();
+			let watcher = block_on(pool.submit_and_watch(&BlockId::Number(0), uxt(Transfer {
+				from: AccountId::from_h256(H256::from_low_u64_be(1)),
+				to: AccountId::from_h256(H256::from_low_u64_be(2)),
+				amount: 5,
+				nonce: 0,
+			}))).unwrap();
+			assert_eq!(pool.status().ready, 1);
+			assert_eq!(pool.status().future, 0);
+
+			// when
+			block_on(pool.prune_tags(&BlockId::Number(2), vec![vec![0u8]], vec![2u64])).unwrap();
+			assert_eq!(pool.status().ready, 0);
+			assert_eq!(pool.status().future, 0);
+
+			// then
+			let mut stream = futures::executor::block_on_stream(watcher.into_stream());
+			assert_eq!(stream.next(), Some(watcher::Status::Ready));
+			assert_eq!(stream.next(), Some(watcher::Status::Finalized(H256::from_low_u64_be(2).into())));
+			assert_eq!(stream.next(), None);
+		}
+
+		#[test]
+		fn should_trigger_future_and_ready_after_promoted() {
+			// given
+			let pool = pool();
+			let watcher = block_on(pool.submit_and_watch(&BlockId::Number(0), uxt(Transfer {
+				from: AccountId::from_h256(H256::from_low_u64_be(1)),
+				to: AccountId::from_h256(H256::from_low_u64_be(2)),
+				amount: 5,
+				nonce: 1,
+			}))).unwrap();
+			assert_eq!(pool.status().ready, 0);
+			assert_eq!(pool.status().future, 1);
+
+			// when
+			block_on(pool.submit_one(&BlockId::Number(0), uxt(Transfer {
+				from: AccountId::from_h256(H256::from_low_u64_be(1)),
+				to: AccountId::from_h256(H256::from_low_u64_be(2)),
+				amount: 5,
+				nonce: 0,
+			}))).unwrap();
+			assert_eq!(pool.status().ready, 2);
+
+			// then
+			let mut stream = futures::executor::block_on_stream(watcher.into_stream());
+			assert_eq!(stream.next(), Some(watcher::Status::Future));
+			assert_eq!(stream.next(), Some(watcher::Status::Ready));
+		}
+
+		#[test]
+		fn should_trigger_invalid_and_ban() {
+			// given
+			let pool = pool();
+			let uxt = uxt(Transfer {
+				from: AccountId::from_h256(H256::from_low_u64_be(1)),
+				to: AccountId::from_h256(H256::from_low_u64_be(2)),
+				amount: 5,
+				nonce: 0,
+			});
+			let watcher = block_on(pool.submit_and_watch(&BlockId::Number(0), uxt)).unwrap();
+			assert_eq!(pool.status().ready, 1);
+
+			// when
+			pool.validated_pool.remove_invalid(&[*watcher.hash()]);
+
+
+			// then
+			let mut stream = futures::executor::block_on_stream(watcher.into_stream());
+			assert_eq!(stream.next(), Some(watcher::Status::Ready));
+			assert_eq!(stream.next(), Some(watcher::Status::Invalid));
+			assert_eq!(stream.next(), None);
+		}
+
+		#[test]
+		fn should_trigger_broadcasted() {
+			// given
+			let pool = pool();
+			let uxt = uxt(Transfer {
+				from: AccountId::from_h256(H256::from_low_u64_be(1)),
+				to: AccountId::from_h256(H256::from_low_u64_be(2)),
+				amount: 5,
+				nonce: 0,
+			});
+			let watcher = block_on(pool.submit_and_watch(&BlockId::Number(0), uxt)).unwrap();
+			assert_eq!(pool.status().ready, 1);
+
+			// when
+			let mut map = HashMap::new();
+			let peers = vec!["a".into(), "b".into(), "c".into()];
+			map.insert(*watcher.hash(), peers.clone());
+			pool.on_broadcasted(map);
+
+
+			// then
+			let mut stream = futures::executor::block_on_stream(watcher.into_stream());
+			assert_eq!(stream.next(), Some(watcher::Status::Ready));
+			assert_eq!(stream.next(), Some(watcher::Status::Broadcast(peers)));
+		}
+
+		#[test]
+		fn should_trigger_dropped() {
+			// given
+			let limit = Limit {
+				count: 1,
+				total_bytes: 1000,
+			};
+			let pool = Pool::new(Options {
+				ready: limit.clone(),
+				future: limit.clone(),
+			}, TestApi::default());
+
+			let xt = uxt(Transfer {
+				from: AccountId::from_h256(H256::from_low_u64_be(1)),
+				to: AccountId::from_h256(H256::from_low_u64_be(2)),
+				amount: 5,
+				nonce: 0,
+			});
+			let watcher = block_on(pool.submit_and_watch(&BlockId::Number(0), xt)).unwrap();
+			assert_eq!(pool.status().ready, 1);
+
+			// when
+			let xt = uxt(Transfer {
+				from: AccountId::from_h256(H256::from_low_u64_be(2)),
+				to: AccountId::from_h256(H256::from_low_u64_be(1)),
+				amount: 4,
+				nonce: 1,
+			});
+			block_on(pool.submit_one(&BlockId::Number(1), xt)).unwrap();
+			assert_eq!(pool.status().ready, 1);
+
+			// then
+			let mut stream = futures::executor::block_on_stream(watcher.into_stream());
+			assert_eq!(stream.next(), Some(watcher::Status::Ready));
+			assert_eq!(stream.next(), Some(watcher::Status::Dropped));
+		}
+
+		#[test]
+		fn should_handle_pruning_in_the_middle_of_import() {
+			let _ = env_logger::try_init();
+			// given
+			let (ready, is_ready) = std::sync::mpsc::sync_channel(0);
+			let (tx, rx) = std::sync::mpsc::sync_channel(1);
+			let mut api = TestApi::default();
+			api.delay = Arc::new(Mutex::new(rx.into()));
+			let pool = Arc::new(Pool::new(Default::default(), api));
+
+			// when
+			let xt = uxt(Transfer {
+				from: AccountId::from_h256(H256::from_low_u64_be(1)),
+				to: AccountId::from_h256(H256::from_low_u64_be(2)),
+				amount: 5,
+				nonce: 1,
+			});
+
+			// This transaction should go to future, since we use `nonce: 1`
+			let pool2 = pool.clone();
+			std::thread::spawn(move || {
+				block_on(pool2.submit_one(&BlockId::Number(0), xt)).unwrap();
+				ready.send(()).unwrap();
+			});
+
+			// But now before the previous one is imported we import
+			// the one that it depends on.
+			let xt = uxt(Transfer {
+				from: AccountId::from_h256(H256::from_low_u64_be(1)),
+				to: AccountId::from_h256(H256::from_low_u64_be(2)),
+				amount: 4,
+				nonce: 0,
+			});
+			// The tag the above transaction provides (TestApi is using just nonce as u8)
+			let provides = vec![0_u8];
+			block_on(pool.submit_one(&BlockId::Number(0), xt)).unwrap();
+			assert_eq!(pool.status().ready, 1);
+
+			// Now block import happens before the second transaction is able to finish verification.
+			block_on(pool.prune_tags(&BlockId::Number(1), vec![provides], vec![])).unwrap();
+			assert_eq!(pool.status().ready, 0);
+
+
+			// so when we release the verification of the previous one it will have
+			// something in `requires`, but should go to ready directly, since the previous transaction was imported
+			// correctly.
+			tx.send(()).unwrap();
+
+			// then
+			is_ready.recv().unwrap(); // wait for finish
+			assert_eq!(pool.status().ready, 1);
+			assert_eq!(pool.status().future, 0);
+		}
+	}
+}
+

substrate/client/transaction-pool/graph/src/ready.rs

@@ -0,0 +1,638 @@
+// Copyright 2018-2019 Parity Technologies (UK) Ltd.
+// This file is part of Substrate.
+
+// Substrate is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// Substrate is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with Substrate.  If not, see <http://www.gnu.org/licenses/>.
+
+use std::{
+	collections::{HashMap, HashSet, BTreeSet},
+	cmp,
+	hash,
+	sync::Arc,
+};
+
+use serde::Serialize;
+use log::debug;
+use parking_lot::RwLock;
+use sr_primitives::traits::Member;
+use sr_primitives::transaction_validity::{
+	TransactionTag as Tag,
+};
+
+use crate::error;
+use crate::future::WaitingTransaction;
+use crate::base_pool::Transaction;
+
+/// An in-pool transaction reference.
+///
+/// Should be cheap to clone.
+#[derive(Debug)]
+pub struct TransactionRef<Hash, Ex> {
+	/// The actual transaction data.
+	pub transaction: Arc<Transaction<Hash, Ex>>,
+	/// Unique id when transaction was inserted into the pool.
+	pub insertion_id: u64,
+}
+
+impl<Hash, Ex> Clone for TransactionRef<Hash, Ex> {
+	fn clone(&self) -> Self {
+		TransactionRef {
+			transaction: self.transaction.clone(),
+			insertion_id: self.insertion_id,
+		}
+	}
+}
+
+impl<Hash, Ex> Ord for TransactionRef<Hash, Ex> {
+	fn cmp(&self, other: &Self) -> cmp::Ordering {
+		self.transaction.priority.cmp(&other.transaction.priority)
+			.then(other.transaction.valid_till.cmp(&self.transaction.valid_till))
+			.then(other.insertion_id.cmp(&self.insertion_id))
+	}
+}
+
+impl<Hash, Ex> PartialOrd for TransactionRef<Hash, Ex> {
+	fn partial_cmp(&self, other: &Self) -> Option<cmp::Ordering> {
+		Some(self.cmp(other))
+	}
+}
+
+impl<Hash, Ex> PartialEq for TransactionRef<Hash, Ex> {
+	fn eq(&self, other: &Self) -> bool {
+		self.cmp(other) == cmp::Ordering::Equal
+	}
+}
+impl<Hash, Ex> Eq for TransactionRef<Hash, Ex> {}
+
+#[derive(Debug)]
+pub struct ReadyTx<Hash, Ex> {
+	/// A reference to a transaction
+	pub transaction: TransactionRef<Hash, Ex>,
+	/// A list of transactions that get unlocked by this one
+	pub unlocks: Vec<Hash>,
+	/// How many required tags are provided inherently
+	///
+	/// Some transactions might be already pruned from the queue,
+	/// so when we compute ready set we may consider this transactions ready earlier.
+	pub requires_offset: usize,
+}
+
+impl<Hash: Clone, Ex> Clone for ReadyTx<Hash, Ex> {
+	fn clone(&self) -> Self {
+		ReadyTx {
+			transaction: self.transaction.clone(),
+			unlocks: self.unlocks.clone(),
+			requires_offset: self.requires_offset,
+		}
+	}
+}
+
+const HASH_READY: &str = r#"
+Every time transaction is imported its hash is placed in `ready` map and tags in `provided_tags`;
+Every time transaction is removed from the queue we remove the hash from `ready` map and from `provided_tags`;
+Hence every hash retrieved from `provided_tags` is always present in `ready`;
+qed
+"#;
+
+#[derive(Debug)]
+pub struct ReadyTransactions<Hash: hash::Hash + Eq, Ex> {
+	/// Insertion id
+	insertion_id: u64,
+	/// tags that are provided by Ready transactions
+	provided_tags: HashMap<Tag, Hash>,
+	/// Transactions that are ready (i.e. don't have any requirements external to the pool)
+	ready: Arc<RwLock<HashMap<Hash, ReadyTx<Hash, Ex>>>>,
+	/// Best transactions that are ready to be included to the block without any other previous transaction.
+	best: BTreeSet<TransactionRef<Hash, Ex>>,
+}
+
+impl<Hash: hash::Hash + Eq, Ex> Default for ReadyTransactions<Hash, Ex> {
+	fn default() -> Self {
+		ReadyTransactions {
+			insertion_id: Default::default(),
+			provided_tags: Default::default(),
+			ready: Default::default(),
+			best: Default::default(),
+		}
+	}
+}
+
+impl<Hash: hash::Hash + Member + Serialize, Ex> ReadyTransactions<Hash, Ex> {
+	/// Borrows a map of tags that are provided by transactions in this queue.
+	pub fn provided_tags(&self) -> &HashMap<Tag, Hash> {
+		&self.provided_tags
+	}
+
+	/// Returns an iterator of ready transactions.
+	///
+	/// Transactions are returned in order:
+	/// 1. First by the dependencies:
+	///	- never return transaction that requires a tag, which was not provided by one of the previously returned transactions
+	/// 2. Then by priority:
+	/// - If there are two transactions with all requirements satisfied the one with higher priority goes first.
+	/// 3. Then by the ttl that's left
+	/// - transactions that are valid for a shorter time go first
+	/// 4. Lastly we sort by the time in the queue
+	/// - transactions that are longer in the queue go first
+	pub fn get(&self) -> impl Iterator<Item=Arc<Transaction<Hash, Ex>>> {
+		BestIterator {
+			all: self.ready.clone(),
+			best: self.best.clone(),
+			awaiting: Default::default(),
+		}
+	}
+
+	/// Imports transactions to the pool of ready transactions.
+	///
+	/// The transaction needs to have all tags satisfied (be ready) by transactions
+	/// that are in this queue.
+	/// Returns transactions that were replaced by the one imported.
+	pub fn import(
+		&mut self,
+		tx: WaitingTransaction<Hash, Ex>,
+	) -> error::Result<Vec<Arc<Transaction<Hash, Ex>>>> {
+		assert!(tx.is_ready(), "Only ready transactions can be imported.");
+		assert!(!self.ready.read().contains_key(&tx.transaction.hash), "Transaction is already imported.");
+
+		self.insertion_id += 1;
+		let insertion_id = self.insertion_id;
+		let hash = tx.transaction.hash.clone();
+		let transaction = tx.transaction;
+
+		let replaced = self.replace_previous(&transaction)?;
+
+		let mut goes_to_best = true;
+		let mut ready = self.ready.write();
+		// Add links to transactions that unlock the current one
+		for tag in &transaction.requires {
+			// Check if the transaction that satisfies the tag is still in the queue.
+			if let Some(other) = self.provided_tags.get(tag) {
+				let tx = ready.get_mut(other).expect(HASH_READY);
+				tx.unlocks.push(hash.clone());
+				// this transaction depends on some other, so it doesn't go to best directly.
+				goes_to_best = false;
+			}
+	 	}
+
+		// update provided_tags
+		for tag in &transaction.provides {
+			self.provided_tags.insert(tag.clone(), hash.clone());
+		}
+
+		let transaction = TransactionRef {
+			insertion_id,
+			transaction
+		};
+
+		// insert to best if it doesn't require any other transaction to be included before it
+		if goes_to_best {
+			self.best.insert(transaction.clone());
+		}
+
+		// insert to Ready
+		ready.insert(hash, ReadyTx {
+			transaction,
+			unlocks: vec![],
+			requires_offset: 0,
+		});
+
+		Ok(replaced)
+	}
+
+	/// Fold a list of ready transactions to compute a single value.
+	pub fn fold<R, F: FnMut(Option<R>, &ReadyTx<Hash, Ex>) -> Option<R>>(&mut self, f: F) -> Option<R> {
+		self.ready
+			.read()
+			.values()
+			.fold(None, f)
+	}
+
+	/// Returns true if given hash is part of the queue.
+	pub fn contains(&self, hash: &Hash) -> bool {
+		self.ready.read().contains_key(hash)
+	}
+
+	/// Retrieve transaction by hash
+	pub fn by_hash(&self, hashes: &[Hash]) -> Vec<Option<Arc<Transaction<Hash, Ex>>>> {
+		let ready = self.ready.read();
+		hashes.iter().map(|hash| {
+			ready.get(hash).map(|x| x.transaction.transaction.clone())
+		}).collect()
+	}
+
+	/// Removes invalid transactions from the ready pool.
+	///
+	/// NOTE removing a transaction will also cause a removal of all transactions that depend on that one
+	/// (i.e. the entire subgraph that this transaction is a start of will be removed).
+	/// All removed transactions are returned.
+	pub fn remove_invalid(&mut self, hashes: &[Hash]) -> Vec<Arc<Transaction<Hash, Ex>>> {
+		let mut removed = vec![];
+		let mut to_remove = hashes.iter().cloned().collect::<Vec<_>>();
+
+		let mut ready = self.ready.write();
+		loop {
+			let hash = match to_remove.pop() {
+				Some(hash) => hash,
+				None => return removed,
+			};
+
+			if let Some(mut tx) = ready.remove(&hash) {
+				// remove entries from provided_tags
+				for tag in &tx.transaction.transaction.provides {
+					self.provided_tags.remove(tag);
+				}
+				// remove from unlocks
+				for tag in &tx.transaction.transaction.requires {
+					if let Some(hash) = self.provided_tags.get(tag) {
+						if let Some(tx) = ready.get_mut(hash) {
+							remove_item(&mut tx.unlocks, &hash);
+						}
+					}
+				}
+
+				// remove from best
+				self.best.remove(&tx.transaction);
+
+				// remove all transactions that the current one unlocks
+				to_remove.append(&mut tx.unlocks);
+
+				// add to removed
+				debug!(target: "txpool", "[{:?}] Removed as invalid: ", hash);
+				removed.push(tx.transaction.transaction);
+			}
+		}
+	}
+
+	/// Removes transactions that provide given tag.
+	///
+	/// All transactions that lead to a transaction, which provides this tag
+	/// are going to be removed from the queue, but no other transactions are touched -
+	/// i.e. all other subgraphs starting from given tag are still considered valid & ready.
+	pub fn prune_tags(&mut self, tag: Tag) -> Vec<Arc<Transaction<Hash, Ex>>> {
+		let mut removed = vec![];
+		let mut to_remove = vec![tag];
+
+		loop {
+			let tag = match to_remove.pop() {
+				Some(tag) => tag,
+				None => return removed,
+			};
+
+			let res = self.provided_tags.remove(&tag)
+					.and_then(|hash| self.ready.write().remove(&hash));
+
+			if let Some(tx) = res {
+				let unlocks = tx.unlocks;
+				let tx = tx.transaction.transaction;
+
+				// prune previous transactions as well
+				{
+					let hash = &tx.hash;
+					let mut ready = self.ready.write();
+					let mut find_previous = |tag| -> Option<Vec<Tag>> {
+						let prev_hash = self.provided_tags.get(tag)?;
+						let tx2 = ready.get_mut(&prev_hash)?;
+						remove_item(&mut tx2.unlocks, hash);
+						// We eagerly prune previous transactions as well.
+						// But it might not always be good.
+						// Possible edge case:
+						// - tx provides two tags
+						// - the second tag enables some subgraph we don't know of yet
+						// - we will prune the transaction
+						// - when we learn about the subgraph it will go to future
+						// - we will have to wait for re-propagation of that transaction
+						// Alternatively the caller may attempt to re-import these transactions.
+						if tx2.unlocks.is_empty() {
+							Some(tx2.transaction.transaction.provides.clone())
+						} else {
+							None
+						}
+					};
+
+					// find previous transactions
+					for tag in &tx.requires {
+						if let Some(mut tags_to_remove) = find_previous(tag) {
+							to_remove.append(&mut tags_to_remove);
+						}
+					}
+				}
+
+				// add the transactions that just got unlocked to `best`
+				for hash in unlocks {
+					if let Some(tx) = self.ready.write().get_mut(&hash) {
+						tx.requires_offset += 1;
+						// this transaction is ready
+						if tx.requires_offset == tx.transaction.transaction.requires.len() {
+							self.best.insert(tx.transaction.clone());
+						}
+					}
+				}
+
+				// we also need to remove all other tags that this transaction provides,
+				// but since all the hard work is done, we only clear the provided_tag -> hash
+				// mapping.
+				let current_tag = &tag;
+				for tag in &tx.provides {
+					let removed = self.provided_tags.remove(tag);
+					assert_eq!(
+						removed.as_ref(),
+						if current_tag == tag { None } else { Some(&tx.hash) },
+						"The pool contains exactly one transaction providing given tag; the removed transaction
+						claims to provide that tag, so it has to be mapped to it's hash; qed"
+					);
+				}
+
+				removed.push(tx);
+			}
+		}
+	}
+
+	/// Checks if the transaction is providing the same tags as other transactions.
+	///
+	/// In case that's true it determines if the priority of transactions that
+	/// we are about to replace is lower than the priority of the replacement transaction.
+	/// We remove/replace old transactions in case they have lower priority.
+	///
+	/// In case replacement is successful returns a list of removed transactions.
+	fn replace_previous(&mut self, tx: &Transaction<Hash, Ex>) -> error::Result<Vec<Arc<Transaction<Hash, Ex>>>> {
+		let mut to_remove = {
+			// check if we are replacing a transaction
+			let replace_hashes = tx.provides
+				.iter()
+				.filter_map(|tag| self.provided_tags.get(tag))
+				.collect::<HashSet<_>>();
+
+			// early exit if we are not replacing anything.
+			if replace_hashes.is_empty() {
+				return Ok(vec![]);
+			}
+
+			// now check if collective priority is lower than the replacement transaction.
+			let old_priority = {
+				let ready = self.ready.read();
+				replace_hashes
+					.iter()
+					.filter_map(|hash| ready.get(hash))
+					.fold(0u64, |total, tx| total.saturating_add(tx.transaction.transaction.priority))
+			};
+
+			// bail - the transaction has too low priority to replace the old ones
+			if old_priority >= tx.priority {
+				return Err(error::Error::TooLowPriority { old: old_priority, new: tx.priority })
+			}
+
+			replace_hashes.into_iter().cloned().collect::<Vec<_>>()
+		};
+
+		let new_provides = tx.provides.iter().cloned().collect::<HashSet<_>>();
+		let mut removed = vec![];
+		loop {
+			let hash = match to_remove.pop() {
+				Some(hash) => hash,
+				None => return Ok(removed),
+			};
+
+			let tx = self.ready.write().remove(&hash).expect(HASH_READY);
+			// check if this transaction provides stuff that is not provided by the new one.
+			let (mut unlocks, tx) = (tx.unlocks, tx.transaction.transaction);
+			{
+				let invalidated = tx.provides
+					.iter()
+					.filter(|tag| !new_provides.contains(&**tag));
+
+				for tag in invalidated {
+					// remove the tag since it's no longer provided by any transaction
+					self.provided_tags.remove(tag);
+					// add more transactions to remove
+					to_remove.append(&mut unlocks);
+				}
+			}
+
+			removed.push(tx);
+		}
+	}
+
+	/// Returns number of transactions in this queue.
+	pub fn len(&self) -> usize {
+		self.ready.read().len()
+	}
+
+	/// Returns sum of encoding lengths of all transactions in this queue.
+	pub fn bytes(&self) -> usize {
+		self.ready.read().values().fold(0, |acc, tx| acc + tx.transaction.transaction.bytes)
+	}
+}
+
+pub struct BestIterator<Hash, Ex> {
+	all: Arc<RwLock<HashMap<Hash, ReadyTx<Hash, Ex>>>>,
+	awaiting: HashMap<Hash, (usize, TransactionRef<Hash, Ex>)>,
+	best: BTreeSet<TransactionRef<Hash, Ex>>,
+}
+
+impl<Hash: hash::Hash + Member, Ex> BestIterator<Hash, Ex> {
+	/// Depending on number of satisfied requirements insert given ref
+	/// either to awaiting set or to best set.
+	fn best_or_awaiting(&mut self, satisfied: usize, tx_ref: TransactionRef<Hash, Ex>) {
+		if satisfied == tx_ref.transaction.requires.len() {
+			// If we have satisfied all deps insert to best
+			self.best.insert(tx_ref);
+
+		} else {
+			// otherwise we're still awaiting for some deps
+			self.awaiting.insert(tx_ref.transaction.hash.clone(), (satisfied, tx_ref));
+		}
+	}
+}
+
+impl<Hash: hash::Hash + Member, Ex> Iterator for BestIterator<Hash, Ex> {
+	type Item = Arc<Transaction<Hash, Ex>>;
+
+	fn next(&mut self) -> Option<Self::Item> {
+		loop {
+			let best = self.best.iter().next_back()?.clone();
+			let best = self.best.take(&best)?;
+
+			let next = self.all.read().get(&best.transaction.hash).cloned();
+			let ready = match next {
+				Some(ready) => ready,
+				// The transaction is not in all, maybe it was removed in the meantime?
+				None => continue,
+			};
+
+			// Insert transactions that just got unlocked.
+			for hash in &ready.unlocks {
+				// first check local awaiting transactions
+				let res = if let Some((mut satisfied, tx_ref)) = self.awaiting.remove(hash) {
+					satisfied += 1;
+					Some((satisfied, tx_ref))
+				// then get from the pool
+				} else if let Some(next) = self.all.read().get(hash) {
+					Some((next.requires_offset + 1, next.transaction.clone()))
+				} else {
+					None
+				};
+
+				if let Some((satisfied, tx_ref)) = res {
+					self.best_or_awaiting(satisfied, tx_ref)
+				}
+			}
+
+			return Some(best.transaction.clone())
+		}
+	}
+}
+
+// See: https://github.com/rust-lang/rust/issues/40062
+fn remove_item<T: PartialEq>(vec: &mut Vec<T>, item: &T) {
+	if let Some(idx) = vec.iter().position(|i| i == item) {
+		vec.swap_remove(idx);
+	}
+}
+
+#[cfg(test)]
+mod tests {
+	use super::*;
+
+	fn tx(id: u8) -> Transaction<u64, Vec<u8>> {
+		Transaction {
+			data: vec![id],
+			bytes: 1,
+			hash: id as u64,
+			priority: 1,
+			valid_till: 2,
+			requires: vec![vec![1], vec![2]],
+			provides: vec![vec![3], vec![4]],
+			propagate: true,
+		}
+	}
+
+	#[test]
+	fn should_replace_transaction_that_provides_the_same_tag() {
+		// given
+		let mut ready = ReadyTransactions::default();
+		let mut tx1 = tx(1);
+		tx1.requires.clear();
+		let mut tx2 = tx(2);
+		tx2.requires.clear();
+		tx2.provides = vec![vec![3]];
+		let mut tx3 = tx(3);
+		tx3.requires.clear();
+		tx3.provides = vec![vec![4]];
+
+		// when
+		let x = WaitingTransaction::new(tx2, &ready.provided_tags(), &[]);
+		ready.import(x).unwrap();
+		let x = WaitingTransaction::new(tx3, &ready.provided_tags(), &[]);
+		ready.import(x).unwrap();
+		assert_eq!(ready.get().count(), 2);
+
+		// too low priority
+		let x = WaitingTransaction::new(tx1.clone(), &ready.provided_tags(), &[]);
+		ready.import(x).unwrap_err();
+
+		tx1.priority = 10;
+		let x = WaitingTransaction::new(tx1.clone(), &ready.provided_tags(), &[]);
+		ready.import(x).unwrap();
+
+		// then
+		assert_eq!(ready.get().count(), 1);
+	}
+
+
+	#[test]
+	fn should_return_best_transactions_in_correct_order() {
+		// given
+		let mut ready = ReadyTransactions::default();
+		let mut tx1 = tx(1);
+		tx1.requires.clear();
+		let mut tx2 = tx(2);
+		tx2.requires = tx1.provides.clone();
+		tx2.provides = vec![vec![106]];
+		let mut tx3 = tx(3);
+		tx3.requires = vec![tx1.provides[0].clone(), vec![106]];
+		tx3.provides = vec![];
+		let mut tx4 = tx(4);
+		tx4.requires = vec![tx1.provides[0].clone()];
+		tx4.provides = vec![];
+		let tx5 = Transaction {
+			data: vec![5],
+			bytes: 1,
+			hash: 5,
+			priority: 1,
+			valid_till: u64::max_value(),	// use the max_value() here for testing.
+			requires: vec![tx1.provides[0].clone()],
+			provides: vec![],
+			propagate: true,
+		};
+
+		// when
+		let x = WaitingTransaction::new(tx1, &ready.provided_tags(), &[]);
+		ready.import(x).unwrap();
+		let x = WaitingTransaction::new(tx2, &ready.provided_tags(), &[]);
+		ready.import(x).unwrap();
+		let x = WaitingTransaction::new(tx3, &ready.provided_tags(), &[]);
+		ready.import(x).unwrap();
+		let x = WaitingTransaction::new(tx4, &ready.provided_tags(), &[]);
+		ready.import(x).unwrap();
+		let x = WaitingTransaction::new(tx5, &ready.provided_tags(), &[]);
+		ready.import(x).unwrap();
+
+		// then
+		assert_eq!(ready.best.len(), 1);
+
+		let mut it = ready.get().map(|tx| tx.data[0]);
+
+		assert_eq!(it.next(), Some(1));
+		assert_eq!(it.next(), Some(2));
+		assert_eq!(it.next(), Some(3));
+		assert_eq!(it.next(), Some(4));
+		assert_eq!(it.next(), Some(5));
+		assert_eq!(it.next(), None);
+	}
+
+	#[test]
+	fn should_order_refs() {
+		let mut id = 1;
+		let mut with_priority = |priority, longevity| {
+			id += 1;
+			let mut tx = tx(id);
+			tx.priority = priority;
+			tx.valid_till = longevity;
+			tx
+		};
+		// higher priority = better
+		assert!(TransactionRef {
+			transaction: Arc::new(with_priority(3, 3)),
+			insertion_id: 1,
+		} > TransactionRef {
+			transaction: Arc::new(with_priority(2, 3)),
+			insertion_id: 2,
+		});
+		// lower validity = better
+		assert!(TransactionRef {
+			transaction: Arc::new(with_priority(3, 2)),
+			insertion_id: 1,
+		} > TransactionRef {
+			transaction: Arc::new(with_priority(3, 3)),
+			insertion_id: 2,
+		});
+		// lower insertion_id = better
+		assert!(TransactionRef {
+			transaction: Arc::new(with_priority(3, 3)),
+			insertion_id: 1,
+		} > TransactionRef {
+			transaction: Arc::new(with_priority(3, 3)),
+			insertion_id: 2,
+		});
+	}
+}

substrate/client/transaction-pool/graph/src/rotator.rs

@@ -0,0 +1,211 @@
+// Copyright 2018-2019 Parity Technologies (UK) Ltd.
+// This file is part of Substrate.
+
+// Substrate is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// Substrate is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with Substrate.  If not, see <http://www.gnu.org/licenses/>.
+
+//! Rotate extrinsic inside the pool.
+//!
+//! Keeps only recent extrinsic and discard the ones kept for a significant amount of time.
+//! Discarded extrinsics are banned so that they don't get re-imported again.
+
+use std::{
+	collections::HashMap,
+	hash,
+	iter,
+	time::{Duration, Instant},
+};
+use parking_lot::RwLock;
+
+use crate::base_pool::Transaction;
+
+/// Expected size of the banned extrinsics cache.
+const EXPECTED_SIZE: usize = 2048;
+
+/// Pool rotator is responsible to only keep fresh extrinsics in the pool.
+///
+/// Extrinsics that occupy the pool for too long are culled and temporarily banned from entering
+/// the pool again.
+pub struct PoolRotator<Hash> {
+	/// How long the extrinsic is banned for.
+	ban_time: Duration,
+	/// Currently banned extrinsics.
+	banned_until: RwLock<HashMap<Hash, Instant>>,
+}
+
+impl<Hash: hash::Hash + Eq> Default for PoolRotator<Hash> {
+	fn default() -> Self {
+		PoolRotator {
+			ban_time: Duration::from_secs(60 * 30),
+			banned_until: Default::default(),
+		}
+	}
+}
+
+impl<Hash: hash::Hash + Eq + Clone> PoolRotator<Hash> {
+	/// Returns `true` if extrinsic hash is currently banned.
+	pub fn is_banned(&self, hash: &Hash) -> bool {
+		self.banned_until.read().contains_key(hash)
+	}
+
+	/// Bans given set of hashes.
+	pub fn ban(&self, now: &Instant, hashes: impl IntoIterator<Item=Hash>) {
+		let mut banned = self.banned_until.write();
+
+		for hash in hashes {
+			banned.insert(hash, *now + self.ban_time);
+		}
+
+		if banned.len() > 2 * EXPECTED_SIZE {
+			while banned.len() > EXPECTED_SIZE {
+				if let Some(key) = banned.keys().next().cloned() {
+					banned.remove(&key);
+				}
+			}
+		}
+	}
+
+
+	/// Bans extrinsic if it's stale.
+	///
+	/// Returns `true` if extrinsic is stale and got banned.
+	pub fn ban_if_stale<Ex>(&self, now: &Instant, current_block: u64, xt: &Transaction<Hash, Ex>) -> bool {
+		if xt.valid_till > current_block {
+			return false;
+		}
+
+		self.ban(now, iter::once(xt.hash.clone()));
+		true
+	}
+
+	/// Removes timed bans.
+	pub fn clear_timeouts(&self, now: &Instant) {
+		let mut banned = self.banned_until.write();
+
+		banned.retain(|_, &mut v| v >= *now);
+	}
+}
+
+#[cfg(test)]
+mod tests {
+	use super::*;
+
+	type Hash = u64;
+	type Ex = ();
+
+	fn rotator() -> PoolRotator<Hash> {
+		PoolRotator {
+			ban_time: Duration::from_millis(10),
+			..Default::default()
+		}
+	}
+
+	fn tx() -> (Hash, Transaction<Hash, Ex>) {
+		let hash = 5u64;
+		let tx = Transaction {
+			data: (),
+			bytes: 1,
+			hash: hash.clone(),
+			priority: 5,
+			valid_till: 1,
+			requires: vec![],
+			provides: vec![],
+			propagate: true,
+		};
+
+		(hash, tx)
+	}
+
+	#[test]
+	fn should_not_ban_if_not_stale() {
+		// given
+		let (hash, tx) = tx();
+		let rotator = rotator();
+		assert!(!rotator.is_banned(&hash));
+		let now = Instant::now();
+		let past_block = 0;
+
+		// when
+		assert!(!rotator.ban_if_stale(&now, past_block, &tx));
+
+		// then
+		assert!(!rotator.is_banned(&hash));
+	}
+
+	#[test]
+	fn should_ban_stale_extrinsic() {
+		// given
+		let (hash, tx) = tx();
+		let rotator = rotator();
+		assert!(!rotator.is_banned(&hash));
+
+		// when
+		assert!(rotator.ban_if_stale(&Instant::now(), 1, &tx));
+
+		// then
+		assert!(rotator.is_banned(&hash));
+	}
+
+
+	#[test]
+	fn should_clear_banned() {
+		// given
+		let (hash, tx) = tx();
+		let rotator = rotator();
+		assert!(rotator.ban_if_stale(&Instant::now(), 1, &tx));
+		assert!(rotator.is_banned(&hash));
+
+		// when
+		let future = Instant::now() + rotator.ban_time + rotator.ban_time;
+		rotator.clear_timeouts(&future);
+
+		// then
+		assert!(!rotator.is_banned(&hash));
+	}
+
+	#[test]
+	fn should_garbage_collect() {
+		// given
+		fn tx_with(i: u64, valid_till: u64) -> Transaction<Hash, Ex> {
+			let hash = i;
+			Transaction {
+				data: (),
+				bytes: 2,
+				hash,
+				priority: 5,
+				valid_till,
+				requires: vec![],
+				provides: vec![],
+				propagate: true,
+			}
+		}
+
+		let rotator = rotator();
+
+		let now = Instant::now();
+		let past_block = 0;
+
+		// when
+		for i in 0..2*EXPECTED_SIZE {
+			let tx = tx_with(i as u64, past_block);
+			assert!(rotator.ban_if_stale(&now, past_block, &tx));
+		}
+		assert_eq!(rotator.banned_until.read().len(), 2*EXPECTED_SIZE);
+
+		// then
+		let tx = tx_with(2*EXPECTED_SIZE as u64, past_block);
+		// trigger a garbage collection
+		assert!(rotator.ban_if_stale(&now, past_block, &tx));
+		assert_eq!(rotator.banned_until.read().len(), EXPECTED_SIZE);
+	}
+}

substrate/client/transaction-pool/graph/src/validated_pool.rs

@@ -0,0 +1,378 @@
+// Copyright 2018-2019 Parity Technologies (UK) Ltd.
+// This file is part of Substrate.
+
+// Substrate is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// Substrate is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with Substrate.  If not, see <http://www.gnu.org/licenses/>.
+
+use std::{
+	collections::{HashSet, HashMap},
+	fmt,
+	hash,
+	time,
+};
+
+use crate::base_pool as base;
+use crate::error;
+use crate::listener::Listener;
+use crate::rotator::PoolRotator;
+use crate::watcher::Watcher;
+use serde::Serialize;
+use log::debug;
+
+use futures::channel::mpsc;
+use parking_lot::{Mutex, RwLock};
+use sr_primitives::{
+	generic::BlockId,
+	traits::{self, SaturatedConversion},
+	transaction_validity::TransactionTag as Tag,
+};
+
+use crate::base_pool::PruneStatus;
+use crate::pool::{EventStream, Options, ChainApi, BlockHash, ExHash, ExtrinsicFor, TransactionFor};
+
+/// Pre-validated transaction. Validated pool only accepts transactions wrapped in this enum.
+#[derive(Debug)]
+pub enum ValidatedTransaction<Hash, Ex, Error> {
+	/// Transaction that has been validated successfully.
+	Valid(base::Transaction<Hash, Ex>),
+	/// Transaction that is invalid.
+	Invalid(Hash, Error),
+	/// Transaction which validity can't be determined.
+	///
+	/// We're notifying watchers about failure, if 'unknown' transaction is submitted.
+	Unknown(Hash, Error),
+}
+
+/// A type of validated transaction stored in the pool.
+pub type ValidatedTransactionFor<B> = ValidatedTransaction<
+	ExHash<B>,
+	ExtrinsicFor<B>,
+	<B as ChainApi>::Error,
+>;
+
+/// Pool that deals with validated transactions.
+pub(crate) struct ValidatedPool<B: ChainApi> {
+	api: B,
+	options: Options,
+	listener: RwLock<Listener<ExHash<B>, BlockHash<B>>>,
+	pool: RwLock<base::BasePool<
+		ExHash<B>,
+		ExtrinsicFor<B>,
+	>>,
+	import_notification_sinks: Mutex<Vec<mpsc::UnboundedSender<()>>>,
+	rotator: PoolRotator<ExHash<B>>,
+}
+
+impl<B: ChainApi> ValidatedPool<B> {
+	/// Create a new transaction pool.
+	pub fn new(options: Options, api: B) -> Self {
+		ValidatedPool {
+			api,
+			options,
+			listener: Default::default(),
+			pool: Default::default(),
+			import_notification_sinks: Default::default(),
+			rotator: Default::default(),
+		}
+	}
+
+	/// Bans given set of hashes.
+	pub fn ban(&self, now: &std::time::Instant, hashes: impl IntoIterator<Item=ExHash<B>>) {
+		self.rotator.ban(now, hashes)
+	}
+
+	/// Returns true if transaction with given hash is currently banned from the pool.
+	pub fn is_banned(&self, hash: &ExHash<B>) -> bool {
+		self.rotator.is_banned(hash)
+	}
+
+	/// Imports a bunch of pre-validated transactions to the pool.
+	pub fn submit<T>(&self, txs: T) -> Vec<Result<ExHash<B>, B::Error>> where
+		T: IntoIterator<Item=ValidatedTransactionFor<B>>
+	{
+		let results = txs.into_iter()
+			.map(|validated_tx| self.submit_one(validated_tx))
+			.collect::<Vec<_>>();
+
+		let removed = self.enforce_limits();
+
+		results.into_iter().map(|res| match res {
+			Ok(ref hash) if removed.contains(hash) => Err(error::Error::ImmediatelyDropped.into()),
+			other => other,
+		}).collect()
+	}
+
+	/// Submit single pre-validated transaction to the pool.
+	fn submit_one(&self, tx: ValidatedTransactionFor<B>) -> Result<ExHash<B>, B::Error> {
+		match tx {
+			ValidatedTransaction::Valid(tx) => {
+				let imported = self.pool.write().import(tx)?;
+
+				if let base::Imported::Ready { .. } = imported {
+					self.import_notification_sinks.lock().retain(|sink| sink.unbounded_send(()).is_ok());
+				}
+
+				let mut listener = self.listener.write();
+				fire_events(&mut *listener, &imported);
+				Ok(imported.hash().clone())
+			}
+			ValidatedTransaction::Invalid(hash, err) => {
+				self.rotator.ban(&std::time::Instant::now(), std::iter::once(hash));
+				Err(err.into())
+			},
+			ValidatedTransaction::Unknown(hash, err) => {
+				self.listener.write().invalid(&hash);
+				Err(err.into())
+			}
+		}
+	}
+
+	fn enforce_limits(&self) -> HashSet<ExHash<B>> {
+		let status = self.pool.read().status();
+		let ready_limit = &self.options.ready;
+		let future_limit = &self.options.future;
+
+		debug!(target: "txpool", "Pool Status: {:?}", status);
+
+		if ready_limit.is_exceeded(status.ready, status.ready_bytes)
+			|| future_limit.is_exceeded(status.future, status.future_bytes) {
+			// clean up the pool
+			let removed = {
+				let mut pool = self.pool.write();
+				let removed = pool.enforce_limits(ready_limit, future_limit)
+					.into_iter().map(|x| x.hash.clone()).collect::<HashSet<_>>();
+				// ban all removed transactions
+				self.rotator.ban(&std::time::Instant::now(), removed.iter().map(|x| x.clone()));
+				removed
+			};
+			// run notifications
+			let mut listener = self.listener.write();
+			for h in &removed {
+				listener.dropped(h, None);
+			}
+
+			removed
+		} else {
+			Default::default()
+		}
+	}
+
+	/// Import a single extrinsic and starts to watch their progress in the pool.
+	pub fn submit_and_watch(
+		&self,
+		tx: ValidatedTransactionFor<B>,
+	) -> Result<Watcher<ExHash<B>, BlockHash<B>>, B::Error> {
+		match tx {
+			ValidatedTransaction::Valid(tx) => {
+				let hash = self.api.hash_and_length(&tx.data).0;
+				let watcher = self.listener.write().create_watcher(hash);
+				self.submit(std::iter::once(ValidatedTransaction::Valid(tx)))
+					.pop()
+					.expect("One extrinsic passed; one result returned; qed")
+					.map(|_| watcher)
+			},
+			ValidatedTransaction::Invalid(hash, err) => {
+				self.rotator.ban(&std::time::Instant::now(), std::iter::once(hash));
+				Err(err.into())
+			},
+			ValidatedTransaction::Unknown(_, err) => Err(err.into()),
+		}
+	}
+
+	/// For each extrinsic, returns tags that it provides (if known), or None (if it is unknown).
+	pub fn extrinsics_tags(&self, extrinsics: &[ExtrinsicFor<B>]) -> (Vec<ExHash<B>>, Vec<Option<Vec<Tag>>>) {
+		let hashes = extrinsics.iter().map(|extrinsic| self.api.hash_and_length(extrinsic).0).collect::<Vec<_>>();
+		let in_pool = self.pool.read().by_hash(&hashes);
+		(
+			hashes,
+			in_pool.into_iter()
+				.map(|existing_in_pool| existing_in_pool
+					.map(|transaction| transaction.provides.iter().cloned()
+					.collect()))
+				.collect(),
+		)
+	}
+
+	/// Prunes ready transactions that provide given list of tags.
+	pub fn prune_tags(
+		&self,
+		tags: impl IntoIterator<Item=Tag>,
+	) -> Result<PruneStatus<ExHash<B>, ExtrinsicFor<B>>, B::Error> {
+		// Perform tag-based pruning in the base pool
+		let status = self.pool.write().prune_tags(tags);
+		// Notify event listeners of all transactions
+		// that were promoted to `Ready` or were dropped.
+		{
+			let mut listener = self.listener.write();
+			for promoted in &status.promoted {
+				fire_events(&mut *listener, promoted);
+			}
+			for f in &status.failed {
+				listener.dropped(f, None);
+			}
+		}
+
+		Ok(status)
+	}
+
+	/// Resubmit transactions that have been revalidated after prune_tags call.
+	pub fn resubmit_pruned(
+		&self,
+		at: &BlockId<B::Block>,
+		known_imported_hashes: impl IntoIterator<Item=ExHash<B>> + Clone,
+		pruned_hashes: Vec<ExHash<B>>,
+		pruned_xts: Vec<ValidatedTransactionFor<B>>,
+	) -> Result<(), B::Error> {
+		debug_assert_eq!(pruned_hashes.len(), pruned_xts.len());
+
+		// Resubmit pruned transactions
+		let results = self.submit(pruned_xts);
+
+		// Collect the hashes of transactions that now became invalid (meaning that they are successfully pruned).
+		let hashes = results
+			.into_iter()
+			.enumerate()
+			.filter_map(|(idx, r)| match r.map_err(error::IntoPoolError::into_pool_error) {
+				Err(Ok(error::Error::InvalidTransaction(_))) => Some(pruned_hashes[idx].clone()),
+				_ => None,
+			});
+		// Fire `pruned` notifications for collected hashes and make sure to include
+		// `known_imported_hashes` since they were just imported as part of the block.
+		let hashes = hashes.chain(known_imported_hashes.into_iter());
+		{
+			let header_hash = self.api.block_id_to_hash(at)?
+				.ok_or_else(|| error::Error::InvalidBlockId(format!("{:?}", at)).into())?;
+			let mut listener = self.listener.write();
+			for h in hashes {
+				listener.pruned(header_hash, &h);
+			}
+		}
+		// perform regular cleanup of old transactions in the pool
+		// and update temporary bans.
+		self.clear_stale(at)?;
+		Ok(())
+	}
+
+	/// Removes stale transactions from the pool.
+	///
+	/// Stale transactions are transaction beyond their longevity period.
+	/// Note this function does not remove transactions that are already included in the chain.
+	/// See `prune_tags` if you want this.
+	pub fn clear_stale(&self, at: &BlockId<B::Block>) -> Result<(), B::Error> {
+		let block_number = self.api.block_id_to_number(at)?
+				.ok_or_else(|| error::Error::InvalidBlockId(format!("{:?}", at)).into())?
+				.saturated_into::<u64>();
+		let now = time::Instant::now();
+		let to_remove = {
+			self.ready()
+				.filter(|tx| self.rotator.ban_if_stale(&now, block_number, &tx))
+				.map(|tx| tx.hash.clone())
+				.collect::<Vec<_>>()
+		};
+		let futures_to_remove: Vec<ExHash<B>> = {
+			let p = self.pool.read();
+			let mut hashes = Vec::new();
+			for tx in p.futures() {
+				if self.rotator.ban_if_stale(&now, block_number, &tx) {
+					hashes.push(tx.hash.clone());
+				}
+			}
+			hashes
+		};
+		// removing old transactions
+		self.remove_invalid(&to_remove);
+		self.remove_invalid(&futures_to_remove);
+		// clear banned transactions timeouts
+		self.rotator.clear_timeouts(&now);
+
+		Ok(())
+	}
+
+	/// Get rotator reference.
+	#[cfg(test)]
+	pub fn rotator(&self) -> &PoolRotator<ExHash<B>> {
+		&self.rotator
+	}
+
+	/// Get api reference.
+	pub fn api(&self) -> &B {
+		&self.api
+	}
+
+	/// Return an event stream of transactions imported to the pool.
+	pub fn import_notification_stream(&self) -> EventStream {
+		let (sink, stream) = mpsc::unbounded();
+		self.import_notification_sinks.lock().push(sink);
+		stream
+	}
+
+	/// Invoked when extrinsics are broadcasted.
+	pub fn on_broadcasted(&self, propagated: HashMap<ExHash<B>, Vec<String>>) {
+		let mut listener = self.listener.write();
+		for (hash, peers) in propagated.into_iter() {
+			listener.broadcasted(&hash, peers);
+		}
+	}
+
+	/// Remove from the pool.
+	pub fn remove_invalid(&self, hashes: &[ExHash<B>]) -> Vec<TransactionFor<B>> {
+		// temporarily ban invalid transactions
+		debug!(target: "txpool", "Banning invalid transactions: {:?}", hashes);
+		self.rotator.ban(&time::Instant::now(), hashes.iter().cloned());
+
+		let invalid = self.pool.write().remove_invalid(hashes);
+
+		let mut listener = self.listener.write();
+		for tx in &invalid {
+			listener.invalid(&tx.hash);
+		}
+
+		invalid
+	}
+
+	/// Get an iterator for ready transactions ordered by priority
+	pub fn ready(&self) -> impl Iterator<Item=TransactionFor<B>> {
+		self.pool.read().ready()
+	}
+
+	/// Returns pool status.
+	pub fn status(&self) -> base::Status {
+		self.pool.read().status()
+	}
+}
+
+fn fire_events<H, H2, Ex>(
+	listener: &mut Listener<H, H2>,
+	imported: &base::Imported<H, Ex>,
+) where
+	H: hash::Hash + Eq + traits::Member + Serialize,
+	H2: Clone + fmt::Debug,
+{
+	match *imported {
+		base::Imported::Ready { ref promoted, ref failed, ref removed, ref hash } => {
+			listener.ready(hash, None);
+			for f in failed {
+				listener.invalid(f);
+			}
+			for r in removed {
+				listener.dropped(&r.hash, Some(hash));
+			}
+			for p in promoted {
+				listener.ready(p, None);
+			}
+		},
+		base::Imported::Future { ref hash } => {
+			listener.future(hash)
+		},
+	}
+}

substrate/client/transaction-pool/graph/src/watcher.rs

@@ -0,0 +1,142 @@
+// Copyright 2018-2019 Parity Technologies (UK) Ltd.
+// This file is part of Substrate.
+
+// Substrate is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// Substrate is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with Substrate.  If not, see <http://www.gnu.org/licenses/>.
+
+//! Extrinsics status updates.
+
+use futures::{
+	Stream,
+	channel::mpsc,
+};
+use serde::{Serialize, Deserialize};
+
+/// Possible extrinsic status events
+#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
+#[serde(rename_all = "camelCase")]
+pub enum Status<H, H2> {
+	/// Extrinsic is part of the future queue.
+	Future,
+	/// Extrinsic is part of the ready queue.
+	Ready,
+	/// Extrinsic has been finalized in block with given hash.
+	Finalized(H2),
+	/// Some state change (perhaps another extrinsic was included) rendered this extrinsic invalid.
+	Usurped(H),
+	/// The extrinsic has been broadcast to the given peers.
+	Broadcast(Vec<String>),
+	/// Extrinsic has been dropped from the pool because of the limit.
+	Dropped,
+	/// Extrinsic was detected as invalid.
+	Invalid,
+}
+
+/// Extrinsic watcher.
+///
+/// Represents a stream of status updates for particular extrinsic.
+#[derive(Debug)]
+pub struct Watcher<H, H2> {
+	receiver: mpsc::UnboundedReceiver<Status<H, H2>>,
+	hash: H,
+}
+
+impl<H, H2> Watcher<H, H2> {
+	/// Returns the transaction hash.
+	pub fn hash(&self) -> &H {
+		&self.hash
+	}
+
+	/// Pipe the notifications to given sink.
+	///
+	/// Make sure to drive the future to completion.
+	pub fn into_stream(self) -> impl Stream<Item=Status<H, H2>> {
+		self.receiver
+	}
+}
+
+/// Sender part of the watcher. Exposed only for testing purposes.
+#[derive(Debug)]
+pub struct Sender<H, H2> {
+	receivers: Vec<mpsc::UnboundedSender<Status<H, H2>>>,
+	finalized: bool,
+}
+
+impl<H, H2> Default for Sender<H, H2> {
+	fn default() -> Self {
+		Sender {
+			receivers: Default::default(),
+			finalized: false,
+		}
+	}
+}
+
+impl<H: Clone, H2: Clone> Sender<H, H2> {
+	/// Add a new watcher to this sender object.
+	pub fn new_watcher(&mut self, hash: H) -> Watcher<H, H2> {
+		let (tx, receiver) = mpsc::unbounded();
+		self.receivers.push(tx);
+		Watcher {
+			receiver,
+			hash,
+		}
+	}
+
+	/// Transaction became ready.
+	pub fn ready(&mut self) {
+		self.send(Status::Ready)
+	}
+
+	/// Transaction was moved to future.
+	pub fn future(&mut self) {
+		self.send(Status::Future)
+	}
+
+	/// Some state change (perhaps another extrinsic was included) rendered this extrinsic invalid.
+	pub fn usurped(&mut self, hash: H) {
+		self.send(Status::Usurped(hash))
+	}
+
+	/// Extrinsic has been finalized in block with given hash.
+	pub fn finalized(&mut self, hash: H2) {
+		self.send(Status::Finalized(hash));
+		self.finalized = true;
+	}
+
+	/// Extrinsic has been marked as invalid by the block builder.
+	pub fn invalid(&mut self) {
+		self.send(Status::Invalid);
+		// we mark as finalized as there are no more notifications
+		self.finalized = true;
+	}
+
+	/// Transaction has been dropped from the pool because of the limit.
+	pub fn dropped(&mut self) {
+		self.send(Status::Dropped);
+	}
+
+	/// The extrinsic has been broadcast to the given peers.
+	pub fn broadcast(&mut self, peers: Vec<String>) {
+		self.send(Status::Broadcast(peers))
+	}
+
+
+	/// Returns true if the are no more listeners for this extrinsic or it was finalized.
+	pub fn is_done(&self) -> bool {
+		self.finalized || self.receivers.is_empty()
+	}
+
+	fn send(&mut self, status: Status<H, H2>) {
+		self.receivers.retain(|sender| sender.unbounded_send(status.clone()).is_ok())
+	}
+}