Adds fork-awareness and finalization notifications to transaction pool watchers. (#4740)

* adds finalization support to sc-transaction-pool using MaintainedTransactionPool for finalization events

* adds TransactionStatus::Retracted, notify watchers of retracted blocks, finalized now finalizes, transactions for current finalized -> last finalized block

* adds last_finalized to ChainApi, use generic BlockT for ChainEvent

* fix tests

* Apply suggestions from code review

Co-Authored-By: Tomasz Drwięga <tomusdrw@users.noreply.github.com>

* tests

* fix tests, docs, lazily dedupe pruned hashes

* fix tests, Cargo.lock

* Apply suggestions from code review

Co-Authored-By: Tomasz Drwięga <tomusdrw@users.noreply.github.com>

* remove tree_route, last_finalized from ChainApi, add block hash to Finalization and Retracted events

* prune finality watchers

* fix tests

* remove HeaderBackend bound from FullChainApi

* code style nits, terminate stream in finality_timeout

Co-authored-by: Tomasz Drwięga <tomusdrw@users.noreply.github.com>

substrate/client/transaction-pool/src/api.rs

@@ -64,7 +64,8 @@ impl<Client, Block> FullChainApi<Client, Block> where
 
 impl<Client, Block> sc_transaction_graph::ChainApi for FullChainApi<Client, Block> where
 	Block: BlockT,
-	Client: ProvideRuntimeApi<Block> + BlockBody<Block> + BlockIdTo<Block> + 'static + Send + Sync,
+	Client: ProvideRuntimeApi<Block> + BlockBody<Block> + BlockIdTo<Block>,
+	Client: Send + Sync + 'static,
 	Client::Api: TaggedTransactionQueue<Block>,
 	sp_api::ApiErrorFor<Client, Block>: Send,
 {

substrate/client/transaction-pool/src/lib.rs

@@ -37,9 +37,8 @@ use sp_runtime::{
 	traits::{Block as BlockT, NumberFor, AtLeast32Bit, Extrinsic},
 };
 use sp_transaction_pool::{
-	TransactionPool, PoolStatus, ImportNotificationStream,
-	TxHash, TransactionFor, TransactionStatusStreamFor, BlockHash,
-	MaintainedTransactionPool, PoolFuture,
+	TransactionPool, PoolStatus, ImportNotificationStream, TxHash, TransactionFor,
+	TransactionStatusStreamFor, MaintainedTransactionPool, PoolFuture, ChainEvent,
 };
 use wasm_timer::Instant;
 
@@ -115,7 +114,6 @@ impl<PoolApi, Block> BasicPool<PoolApi, Block>
 				}
 			)),
 		}
-
 	}
 
 	/// Gets shared reference to the underlying pool.
@@ -174,19 +172,19 @@ impl<PoolApi, Block> TransactionPool for BasicPool<PoolApi, Block>
 	}
 
 	fn remove_invalid(&self, hashes: &[TxHash<Self>]) -> Vec<Arc<Self::InPoolTransaction>> {
-		self.pool.remove_invalid(hashes)
+		self.pool.validated_pool().remove_invalid(hashes)
 	}
 
 	fn status(&self) -> PoolStatus {
-		self.pool.status()
+		self.pool.validated_pool().status()
 	}
 
 	fn ready(&self) -> Box<dyn Iterator<Item=Arc<Self::InPoolTransaction>>> {
-		Box::new(self.pool.ready())
+		Box::new(self.pool.validated_pool().ready())
 	}
 
 	fn import_notification_stream(&self) -> ImportNotificationStream<TxHash<Self>> {
-		self.pool.import_notification_stream()
+		self.pool.validated_pool().import_notification_stream()
 	}
 
 	fn hash_of(&self, xt: &TransactionFor<Self>) -> TxHash<Self> {
@@ -194,11 +192,11 @@ impl<PoolApi, Block> TransactionPool for BasicPool<PoolApi, Block>
 	}
 
 	fn on_broadcasted(&self, propagations: HashMap<TxHash<Self>, Vec<String>>) {
-		self.pool.on_broadcasted(propagations)
+		self.pool.validated_pool().on_broadcasted(propagations)
 	}
 
 	fn ready_transaction(&self, hash: &TxHash<Self>) -> Option<Arc<Self::InPoolTransaction>> {
-		self.pool.ready_transaction(hash)
+		self.pool.validated_pool().ready_by_hash(hash)
 	}
 }
 
@@ -214,7 +212,7 @@ enum RevalidationStatus<N> {
 
 enum RevalidationStrategy<N> {
 	Always,
-	Light(RevalidationStatus<N>)
+	Light(RevalidationStatus<N>),
 }
 
 struct RevalidationAction {
@@ -241,7 +239,7 @@ impl<N: Clone + Copy + AtLeast32Bit> RevalidationStrategy<N> {
 				revalidate: status.next_required(
 					block,
 					revalidate_time_period,
-					revalidate_block_period
+					revalidate_block_period,
 				),
 				resubmit: false,
 				revalidate_amount: None,
@@ -275,7 +273,7 @@ impl<N: Clone + Copy + AtLeast32Bit> RevalidationStatus<N> {
 					revalidate_block_period.map(|period| block + period),
 				);
 				false
-			},
+			}
 			Self::Scheduled(revalidate_at_time, revalidate_at_block) => {
 				let is_required = revalidate_at_time.map(|at| Instant::now() >= at).unwrap_or(false)
 					|| revalidate_at_block.map(|at| block >= at).unwrap_or(false);
@@ -283,87 +281,105 @@ impl<N: Clone + Copy + AtLeast32Bit> RevalidationStatus<N> {
 					*self = Self::InProgress;
 				}
 				is_required
-			},
+			}
 			Self::InProgress => false,
 		}
 	}
 }
 
 impl<PoolApi, Block> MaintainedTransactionPool for BasicPool<PoolApi, Block>
-where
-	Block: BlockT,
-	PoolApi: 'static + sc_transaction_graph::ChainApi<Block=Block, Hash=Block::Hash>,
+	where
+		Block: BlockT,
+		PoolApi: 'static + sc_transaction_graph::ChainApi<Block=Block, Hash=Block::Hash>,
 {
-	fn maintain(&self, id: &BlockId<Self::Block>, retracted: &[BlockHash<Self>])
-		-> Pin<Box<dyn Future<Output=()> + Send>>
-	{
-		let id = id.clone();
-		let pool = self.pool.clone();
-		let api = self.api.clone();
+	fn maintain(&self, event: ChainEvent<Self::Block>) -> Pin<Box<dyn Future<Output=()> + Send>> {
+		match event {
+			ChainEvent::NewBlock { id, retracted, .. } => {
+				let id = id.clone();
+				let pool = self.pool.clone();
+				let api = self.api.clone();
 
-		let block_number = match api.block_id_to_number(&id) {
-			Ok(Some(number)) => number,
-			_ => {
-				log::trace!(target: "txqueue", "Skipping chain event - no number for that block {:?}", id);
-				return Box::pin(ready(()));
+				let block_number = match api.block_id_to_number(&id) {
+					Ok(Some(number)) => number,
+					_ => {
+						log::trace!(target: "txqueue", "Skipping chain event - no number for that block {:?}", id);
+						return Box::pin(ready(()));
+					}
+				};
+
+				let next_action = self.revalidation_strategy.lock().next(
+					block_number,
+					Some(std::time::Duration::from_secs(60)),
+					Some(20.into()),
+				);
+				let revalidation_strategy = self.revalidation_strategy.clone();
+				let retracted = retracted.clone();
+
+				async move {
+					// We don't query block if we won't prune anything
+					if !pool.validated_pool().status().is_empty() {
+						let hashes = api.block_body(&id).await
+							.unwrap_or_else(|e| {
+								log::warn!("Prune known transactions: error request {:?}!", e);
+								None
+							})
+							.unwrap_or_default()
+							.into_iter()
+							.map(|tx| pool.hash_of(&tx))
+							.collect::<Vec<_>>();
+
+						if let Err(e) = pool.prune_known(&id, &hashes) {
+							log::error!("Cannot prune known in the pool {:?}!", e);
+						}
+					}
+
+					if next_action.resubmit {
+						let mut resubmit_transactions = Vec::new();
+
+						for retracted_hash in retracted {
+							// notify txs awaiting finality that it has been retracted
+							pool.validated_pool().on_block_retracted(retracted_hash.clone());
+
+							let block_transactions = api.block_body(&BlockId::hash(retracted_hash.clone())).await
+								.unwrap_or_else(|e| {
+									log::warn!("Failed to fetch block body {:?}!", e);
+									None
+								})
+								.unwrap_or_default()
+								.into_iter()
+								.filter(|tx| tx.is_signed().unwrap_or(true));
+
+							resubmit_transactions.extend(block_transactions);
+						}
+						if let Err(e) = pool.submit_at(&id, resubmit_transactions, true).await {
+							log::debug!(
+								target: "txpool",
+								"[{:?}] Error re-submitting transactions: {:?}", id, e
+							)
+						}
+					}
+
+					if next_action.revalidate {
+						if let Err(e) = pool.revalidate_ready(&id, next_action.revalidate_amount).await {
+							log::warn!("Revalidate ready failed {:?}", e);
+						}
+					}
+
+					revalidation_strategy.lock().clear();
+				}.boxed()
 			}
-		};
-
-		let next_action = self.revalidation_strategy.lock().next(
-			block_number,
-			Some(std::time::Duration::from_secs(60)),
-			Some(20.into()),
-		);
-		let revalidation_strategy = self.revalidation_strategy.clone();
-		let retracted = retracted.to_vec();
-
-		async move {
-			// We don't query block if we won't prune anything
-			if !pool.status().is_empty() {
-				let hashes = api.block_body(&id).await
-					.unwrap_or_else(|e| {
-						log::warn!("Prune known transactions: error request {:?}!", e);
-						None
-					})
-				.unwrap_or_default()
-				.into_iter()
-				.map(|tx| pool.hash_of(&tx))
-				.collect::<Vec<_>>();
-
-				if let Err(e) = pool.prune_known(&id, &hashes) {
-					log::error!("Cannot prune known in the pool {:?}!", e);
-				}
+			ChainEvent::Finalized { hash } => {
+				let pool = self.pool.clone();
+				async move {
+					if let Err(e) = pool.validated_pool().on_block_finalized(hash).await {
+						log::warn!(
+							target: "txpool",
+							"Error [{}] occurred while attempting to notify watchers of finalization {}",
+							e, hash
+						)
+					}
+				}.boxed()
 			}
-
-			if next_action.resubmit {
-				let mut resubmit_transactions = Vec::new();
-
-				for retracted_hash in retracted {
-					let block_transactions = api.block_body(&BlockId::hash(retracted_hash.clone())).await
-						.unwrap_or_else(|e| {
-							log::warn!("Failed to fetch block body {:?}!", e);
-							None
-						})
-						.unwrap_or_default()
-						.into_iter()
-						.filter(|tx| tx.is_signed().unwrap_or(true));
-
-					resubmit_transactions.extend(block_transactions);
-				}
-				if let Err(e) = pool.submit_at(&id, resubmit_transactions, true).await {
-					log::debug!(target: "txpool",
-						"[{:?}] Error re-submitting transactions: {:?}", id, e
-					)
-				}
-			}
-
-			if next_action.revalidate {
-				if let Err(e) = pool.revalidate_ready(&id, next_action.revalidate_amount).await {
-					log::warn!("Revalidate ready failed {:?}", e);
-				}
-			}
-
-			revalidation_strategy.lock().clear();
-		}.boxed()
+		}
 	}
 }

substrate/client/transaction-pool/src/testing/pool.rs

@@ -15,17 +15,18 @@
 // along with Substrate.  If not, see <http://www.gnu.org/licenses/>.
 
 use crate::*;
-use sc_transaction_graph::Pool;
 use futures::executor::block_on;
+use txpool::{self, Pool};
 use sp_runtime::{
 	generic::BlockId,
 	transaction_validity::ValidTransaction,
 };
 use substrate_test_runtime_client::{
-	runtime::{Block, Hash, Index},
+	runtime::{Block, Hash, Index, Header},
 	AccountKeyring::*,
 };
 use substrate_test_runtime_transaction_pool::{TestApi, uxt};
+use sp_transaction_pool::TransactionStatus;
 
 fn pool() -> Pool<TestApi> {
 	Pool::new(Default::default(), TestApi::with_alice_nonce(209).into())
@@ -35,12 +36,22 @@ fn maintained_pool() -> BasicPool<TestApi, Block> {
 	BasicPool::new(Default::default(), std::sync::Arc::new(TestApi::with_alice_nonce(209)))
 }
 
+fn header(number: u64) -> Header {
+	Header {
+		number,
+		digest: Default::default(),
+		extrinsics_root:  Default::default(),
+		parent_hash: Default::default(),
+		state_root: Default::default(),
+	}
+}
+
 #[test]
 fn submission_should_work() {
 	let pool = pool();
 	block_on(pool.submit_one(&BlockId::number(0), uxt(Alice, 209))).unwrap();
 
-	let pending: Vec<_> = pool.ready().map(|a| a.data.transfer().nonce).collect();
+	let pending: Vec<_> = pool.validated_pool().ready().map(|a| a.data.transfer().nonce).collect();
 	assert_eq!(pending, vec![209]);
 }
 
@@ -50,7 +61,7 @@ fn multiple_submission_should_work() {
 	block_on(pool.submit_one(&BlockId::number(0), uxt(Alice, 209))).unwrap();
 	block_on(pool.submit_one(&BlockId::number(0), uxt(Alice, 210))).unwrap();
 
-	let pending: Vec<_> = pool.ready().map(|a| a.data.transfer().nonce).collect();
+	let pending: Vec<_> = pool.validated_pool().ready().map(|a| a.data.transfer().nonce).collect();
 	assert_eq!(pending, vec![209, 210]);
 }
 
@@ -59,7 +70,7 @@ fn early_nonce_should_be_culled() {
 	let pool = pool();
 	block_on(pool.submit_one(&BlockId::number(0), uxt(Alice, 208))).unwrap();
 
-	let pending: Vec<_> = pool.ready().map(|a| a.data.transfer().nonce).collect();
+	let pending: Vec<_> = pool.validated_pool().ready().map(|a| a.data.transfer().nonce).collect();
 	assert_eq!(pending, Vec::<Index>::new());
 }
 
@@ -68,11 +79,11 @@ fn late_nonce_should_be_queued() {
 	let pool = pool();
 
 	block_on(pool.submit_one(&BlockId::number(0), uxt(Alice, 210))).unwrap();
-	let pending: Vec<_> = pool.ready().map(|a| a.data.transfer().nonce).collect();
+	let pending: Vec<_> = pool.validated_pool().ready().map(|a| a.data.transfer().nonce).collect();
 	assert_eq!(pending, Vec::<Index>::new());
 
 	block_on(pool.submit_one(&BlockId::number(0), uxt(Alice, 209))).unwrap();
-	let pending: Vec<_> = pool.ready().map(|a| a.data.transfer().nonce).collect();
+	let pending: Vec<_> = pool.validated_pool().ready().map(|a| a.data.transfer().nonce).collect();
 	assert_eq!(pending, vec![209, 210]);
 }
 
@@ -82,7 +93,7 @@ fn prune_tags_should_work() {
 	let hash209 = block_on(pool.submit_one(&BlockId::number(0), uxt(Alice, 209))).unwrap();
 	block_on(pool.submit_one(&BlockId::number(0), uxt(Alice, 210))).unwrap();
 
-	let pending: Vec<_> = pool.ready().map(|a| a.data.transfer().nonce).collect();
+	let pending: Vec<_> = pool.validated_pool().ready().map(|a| a.data.transfer().nonce).collect();
 	assert_eq!(pending, vec![209, 210]);
 
 	block_on(
@@ -93,7 +104,7 @@ fn prune_tags_should_work() {
 		)
 	).expect("Prune tags");
 
-	let pending: Vec<_> = pool.ready().map(|a| a.data.transfer().nonce).collect();
+	let pending: Vec<_> = pool.validated_pool().ready().map(|a| a.data.transfer().nonce).collect();
 	assert_eq!(pending, vec![210]);
 }
 
@@ -102,11 +113,11 @@ fn should_ban_invalid_transactions() {
 	let pool = pool();
 	let uxt = uxt(Alice, 209);
 	let hash = block_on(pool.submit_one(&BlockId::number(0), uxt.clone())).unwrap();
-	pool.remove_invalid(&[hash]);
+	pool.validated_pool().remove_invalid(&[hash]);
 	block_on(pool.submit_one(&BlockId::number(0), uxt.clone())).unwrap_err();
 
 	// when
-	let pending: Vec<_> = pool.ready().map(|a| a.data.transfer().nonce).collect();
+	let pending: Vec<_> = pool.validated_pool().ready().map(|a| a.data.transfer().nonce).collect();
 	assert_eq!(pending, Vec::<Index>::new());
 
 	// then
@@ -122,29 +133,29 @@ fn should_correctly_prune_transactions_providing_more_than_one_tag() {
 	let pool = Pool::new(Default::default(), api.clone());
 	let xt = uxt(Alice, 209);
 	block_on(pool.submit_one(&BlockId::number(0), xt.clone())).expect("1. Imported");
-	assert_eq!(pool.status().ready, 1);
+	assert_eq!(pool.validated_pool().status().ready, 1);
 
 	// remove the transaction that just got imported.
 	api.increment_nonce(Alice.into());
 	block_on(pool.prune_tags(&BlockId::number(1), vec![vec![209]], vec![])).expect("1. Pruned");
-	assert_eq!(pool.status().ready, 0);
+	assert_eq!(pool.validated_pool().status().ready, 0);
 	// it's re-imported to future
-	assert_eq!(pool.status().future, 1);
+	assert_eq!(pool.validated_pool().status().future, 1);
 
 	// so now let's insert another transaction that also provides the 155
 	api.increment_nonce(Alice.into());
 	let xt = uxt(Alice, 211);
 	block_on(pool.submit_one(&BlockId::number(2), xt.clone())).expect("2. Imported");
-	assert_eq!(pool.status().ready, 1);
-	assert_eq!(pool.status().future, 1);
-	let pending: Vec<_> = pool.ready().map(|a| a.data.transfer().nonce).collect();
+	assert_eq!(pool.validated_pool().status().ready, 1);
+	assert_eq!(pool.validated_pool().status().future, 1);
+	let pending: Vec<_> = pool.validated_pool().ready().map(|a| a.data.transfer().nonce).collect();
 	assert_eq!(pending, vec![211]);
 
 	// prune it and make sure the pool is empty
 	api.increment_nonce(Alice.into());
 	block_on(pool.prune_tags(&BlockId::number(3), vec![vec![155]], vec![])).expect("2. Pruned");
-	assert_eq!(pool.status().ready, 0);
-	assert_eq!(pool.status().future, 2);
+	assert_eq!(pool.validated_pool().status().ready, 0);
+	assert_eq!(pool.validated_pool().status().future, 2);
 }
 
 #[test]
@@ -158,7 +169,14 @@ fn should_prune_old_during_maintenance() {
 
 	pool.api.push_block(1, vec![xt.clone()]);
 
-	block_on(pool.maintain(&BlockId::number(1), &[]));
+	let event = ChainEvent::NewBlock {
+		id: BlockId::number(1),
+		is_new_best: true,
+		retracted: vec![],
+		header: header(1),
+	};
+
+	block_on(pool.maintain(event));
 	assert_eq!(pool.status().ready, 0);
 }
 
@@ -174,8 +192,14 @@ fn should_revalidate_during_maintenance() {
 	assert_eq!(pool.api.validation_requests().len(), 2);
 
 	pool.api.push_block(1, vec![xt1.clone()]);
+	let event = ChainEvent::NewBlock {
+		id: BlockId::number(1),
+		is_new_best: true,
+		retracted: vec![],
+		header: header(1),
+	};
 
-	block_on(pool.maintain(&BlockId::number(1), &[]));
+	block_on(pool.maintain(event));
 	assert_eq!(pool.status().ready, 1);
 	// test that pool revalidated transaction that left ready and not included in the block
 	assert_eq!(pool.api.validation_requests().len(), 3);
@@ -193,8 +217,14 @@ fn should_resubmit_from_retracted_during_maintaince() {
 
 	pool.api.push_block(1, vec![]);
 	pool.api.push_fork_block(retracted_hash, vec![xt.clone()]);
+	let event = ChainEvent::NewBlock {
+		id: BlockId::Number(1),
+		is_new_best: true,
+		header: header(1),
+		retracted: vec![retracted_hash]
+	};
 
-	block_on(pool.maintain(&BlockId::number(1), &[retracted_hash]));
+	block_on(pool.maintain(event));
 	assert_eq!(pool.status().ready, 1);
 }
 
@@ -212,7 +242,14 @@ fn should_not_retain_invalid_hashes_from_retracted() {
 	pool.api.push_fork_block(retracted_hash, vec![xt.clone()]);
 	pool.api.add_invalid(&xt);
 
-	block_on(pool.maintain(&BlockId::number(1), &[retracted_hash]));
+	let event = ChainEvent::NewBlock {
+		id: BlockId::Number(1),
+		is_new_best: true,
+		header: header(1),
+		retracted: vec![retracted_hash]
+	};
+
+	block_on(pool.maintain(event));
 	assert_eq!(pool.status().ready, 0);
 }
 
@@ -225,4 +262,188 @@ fn can_track_heap_size() {
 	block_on(pool.submit_one(&BlockId::number(0), uxt(Alice, 212))).expect("1. Imported");
 
 	assert!(parity_util_mem::malloc_size(&pool) > 3000);
-}
+}
+
+#[test]
+fn finalization() {
+	let xt = uxt(Alice, 209);
+	let api = TestApi::with_alice_nonce(209);
+	api.push_block(1, vec![]);
+	let pool = BasicPool::new(Default::default(), api.into());
+	let watcher = block_on(pool.submit_and_watch(&BlockId::number(1), xt.clone())).expect("1. Imported");
+	pool.api.push_block(2, vec![xt.clone()]);
+
+	let header = pool.api.chain().read().header_by_number.get(&2).cloned().unwrap();
+	let event = ChainEvent::NewBlock {
+		id: BlockId::Hash(header.hash()),
+		is_new_best: true,
+		header: header.clone(),
+		retracted: vec![]
+	};
+	block_on(pool.maintain(event));
+
+	let event = ChainEvent::Finalized { hash: header.hash() };
+	block_on(pool.maintain(event));
+
+	let mut stream = futures::executor::block_on_stream(watcher);
+	assert_eq!(stream.next(), Some(TransactionStatus::Ready));
+	assert_eq!(stream.next(), Some(TransactionStatus::InBlock(header.hash())));
+	assert_eq!(stream.next(), Some(TransactionStatus::Finalized(header.hash())));
+	assert_eq!(stream.next(), None);
+}
+
+#[test]
+fn fork_aware_finalization() {
+	let api = TestApi::empty();
+	// starting block A1 (last finalized.)
+	api.push_block(1, vec![]);
+
+	let pool = BasicPool::new(Default::default(), api.into());
+	let mut canon_watchers = vec![];
+
+	let from_alice = uxt(Alice, 1);
+	let from_dave = uxt(Dave, 1);
+	let from_bob = uxt(Bob, 1);
+	let from_charlie = uxt(Charlie, 1);
+	pool.api.increment_nonce(Alice.into());
+	pool.api.increment_nonce(Dave.into());
+	pool.api.increment_nonce(Charlie.into());
+	pool.api.increment_nonce(Bob.into());
+
+	let from_dave_watcher;
+	let from_bob_watcher;
+	let b1;
+	let d1;
+	let c2;
+	let d2;
+
+
+	// block B1
+	{
+		let watcher = block_on(pool.submit_and_watch(&BlockId::number(1), from_alice.clone())).expect("1. Imported");
+		let header = pool.api.push_block(2, vec![from_alice.clone()]);
+		canon_watchers.push((watcher, header.hash()));
+
+		let event = ChainEvent::NewBlock {
+			id: BlockId::Number(2),
+			is_new_best: true,
+			header: header.clone(),
+			retracted: vec![],
+		};
+		b1 = header.hash();
+		block_on(pool.maintain(event));
+		let event = ChainEvent::Finalized { hash: b1 };
+		block_on(pool.maintain(event));
+	}
+
+	// block C2
+	{
+		let header = pool.api.push_fork_block_with_parent(b1, vec![from_dave.clone()]);
+		from_dave_watcher = block_on(pool.submit_and_watch(&BlockId::number(1), from_dave.clone()))
+			.expect("1. Imported");
+		let event = ChainEvent::NewBlock {
+			id: BlockId::Hash(header.hash()),
+			is_new_best: true,
+			header: header.clone(),
+			retracted: vec![]
+		};
+		c2 = header.hash();
+		block_on(pool.maintain(event));
+	}
+
+	// block D2
+	{
+		from_bob_watcher = block_on(pool.submit_and_watch(&BlockId::number(1), from_bob.clone())).expect("1. Imported");
+		let header = pool.api.push_fork_block_with_parent(c2, vec![from_bob.clone()]);
+
+		let event = ChainEvent::NewBlock {
+			id: BlockId::Hash(header.hash()),
+			is_new_best: true,
+			header: header.clone(),
+			retracted: vec![]
+		};
+		d2 = header.hash();
+		block_on(pool.maintain(event));
+	}
+
+	// block C1
+	{
+		let watcher = block_on(pool.submit_and_watch(&BlockId::number(1), from_charlie.clone())).expect("1.Imported");
+		let header = pool.api.push_block(3, vec![from_charlie.clone()]);
+
+		canon_watchers.push((watcher, header.hash()));
+		let event = ChainEvent::NewBlock {
+			id: BlockId::Number(3),
+			is_new_best: true,
+			header: header.clone(),
+			retracted: vec![c2, d2],
+		};
+		block_on(pool.maintain(event));
+		let event = ChainEvent::Finalized { hash: header.hash() };
+		block_on(pool.maintain(event));
+	}
+
+	// block D1
+	{
+		let xt = uxt(Eve, 0);
+		let w = block_on(pool.submit_and_watch(&BlockId::number(1), xt.clone())).expect("1. Imported");
+		let header = pool.api.push_block(4, vec![xt.clone()]);
+		canon_watchers.push((w, header.hash()));
+
+		let event = ChainEvent::NewBlock {
+			id: BlockId::Hash(header.hash()),
+			is_new_best: true,
+			header: header.clone(),
+			retracted: vec![]
+		};
+		d1 = header.hash();
+		block_on(pool.maintain(event));
+		let event = ChainEvent::Finalized { hash: d1 };
+		block_on(pool.maintain(event));
+	}
+
+	let e1;
+
+	// block e1
+	{
+		let header = pool.api.push_block(5, vec![from_dave]);
+		e1 = header.hash();
+		let event = ChainEvent::NewBlock {
+			id: BlockId::Hash(header.hash()),
+			is_new_best: true,
+			header: header.clone(),
+			retracted: vec![]
+		};
+		block_on(pool.maintain(event));
+		block_on(pool.maintain(ChainEvent::Finalized { hash: e1 }));
+	}
+
+
+	for (canon_watcher, h) in canon_watchers {
+		let mut stream = futures::executor::block_on_stream(canon_watcher);
+		assert_eq!(stream.next(), Some(TransactionStatus::Ready));
+		assert_eq!(stream.next(), Some(TransactionStatus::InBlock(h.clone())));
+		assert_eq!(stream.next(), Some(TransactionStatus::Finalized(h)));
+		assert_eq!(stream.next(), None);
+	}
+
+
+	{
+		let mut stream= futures::executor::block_on_stream(from_dave_watcher);
+		assert_eq!(stream.next(), Some(TransactionStatus::Ready));
+		assert_eq!(stream.next(), Some(TransactionStatus::InBlock(c2.clone())));
+		assert_eq!(stream.next(), Some(TransactionStatus::Retracted(c2)));
+		assert_eq!(stream.next(), Some(TransactionStatus::Ready));
+		assert_eq!(stream.next(), Some(TransactionStatus::InBlock(e1)));
+		assert_eq!(stream.next(), Some(TransactionStatus::Finalized(e1.clone())));
+		assert_eq!(stream.next(), None);
+	}
+
+	{
+		let mut stream= futures::executor::block_on_stream(from_bob_watcher);
+		assert_eq!(stream.next(), Some(TransactionStatus::Ready));
+		assert_eq!(stream.next(), Some(TransactionStatus::InBlock(d2.clone())));
+		assert_eq!(stream.next(), Some(TransactionStatus::Retracted(d2)));
+	}
+
+}