Handle AccountIndices in transaction pool (#225)

* Merge remote-tracking branch 'origin/master' into gav-xts-dont-panic

* Update wasm.

* consensus, session and staking all panic-safe.

* Democracy doesn't panic in apply.

* Fix tests.

* Extra helper macro, council depanicked.

* Fix one test.

* Fix up all council tests. No panics!

* Council voting depanicked.

* Dispatch returns result.

* session & staking tests updated

* Fix democracy tests.

* Fix council tests.

* Fix up polkadot parachains in runtime

* Fix borked merge

* More Slicable support

Support general `Option` and array types.

* Basic storage types.

* Existential deposit for contract creation

* Basic implemnetation along with removals

* Fix tests.

* externalities builder fix.

* Tests.

* Fix up the runtime.

* Fix tests.

* Add generic `Address` type.

* Initial function integration of Address into Extrinsic.

* Fix build

* All tests compile.

* Fix (some) tests.

* Fix signing.

* Push error.

* transfer can accept Address

* Make Address generic over AccountIndex

* Fix test

* Make Council use Address for dispatch.

* Fix build

* Bend over backwards to support braindead derive.

* Repot some files.

* Fix tests.

* Fix grumbles

* Remove Default bound

* Fix build for new nightly.

* Make `apply_extrinsic` never panic, return useful Result.

* More merge hell

* Doesn't build, but might do soon

* Serde woes

* get substrate-runtime-staking compiling

* Polkadot builds again!

* Fix all build.

* Fix tests & binaries.

* Reserve some extra initial byte values of address for future format changes

* Make semantic of `ReservedBalance` clear.

* Fix panic handler.

* Integrate other balance transformations into the new model

Fix up staking tests.

* Fix runtime tests.

* Fix panic build.

* Tests for demonstrating interaction between balance types.

* Repot some runtime code

* Fix checkedblock in non-std builds

* Get rid of `DoLookup` phantom.

* Attempt to make transaction_pool work with lookups.

* Remove vscode settings

* New attempt at making transaction pool work.

* It builds again!

* --all builds

* Fix tests.

* New build.

* Test account nonce reset.

* polkadot transaction pool tests/framework.

* Address grumbles.

* Pool support non-verified transactions.

* Revert bad `map_or`

* Rebuild binaries, workaround.

* Avoid casting to usize early.

* Make verification use provided block_id.

* Fix tests.

* Alter tests to use retry.

* Fix tests & add call to re-verify.

* Semi-refactor.

* Integrate new queue with the rest of the code.

* Fix tests.

* Add reverify_transaction method.

* Use result.

substrate/polkadot/consensus/src/lib.rs

@@ -74,7 +74,7 @@ use polkadot_primitives::{Hash, Block, BlockId, BlockNumber, Header, Timestamp};
 use polkadot_primitives::parachain::{Id as ParaId, Chain, DutyRoster, BlockData, Extrinsic as ParachainExtrinsic, CandidateReceipt};
 use polkadot_runtime::BareExtrinsic;
 use primitives::AuthorityId;
-use transaction_pool::{Ready, TransactionPool};
+use transaction_pool::{TransactionPool};
 use tokio_core::reactor::{Handle, Timeout, Interval};
 
 use futures::prelude::*;
@@ -226,7 +226,7 @@ pub struct ProposerFactory<C, N, P> {
 	/// The client instance.
 	pub client: Arc<C>,
 	/// The transaction pool.
-	pub transaction_pool: Arc<TransactionPool>,
+	pub transaction_pool: Arc<TransactionPool<C>>,
 	/// The backing network handle.
 	pub network: N,
 	/// Parachain collators.
@@ -239,7 +239,8 @@ pub struct ProposerFactory<C, N, P> {
 
 impl<C, N, P> bft::ProposerFactory<Block> for ProposerFactory<C, N, P>
 	where
-		C: PolkadotApi,
+		C: PolkadotApi + Send + Sync,
+		C::CheckedBlockId: Sync,
 		N: Network,
 		P: Collators,
 {
@@ -319,12 +320,13 @@ pub struct Proposer<C: PolkadotApi, R, P> {
 	random_seed: Hash,
 	router: R,
 	table: Arc<SharedTable>,
-	transaction_pool: Arc<TransactionPool>,
+	transaction_pool: Arc<TransactionPool<C>>,
 }
 
 impl<C, R, P> bft::Proposer<Block> for Proposer<C, R, P>
 	where
-		C: PolkadotApi,
+		C: PolkadotApi + Send + Sync,
+		C::CheckedBlockId: Sync,
 		R: TableRouter,
 		P: Collators,
 {
@@ -501,8 +503,7 @@ impl<C, R, P> bft::Proposer<Block> for Proposer<C, R, P>
 
 		let local_id = self.local_key.public().0.into();
 		let mut next_index = {
-			let readiness_evaluator = Ready::create(self.parent_id.clone(), &*self.client);
-			let cur_index = self.transaction_pool.cull_and_get_pending(readiness_evaluator, |pending| pending
+			let cur_index = self.transaction_pool.cull_and_get_pending(BlockId::hash(self.parent_hash), |pending| pending
 				.filter(|tx| tx.sender().map(|s| s == local_id).unwrap_or(false))
 				.last()
 				.map(|tx| Ok(tx.index()))
@@ -510,7 +511,11 @@ impl<C, R, P> bft::Proposer<Block> for Proposer<C, R, P>
 			);
 
 			match cur_index {
-				Ok(cur_index) => cur_index + 1,
+				Ok(Ok(cur_index)) => cur_index + 1,
+				Ok(Err(e)) => {
+					warn!(target: "consensus", "Error computing next transaction index: {}", e);
+					return;
+				}
 				Err(e) => {
 					warn!(target: "consensus", "Error computing next transaction index: {}", e);
 					return;
@@ -549,7 +554,7 @@ impl<C, R, P> bft::Proposer<Block> for Proposer<C, R, P>
 			};
 			let uxt = UncheckedExtrinsic::new(extrinsic, signature);
 
-			self.transaction_pool.import_unchecked_extrinsic(uxt)
+			self.transaction_pool.import_unchecked_extrinsic(BlockId::hash(self.parent_hash), uxt)
 				.expect("locally signed extrinsic is valid; qed");
 		}
 	}
@@ -618,7 +623,7 @@ pub struct CreateProposal<C: PolkadotApi, R, P: Collators>  {
 	parent_number: BlockNumber,
 	parent_id: C::CheckedBlockId,
 	client: Arc<C>,
-	transaction_pool: Arc<TransactionPool>,
+	transaction_pool: Arc<TransactionPool<C>>,
 	collation: CollationFetch<P, C>,
 	router: R,
 	table: Arc<SharedTable>,
@@ -640,9 +645,8 @@ impl<C, R, P> CreateProposal<C, R, P>
 		let mut block_builder = self.client.build_block(&self.parent_id, timestamp, candidates)?;
 
 		{
-			let readiness_evaluator = Ready::create(self.parent_id.clone(), &*self.client);
 			let mut unqueue_invalid = Vec::new();
-			self.transaction_pool.cull_and_get_pending(readiness_evaluator, |pending_iterator| {
+			let result = self.transaction_pool.cull_and_get_pending(BlockId::hash(self.parent_hash), |pending_iterator| {
 				let mut pending_size = 0;
 				for pending in pending_iterator {
 					// skip and cull transactions which are too large.
@@ -664,6 +668,9 @@ impl<C, R, P> CreateProposal<C, R, P>
 					}
 				}
 			});
+			if let Err(e) = result {
+				warn!("Unable to get the pending set: {:?}", e);
+			}
 
 			self.transaction_pool.remove(&unqueue_invalid, false);
 		}

substrate/polkadot/consensus/src/service.rs

@@ -235,13 +235,14 @@ impl Service {
 		client: Arc<C>,
 		api: Arc<A>,
 		network: Arc<net::ConsensusService<Block>>,
-		transaction_pool: Arc<TransactionPool>,
+		transaction_pool: Arc<TransactionPool<A>>,
 		parachain_empty_duration: Duration,
 		key: ed25519::Pair,
 	) -> Service
 		where
 			A: LocalPolkadotApi + Send + Sync + 'static,
 			C: BlockchainEvents<Block> + ChainHead<Block> + bft::BlockImport<Block> + bft::Authorities<Block> + Send + Sync + 'static,
+			A::CheckedBlockId: Sync,
 	{
 		let (signal, exit) = ::exit_future::signal();
 		let thread = thread::spawn(move || {