Dispute Coordinator: Batch queries (#3459)

* disputes: Allow batch queries in dispute-coordinator

This commit moves to batch queries when responding to QueryCandidateVotes
messages. This simplifies the code in the provisioner and dispute-coordinator
by no longer requiring to make use of a FuturesOrdered when awaiting multiple
quries. Instead, the provisioner need only request the batch itself.

* node/approval-voting: Address Feedback to fail on query element missing.

* Address feedback

* Fix implementer's guide

polkadot/node/core/dispute-coordinator/src/lib.rs

@@ -444,18 +444,27 @@ async fn handle_incoming(
 			let _ = rx.send(collect_active(recent_disputes, now));
 		}
 		DisputeCoordinatorMessage::QueryCandidateVotes(
-			session,
+			query,
-			candidate_hash,
 			rx
 		) => {
-			let candidate_votes = db::v1::load_candidate_votes(
+			let mut query_output = Vec::new();
-				store,
+			for (session_index, candidate_hash) in query.into_iter() {
-				&config.column_config(),
+				if let Some(v) = db::v1::load_candidate_votes(
-				session,
+					store,
-				&candidate_hash,
+					&config.column_config(),
-			)?;
+					session_index,
-
+					&candidate_hash,
-			let _ = rx.send(candidate_votes.map(Into::into));
+				)? {
+					query_output.push((session_index, candidate_hash, v.into()));
+				} else {
+					tracing::debug!(
+						target: LOG_TARGET,
+						session_index,
+						"No votes found for candidate",
+					);
+				}
+			}
+			let _ = rx.send(query_output);
 		}
 		DisputeCoordinatorMessage::IssueLocalStatement(
 			session,

polkadot/node/core/dispute-coordinator/src/tests.rs

@@ -332,13 +332,12 @@ fn conflicting_votes_lead_to_dispute_participation() {
 			let (tx, rx) = oneshot::channel();
 			virtual_overseer.send(FromOverseer::Communication {
 				msg: DisputeCoordinatorMessage::QueryCandidateVotes(
-					session,
+					vec![(session, candidate_hash)],
-					candidate_hash,
 					tx,
 				),
 			}).await;
 
-			let votes = rx.await.unwrap().unwrap();
+			let (_, _, votes) = rx.await.unwrap().get(0).unwrap().clone();
 			assert_eq!(votes.valid.len(), 1);
 			assert_eq!(votes.invalid.len(), 1);
 		}
@@ -360,13 +359,12 @@ fn conflicting_votes_lead_to_dispute_participation() {
 			let (tx, rx) = oneshot::channel();
 			virtual_overseer.send(FromOverseer::Communication {
 				msg: DisputeCoordinatorMessage::QueryCandidateVotes(
-					session,
+					vec![(session, candidate_hash)],
-					candidate_hash,
 					tx,
 				),
 			}).await;
 
-			let votes = rx.await.unwrap().unwrap();
+			let (_, _, votes) = rx.await.unwrap().get(0).unwrap().clone();
 			assert_eq!(votes.valid.len(), 1);
 			assert_eq!(votes.invalid.len(), 2);
 		}
@@ -430,13 +428,12 @@ fn positive_votes_dont_trigger_participation() {
 			let (tx, rx) = oneshot::channel();
 			virtual_overseer.send(FromOverseer::Communication {
 				msg: DisputeCoordinatorMessage::QueryCandidateVotes(
-					session,
+					vec![(session, candidate_hash)],
-					candidate_hash,
 					tx,
 				),
 			}).await;
 
-			let votes = rx.await.unwrap().unwrap();
+			let (_, _, votes) = rx.await.unwrap().get(0).unwrap().clone();
 			assert_eq!(votes.valid.len(), 1);
 			assert!(votes.invalid.is_empty());
 		}
@@ -465,13 +462,12 @@ fn positive_votes_dont_trigger_participation() {
 			let (tx, rx) = oneshot::channel();
 			virtual_overseer.send(FromOverseer::Communication {
 				msg: DisputeCoordinatorMessage::QueryCandidateVotes(
-					session,
+					vec![(session, candidate_hash)],
-					candidate_hash,
 					tx,
 				),
 			}).await;
 
-			let votes = rx.await.unwrap().unwrap();
+			let (_, _, votes) = rx.await.unwrap().get(0).unwrap().clone();
 			assert_eq!(votes.valid.len(), 2);
 			assert!(votes.invalid.is_empty());
 		}
@@ -536,13 +532,12 @@ fn wrong_validator_index_is_ignored() {
 			let (tx, rx) = oneshot::channel();
 			virtual_overseer.send(FromOverseer::Communication {
 				msg: DisputeCoordinatorMessage::QueryCandidateVotes(
-					session,
+					vec![(session, candidate_hash)],
-					candidate_hash,
 					tx,
 				),
 			}).await;
 
-			let votes = rx.await.unwrap().unwrap();
+			let (_, _, votes) = rx.await.unwrap().get(0).unwrap().clone();
 			assert!(votes.valid.is_empty());
 			assert!(votes.invalid.is_empty());
 		}

polkadot/node/core/provisioner/src/lib.rs

@@ -23,7 +23,6 @@ use bitvec::vec::BitVec;
 use futures::{
 	channel::{mpsc, oneshot},
 	prelude::*,
-	stream::FuturesOrdered,
 };
 use polkadot_node_subsystem::{
 	errors::{ChainApiError, RuntimeApiError}, PerLeafSpan, SubsystemSender, jaeger,
@@ -568,37 +567,22 @@ async fn select_disputes(
 
 	// Load all votes for all disputes from the coordinator.
 	let dispute_candidate_votes = {
-		let mut awaited_votes = FuturesOrdered::new();
+		let (tx, rx) = oneshot::channel();
+		sender.send_message(DisputeCoordinatorMessage::QueryCandidateVotes(
+			recent_disputes,
+			tx,
+		).into()).await;
 
-		let n_disputes = recent_disputes.len();
+		match rx.await {
-		for (session_index, candidate_hash) in recent_disputes {
+			Ok(v) => v,
-			let (tx, rx) = oneshot::channel();
+			Err(oneshot::Canceled) => {
-			sender.send_message(DisputeCoordinatorMessage::QueryCandidateVotes(
+				tracing::debug!(
-				session_index,
+					target: LOG_TARGET,
-				candidate_hash,
+					"Unable to query candidate votes - subsystem disconnected?",
-				tx,
+				);
-			).into()).await;
+				Vec::new()
-
-			awaited_votes.push(async move {
-				rx.await
-					.map_err(Error::CanceledCandidateVotes)
-					.map(|maybe_votes| maybe_votes.map(|v| (session_index, candidate_hash, v)))
-			});
-		}
-
-		// Sadly `StreamExt::collect` requires `Default`, so we have to do this more
-		// manually.
-		let mut vote_sets = Vec::with_capacity(n_disputes);
-		while let Some(res) = awaited_votes.next().await {
-			// sanity check - anything present in recent disputes should have
-			// candidate votes. but we might race with block import on
-			// session boundaries.
-			if let Some(vote_set) = res? {
-				vote_sets.push(vote_set);
 			}
 		}
-
-		vote_sets
 	};
 
 	// Transform all `CandidateVotes` into `MultiDisputeStatementSet`.

polkadot/node/network/dispute-distribution/src/lib.rs

@@ -14,7 +14,6 @@
 // You should have received a copy of the GNU General Public License
 // along with Polkadot.  If not, see <http://www.gnu.org/licenses/>.
 
-
 //! # Sending and receiving of `DisputeRequest`s.
 //!
 //! This subsystem essentially consists of two parts:

polkadot/node/network/dispute-distribution/src/sender/mod.rs

@@ -352,11 +352,12 @@ async fn get_candidate_votes<Context: SubsystemContext>(
 	let (tx, rx) = oneshot::channel();
 	ctx.send_message(AllMessages::DisputeCoordinator(
 		DisputeCoordinatorMessage::QueryCandidateVotes(
-			session_index,
+			vec![(session_index, candidate_hash)],
-			candidate_hash,
 			tx
 		)
 	))
 	.await;
-	rx.await.map_err(|_| NonFatal::AskCandidateVotesCanceled)
+	rx.await
+		.map(|v| v.get(0).map(|inner| inner.to_owned().2))
+		.map_err(|_| NonFatal::AskCandidateVotesCanceled)
 }

polkadot/node/network/dispute-distribution/src/tests/mod.rs

@@ -258,15 +258,15 @@ fn disputes_are_recovered_at_startup() {
 				handle.recv().await,
 				AllMessages::DisputeCoordinator(
 					DisputeCoordinatorMessage::QueryCandidateVotes(
-						session_index,
+						query,
-						candidate_hash,
 						tx,
 					)
 				) => {
+					let (session_index, candidate_hash) = query.get(0).unwrap().clone();
 					assert_eq!(session_index, MOCK_SESSION_INDEX);
 					assert_eq!(candidate_hash, candidate.hash());
 					let unchecked: UncheckedDisputeMessage = message.into();
-					tx.send(Some(CandidateVotes {
+					tx.send(vec![(session_index, candidate_hash, CandidateVotes {
 						candidate_receipt: candidate,
 						valid: vec![(
 							unchecked.valid_vote.kind,
@@ -278,7 +278,7 @@ fn disputes_are_recovered_at_startup() {
 							unchecked.invalid_vote.validator_index,
 							unchecked.invalid_vote.signature
 						)],
-					}))
+					})])
 					.expect("Receiver should stay alive.");
 				}
 			);

polkadot/node/subsystem-types/src/messages.rs

@@ -232,7 +232,10 @@ pub enum DisputeCoordinatorMessage {
 	/// These disputes are either unconcluded or recently concluded.
 	ActiveDisputes(oneshot::Sender<Vec<(SessionIndex, CandidateHash)>>),
 	/// Get candidate votes for a candidate.
-	QueryCandidateVotes(SessionIndex, CandidateHash, oneshot::Sender<Option<CandidateVotes>>),
+	QueryCandidateVotes(
+		Vec<(SessionIndex, CandidateHash)>,
+		oneshot::Sender<Vec<(SessionIndex, CandidateHash, CandidateVotes)>>,
+	),
 	/// Sign and issue local dispute votes. A value of `true` indicates validity, and `false` invalidity.
 	IssueLocalStatement(SessionIndex, CandidateHash, CandidateReceipt, bool),
 	/// Determine the highest undisputed block within the given chain, based on where candidates

polkadot/roadmap/implementers-guide/src/node/disputes/dispute-coordinator.md

@@ -143,7 +143,8 @@ Do nothing.
 
 ### On `DisputeCoordinatorMessage::QueryCandidateVotes`
 
-* Load `"candidate-votes"` and return the data within or `None` if missing.
+* Load `"candidate-votes"` for every `(SessionIndex, CandidateHash)` in the query and return data within each `CandidateVote`.
+  If a particular `candidate-vote` is missing, that particular request is ommitted from the response.
 
 ### On `DisputeCoordinatorMessage::IssueLocalStatement`