collect included disputes from on-chain (#3924)

* dummy: impl another runtime API * query the on chain disputes, and inform self * make use of the refactor * minro * SPLIT ME * write dispute values * wip * impl for all runtimes * chore: fmt * [] -> get * fixup mock runtime * fixup * fixup discovery for overseer init * chore: fmt * spellcheck * rename imported_on_chain_disputes -> on_chain_votes * reduction * make it mockable * rename and refactor * don't query on chain info if it's not needed * yikes * fmt * fix test * minimal fix for existing tests * attempt to fetch the session info from the rolling window before falling back * moved * comments * comments * test for backing votes * rename * Update runtime/polkadot/src/lib.rs * chore: spellcheck + dict * chore: fmt * fixup cache size * add warning * logging, rationale, less defense * introduce new unchecked, that still checks in debug builds * fix * draft alt approach * fix unused imports * include the session * Update node/core/dispute-coordinator/src/real/mod.rs Co-authored-by: Robert Habermeier <rphmeier@gmail.com> * provide where possible * expand comment * fixin * fixup * ValidityVote <-> ValidityAttestation <-> CompactStatement has a 1:1 representation * mark TODO * Update primitives/src/v1/mod.rs Co-authored-by: Robert Habermeier <rphmeier@gmail.com> * address review comments * update docs Co-authored-by: Robert Habermeier <rphmeier@gmail.com>
2026-05-30 10:31:03 +00:00 · 2021-10-06 21:16:23 +02:00
parent 561219eef6
commit 30bdc8f5d4
23 changed files with 606 additions and 84 deletions
@@ -25,8 +25,9 @@ use frame_support::pallet_prelude::*;
 use parity_scale_codec::{Decode, Encode};
 use primitives::v1::{
 	AvailabilityBitfield, BackedCandidate, CandidateCommitments, CandidateDescriptor,
-	CandidateHash, CandidateReceipt, CommittedCandidateReceipt, CoreIndex, GroupIndex, HeadData,
-	Id as ParaId, SigningContext, UncheckedSignedAvailabilityBitfields, ValidatorIndex,
+	CandidateHash, CandidateReceipt, CommittedCandidateReceipt, CoreIndex, GroupIndex, Hash,
+	HeadData, Id as ParaId, SigningContext, UncheckedSignedAvailabilityBitfields, ValidatorIndex,
+	ValidityAttestation,
 };
 use scale_info::TypeInfo;
 use sp_runtime::{
@@ -110,6 +111,24 @@ pub trait RewardValidators {
 	fn reward_bitfields(validators: impl IntoIterator<Item = ValidatorIndex>);
 }

+/// Helper return type for `process_candidates`.
+#[derive(Encode, Decode, PartialEq, TypeInfo)]
+#[cfg_attr(test, derive(Debug))]
+pub(crate) struct ProcessedCandidates<H = Hash> {
+	pub(crate) core_indices: Vec<CoreIndex>,
+	pub(crate) candidate_receipt_with_backing_validator_indices:
+		Vec<(CandidateReceipt<H>, Vec<(ValidatorIndex, ValidityAttestation)>)>,
+}
+
+impl<H> Default for ProcessedCandidates<H> {
+	fn default() -> Self {
+		Self {
+			core_indices: Vec::new(),
+			candidate_receipt_with_backing_validator_indices: Vec::new(),
+		}
+	}
+}
+
 #[frame_support::pallet]
 pub mod pallet {
 	use super::*;
@@ -392,11 +411,11 @@ impl<T: Config> Pallet<T> {
 		candidates: Vec<BackedCandidate<T::Hash>>,
 		scheduled: Vec<CoreAssignment>,
 		group_validators: impl Fn(GroupIndex) -> Option<Vec<ValidatorIndex>>,
-	) -> Result<Vec<CoreIndex>, DispatchError> {
+	) -> Result<ProcessedCandidates<T::Hash>, DispatchError> {
 		ensure!(candidates.len() <= scheduled.len(), Error::<T>::UnscheduledCandidate);

 		if scheduled.is_empty() {
-			return Ok(Vec::new())
+			return Ok(ProcessedCandidates::default())
 		}

 		let validators = shared::Pallet::<T>::active_validator_keys();
@@ -408,7 +427,11 @@ impl<T: Config> Pallet<T> {
 		let relay_parent_number = now - One::one();
 		let check_cx = CandidateCheckContext::<T>::new(now, relay_parent_number);

-		// do all checks before writing storage.
+		// Collect candidate receipts with backers.
+		let mut candidate_receipt_with_backing_validator_indices =
+			Vec::with_capacity(candidates.len());
+
+		// Do all checks before writing storage.
 		let core_indices_and_backers = {
 			let mut skip = 0;
 			let mut core_indices_and_backers = Vec::with_capacity(candidates.len());
@@ -437,17 +460,17 @@ impl<T: Config> Pallet<T> {
 			//
 			// In the meantime, we do certain sanity checks on the candidates and on the scheduled
 			// list.
-			'a: for (candidate_idx, candidate) in candidates.iter().enumerate() {
-				let para_id = candidate.descriptor().para_id;
+			'a: for (candidate_idx, backed_candidate) in candidates.iter().enumerate() {
+				let para_id = backed_candidate.descriptor().para_id;
 				let mut backers = bitvec::bitvec![BitOrderLsb0, u8; 0; validators.len()];

 				// we require that the candidate is in the context of the parent block.
 				ensure!(
-					candidate.descriptor().relay_parent == parent_hash,
+					backed_candidate.descriptor().relay_parent == parent_hash,
 					Error::<T>::CandidateNotInParentContext,
 				);
 				ensure!(
-					candidate.descriptor().check_collator_signature().is_ok(),
+					backed_candidate.descriptor().check_collator_signature().is_ok(),
 					Error::<T>::NotCollatorSigned,
 				);

@@ -456,24 +479,24 @@ impl<T: Config> Pallet<T> {
 						// A candidate for a parachain without current validation code is not scheduled.
 						.ok_or_else(|| Error::<T>::UnscheduledCandidate)?;
 				ensure!(
-					candidate.descriptor().validation_code_hash == validation_code_hash,
+					backed_candidate.descriptor().validation_code_hash == validation_code_hash,
 					Error::<T>::InvalidValidationCodeHash,
 				);

 				ensure!(
-					candidate.descriptor().para_head ==
-						candidate.candidate.commitments.head_data.hash(),
+					backed_candidate.descriptor().para_head ==
+						backed_candidate.candidate.commitments.head_data.hash(),
 					Error::<T>::ParaHeadMismatch,
 				);

 				if let Err(err) = check_cx.check_validation_outputs(
 					para_id,
-					&candidate.candidate.commitments.head_data,
-					&candidate.candidate.commitments.new_validation_code,
-					candidate.candidate.commitments.processed_downward_messages,
-					&candidate.candidate.commitments.upward_messages,
-					T::BlockNumber::from(candidate.candidate.commitments.hrmp_watermark),
-					&candidate.candidate.commitments.horizontal_messages,
+					&backed_candidate.candidate.commitments.head_data,
+					&backed_candidate.candidate.commitments.new_validation_code,
+					backed_candidate.candidate.commitments.processed_downward_messages,
+					&backed_candidate.candidate.commitments.upward_messages,
+					T::BlockNumber::from(backed_candidate.candidate.commitments.hrmp_watermark),
+					&backed_candidate.candidate.commitments.horizontal_messages,
 				) {
 					log::debug!(
 						target: LOG_TARGET,
@@ -491,7 +514,7 @@ impl<T: Config> Pallet<T> {
 					if para_id == assignment.para_id {
 						if let Some(required_collator) = assignment.required_collator() {
 							ensure!(
-								required_collator == &candidate.descriptor().collator,
+								required_collator == &backed_candidate.descriptor().collator,
 								Error::<T>::WrongCollator,
 							);
 						}
@@ -509,14 +532,15 @@ impl<T: Config> Pallet<T> {
 										// We don't want to error out here because it will
 										// brick the relay-chain. So we return early without
 										// doing anything.
-										return Ok(Vec::new())
+										return Ok(ProcessedCandidates::default())
 									},
 								};

 							let expected = persisted_validation_data.hash();

 							ensure!(
-								expected == candidate.descriptor().persisted_validation_data_hash,
+								expected ==
+									backed_candidate.descriptor().persisted_validation_data_hash,
 								Error::<T>::ValidationDataHashMismatch,
 							);
 						}
@@ -536,7 +560,7 @@ impl<T: Config> Pallet<T> {
 						// check the signatures in the backing and that it is a majority.
 						{
 							let maybe_amount_validated = primitives::v1::check_candidate_backing(
-								&candidate,
+								&backed_candidate,
 								&signing_context,
 								group_vals.len(),
 								|idx| {
@@ -557,17 +581,28 @@ impl<T: Config> Pallet<T> {
 								},
 							}

-							for (bit_idx, _) in candidate
+							let mut backer_idx_and_attestation =
+								Vec::<(ValidatorIndex, ValidityAttestation)>::with_capacity(
+									backed_candidate.validator_indices.count_ones(),
+								);
+							let candidate_receipt = backed_candidate.receipt();
+
+							for ((bit_idx, _), attestation) in backed_candidate
 								.validator_indices
 								.iter()
 								.enumerate()
 								.filter(|(_, signed)| **signed)
+								.zip(backed_candidate.validity_votes.iter().cloned())
 							{
-								let val_idx =
-									group_vals.get(bit_idx).expect("this query done above; qed");
+								let val_idx = group_vals
+									.get(bit_idx)
+									.expect("this query succeeded above; qed");
+								backer_idx_and_attestation.push((*val_idx, attestation));

 								backers.set(val_idx.0 as _, true);
 							}
+							candidate_receipt_with_backing_validator_indices
+								.push((candidate_receipt, backer_idx_and_attestation));
 						}

 						core_indices_and_backers.push((
@@ -625,7 +660,7 @@ impl<T: Config> Pallet<T> {
 					descriptor,
 					availability_votes,
 					relay_parent_number,
-					backers,
+					backers: backers.to_bitvec(),
 					backed_in_number: check_cx.now,
 					backing_group: group,
 				},
@@ -633,7 +668,10 @@ impl<T: Config> Pallet<T> {
 			<PendingAvailabilityCommitments<T>>::insert(&para_id, commitments);
 		}

-		Ok(core_indices)
+		Ok(ProcessedCandidates::<T::Hash> {
+			core_indices,
+			candidate_receipt_with_backing_validator_indices,
+		})
 	}

 	/// Run the acceptance criteria checks on the given candidate commitments.
@@ -2386,9 +2424,32 @@ mod tests {
 				BackingKind::Threshold,
 			));

-			let occupied_cores = ParaInclusion::process_candidates(
+			let backed_candidates = vec![backed_a, backed_b, backed_c];
+			let get_backing_group_idx = {
+				// the order defines the group implicitly for this test case
+				let backed_candidates_with_groups = backed_candidates
+					.iter()
+					.enumerate()
+					.map(|(idx, backed_candidate)| (backed_candidate.hash(), GroupIndex(idx as _)))
+					.collect::<Vec<_>>();
+
+				move |candidate_hash_x: CandidateHash| -> Option<GroupIndex> {
+					backed_candidates_with_groups.iter().find_map(|(candidate_hash, grp)| {
+						if *candidate_hash == candidate_hash_x {
+							Some(*grp)
+						} else {
+							None
+						}
+					})
+				}
+			};
+
+			let ProcessedCandidates {
+				core_indices: occupied_cores,
+				candidate_receipt_with_backing_validator_indices,
+			} = ParaInclusion::process_candidates(
 				Default::default(),
-				vec![backed_a, backed_b, backed_c],
+				backed_candidates.clone(),
 				vec![
 					chain_a_assignment.clone(),
 					chain_b_assignment.clone(),
@@ -2403,6 +2464,55 @@ mod tests {
 				vec![CoreIndex::from(0), CoreIndex::from(1), CoreIndex::from(2)]
 			);

+			// Transform the votes into the setup we expect
+			let expected = {
+				let mut intermediate = std::collections::HashMap::<
+					CandidateHash,
+					(CandidateReceipt, Vec<(ValidatorIndex, ValidityAttestation)>),
+				>::new();
+				backed_candidates.into_iter().for_each(|backed_candidate| {
+					let candidate_receipt_with_backers = intermediate
+						.entry(backed_candidate.hash())
+						.or_insert_with(|| (backed_candidate.receipt(), Vec::new()));
+
+					assert_eq!(
+						backed_candidate.validity_votes.len(),
+						backed_candidate.validator_indices.count_ones()
+					);
+					candidate_receipt_with_backers.1.extend(
+						backed_candidate
+							.validator_indices
+							.iter()
+							.enumerate()
+							.filter(|(_, signed)| **signed)
+							.zip(backed_candidate.validity_votes.iter().cloned())
+							.filter_map(|((validator_index_within_group, _), attestation)| {
+								let grp_idx =
+									get_backing_group_idx(backed_candidate.hash()).unwrap();
+								group_validators(grp_idx).map(|validator_indices| {
+									(validator_indices[validator_index_within_group], attestation)
+								})
+							}),
+					);
+				});
+				intermediate.into_values().collect::<Vec<_>>()
+			};
+
+			// sort, since we use a hashmap above
+			let assure_candidate_sorting = |mut candidate_receipts_with_backers: Vec<(
+				CandidateReceipt,
+				Vec<(ValidatorIndex, ValidityAttestation)>,
+			)>| {
+				candidate_receipts_with_backers.sort_by(|(cr1, _), (cr2, _)| {
+					cr1.descriptor().para_id.cmp(&cr2.descriptor().para_id)
+				});
+				candidate_receipts_with_backers
+			};
+			assert_eq!(
+				assure_candidate_sorting(expected),
+				assure_candidate_sorting(candidate_receipt_with_backing_validator_indices)
+			);
+
 			assert_eq!(
 				<PendingAvailability<Test>>::get(&chain_a),
 				Some(CandidatePendingAvailability {
@@ -2530,13 +2640,14 @@ mod tests {
 				BackingKind::Threshold,
 			));

-			let occupied_cores = ParaInclusion::process_candidates(
-				Default::default(),
-				vec![backed_a],
-				vec![chain_a_assignment.clone()],
-				&group_validators,
-			)
-			.expect("candidates scheduled, in order, and backed");
+			let ProcessedCandidates { core_indices: occupied_cores, .. } =
+				ParaInclusion::process_candidates(
+					Default::default(),
+					vec![backed_a],
+					vec![chain_a_assignment.clone()],
+					&group_validators,
+				)
+				.expect("candidates scheduled, in order, and backed");

 			assert_eq!(occupied_cores, vec![CoreIndex::from(0)]);

@@ -33,7 +33,8 @@ use frame_support::{
 };
 use frame_system::pallet_prelude::*;
 use primitives::v1::{
-	BackedCandidate, InherentData as ParachainsInherentData, PARACHAINS_INHERENT_IDENTIFIER,
+	BackedCandidate, InherentData as ParachainsInherentData, ScrapedOnChainVotes,
+	PARACHAINS_INHERENT_IDENTIFIER,
 };
 use sp_runtime::traits::Header as HeaderT;
 use sp_std::prelude::*;
@@ -79,6 +80,11 @@ pub mod pallet {
 	#[pallet::storage]
 	pub(crate) type Included<T> = StorageValue<_, ()>;

+	/// Scraped on chain data for extracting resolved disputes as well as backing votes.
+	#[pallet::storage]
+	#[pallet::getter(fn on_chain_votes)]
+	pub(crate) type OnChainVotes<T: Config> = StorageValue<_, ScrapedOnChainVotes<T::Hash>>;
+
 	#[pallet::hooks]
 	impl<T: Config> Hooks<BlockNumberFor<T>> for Pallet<T> {
 		fn on_initialize(_: T::BlockNumber) -> Weight {
@@ -180,7 +186,7 @@ pub mod pallet {
 					.map(|s| (s.session, s.candidate_hash))
 					.collect();

-				let _ = T::DisputesHandler::provide_multi_dispute_data(disputes)?;
+				let _ = T::DisputesHandler::provide_multi_dispute_data(disputes.clone())?;
 				if T::DisputesHandler::is_frozen() {
 					// The relay chain we are currently on is invalid. Proceed no further on parachains.
 					Included::<T>::set(Some(()));
@@ -262,13 +268,24 @@ pub mod pallet {

 			// Process backed candidates according to scheduled cores.
 			let parent_storage_root = parent_header.state_root().clone();
-			let occupied = <inclusion::Pallet<T>>::process_candidates(
+			let inclusion::ProcessedCandidates::<<T::Header as HeaderT>::Hash> {
+				core_indices: occupied,
+				candidate_receipt_with_backing_validator_indices,
+			} = <inclusion::Pallet<T>>::process_candidates(
 				parent_storage_root,
 				backed_candidates,
 				<scheduler::Pallet<T>>::scheduled(),
 				<scheduler::Pallet<T>>::group_validators,
 			)?;

+			// The number of disputes included in a block is
+			// limited by the weight as well as the number of candidate blocks.
+			OnChainVotes::<T>::put(ScrapedOnChainVotes::<<T::Header as HeaderT>::Hash> {
+				session: current_session,
+				backing_validators_per_candidate: candidate_receipt_with_backing_validator_indices,
+				disputes,
+			});
+
 			// Note which of the scheduled cores were actually occupied by a backed candidate.
 			<scheduler::Pallet<T>>::occupied(&occupied);

@@ -18,14 +18,15 @@
 //! functions.

 use crate::{
-	configuration, dmp, hrmp, inclusion, initializer, paras, scheduler, session_info, shared,
+	configuration, dmp, hrmp, inclusion, initializer, paras, paras_inherent, scheduler,
+	session_info, shared,
 };
 use primitives::v1::{
 	AuthorityDiscoveryId, CandidateEvent, CommittedCandidateReceipt, CoreIndex, CoreOccupied,
 	CoreState, GroupIndex, GroupRotationInfo, Id as ParaId, InboundDownwardMessage,
 	InboundHrmpMessage, OccupiedCore, OccupiedCoreAssumption, PersistedValidationData,
-	ScheduledCore, SessionIndex, SessionInfo, ValidationCode, ValidationCodeHash, ValidatorId,
-	ValidatorIndex,
+	ScheduledCore, ScrapedOnChainVotes, SessionIndex, SessionInfo, ValidationCode,
+	ValidationCodeHash, ValidatorId, ValidatorIndex,
 };
 use sp_runtime::traits::One;
 use sp_std::{collections::btree_map::BTreeMap, prelude::*};
@@ -329,3 +330,8 @@ pub fn validation_code_by_hash<T: paras::Config>(
 ) -> Option<ValidationCode> {
 	<paras::Pallet<T>>::code_by_hash(hash)
 }
+
+/// Disputes imported via means of on-chain imports.
+pub fn on_chain_votes<T: paras_inherent::Config>() -> Option<ScrapedOnChainVotes<T::Hash>> {
+	<paras_inherent::Pallet<T>>::on_chain_votes()
+}