diff --git a/polkadot/runtime/parachains/src/inclusion.rs b/polkadot/runtime/parachains/src/inclusion.rs deleted file mode 100644 index 23ae25ab14..0000000000 --- a/polkadot/runtime/parachains/src/inclusion.rs +++ /dev/null @@ -1,2954 +0,0 @@ -// Copyright 2020 Parity Technologies (UK) Ltd. -// This file is part of Polkadot. - -// Polkadot is free software: you can redistribute it and/or modify -// it under the terms of the GNU General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. - -// Polkadot is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU General Public License for more details. - -// You should have received a copy of the GNU General Public License -// along with Polkadot. If not, see . - -//! The inclusion pallet is responsible for inclusion and availability of scheduled parachains -//! and parathreads. -//! -//! It is responsible for carrying candidates from being backable to being backed, and then from backed -//! to included. - -use crate::{ - configuration, disputes, dmp, hrmp, paras, - paras_inherent::{sanitize_bitfields, DisputedBitfield}, - scheduler::CoreAssignment, - shared, ump, -}; -use bitvec::{order::Lsb0 as BitOrderLsb0, vec::BitVec}; -use frame_support::pallet_prelude::*; -use parity_scale_codec::{Decode, Encode}; -use primitives::v1::{ - AvailabilityBitfield, BackedCandidate, CandidateCommitments, CandidateDescriptor, - CandidateHash, CandidateReceipt, CommittedCandidateReceipt, CoreIndex, GroupIndex, Hash, - HeadData, Id as ParaId, SigningContext, UncheckedSignedAvailabilityBitfields, ValidatorId, - ValidatorIndex, ValidityAttestation, -}; -use scale_info::TypeInfo; -use sp_runtime::{ - traits::{One, Saturating}, - DispatchError, -}; -use sp_std::{collections::btree_set::BTreeSet, prelude::*}; - -pub use pallet::*; - -/// A bitfield signed by a validator indicating that it is keeping its piece of the erasure-coding -/// for any backed candidates referred to by a `1` bit available. -/// -/// The bitfield's signature should be checked at the point of submission. Afterwards it can be -/// dropped. -#[derive(Encode, Decode, TypeInfo)] -#[cfg_attr(test, derive(Debug))] -pub struct AvailabilityBitfieldRecord { - bitfield: AvailabilityBitfield, // one bit per core. - submitted_at: N, // for accounting, as meaning of bits may change over time. -} - -/// Determines if all checks should be applied or if a subset was already completed -/// in a code path that will be executed afterwards or was already executed before. -#[derive(Encode, Decode, PartialEq, Eq, RuntimeDebug, TypeInfo)] -pub(crate) enum FullCheck { - /// Yes, do a full check, skip nothing. - Yes, - /// Skip a subset of checks that are already completed before. - /// - /// Attention: Should only be used when absolutely sure that the required - /// checks are completed before. - Skip, -} - -/// A backed candidate pending availability. -#[derive(Encode, Decode, PartialEq, TypeInfo)] -#[cfg_attr(test, derive(Debug, Default))] -pub struct CandidatePendingAvailability { - /// The availability core this is assigned to. - core: CoreIndex, - /// The candidate hash. - hash: CandidateHash, - /// The candidate descriptor. - descriptor: CandidateDescriptor, - /// The received availability votes. One bit per validator. - availability_votes: BitVec, - /// The backers of the candidate pending availability. - backers: BitVec, - /// The block number of the relay-parent of the receipt. - relay_parent_number: N, - /// The block number of the relay-chain block this was backed in. - backed_in_number: N, - /// The group index backing this block. - backing_group: GroupIndex, -} - -impl CandidatePendingAvailability { - /// Get the availability votes on the candidate. - pub(crate) fn availability_votes(&self) -> &BitVec { - &self.availability_votes - } - - /// Get the relay-chain block number this was backed in. - pub(crate) fn backed_in_number(&self) -> &N { - &self.backed_in_number - } - - /// Get the core index. - pub(crate) fn core_occupied(&self) -> CoreIndex { - self.core.clone() - } - - /// Get the candidate hash. - pub(crate) fn candidate_hash(&self) -> CandidateHash { - self.hash - } - - /// Get the candidate descriptor. - pub(crate) fn candidate_descriptor(&self) -> &CandidateDescriptor { - &self.descriptor - } - - #[cfg(any(feature = "runtime-benchmarks", test))] - pub(crate) fn new( - core: CoreIndex, - hash: CandidateHash, - descriptor: CandidateDescriptor, - availability_votes: BitVec, - backers: BitVec, - relay_parent_number: N, - backed_in_number: N, - backing_group: GroupIndex, - ) -> Self { - Self { - core, - hash, - descriptor, - availability_votes, - backers, - relay_parent_number, - backed_in_number, - backing_group, - } - } -} - -/// A hook for applying validator rewards -pub trait RewardValidators { - // Reward the validators with the given indices for issuing backing statements. - fn reward_backing(validators: impl IntoIterator); - // Reward the validators with the given indices for issuing availability bitfields. - // Validators are sent to this hook when they have contributed to the availability - // of a candidate by setting a bit in their bitfield. - fn reward_bitfields(validators: impl IntoIterator); -} - -/// Helper return type for `process_candidates`. -#[derive(Encode, Decode, PartialEq, TypeInfo)] -#[cfg_attr(test, derive(Debug))] -pub(crate) struct ProcessedCandidates { - pub(crate) core_indices: Vec, - pub(crate) candidate_receipt_with_backing_validator_indices: - Vec<(CandidateReceipt, Vec<(ValidatorIndex, ValidityAttestation)>)>, -} - -impl Default for ProcessedCandidates { - fn default() -> Self { - Self { - core_indices: Vec::new(), - candidate_receipt_with_backing_validator_indices: Vec::new(), - } - } -} - -#[frame_support::pallet] -pub mod pallet { - use super::*; - - #[pallet::pallet] - #[pallet::generate_store(pub(super) trait Store)] - pub struct Pallet(_); - - #[pallet::config] - pub trait Config: - frame_system::Config - + shared::Config - + paras::Config - + dmp::Config - + ump::Config - + hrmp::Config - + configuration::Config - { - type Event: From> + IsType<::Event>; - type DisputesHandler: disputes::DisputesHandler; - type RewardValidators: RewardValidators; - } - - #[pallet::event] - #[pallet::generate_deposit(pub(super) fn deposit_event)] - pub enum Event { - /// A candidate was backed. `[candidate, head_data]` - CandidateBacked(CandidateReceipt, HeadData, CoreIndex, GroupIndex), - /// A candidate was included. `[candidate, head_data]` - CandidateIncluded(CandidateReceipt, HeadData, CoreIndex, GroupIndex), - /// A candidate timed out. `[candidate, head_data]` - CandidateTimedOut(CandidateReceipt, HeadData, CoreIndex), - } - - #[pallet::error] - pub enum Error { - /// Availability bitfield has unexpected size. - WrongBitfieldSize, - /// Multiple bitfields submitted by same validator or validators out of order by index. - BitfieldDuplicateOrUnordered, - /// Validator index out of bounds. - ValidatorIndexOutOfBounds, - /// Invalid signature - InvalidBitfieldSignature, - /// Candidate submitted but para not scheduled. - UnscheduledCandidate, - /// Candidate scheduled despite pending candidate already existing for the para. - CandidateScheduledBeforeParaFree, - /// Candidate included with the wrong collator. - WrongCollator, - /// Scheduled cores out of order. - ScheduledOutOfOrder, - /// Head data exceeds the configured maximum. - HeadDataTooLarge, - /// Code upgrade prematurely. - PrematureCodeUpgrade, - /// Output code is too large - NewCodeTooLarge, - /// Candidate not in parent context. - CandidateNotInParentContext, - /// Invalid group index in core assignment. - InvalidGroupIndex, - /// Insufficient (non-majority) backing. - InsufficientBacking, - /// Invalid (bad signature, unknown validator, etc.) backing. - InvalidBacking, - /// Collator did not sign PoV. - NotCollatorSigned, - /// The validation data hash does not match expected. - ValidationDataHashMismatch, - /// The downward message queue is not processed correctly. - IncorrectDownwardMessageHandling, - /// At least one upward message sent does not pass the acceptance criteria. - InvalidUpwardMessages, - /// The candidate didn't follow the rules of HRMP watermark advancement. - HrmpWatermarkMishandling, - /// The HRMP messages sent by the candidate is not valid. - InvalidOutboundHrmp, - /// The validation code hash of the candidate is not valid. - InvalidValidationCodeHash, - /// The `para_head` hash in the candidate descriptor doesn't match the hash of the actual para head in the - /// commitments. - ParaHeadMismatch, - /// A bitfield that references a freed core, - /// either intentionally or as part of a concluded - /// invalid dispute. - BitfieldReferencesFreedCore, - } - - /// The latest bitfield for each validator, referred to by their index in the validator set. - #[pallet::storage] - pub(crate) type AvailabilityBitfields = - StorageMap<_, Twox64Concat, ValidatorIndex, AvailabilityBitfieldRecord>; - - /// Candidates pending availability by `ParaId`. - #[pallet::storage] - pub(crate) type PendingAvailability = - StorageMap<_, Twox64Concat, ParaId, CandidatePendingAvailability>; - - /// The commitments of candidates pending availability, by `ParaId`. - #[pallet::storage] - pub(crate) type PendingAvailabilityCommitments = - StorageMap<_, Twox64Concat, ParaId, CandidateCommitments>; - - #[pallet::call] - impl Pallet {} -} - -const LOG_TARGET: &str = "runtime::inclusion"; - -impl Pallet { - /// Block initialization logic, called by initializer. - pub(crate) fn initializer_initialize(_now: T::BlockNumber) -> Weight { - 0 - } - - /// Block finalization logic, called by initializer. - pub(crate) fn initializer_finalize() {} - - /// Handle an incoming session change. - pub(crate) fn initializer_on_new_session( - _notification: &crate::initializer::SessionChangeNotification, - ) { - // unlike most drain methods, drained elements are not cleared on `Drop` of the iterator - // and require consumption. - for _ in >::drain() {} - for _ in >::drain() {} - for _ in >::drain() {} - } - - /// Extract the freed cores based on cores that became available. - /// - /// Updates storage items `PendingAvailability` and `AvailabilityBitfields`. - pub(crate) fn update_pending_availability_and_get_freed_cores( - expected_bits: usize, - validators: &[ValidatorId], - signed_bitfields: UncheckedSignedAvailabilityBitfields, - core_lookup: F, - ) -> Vec<(CoreIndex, CandidateHash)> - where - F: Fn(CoreIndex) -> Option, - { - let mut assigned_paras_record = (0..expected_bits) - .map(|bit_index| core_lookup(CoreIndex::from(bit_index as u32))) - .map(|opt_para_id| { - opt_para_id.map(|para_id| (para_id, PendingAvailability::::get(¶_id))) - }) - .collect::>(); - - let now = >::block_number(); - for (checked_bitfield, validator_index) in - signed_bitfields.into_iter().map(|signed_bitfield| { - // extracting unchecked data, since it's checked in `fn sanitize_bitfields` already. - let validator_idx = signed_bitfield.unchecked_validator_index(); - let checked_bitfield = signed_bitfield.unchecked_into_payload(); - (checked_bitfield, validator_idx) - }) { - for (bit_idx, _) in checked_bitfield.0.iter().enumerate().filter(|(_, is_av)| **is_av) { - let pending_availability = if let Some((_, pending_availability)) = - assigned_paras_record[bit_idx].as_mut() - { - pending_availability - } else { - // For honest validators, this happens in case of unoccupied cores, - // which in turn happens in case of a disputed candidate. - // A malicious one might include arbitrary indices, but they are represented - // by `None` values and will be sorted out in the next if case. - continue - }; - - // defensive check - this is constructed by loading the availability bitfield record, - // which is always `Some` if the core is occupied - that's why we're here. - let validator_index = validator_index.0 as usize; - if let Some(mut bit) = - pending_availability.as_mut().and_then(|candidate_pending_availability| { - candidate_pending_availability.availability_votes.get_mut(validator_index) - }) { - *bit = true; - } - } - - let record = - AvailabilityBitfieldRecord { bitfield: checked_bitfield, submitted_at: now }; - - >::insert(&validator_index, record); - } - - let threshold = availability_threshold(validators.len()); - - let mut freed_cores = Vec::with_capacity(expected_bits); - for (para_id, pending_availability) in assigned_paras_record - .into_iter() - .filter_map(|x| x) - .filter_map(|(id, p)| p.map(|p| (id, p))) - { - if pending_availability.availability_votes.count_ones() >= threshold { - >::remove(¶_id); - let commitments = match PendingAvailabilityCommitments::::take(¶_id) { - Some(commitments) => commitments, - None => { - log::warn!( - target: LOG_TARGET, - "Inclusion::process_bitfields: PendingAvailability and PendingAvailabilityCommitments - are out of sync, did someone mess with the storage?", - ); - continue - }, - }; - - if ON_CHAIN_USE { - let receipt = CommittedCandidateReceipt { - descriptor: pending_availability.descriptor, - commitments, - }; - let _weight = Self::enact_candidate( - pending_availability.relay_parent_number, - receipt, - pending_availability.backers, - pending_availability.availability_votes, - pending_availability.core, - pending_availability.backing_group, - ); - } - - freed_cores.push((pending_availability.core, pending_availability.hash)); - } else { - >::insert(¶_id, &pending_availability); - } - } - - freed_cores - } - - /// Process a set of incoming bitfields. - /// - /// Returns a `Vec` of `CandidateHash`es and their respective `AvailabilityCore`s that became available, - /// and cores free. - pub(crate) fn process_bitfields( - expected_bits: usize, - signed_bitfields: UncheckedSignedAvailabilityBitfields, - disputed_bitfield: DisputedBitfield, - core_lookup: impl Fn(CoreIndex) -> Option, - ) -> Vec<(CoreIndex, CandidateHash)> { - let validators = shared::Pallet::::active_validator_keys(); - let session_index = shared::Pallet::::session_index(); - let parent_hash = frame_system::Pallet::::parent_hash(); - - let checked_bitfields = sanitize_bitfields::( - signed_bitfields, - disputed_bitfield, - expected_bits, - parent_hash, - session_index, - &validators[..], - FullCheck::Yes, - ); - - let freed_cores = Self::update_pending_availability_and_get_freed_cores::<_, true>( - expected_bits, - &validators[..], - checked_bitfields, - core_lookup, - ); - - freed_cores - } - - /// Process candidates that have been backed. Provide the relay storage root, a set of candidates - /// and scheduled cores. - /// - /// Both should be sorted ascending by core index, and the candidates should be a subset of - /// scheduled cores. If these conditions are not met, the execution of the function fails. - pub(crate) fn process_candidates( - parent_storage_root: T::Hash, - candidates: Vec>, - scheduled: Vec, - group_validators: GV, - full_check: FullCheck, - ) -> Result, DispatchError> - where - GV: Fn(GroupIndex) -> Option>, - { - ensure!(candidates.len() <= scheduled.len(), Error::::UnscheduledCandidate); - - if scheduled.is_empty() { - return Ok(ProcessedCandidates::default()) - } - - let validators = shared::Pallet::::active_validator_keys(); - let parent_hash = >::parent_hash(); - - // At the moment we assume (and in fact enforce, below) that the relay-parent is always one - // before of the block where we include a candidate (i.e. this code path). - let now = >::block_number(); - let relay_parent_number = now - One::one(); - let check_ctx = CandidateCheckContext::::new(now, relay_parent_number); - - // Collect candidate receipts with backers. - let mut candidate_receipt_with_backing_validator_indices = - Vec::with_capacity(candidates.len()); - - // Do all checks before writing storage. - let core_indices_and_backers = { - let mut skip = 0; - let mut core_indices_and_backers = Vec::with_capacity(candidates.len()); - let mut last_core = None; - - let mut check_assignment_in_order = |assignment: &CoreAssignment| -> DispatchResult { - ensure!( - last_core.map_or(true, |core| assignment.core > core), - Error::::ScheduledOutOfOrder, - ); - - last_core = Some(assignment.core); - Ok(()) - }; - - let signing_context = - SigningContext { parent_hash, session_index: shared::Pallet::::session_index() }; - - // We combine an outer loop over candidates with an inner loop over the scheduled, - // where each iteration of the outer loop picks up at the position - // in scheduled just after the past iteration left off. - // - // If the candidates appear in the same order as they appear in `scheduled`, - // then they should always be found. If the end of `scheduled` is reached, - // then the candidate was either not scheduled or out-of-order. - // - // In the meantime, we do certain sanity checks on the candidates and on the scheduled - // list. - 'next_backed_candidate: for (candidate_idx, backed_candidate) in - candidates.iter().enumerate() - { - if let FullCheck::Yes = full_check { - check_ctx.verify_backed_candidate( - parent_hash, - candidate_idx, - backed_candidate, - )?; - } - - let para_id = backed_candidate.descriptor().para_id; - let mut backers = bitvec::bitvec![BitOrderLsb0, u8; 0; validators.len()]; - - for (i, assignment) in scheduled[skip..].iter().enumerate() { - check_assignment_in_order(assignment)?; - - if para_id == assignment.para_id { - if let Some(required_collator) = assignment.required_collator() { - ensure!( - required_collator == &backed_candidate.descriptor().collator, - Error::::WrongCollator, - ); - } - - { - // this should never fail because the para is registered - let persisted_validation_data = - match crate::util::make_persisted_validation_data::( - para_id, - relay_parent_number, - parent_storage_root, - ) { - Some(l) => l, - None => { - // We don't want to error out here because it will - // brick the relay-chain. So we return early without - // doing anything. - return Ok(ProcessedCandidates::default()) - }, - }; - - let expected = persisted_validation_data.hash(); - - ensure!( - expected == - backed_candidate.descriptor().persisted_validation_data_hash, - Error::::ValidationDataHashMismatch, - ); - } - - ensure!( - >::get(¶_id).is_none() && - >::get(¶_id).is_none(), - Error::::CandidateScheduledBeforeParaFree, - ); - - // account for already skipped, and then skip this one. - skip = i + skip + 1; - - let group_vals = group_validators(assignment.group_idx) - .ok_or_else(|| Error::::InvalidGroupIndex)?; - - // check the signatures in the backing and that it is a majority. - { - let maybe_amount_validated = primitives::v1::check_candidate_backing( - &backed_candidate, - &signing_context, - group_vals.len(), - |intra_group_vi| { - group_vals - .get(intra_group_vi) - .and_then(|vi| validators.get(vi.0 as usize)) - .map(|v| v.clone()) - }, - ); - - match maybe_amount_validated { - Ok(amount_validated) => ensure!( - amount_validated * 2 > group_vals.len(), - Error::::InsufficientBacking, - ), - Err(()) => { - Err(Error::::InvalidBacking)?; - }, - } - - let mut backer_idx_and_attestation = - Vec::<(ValidatorIndex, ValidityAttestation)>::with_capacity( - backed_candidate.validator_indices.count_ones(), - ); - let candidate_receipt = backed_candidate.receipt(); - - for ((bit_idx, _), attestation) in backed_candidate - .validator_indices - .iter() - .enumerate() - .filter(|(_, signed)| **signed) - .zip(backed_candidate.validity_votes.iter().cloned()) - { - let val_idx = group_vals - .get(bit_idx) - .expect("this query succeeded above; qed"); - backer_idx_and_attestation.push((*val_idx, attestation)); - - backers.set(val_idx.0 as _, true); - } - candidate_receipt_with_backing_validator_indices - .push((candidate_receipt, backer_idx_and_attestation)); - } - - core_indices_and_backers.push(( - assignment.core, - backers, - assignment.group_idx, - )); - continue 'next_backed_candidate - } - } - - // end of loop reached means that the candidate didn't appear in the non-traversed - // section of the `scheduled` slice. either it was not scheduled or didn't appear in - // `candidates` in the correct order. - ensure!(false, Error::::UnscheduledCandidate); - } - - // check remainder of scheduled cores, if any. - for assignment in scheduled[skip..].iter() { - check_assignment_in_order(assignment)?; - } - - core_indices_and_backers - }; - - // one more sweep for actually writing to storage. - let core_indices = - core_indices_and_backers.iter().map(|&(ref c, _, _)| c.clone()).collect(); - for (candidate, (core, backers, group)) in - candidates.into_iter().zip(core_indices_and_backers) - { - let para_id = candidate.descriptor().para_id; - - // initialize all availability votes to 0. - let availability_votes: BitVec = - bitvec::bitvec![BitOrderLsb0, u8; 0; validators.len()]; - - Self::deposit_event(Event::::CandidateBacked( - candidate.candidate.to_plain(), - candidate.candidate.commitments.head_data.clone(), - core, - group, - )); - - let candidate_hash = candidate.candidate.hash(); - - let (descriptor, commitments) = - (candidate.candidate.descriptor, candidate.candidate.commitments); - - >::insert( - ¶_id, - CandidatePendingAvailability { - core, - hash: candidate_hash, - descriptor, - availability_votes, - relay_parent_number, - backers: backers.to_bitvec(), - backed_in_number: check_ctx.now, - backing_group: group, - }, - ); - >::insert(¶_id, commitments); - } - - Ok(ProcessedCandidates:: { - core_indices, - candidate_receipt_with_backing_validator_indices, - }) - } - - /// Run the acceptance criteria checks on the given candidate commitments. - pub(crate) fn check_validation_outputs_for_runtime_api( - para_id: ParaId, - validation_outputs: primitives::v1::CandidateCommitments, - ) -> bool { - // This function is meant to be called from the runtime APIs against the relay-parent, hence - // `relay_parent_number` is equal to `now`. - let now = >::block_number(); - let relay_parent_number = now; - let check_ctx = CandidateCheckContext::::new(now, relay_parent_number); - - if let Err(err) = check_ctx.check_validation_outputs( - para_id, - &validation_outputs.head_data, - &validation_outputs.new_validation_code, - validation_outputs.processed_downward_messages, - &validation_outputs.upward_messages, - T::BlockNumber::from(validation_outputs.hrmp_watermark), - &validation_outputs.horizontal_messages, - ) { - log::debug!( - target: LOG_TARGET, - "Validation outputs checking for parachain `{}` failed: {:?}", - u32::from(para_id), - err, - ); - false - } else { - true - } - } - - fn enact_candidate( - relay_parent_number: T::BlockNumber, - receipt: CommittedCandidateReceipt, - backers: BitVec, - availability_votes: BitVec, - core_index: CoreIndex, - backing_group: GroupIndex, - ) -> Weight { - let plain = receipt.to_plain(); - let commitments = receipt.commitments; - let config = >::config(); - - T::RewardValidators::reward_backing( - backers - .iter() - .enumerate() - .filter(|(_, backed)| **backed) - .map(|(i, _)| ValidatorIndex(i as _)), - ); - - T::RewardValidators::reward_bitfields( - availability_votes - .iter() - .enumerate() - .filter(|(_, voted)| **voted) - .map(|(i, _)| ValidatorIndex(i as _)), - ); - - // initial weight is config read. - let mut weight = T::DbWeight::get().reads_writes(1, 0); - if let Some(new_code) = commitments.new_validation_code { - weight += >::schedule_code_upgrade( - receipt.descriptor.para_id, - new_code, - relay_parent_number, - &config, - ); - } - - // enact the messaging facet of the candidate. - weight += >::prune_dmq( - receipt.descriptor.para_id, - commitments.processed_downward_messages, - ); - weight += >::receive_upward_messages( - receipt.descriptor.para_id, - commitments.upward_messages, - ); - weight += >::prune_hrmp( - receipt.descriptor.para_id, - T::BlockNumber::from(commitments.hrmp_watermark), - ); - weight += >::queue_outbound_hrmp( - receipt.descriptor.para_id, - commitments.horizontal_messages, - ); - - Self::deposit_event(Event::::CandidateIncluded( - plain, - commitments.head_data.clone(), - core_index, - backing_group, - )); - - weight + - >::note_new_head( - receipt.descriptor.para_id, - commitments.head_data, - relay_parent_number, - ) - } - - /// Cleans up all paras pending availability that the predicate returns true for. - /// - /// The predicate accepts the index of the core and the block number the core has been occupied - /// since (i.e. the block number the candidate was backed at in this fork of the relay chain). - /// - /// Returns a vector of cleaned-up core IDs. - pub(crate) fn collect_pending( - pred: impl Fn(CoreIndex, T::BlockNumber) -> bool, - ) -> Vec { - let mut cleaned_up_ids = Vec::new(); - let mut cleaned_up_cores = Vec::new(); - - for (para_id, pending_record) in >::iter() { - if pred(pending_record.core, pending_record.backed_in_number) { - cleaned_up_ids.push(para_id); - cleaned_up_cores.push(pending_record.core); - } - } - - for para_id in cleaned_up_ids { - let pending = >::take(¶_id); - let commitments = >::take(¶_id); - - if let (Some(pending), Some(commitments)) = (pending, commitments) { - // defensive: this should always be true. - let candidate = CandidateReceipt { - descriptor: pending.descriptor, - commitments_hash: commitments.hash(), - }; - - Self::deposit_event(Event::::CandidateTimedOut( - candidate, - commitments.head_data, - pending.core, - )); - } - } - - cleaned_up_cores - } - - /// Cleans up all paras pending availability that are in the given list of disputed candidates. - /// - /// Returns a vector of cleaned-up core IDs. - pub(crate) fn collect_disputed(disputed: &BTreeSet) -> Vec { - let mut cleaned_up_ids = Vec::new(); - let mut cleaned_up_cores = Vec::new(); - - for (para_id, pending_record) in >::iter() { - if disputed.contains(&pending_record.hash) { - cleaned_up_ids.push(para_id); - cleaned_up_cores.push(pending_record.core); - } - } - - for para_id in cleaned_up_ids { - let _ = >::take(¶_id); - let _ = >::take(¶_id); - } - - cleaned_up_cores - } - - /// Forcibly enact the candidate with the given ID as though it had been deemed available - /// by bitfields. - /// - /// Is a no-op if there is no candidate pending availability for this para-id. - /// This should generally not be used but it is useful during execution of Runtime APIs, - /// where the changes to the state are expected to be discarded directly after. - pub(crate) fn force_enact(para: ParaId) { - let pending = >::take(¶); - let commitments = >::take(¶); - - if let (Some(pending), Some(commitments)) = (pending, commitments) { - let candidate = - CommittedCandidateReceipt { descriptor: pending.descriptor, commitments }; - - Self::enact_candidate( - pending.relay_parent_number, - candidate, - pending.backers, - pending.availability_votes, - pending.core, - pending.backing_group, - ); - } - } - - /// Returns the `CommittedCandidateReceipt` pending availability for the para provided, if any. - pub(crate) fn candidate_pending_availability( - para: ParaId, - ) -> Option> { - >::get(¶) - .map(|p| p.descriptor) - .and_then(|d| >::get(¶).map(move |c| (d, c))) - .map(|(d, c)| CommittedCandidateReceipt { descriptor: d, commitments: c }) - } - - /// Returns the metadata around the candidate pending availability for the - /// para provided, if any. - pub(crate) fn pending_availability( - para: ParaId, - ) -> Option> { - >::get(¶) - } -} - -const fn availability_threshold(n_validators: usize) -> usize { - let mut threshold = (n_validators * 2) / 3; - threshold += (n_validators * 2) % 3; - threshold -} - -#[derive(derive_more::From, Debug)] -enum AcceptanceCheckErr { - HeadDataTooLarge, - PrematureCodeUpgrade, - NewCodeTooLarge, - ProcessedDownwardMessages(dmp::ProcessedDownwardMessagesAcceptanceErr), - UpwardMessages(ump::AcceptanceCheckErr), - HrmpWatermark(hrmp::HrmpWatermarkAcceptanceErr), - OutboundHrmp(hrmp::OutboundHrmpAcceptanceErr), -} - -impl AcceptanceCheckErr { - /// Returns the same error so that it can be threaded through a needle of `DispatchError` and - /// ultimately returned from a `Dispatchable`. - fn strip_into_dispatch_err(self) -> Error { - use AcceptanceCheckErr::*; - match self { - HeadDataTooLarge => Error::::HeadDataTooLarge, - PrematureCodeUpgrade => Error::::PrematureCodeUpgrade, - NewCodeTooLarge => Error::::NewCodeTooLarge, - ProcessedDownwardMessages(_) => Error::::IncorrectDownwardMessageHandling, - UpwardMessages(_) => Error::::InvalidUpwardMessages, - HrmpWatermark(_) => Error::::HrmpWatermarkMishandling, - OutboundHrmp(_) => Error::::InvalidOutboundHrmp, - } - } -} - -/// A collection of data required for checking a candidate. -pub(crate) struct CandidateCheckContext { - config: configuration::HostConfiguration, - now: T::BlockNumber, - relay_parent_number: T::BlockNumber, -} - -impl CandidateCheckContext { - pub(crate) fn new(now: T::BlockNumber, relay_parent_number: T::BlockNumber) -> Self { - Self { config: >::config(), now, relay_parent_number } - } - - /// Execute verification of the candidate. - /// - /// Assures: - /// * correct expected relay parent reference - /// * collator signature check passes - /// * code hash of commitments matches current code hash - /// * para head in the descriptor and commitments match - pub(crate) fn verify_backed_candidate( - &self, - parent_hash: ::Hash, - candidate_idx: usize, - backed_candidate: &BackedCandidate<::Hash>, - ) -> Result<(), Error> { - let para_id = backed_candidate.descriptor().para_id; - let now = self.now; - - // we require that the candidate is in the context of the parent block. - ensure!( - backed_candidate.descriptor().relay_parent == parent_hash, - Error::::CandidateNotInParentContext, - ); - ensure!( - backed_candidate.descriptor().check_collator_signature().is_ok(), - Error::::NotCollatorSigned, - ); - - let validation_code_hash = >::validation_code_hash_at(para_id, now, None) - // A candidate for a parachain without current validation code is not scheduled. - .ok_or_else(|| Error::::UnscheduledCandidate)?; - ensure!( - backed_candidate.descriptor().validation_code_hash == validation_code_hash, - Error::::InvalidValidationCodeHash, - ); - - ensure!( - backed_candidate.descriptor().para_head == - backed_candidate.candidate.commitments.head_data.hash(), - Error::::ParaHeadMismatch, - ); - - if let Err(err) = self.check_validation_outputs( - para_id, - &backed_candidate.candidate.commitments.head_data, - &backed_candidate.candidate.commitments.new_validation_code, - backed_candidate.candidate.commitments.processed_downward_messages, - &backed_candidate.candidate.commitments.upward_messages, - T::BlockNumber::from(backed_candidate.candidate.commitments.hrmp_watermark), - &backed_candidate.candidate.commitments.horizontal_messages, - ) { - log::debug!( - target: LOG_TARGET, - "Validation outputs checking during inclusion of a candidate {} for parachain `{}` failed: {:?}", - candidate_idx, - u32::from(para_id), - err, - ); - Err(err.strip_into_dispatch_err::())?; - }; - Ok(()) - } - - /// Check the given outputs after candidate validation on whether it passes the acceptance - /// criteria. - fn check_validation_outputs( - &self, - para_id: ParaId, - head_data: &HeadData, - new_validation_code: &Option, - processed_downward_messages: u32, - upward_messages: &[primitives::v1::UpwardMessage], - hrmp_watermark: T::BlockNumber, - horizontal_messages: &[primitives::v1::OutboundHrmpMessage], - ) -> Result<(), AcceptanceCheckErr> { - ensure!( - head_data.0.len() <= self.config.max_head_data_size as _, - AcceptanceCheckErr::HeadDataTooLarge, - ); - - // if any, the code upgrade attempt is allowed. - if let Some(new_validation_code) = new_validation_code { - let valid_upgrade_attempt = >::last_code_upgrade(para_id, true) - .map_or(true, |last| { - last <= self.relay_parent_number && - self.relay_parent_number.saturating_sub(last) >= - self.config.validation_upgrade_frequency - }); - ensure!(valid_upgrade_attempt, AcceptanceCheckErr::PrematureCodeUpgrade); - ensure!( - new_validation_code.0.len() <= self.config.max_code_size as _, - AcceptanceCheckErr::NewCodeTooLarge, - ); - } - - // check if the candidate passes the messaging acceptance criteria - >::check_processed_downward_messages(para_id, processed_downward_messages)?; - >::check_upward_messages(&self.config, para_id, upward_messages)?; - >::check_hrmp_watermark(para_id, self.relay_parent_number, hrmp_watermark)?; - >::check_outbound_hrmp(&self.config, para_id, horizontal_messages)?; - - Ok(()) - } -} - -#[cfg(test)] -pub(crate) mod tests { - use super::*; - use crate::{ - configuration::HostConfiguration, - initializer::SessionChangeNotification, - mock::{ - new_test_ext, Configuration, MockGenesisConfig, ParaInclusion, Paras, ParasShared, - System, Test, - }, - paras::ParaGenesisArgs, - paras_inherent::DisputedBitfield, - scheduler::AssignmentKind, - }; - use frame_support::assert_noop; - use futures::executor::block_on; - use keyring::Sr25519Keyring; - use primitives::{ - v0::PARACHAIN_KEY_TYPE_ID, - v1::{ - BlockNumber, CandidateCommitments, CandidateDescriptor, CollatorId, - CompactStatement as Statement, Hash, SignedAvailabilityBitfield, SignedStatement, - UncheckedSignedAvailabilityBitfield, ValidationCode, ValidatorId, ValidityAttestation, - }, - }; - use sc_keystore::LocalKeystore; - use sp_keystore::{SyncCryptoStore, SyncCryptoStorePtr}; - use std::sync::Arc; - - fn default_config() -> HostConfiguration { - let mut config = HostConfiguration::default(); - config.parathread_cores = 1; - config.max_code_size = 3; - config - } - - pub(crate) fn genesis_config(paras: Vec<(ParaId, bool)>) -> MockGenesisConfig { - MockGenesisConfig { - paras: paras::GenesisConfig { - paras: paras - .into_iter() - .map(|(id, is_chain)| { - ( - id, - ParaGenesisArgs { - genesis_head: Vec::new().into(), - validation_code: Vec::new().into(), - parachain: is_chain, - }, - ) - }) - .collect(), - ..Default::default() - }, - configuration: configuration::GenesisConfig { - config: default_config(), - ..Default::default() - }, - ..Default::default() - } - } - - #[derive(Debug, Clone, Copy, PartialEq)] - pub(crate) enum BackingKind { - #[allow(unused)] - Unanimous, - Threshold, - Lacking, - } - - pub(crate) fn collator_sign_candidate( - collator: Sr25519Keyring, - candidate: &mut CommittedCandidateReceipt, - ) { - candidate.descriptor.collator = collator.public().into(); - - let payload = primitives::v1::collator_signature_payload( - &candidate.descriptor.relay_parent, - &candidate.descriptor.para_id, - &candidate.descriptor.persisted_validation_data_hash, - &candidate.descriptor.pov_hash, - &candidate.descriptor.validation_code_hash, - ); - - candidate.descriptor.signature = collator.sign(&payload[..]).into(); - assert!(candidate.descriptor().check_collator_signature().is_ok()); - } - - pub(crate) async fn back_candidate( - candidate: CommittedCandidateReceipt, - validators: &[Sr25519Keyring], - group: &[ValidatorIndex], - keystore: &SyncCryptoStorePtr, - signing_context: &SigningContext, - kind: BackingKind, - ) -> BackedCandidate { - let mut validator_indices = bitvec::bitvec![BitOrderLsb0, u8; 0; group.len()]; - let threshold = (group.len() / 2) + 1; - - let signing = match kind { - BackingKind::Unanimous => group.len(), - BackingKind::Threshold => threshold, - BackingKind::Lacking => threshold.saturating_sub(1), - }; - - let mut validity_votes = Vec::with_capacity(signing); - let candidate_hash = candidate.hash(); - - for (idx_in_group, val_idx) in group.iter().enumerate().take(signing) { - let key: Sr25519Keyring = validators[val_idx.0 as usize]; - *validator_indices.get_mut(idx_in_group).unwrap() = true; - - let signature = SignedStatement::sign( - &keystore, - Statement::Valid(candidate_hash), - signing_context, - *val_idx, - &key.public().into(), - ) - .await - .unwrap() - .unwrap() - .signature() - .clone(); - - validity_votes.push(ValidityAttestation::Explicit(signature).into()); - } - - let backed = BackedCandidate { candidate, validity_votes, validator_indices }; - - let successfully_backed = - primitives::v1::check_candidate_backing(&backed, signing_context, group.len(), |i| { - Some(validators[group[i].0 as usize].public().into()) - }) - .ok() - .unwrap_or(0) * 2 > - group.len(); - - match kind { - BackingKind::Unanimous | BackingKind::Threshold => assert!(successfully_backed), - BackingKind::Lacking => assert!(!successfully_backed), - }; - - backed - } - - pub(crate) fn run_to_block( - to: BlockNumber, - new_session: impl Fn(BlockNumber) -> Option>, - ) { - while System::block_number() < to { - let b = System::block_number(); - - ParaInclusion::initializer_finalize(); - Paras::initializer_finalize(); - ParasShared::initializer_finalize(); - - if let Some(notification) = new_session(b + 1) { - ParasShared::initializer_on_new_session( - notification.session_index, - notification.random_seed, - ¬ification.new_config, - notification.validators.clone(), - ); - Paras::initializer_on_new_session(¬ification); - ParaInclusion::initializer_on_new_session(¬ification); - } - - System::on_finalize(b); - - System::on_initialize(b + 1); - System::set_block_number(b + 1); - - ParasShared::initializer_initialize(b + 1); - Paras::initializer_initialize(b + 1); - ParaInclusion::initializer_initialize(b + 1); - } - } - - pub(crate) fn expected_bits() -> usize { - Paras::parachains().len() + Configuration::config().parathread_cores as usize - } - - fn default_bitfield() -> AvailabilityBitfield { - AvailabilityBitfield(bitvec::bitvec![BitOrderLsb0, u8; 0; expected_bits()]) - } - - fn default_availability_votes() -> BitVec { - bitvec::bitvec![BitOrderLsb0, u8; 0; ParasShared::active_validator_keys().len()] - } - - fn default_backing_bitfield() -> BitVec { - bitvec::bitvec![BitOrderLsb0, u8; 0; ParasShared::active_validator_keys().len()] - } - - fn backing_bitfield(v: &[usize]) -> BitVec { - let mut b = default_backing_bitfield(); - for i in v { - b.set(*i, true); - } - b - } - - pub(crate) fn validator_pubkeys(val_ids: &[Sr25519Keyring]) -> Vec { - val_ids.iter().map(|v| v.public().into()).collect() - } - - pub(crate) async fn sign_bitfield( - keystore: &SyncCryptoStorePtr, - key: &Sr25519Keyring, - validator_index: ValidatorIndex, - bitfield: AvailabilityBitfield, - signing_context: &SigningContext, - ) -> SignedAvailabilityBitfield { - SignedAvailabilityBitfield::sign( - &keystore, - bitfield, - &signing_context, - validator_index, - &key.public().into(), - ) - .await - .unwrap() - .unwrap() - } - - #[derive(Default)] - pub(crate) struct TestCandidateBuilder { - pub(crate) para_id: ParaId, - pub(crate) head_data: HeadData, - pub(crate) para_head_hash: Option, - pub(crate) pov_hash: Hash, - pub(crate) relay_parent: Hash, - pub(crate) persisted_validation_data_hash: Hash, - pub(crate) new_validation_code: Option, - pub(crate) validation_code: ValidationCode, - pub(crate) hrmp_watermark: BlockNumber, - } - - impl TestCandidateBuilder { - pub(crate) fn build(self) -> CommittedCandidateReceipt { - CommittedCandidateReceipt { - descriptor: CandidateDescriptor { - para_id: self.para_id, - pov_hash: self.pov_hash, - relay_parent: self.relay_parent, - persisted_validation_data_hash: self.persisted_validation_data_hash, - validation_code_hash: self.validation_code.hash(), - para_head: self.para_head_hash.unwrap_or_else(|| self.head_data.hash()), - ..Default::default() - }, - commitments: CandidateCommitments { - head_data: self.head_data, - new_validation_code: self.new_validation_code, - hrmp_watermark: self.hrmp_watermark, - ..Default::default() - }, - } - } - } - - pub(crate) fn make_vdata_hash(para_id: ParaId) -> Option { - let relay_parent_number = >::block_number() - 1; - let persisted_validation_data = crate::util::make_persisted_validation_data::( - para_id, - relay_parent_number, - Default::default(), - )?; - Some(persisted_validation_data.hash()) - } - - #[test] - fn collect_pending_cleans_up_pending() { - let chain_a = ParaId::from(1); - let chain_b = ParaId::from(2); - let thread_a = ParaId::from(3); - - let paras = vec![(chain_a, true), (chain_b, true), (thread_a, false)]; - new_test_ext(genesis_config(paras)).execute_with(|| { - let default_candidate = TestCandidateBuilder::default().build(); - >::insert( - chain_a, - CandidatePendingAvailability { - core: CoreIndex::from(0), - hash: default_candidate.hash(), - descriptor: default_candidate.descriptor.clone(), - availability_votes: default_availability_votes(), - relay_parent_number: 0, - backed_in_number: 0, - backers: default_backing_bitfield(), - backing_group: GroupIndex::from(0), - }, - ); - PendingAvailabilityCommitments::::insert( - chain_a, - default_candidate.commitments.clone(), - ); - - >::insert( - &chain_b, - CandidatePendingAvailability { - core: CoreIndex::from(1), - hash: default_candidate.hash(), - descriptor: default_candidate.descriptor, - availability_votes: default_availability_votes(), - relay_parent_number: 0, - backed_in_number: 0, - backers: default_backing_bitfield(), - backing_group: GroupIndex::from(1), - }, - ); - PendingAvailabilityCommitments::::insert(chain_b, default_candidate.commitments); - - run_to_block(5, |_| None); - - assert!(>::get(&chain_a).is_some()); - assert!(>::get(&chain_b).is_some()); - assert!(>::get(&chain_a).is_some()); - assert!(>::get(&chain_b).is_some()); - - ParaInclusion::collect_pending(|core, _since| core == CoreIndex::from(0)); - - assert!(>::get(&chain_a).is_none()); - assert!(>::get(&chain_b).is_some()); - assert!(>::get(&chain_a).is_none()); - assert!(>::get(&chain_b).is_some()); - }); - } - - #[test] - fn bitfield_checks() { - let chain_a = ParaId::from(1); - let chain_b = ParaId::from(2); - let thread_a = ParaId::from(3); - - let paras = vec![(chain_a, true), (chain_b, true), (thread_a, false)]; - let validators = vec![ - Sr25519Keyring::Alice, - Sr25519Keyring::Bob, - Sr25519Keyring::Charlie, - Sr25519Keyring::Dave, - Sr25519Keyring::Ferdie, - ]; - let keystore: SyncCryptoStorePtr = Arc::new(LocalKeystore::in_memory()); - for validator in validators.iter() { - SyncCryptoStore::sr25519_generate_new( - &*keystore, - PARACHAIN_KEY_TYPE_ID, - Some(&validator.to_seed()), - ) - .unwrap(); - } - let validator_public = validator_pubkeys(&validators); - - new_test_ext(genesis_config(paras.clone())).execute_with(|| { - shared::Pallet::::set_active_validators_ascending(validator_public.clone()); - shared::Pallet::::set_session_index(5); - - let signing_context = - SigningContext { parent_hash: System::parent_hash(), session_index: 5 }; - - let core_lookup = |core| match core { - core if core == CoreIndex::from(0) => Some(chain_a), - core if core == CoreIndex::from(1) => Some(chain_b), - core if core == CoreIndex::from(2) => Some(thread_a), - core if core == CoreIndex::from(3) => None, // for the expected_cores() + 1 test below. - _ => panic!("out of bounds for testing"), - }; - - // mark all candidates as pending availability - let set_pending_av = || { - for (p_id, _) in paras { - PendingAvailability::::insert( - p_id, - CandidatePendingAvailability { - availability_votes: default_availability_votes(), - ..Default::default() - }, - ) - } - }; - - // too many bits in bitfield - { - let mut bare_bitfield = default_bitfield(); - bare_bitfield.0.push(false); - let signed = block_on(sign_bitfield( - &keystore, - &validators[0], - ValidatorIndex(0), - bare_bitfield, - &signing_context, - )); - - assert_eq!( - ParaInclusion::process_bitfields( - expected_bits(), - vec![signed.into()], - DisputedBitfield::zeros(expected_bits()), - &core_lookup, - ), - vec![] - ); - } - - // not enough bits - { - let bare_bitfield = default_bitfield(); - let signed = block_on(sign_bitfield( - &keystore, - &validators[0], - ValidatorIndex(0), - bare_bitfield, - &signing_context, - )); - - assert_eq!( - ParaInclusion::process_bitfields( - expected_bits() + 1, - vec![signed.into()], - DisputedBitfield::zeros(expected_bits()), - &core_lookup, - ), - vec![] - ); - } - - // duplicate. - { - set_pending_av.clone()(); - let back_core_0_bitfield = { - let mut b = default_bitfield(); - b.0.set(0, true); - b - }; - let signed: UncheckedSignedAvailabilityBitfield = block_on(sign_bitfield( - &keystore, - &validators[0], - ValidatorIndex(0), - back_core_0_bitfield, - &signing_context, - )) - .into(); - - assert_eq!( - >::get(chain_a) - .unwrap() - .availability_votes - .count_ones(), - 0 - ); - - // the threshold to free a core is 4 availability votes, but we only expect 1 valid - // valid bitfield. - assert!(ParaInclusion::process_bitfields( - expected_bits(), - vec![signed.clone(), signed], - DisputedBitfield::zeros(expected_bits()), - &core_lookup, - ) - .is_empty()); - - assert_eq!( - >::get(chain_a) - .unwrap() - .availability_votes - .count_ones(), - 1 - ); - - // clean up - PendingAvailability::::remove_all(None); - } - - // out of order. - { - set_pending_av.clone()(); - let back_core_0_bitfield = { - let mut b = default_bitfield(); - b.0.set(0, true); - b - }; - let signed_0 = block_on(sign_bitfield( - &keystore, - &validators[0], - ValidatorIndex(0), - back_core_0_bitfield.clone(), - &signing_context, - )) - .into(); - - let signed_1 = block_on(sign_bitfield( - &keystore, - &validators[1], - ValidatorIndex(1), - back_core_0_bitfield, - &signing_context, - )) - .into(); - - assert_eq!( - >::get(chain_a) - .unwrap() - .availability_votes - .count_ones(), - 0 - ); - - // the threshold to free a core is 4 availability votes, but we only expect 1 valid - // valid bitfield because `signed_0` will get skipped for being out of order. - assert!(ParaInclusion::process_bitfields( - expected_bits(), - vec![signed_1, signed_0], - DisputedBitfield::zeros(expected_bits()), - &core_lookup, - ) - .is_empty()); - - assert_eq!( - >::get(chain_a) - .unwrap() - .availability_votes - .count_ones(), - 1 - ); - - PendingAvailability::::remove_all(None); - } - - // non-pending bit set. - { - let mut bare_bitfield = default_bitfield(); - *bare_bitfield.0.get_mut(0).unwrap() = true; - let signed = block_on(sign_bitfield( - &keystore, - &validators[0], - ValidatorIndex(0), - bare_bitfield, - &signing_context, - )); - - assert!(ParaInclusion::process_bitfields( - expected_bits(), - vec![signed.into()], - DisputedBitfield::zeros(expected_bits()), - &core_lookup, - ) - .is_empty()); - } - - // empty bitfield signed: always ok, but kind of useless. - { - let bare_bitfield = default_bitfield(); - let signed = block_on(sign_bitfield( - &keystore, - &validators[0], - ValidatorIndex(0), - bare_bitfield, - &signing_context, - )); - - assert!(ParaInclusion::process_bitfields( - expected_bits(), - vec![signed.into()], - DisputedBitfield::zeros(expected_bits()), - &core_lookup, - ) - .is_empty()); - } - - // bitfield signed with pending bit signed. - { - let mut bare_bitfield = default_bitfield(); - - assert_eq!(core_lookup(CoreIndex::from(0)), Some(chain_a)); - - let default_candidate = TestCandidateBuilder::default().build(); - >::insert( - chain_a, - CandidatePendingAvailability { - core: CoreIndex::from(0), - hash: default_candidate.hash(), - descriptor: default_candidate.descriptor, - availability_votes: default_availability_votes(), - relay_parent_number: 0, - backed_in_number: 0, - backers: default_backing_bitfield(), - backing_group: GroupIndex::from(0), - }, - ); - PendingAvailabilityCommitments::::insert( - chain_a, - default_candidate.commitments, - ); - - *bare_bitfield.0.get_mut(0).unwrap() = true; - let signed = block_on(sign_bitfield( - &keystore, - &validators[0], - ValidatorIndex(0), - bare_bitfield, - &signing_context, - )); - - assert!(ParaInclusion::process_bitfields( - expected_bits(), - vec![signed.into()], - DisputedBitfield::zeros(expected_bits()), - &core_lookup, - ) - .is_empty()); - - >::remove(chain_a); - PendingAvailabilityCommitments::::remove(chain_a); - } - - // bitfield signed with pending bit signed, but no commitments. - { - let mut bare_bitfield = default_bitfield(); - - assert_eq!(core_lookup(CoreIndex::from(0)), Some(chain_a)); - - let default_candidate = TestCandidateBuilder::default().build(); - >::insert( - chain_a, - CandidatePendingAvailability { - core: CoreIndex::from(0), - hash: default_candidate.hash(), - descriptor: default_candidate.descriptor, - availability_votes: default_availability_votes(), - relay_parent_number: 0, - backed_in_number: 0, - backers: default_backing_bitfield(), - backing_group: GroupIndex::from(0), - }, - ); - - *bare_bitfield.0.get_mut(0).unwrap() = true; - let signed = block_on(sign_bitfield( - &keystore, - &validators[0], - ValidatorIndex(0), - bare_bitfield, - &signing_context, - )); - - // no core is freed - assert!(ParaInclusion::process_bitfields( - expected_bits(), - vec![signed.into()], - DisputedBitfield::zeros(expected_bits()), - &core_lookup, - ) - .is_empty()); - } - }); - } - - #[test] - fn supermajority_bitfields_trigger_availability() { - let chain_a = ParaId::from(1); - let chain_b = ParaId::from(2); - let thread_a = ParaId::from(3); - - let paras = vec![(chain_a, true), (chain_b, true), (thread_a, false)]; - let validators = vec![ - Sr25519Keyring::Alice, - Sr25519Keyring::Bob, - Sr25519Keyring::Charlie, - Sr25519Keyring::Dave, - Sr25519Keyring::Ferdie, - ]; - let keystore: SyncCryptoStorePtr = Arc::new(LocalKeystore::in_memory()); - for validator in validators.iter() { - SyncCryptoStore::sr25519_generate_new( - &*keystore, - PARACHAIN_KEY_TYPE_ID, - Some(&validator.to_seed()), - ) - .unwrap(); - } - let validator_public = validator_pubkeys(&validators); - - new_test_ext(genesis_config(paras)).execute_with(|| { - shared::Pallet::::set_active_validators_ascending(validator_public.clone()); - shared::Pallet::::set_session_index(5); - - let signing_context = - SigningContext { parent_hash: System::parent_hash(), session_index: 5 }; - - let core_lookup = |core| match core { - core if core == CoreIndex::from(0) => Some(chain_a), - core if core == CoreIndex::from(1) => Some(chain_b), - core if core == CoreIndex::from(2) => Some(thread_a), - _ => panic!("Core out of bounds for 2 parachains and 1 parathread core."), - }; - - let candidate_a = TestCandidateBuilder { - para_id: chain_a, - head_data: vec![1, 2, 3, 4].into(), - ..Default::default() - } - .build(); - - >::insert( - chain_a, - CandidatePendingAvailability { - core: CoreIndex::from(0), - hash: candidate_a.hash(), - descriptor: candidate_a.clone().descriptor, - availability_votes: default_availability_votes(), - relay_parent_number: 0, - backed_in_number: 0, - backers: backing_bitfield(&[3, 4]), - backing_group: GroupIndex::from(0), - }, - ); - PendingAvailabilityCommitments::::insert( - chain_a, - candidate_a.clone().commitments, - ); - - let candidate_b = TestCandidateBuilder { - para_id: chain_b, - head_data: vec![5, 6, 7, 8].into(), - ..Default::default() - } - .build(); - - >::insert( - chain_b, - CandidatePendingAvailability { - core: CoreIndex::from(1), - hash: candidate_b.hash(), - descriptor: candidate_b.descriptor, - availability_votes: default_availability_votes(), - relay_parent_number: 0, - backed_in_number: 0, - backers: backing_bitfield(&[0, 2]), - backing_group: GroupIndex::from(1), - }, - ); - PendingAvailabilityCommitments::::insert(chain_b, candidate_b.commitments); - - // this bitfield signals that a and b are available. - let a_and_b_available = { - let mut bare_bitfield = default_bitfield(); - *bare_bitfield.0.get_mut(0).unwrap() = true; - *bare_bitfield.0.get_mut(1).unwrap() = true; - - bare_bitfield - }; - - // this bitfield signals that only a is available. - let a_available = { - let mut bare_bitfield = default_bitfield(); - *bare_bitfield.0.get_mut(0).unwrap() = true; - - bare_bitfield - }; - - let threshold = availability_threshold(validators.len()); - - // 4 of 5 first value >= 2/3 - assert_eq!(threshold, 4); - - let signed_bitfields = validators - .iter() - .enumerate() - .filter_map(|(i, key)| { - let to_sign = if i < 3 { - a_and_b_available.clone() - } else if i < 4 { - a_available.clone() - } else { - // sign nothing. - return None - }; - - Some( - block_on(sign_bitfield( - &keystore, - key, - ValidatorIndex(i as _), - to_sign, - &signing_context, - )) - .into(), - ) - }) - .collect(); - - // only chain A's core is freed. - assert_eq!( - ParaInclusion::process_bitfields( - expected_bits(), - signed_bitfields, - DisputedBitfield::zeros(expected_bits()), - &core_lookup, - ), - vec![(CoreIndex(0), candidate_a.hash())] - ); - - // chain A had 4 signing off, which is >= threshold. - // chain B has 3 signing off, which is < threshold. - assert!(>::get(&chain_a).is_none()); - assert!(>::get(&chain_a).is_none()); - assert!(>::get(&chain_b).is_some()); - assert_eq!(>::get(&chain_b).unwrap().availability_votes, { - // check that votes from first 3 were tracked. - - let mut votes = default_availability_votes(); - *votes.get_mut(0).unwrap() = true; - *votes.get_mut(1).unwrap() = true; - *votes.get_mut(2).unwrap() = true; - - votes - }); - - // and check that chain head was enacted. - assert_eq!(Paras::para_head(&chain_a), Some(vec![1, 2, 3, 4].into())); - - // Check that rewards are applied. - { - let rewards = crate::mock::availability_rewards(); - - assert_eq!(rewards.len(), 4); - assert_eq!(rewards.get(&ValidatorIndex(0)).unwrap(), &1); - assert_eq!(rewards.get(&ValidatorIndex(1)).unwrap(), &1); - assert_eq!(rewards.get(&ValidatorIndex(2)).unwrap(), &1); - assert_eq!(rewards.get(&ValidatorIndex(3)).unwrap(), &1); - } - - { - let rewards = crate::mock::backing_rewards(); - - assert_eq!(rewards.len(), 2); - assert_eq!(rewards.get(&ValidatorIndex(3)).unwrap(), &1); - assert_eq!(rewards.get(&ValidatorIndex(4)).unwrap(), &1); - } - }); - } - - #[test] - fn candidate_checks() { - let chain_a = ParaId::from(1); - let chain_b = ParaId::from(2); - let thread_a = ParaId::from(3); - - // The block number of the relay-parent for testing. - const RELAY_PARENT_NUM: BlockNumber = 4; - - let paras = vec![(chain_a, true), (chain_b, true), (thread_a, false)]; - let validators = vec![ - Sr25519Keyring::Alice, - Sr25519Keyring::Bob, - Sr25519Keyring::Charlie, - Sr25519Keyring::Dave, - Sr25519Keyring::Ferdie, - ]; - let keystore: SyncCryptoStorePtr = Arc::new(LocalKeystore::in_memory()); - for validator in validators.iter() { - SyncCryptoStore::sr25519_generate_new( - &*keystore, - PARACHAIN_KEY_TYPE_ID, - Some(&validator.to_seed()), - ) - .unwrap(); - } - let validator_public = validator_pubkeys(&validators); - - new_test_ext(genesis_config(paras)).execute_with(|| { - shared::Pallet::::set_active_validators_ascending(validator_public.clone()); - shared::Pallet::::set_session_index(5); - - run_to_block(5, |_| None); - - let signing_context = - SigningContext { parent_hash: System::parent_hash(), session_index: 5 }; - - let group_validators = |group_index: GroupIndex| { - match group_index { - group_index if group_index == GroupIndex::from(0) => Some(vec![0, 1]), - group_index if group_index == GroupIndex::from(1) => Some(vec![2, 3]), - group_index if group_index == GroupIndex::from(2) => Some(vec![4]), - _ => panic!("Group index out of bounds for 2 parachains and 1 parathread core"), - } - .map(|m| m.into_iter().map(ValidatorIndex).collect::>()) - }; - - let thread_collator: CollatorId = Sr25519Keyring::Two.public().into(); - - let chain_a_assignment = CoreAssignment { - core: CoreIndex::from(0), - para_id: chain_a, - kind: AssignmentKind::Parachain, - group_idx: GroupIndex::from(0), - }; - - let chain_b_assignment = CoreAssignment { - core: CoreIndex::from(1), - para_id: chain_b, - kind: AssignmentKind::Parachain, - group_idx: GroupIndex::from(1), - }; - - let thread_a_assignment = CoreAssignment { - core: CoreIndex::from(2), - para_id: thread_a, - kind: AssignmentKind::Parathread(thread_collator.clone(), 0), - group_idx: GroupIndex::from(2), - }; - - // unscheduled candidate. - { - let mut candidate = TestCandidateBuilder { - para_id: chain_a, - relay_parent: System::parent_hash(), - pov_hash: Hash::repeat_byte(1), - persisted_validation_data_hash: make_vdata_hash(chain_a).unwrap(), - hrmp_watermark: RELAY_PARENT_NUM, - ..Default::default() - } - .build(); - collator_sign_candidate(Sr25519Keyring::One, &mut candidate); - - let backed = block_on(back_candidate( - candidate, - &validators, - group_validators(GroupIndex::from(0)).unwrap().as_ref(), - &keystore, - &signing_context, - BackingKind::Threshold, - )); - - assert_noop!( - ParaInclusion::process_candidates( - Default::default(), - vec![backed], - vec![chain_b_assignment.clone()], - &group_validators, - FullCheck::Yes, - ), - Error::::UnscheduledCandidate - ); - } - - // candidates out of order. - { - let mut candidate_a = TestCandidateBuilder { - para_id: chain_a, - relay_parent: System::parent_hash(), - pov_hash: Hash::repeat_byte(1), - persisted_validation_data_hash: make_vdata_hash(chain_a).unwrap(), - hrmp_watermark: RELAY_PARENT_NUM, - ..Default::default() - } - .build(); - let mut candidate_b = TestCandidateBuilder { - para_id: chain_b, - relay_parent: System::parent_hash(), - pov_hash: Hash::repeat_byte(2), - persisted_validation_data_hash: make_vdata_hash(chain_b).unwrap(), - hrmp_watermark: RELAY_PARENT_NUM, - ..Default::default() - } - .build(); - - collator_sign_candidate(Sr25519Keyring::One, &mut candidate_a); - - collator_sign_candidate(Sr25519Keyring::Two, &mut candidate_b); - - let backed_a = block_on(back_candidate( - candidate_a, - &validators, - group_validators(GroupIndex::from(0)).unwrap().as_ref(), - &keystore, - &signing_context, - BackingKind::Threshold, - )); - - let backed_b = block_on(back_candidate( - candidate_b, - &validators, - group_validators(GroupIndex::from(1)).unwrap().as_ref(), - &keystore, - &signing_context, - BackingKind::Threshold, - )); - - // out-of-order manifests as unscheduled. - assert_noop!( - ParaInclusion::process_candidates( - Default::default(), - vec![backed_b, backed_a], - vec![chain_a_assignment.clone(), chain_b_assignment.clone()], - &group_validators, - FullCheck::Yes, - ), - Error::::UnscheduledCandidate - ); - } - - // candidate not backed. - { - let mut candidate = TestCandidateBuilder { - para_id: chain_a, - relay_parent: System::parent_hash(), - pov_hash: Hash::repeat_byte(1), - persisted_validation_data_hash: make_vdata_hash(chain_a).unwrap(), - hrmp_watermark: RELAY_PARENT_NUM, - ..Default::default() - } - .build(); - collator_sign_candidate(Sr25519Keyring::One, &mut candidate); - - let backed = block_on(back_candidate( - candidate, - &validators, - group_validators(GroupIndex::from(0)).unwrap().as_ref(), - &keystore, - &signing_context, - BackingKind::Lacking, - )); - - assert_noop!( - ParaInclusion::process_candidates( - Default::default(), - vec![backed], - vec![chain_a_assignment.clone()], - &group_validators, - FullCheck::Yes, - ), - Error::::InsufficientBacking - ); - } - - // candidate not in parent context. - { - let wrong_parent_hash = Hash::repeat_byte(222); - assert!(System::parent_hash() != wrong_parent_hash); - - let mut candidate = TestCandidateBuilder { - para_id: chain_a, - relay_parent: wrong_parent_hash, - pov_hash: Hash::repeat_byte(1), - persisted_validation_data_hash: make_vdata_hash(chain_a).unwrap(), - ..Default::default() - } - .build(); - collator_sign_candidate(Sr25519Keyring::One, &mut candidate); - - let backed = block_on(back_candidate( - candidate, - &validators, - group_validators(GroupIndex::from(0)).unwrap().as_ref(), - &keystore, - &signing_context, - BackingKind::Threshold, - )); - - assert_noop!( - ParaInclusion::process_candidates( - Default::default(), - vec![backed], - vec![chain_a_assignment.clone()], - &group_validators, - FullCheck::Yes, - ), - Error::::CandidateNotInParentContext - ); - } - - // candidate has wrong collator. - { - let mut candidate = TestCandidateBuilder { - para_id: thread_a, - relay_parent: System::parent_hash(), - pov_hash: Hash::repeat_byte(1), - persisted_validation_data_hash: make_vdata_hash(thread_a).unwrap(), - hrmp_watermark: RELAY_PARENT_NUM, - ..Default::default() - } - .build(); - - assert!(CollatorId::from(Sr25519Keyring::One.public()) != thread_collator); - collator_sign_candidate(Sr25519Keyring::One, &mut candidate); - - let backed = block_on(back_candidate( - candidate, - &validators, - group_validators(GroupIndex::from(2)).unwrap().as_ref(), - &keystore, - &signing_context, - BackingKind::Threshold, - )); - - assert_noop!( - ParaInclusion::process_candidates( - Default::default(), - vec![backed], - vec![ - chain_a_assignment.clone(), - chain_b_assignment.clone(), - thread_a_assignment.clone(), - ], - &group_validators, - FullCheck::Yes, - ), - Error::::WrongCollator, - ); - } - - // candidate not well-signed by collator. - { - let mut candidate = TestCandidateBuilder { - para_id: thread_a, - relay_parent: System::parent_hash(), - pov_hash: Hash::repeat_byte(1), - persisted_validation_data_hash: make_vdata_hash(thread_a).unwrap(), - hrmp_watermark: RELAY_PARENT_NUM, - ..Default::default() - } - .build(); - - assert_eq!(CollatorId::from(Sr25519Keyring::Two.public()), thread_collator); - collator_sign_candidate(Sr25519Keyring::Two, &mut candidate); - - // change the candidate after signing. - candidate.descriptor.pov_hash = Hash::repeat_byte(2); - - let backed = block_on(back_candidate( - candidate, - &validators, - group_validators(GroupIndex::from(2)).unwrap().as_ref(), - &keystore, - &signing_context, - BackingKind::Threshold, - )); - - assert_noop!( - ParaInclusion::process_candidates( - Default::default(), - vec![backed], - vec![thread_a_assignment.clone()], - &group_validators, - FullCheck::Yes, - ), - Error::::NotCollatorSigned - ); - } - - // para occupied - reject. - { - let mut candidate = TestCandidateBuilder { - para_id: chain_a, - relay_parent: System::parent_hash(), - pov_hash: Hash::repeat_byte(1), - persisted_validation_data_hash: make_vdata_hash(chain_a).unwrap(), - hrmp_watermark: RELAY_PARENT_NUM, - ..Default::default() - } - .build(); - - collator_sign_candidate(Sr25519Keyring::One, &mut candidate); - - let backed = block_on(back_candidate( - candidate, - &validators, - group_validators(GroupIndex::from(0)).unwrap().as_ref(), - &keystore, - &signing_context, - BackingKind::Threshold, - )); - - let candidate = TestCandidateBuilder::default().build(); - >::insert( - &chain_a, - CandidatePendingAvailability { - core: CoreIndex::from(0), - hash: candidate.hash(), - descriptor: candidate.descriptor, - availability_votes: default_availability_votes(), - relay_parent_number: 3, - backed_in_number: 4, - backers: default_backing_bitfield(), - backing_group: GroupIndex::from(0), - }, - ); - >::insert(&chain_a, candidate.commitments); - - assert_noop!( - ParaInclusion::process_candidates( - Default::default(), - vec![backed], - vec![chain_a_assignment.clone()], - &group_validators, - FullCheck::Yes, - ), - Error::::CandidateScheduledBeforeParaFree - ); - - >::remove(&chain_a); - >::remove(&chain_a); - } - - // messed up commitments storage - do not panic - reject. - { - let mut candidate = TestCandidateBuilder { - para_id: chain_a, - relay_parent: System::parent_hash(), - pov_hash: Hash::repeat_byte(1), - persisted_validation_data_hash: make_vdata_hash(chain_a).unwrap(), - hrmp_watermark: RELAY_PARENT_NUM, - ..Default::default() - } - .build(); - - collator_sign_candidate(Sr25519Keyring::One, &mut candidate); - - // this is not supposed to happen - >::insert( - &chain_a, - candidate.commitments.clone(), - ); - - let backed = block_on(back_candidate( - candidate, - &validators, - group_validators(GroupIndex::from(0)).unwrap().as_ref(), - &keystore, - &signing_context, - BackingKind::Threshold, - )); - - assert_noop!( - ParaInclusion::process_candidates( - Default::default(), - vec![backed], - vec![chain_a_assignment.clone()], - &group_validators, - FullCheck::Yes, - ), - Error::::CandidateScheduledBeforeParaFree - ); - - >::remove(&chain_a); - } - - // interfering code upgrade - reject - { - let mut candidate = TestCandidateBuilder { - para_id: chain_a, - relay_parent: System::parent_hash(), - pov_hash: Hash::repeat_byte(1), - new_validation_code: Some(vec![5, 6, 7, 8].into()), - persisted_validation_data_hash: make_vdata_hash(chain_a).unwrap(), - hrmp_watermark: RELAY_PARENT_NUM, - ..Default::default() - } - .build(); - - collator_sign_candidate(Sr25519Keyring::One, &mut candidate); - - let backed = block_on(back_candidate( - candidate, - &validators, - group_validators(GroupIndex::from(0)).unwrap().as_ref(), - &keystore, - &signing_context, - BackingKind::Threshold, - )); - - { - let cfg = Configuration::config(); - let expected_at = 10 + cfg.validation_upgrade_delay; - assert_eq!(expected_at, 10); - Paras::schedule_code_upgrade( - chain_a, - vec![1, 2, 3, 4].into(), - expected_at, - &cfg, - ); - - assert_eq!(Paras::last_code_upgrade(chain_a, true), Some(expected_at)); - } - - assert_noop!( - ParaInclusion::process_candidates( - Default::default(), - vec![backed], - vec![chain_a_assignment.clone()], - &group_validators, - FullCheck::Yes, - ), - Error::::PrematureCodeUpgrade - ); - } - - // Bad validation data hash - reject - { - let mut candidate = TestCandidateBuilder { - para_id: chain_a, - relay_parent: System::parent_hash(), - pov_hash: Hash::repeat_byte(1), - persisted_validation_data_hash: [42u8; 32].into(), - hrmp_watermark: RELAY_PARENT_NUM, - ..Default::default() - } - .build(); - - collator_sign_candidate(Sr25519Keyring::One, &mut candidate); - - let backed = block_on(back_candidate( - candidate, - &validators, - group_validators(GroupIndex::from(0)).unwrap().as_ref(), - &keystore, - &signing_context, - BackingKind::Threshold, - )); - - assert_eq!( - ParaInclusion::process_candidates( - Default::default(), - vec![backed], - vec![chain_a_assignment.clone()], - &group_validators, - FullCheck::Yes, - ), - Err(Error::::ValidationDataHashMismatch.into()), - ); - } - - // bad validation code hash - { - let mut candidate = TestCandidateBuilder { - para_id: chain_a, - relay_parent: System::parent_hash(), - pov_hash: Hash::repeat_byte(1), - persisted_validation_data_hash: make_vdata_hash(chain_a).unwrap(), - hrmp_watermark: RELAY_PARENT_NUM, - validation_code: ValidationCode(vec![1]), - ..Default::default() - } - .build(); - - collator_sign_candidate(Sr25519Keyring::One, &mut candidate); - - let backed = block_on(back_candidate( - candidate, - &validators, - group_validators(GroupIndex::from(0)).unwrap().as_ref(), - &keystore, - &signing_context, - BackingKind::Threshold, - )); - - assert_noop!( - ParaInclusion::process_candidates( - Default::default(), - vec![backed], - vec![chain_a_assignment.clone()], - &group_validators, - FullCheck::Yes, - ), - Error::::InvalidValidationCodeHash - ); - } - - // Para head hash in descriptor doesn't match head data - { - let mut candidate = TestCandidateBuilder { - para_id: chain_a, - relay_parent: System::parent_hash(), - pov_hash: Hash::repeat_byte(1), - persisted_validation_data_hash: make_vdata_hash(chain_a).unwrap(), - hrmp_watermark: RELAY_PARENT_NUM, - para_head_hash: Some(Hash::random()), - ..Default::default() - } - .build(); - - collator_sign_candidate(Sr25519Keyring::One, &mut candidate); - - let backed = block_on(back_candidate( - candidate, - &validators, - group_validators(GroupIndex::from(0)).unwrap().as_ref(), - &keystore, - &signing_context, - BackingKind::Threshold, - )); - - assert_noop!( - ParaInclusion::process_candidates( - Default::default(), - vec![backed], - vec![chain_a_assignment.clone()], - &group_validators, - FullCheck::Yes, - ), - Error::::ParaHeadMismatch - ); - } - }); - } - - #[test] - fn backing_works() { - let chain_a = ParaId::from(1); - let chain_b = ParaId::from(2); - let thread_a = ParaId::from(3); - - // The block number of the relay-parent for testing. - const RELAY_PARENT_NUM: BlockNumber = 4; - - let paras = vec![(chain_a, true), (chain_b, true), (thread_a, false)]; - let validators = vec![ - Sr25519Keyring::Alice, - Sr25519Keyring::Bob, - Sr25519Keyring::Charlie, - Sr25519Keyring::Dave, - Sr25519Keyring::Ferdie, - ]; - let keystore: SyncCryptoStorePtr = Arc::new(LocalKeystore::in_memory()); - for validator in validators.iter() { - SyncCryptoStore::sr25519_generate_new( - &*keystore, - PARACHAIN_KEY_TYPE_ID, - Some(&validator.to_seed()), - ) - .unwrap(); - } - let validator_public = validator_pubkeys(&validators); - - new_test_ext(genesis_config(paras)).execute_with(|| { - shared::Pallet::::set_active_validators_ascending(validator_public.clone()); - shared::Pallet::::set_session_index(5); - - run_to_block(5, |_| None); - - let signing_context = - SigningContext { parent_hash: System::parent_hash(), session_index: 5 }; - - let group_validators = |group_index: GroupIndex| { - match group_index { - group_index if group_index == GroupIndex::from(0) => Some(vec![0, 1]), - group_index if group_index == GroupIndex::from(1) => Some(vec![2, 3]), - group_index if group_index == GroupIndex::from(2) => Some(vec![4]), - _ => panic!("Group index out of bounds for 2 parachains and 1 parathread core"), - } - .map(|vs| vs.into_iter().map(ValidatorIndex).collect::>()) - }; - - let thread_collator: CollatorId = Sr25519Keyring::Two.public().into(); - - let chain_a_assignment = CoreAssignment { - core: CoreIndex::from(0), - para_id: chain_a, - kind: AssignmentKind::Parachain, - group_idx: GroupIndex::from(0), - }; - - let chain_b_assignment = CoreAssignment { - core: CoreIndex::from(1), - para_id: chain_b, - kind: AssignmentKind::Parachain, - group_idx: GroupIndex::from(1), - }; - - let thread_a_assignment = CoreAssignment { - core: CoreIndex::from(2), - para_id: thread_a, - kind: AssignmentKind::Parathread(thread_collator.clone(), 0), - group_idx: GroupIndex::from(2), - }; - - let mut candidate_a = TestCandidateBuilder { - para_id: chain_a, - relay_parent: System::parent_hash(), - pov_hash: Hash::repeat_byte(1), - persisted_validation_data_hash: make_vdata_hash(chain_a).unwrap(), - hrmp_watermark: RELAY_PARENT_NUM, - ..Default::default() - } - .build(); - collator_sign_candidate(Sr25519Keyring::One, &mut candidate_a); - - let mut candidate_b = TestCandidateBuilder { - para_id: chain_b, - relay_parent: System::parent_hash(), - pov_hash: Hash::repeat_byte(2), - persisted_validation_data_hash: make_vdata_hash(chain_b).unwrap(), - hrmp_watermark: RELAY_PARENT_NUM, - ..Default::default() - } - .build(); - collator_sign_candidate(Sr25519Keyring::One, &mut candidate_b); - - let mut candidate_c = TestCandidateBuilder { - para_id: thread_a, - relay_parent: System::parent_hash(), - pov_hash: Hash::repeat_byte(3), - persisted_validation_data_hash: make_vdata_hash(thread_a).unwrap(), - hrmp_watermark: RELAY_PARENT_NUM, - ..Default::default() - } - .build(); - collator_sign_candidate(Sr25519Keyring::Two, &mut candidate_c); - - let backed_a = block_on(back_candidate( - candidate_a.clone(), - &validators, - group_validators(GroupIndex::from(0)).unwrap().as_ref(), - &keystore, - &signing_context, - BackingKind::Threshold, - )); - - let backed_b = block_on(back_candidate( - candidate_b.clone(), - &validators, - group_validators(GroupIndex::from(1)).unwrap().as_ref(), - &keystore, - &signing_context, - BackingKind::Threshold, - )); - - let backed_c = block_on(back_candidate( - candidate_c.clone(), - &validators, - group_validators(GroupIndex::from(2)).unwrap().as_ref(), - &keystore, - &signing_context, - BackingKind::Threshold, - )); - - let backed_candidates = vec![backed_a, backed_b, backed_c]; - let get_backing_group_idx = { - // the order defines the group implicitly for this test case - let backed_candidates_with_groups = backed_candidates - .iter() - .enumerate() - .map(|(idx, backed_candidate)| (backed_candidate.hash(), GroupIndex(idx as _))) - .collect::>(); - - move |candidate_hash_x: CandidateHash| -> Option { - backed_candidates_with_groups.iter().find_map(|(candidate_hash, grp)| { - if *candidate_hash == candidate_hash_x { - Some(*grp) - } else { - None - } - }) - } - }; - - let ProcessedCandidates { - core_indices: occupied_cores, - candidate_receipt_with_backing_validator_indices, - } = ParaInclusion::process_candidates( - Default::default(), - backed_candidates.clone(), - vec![ - chain_a_assignment.clone(), - chain_b_assignment.clone(), - thread_a_assignment.clone(), - ], - &group_validators, - FullCheck::Yes, - ) - .expect("candidates scheduled, in order, and backed"); - - assert_eq!( - occupied_cores, - vec![CoreIndex::from(0), CoreIndex::from(1), CoreIndex::from(2)] - ); - - // Transform the votes into the setup we expect - let expected = { - let mut intermediate = std::collections::HashMap::< - CandidateHash, - (CandidateReceipt, Vec<(ValidatorIndex, ValidityAttestation)>), - >::new(); - backed_candidates.into_iter().for_each(|backed_candidate| { - let candidate_receipt_with_backers = intermediate - .entry(backed_candidate.hash()) - .or_insert_with(|| (backed_candidate.receipt(), Vec::new())); - - assert_eq!( - backed_candidate.validity_votes.len(), - backed_candidate.validator_indices.count_ones() - ); - candidate_receipt_with_backers.1.extend( - backed_candidate - .validator_indices - .iter() - .enumerate() - .filter(|(_, signed)| **signed) - .zip(backed_candidate.validity_votes.iter().cloned()) - .filter_map(|((validator_index_within_group, _), attestation)| { - let grp_idx = - get_backing_group_idx(backed_candidate.hash()).unwrap(); - group_validators(grp_idx).map(|validator_indices| { - (validator_indices[validator_index_within_group], attestation) - }) - }), - ); - }); - intermediate.into_values().collect::>() - }; - - // sort, since we use a hashmap above - let assure_candidate_sorting = |mut candidate_receipts_with_backers: Vec<( - CandidateReceipt, - Vec<(ValidatorIndex, ValidityAttestation)>, - )>| { - candidate_receipts_with_backers.sort_by(|(cr1, _), (cr2, _)| { - cr1.descriptor().para_id.cmp(&cr2.descriptor().para_id) - }); - candidate_receipts_with_backers - }; - assert_eq!( - assure_candidate_sorting(expected), - assure_candidate_sorting(candidate_receipt_with_backing_validator_indices) - ); - - assert_eq!( - >::get(&chain_a), - Some(CandidatePendingAvailability { - core: CoreIndex::from(0), - hash: candidate_a.hash(), - descriptor: candidate_a.descriptor, - availability_votes: default_availability_votes(), - relay_parent_number: System::block_number() - 1, - backed_in_number: System::block_number(), - backers: backing_bitfield(&[0, 1]), - backing_group: GroupIndex::from(0), - }) - ); - assert_eq!( - >::get(&chain_a), - Some(candidate_a.commitments), - ); - - assert_eq!( - >::get(&chain_b), - Some(CandidatePendingAvailability { - core: CoreIndex::from(1), - hash: candidate_b.hash(), - descriptor: candidate_b.descriptor, - availability_votes: default_availability_votes(), - relay_parent_number: System::block_number() - 1, - backed_in_number: System::block_number(), - backers: backing_bitfield(&[2, 3]), - backing_group: GroupIndex::from(1), - }) - ); - assert_eq!( - >::get(&chain_b), - Some(candidate_b.commitments), - ); - - assert_eq!( - >::get(&thread_a), - Some(CandidatePendingAvailability { - core: CoreIndex::from(2), - hash: candidate_c.hash(), - descriptor: candidate_c.descriptor, - availability_votes: default_availability_votes(), - relay_parent_number: System::block_number() - 1, - backed_in_number: System::block_number(), - backers: backing_bitfield(&[4]), - backing_group: GroupIndex::from(2), - }) - ); - assert_eq!( - >::get(&thread_a), - Some(candidate_c.commitments), - ); - }); - } - - #[test] - fn can_include_candidate_with_ok_code_upgrade() { - let chain_a = ParaId::from(1); - - // The block number of the relay-parent for testing. - const RELAY_PARENT_NUM: BlockNumber = 4; - - let paras = vec![(chain_a, true)]; - let validators = vec![ - Sr25519Keyring::Alice, - Sr25519Keyring::Bob, - Sr25519Keyring::Charlie, - Sr25519Keyring::Dave, - Sr25519Keyring::Ferdie, - ]; - let keystore: SyncCryptoStorePtr = Arc::new(LocalKeystore::in_memory()); - for validator in validators.iter() { - SyncCryptoStore::sr25519_generate_new( - &*keystore, - PARACHAIN_KEY_TYPE_ID, - Some(&validator.to_seed()), - ) - .unwrap(); - } - let validator_public = validator_pubkeys(&validators); - - new_test_ext(genesis_config(paras)).execute_with(|| { - shared::Pallet::::set_active_validators_ascending(validator_public.clone()); - shared::Pallet::::set_session_index(5); - - run_to_block(5, |_| None); - - let signing_context = - SigningContext { parent_hash: System::parent_hash(), session_index: 5 }; - - let group_validators = |group_index: GroupIndex| { - match group_index { - group_index if group_index == GroupIndex::from(0) => Some(vec![0, 1, 2, 3, 4]), - _ => panic!("Group index out of bounds for 1 parachain"), - } - .map(|vs| vs.into_iter().map(ValidatorIndex).collect::>()) - }; - - let chain_a_assignment = CoreAssignment { - core: CoreIndex::from(0), - para_id: chain_a, - kind: AssignmentKind::Parachain, - group_idx: GroupIndex::from(0), - }; - - let mut candidate_a = TestCandidateBuilder { - para_id: chain_a, - relay_parent: System::parent_hash(), - pov_hash: Hash::repeat_byte(1), - persisted_validation_data_hash: make_vdata_hash(chain_a).unwrap(), - new_validation_code: Some(vec![1, 2, 3].into()), - hrmp_watermark: RELAY_PARENT_NUM, - ..Default::default() - } - .build(); - collator_sign_candidate(Sr25519Keyring::One, &mut candidate_a); - - let backed_a = block_on(back_candidate( - candidate_a.clone(), - &validators, - group_validators(GroupIndex::from(0)).unwrap().as_ref(), - &keystore, - &signing_context, - BackingKind::Threshold, - )); - - let ProcessedCandidates { core_indices: occupied_cores, .. } = - ParaInclusion::process_candidates( - Default::default(), - vec![backed_a], - vec![chain_a_assignment.clone()], - &group_validators, - FullCheck::Yes, - ) - .expect("candidates scheduled, in order, and backed"); - - assert_eq!(occupied_cores, vec![CoreIndex::from(0)]); - - assert_eq!( - >::get(&chain_a), - Some(CandidatePendingAvailability { - core: CoreIndex::from(0), - hash: candidate_a.hash(), - descriptor: candidate_a.descriptor, - availability_votes: default_availability_votes(), - relay_parent_number: System::block_number() - 1, - backed_in_number: System::block_number(), - backers: backing_bitfield(&[0, 1, 2]), - backing_group: GroupIndex::from(0), - }) - ); - assert_eq!( - >::get(&chain_a), - Some(candidate_a.commitments), - ); - }); - } - - #[test] - fn session_change_wipes() { - let chain_a = ParaId::from(1); - let chain_b = ParaId::from(2); - let thread_a = ParaId::from(3); - - let paras = vec![(chain_a, true), (chain_b, true), (thread_a, false)]; - let validators = vec![ - Sr25519Keyring::Alice, - Sr25519Keyring::Bob, - Sr25519Keyring::Charlie, - Sr25519Keyring::Dave, - Sr25519Keyring::Ferdie, - ]; - let keystore: SyncCryptoStorePtr = Arc::new(LocalKeystore::in_memory()); - for validator in validators.iter() { - SyncCryptoStore::sr25519_generate_new( - &*keystore, - PARACHAIN_KEY_TYPE_ID, - Some(&validator.to_seed()), - ) - .unwrap(); - } - let validator_public = validator_pubkeys(&validators); - - new_test_ext(genesis_config(paras)).execute_with(|| { - shared::Pallet::::set_active_validators_ascending(validator_public.clone()); - shared::Pallet::::set_session_index(5); - - let validators_new = - vec![Sr25519Keyring::Alice, Sr25519Keyring::Bob, Sr25519Keyring::Charlie]; - - let validator_public_new = validator_pubkeys(&validators_new); - - run_to_block(10, |_| None); - - >::insert( - &ValidatorIndex(0), - AvailabilityBitfieldRecord { bitfield: default_bitfield(), submitted_at: 9 }, - ); - - >::insert( - &ValidatorIndex(1), - AvailabilityBitfieldRecord { bitfield: default_bitfield(), submitted_at: 9 }, - ); - - >::insert( - &ValidatorIndex(4), - AvailabilityBitfieldRecord { bitfield: default_bitfield(), submitted_at: 9 }, - ); - - let candidate = TestCandidateBuilder::default().build(); - >::insert( - &chain_a, - CandidatePendingAvailability { - core: CoreIndex::from(0), - hash: candidate.hash(), - descriptor: candidate.descriptor.clone(), - availability_votes: default_availability_votes(), - relay_parent_number: 5, - backed_in_number: 6, - backers: default_backing_bitfield(), - backing_group: GroupIndex::from(0), - }, - ); - >::insert(&chain_a, candidate.commitments.clone()); - - >::insert( - &chain_b, - CandidatePendingAvailability { - core: CoreIndex::from(1), - hash: candidate.hash(), - descriptor: candidate.descriptor, - availability_votes: default_availability_votes(), - relay_parent_number: 6, - backed_in_number: 7, - backers: default_backing_bitfield(), - backing_group: GroupIndex::from(1), - }, - ); - >::insert(&chain_b, candidate.commitments); - - run_to_block(11, |_| None); - - assert_eq!(shared::Pallet::::session_index(), 5); - - assert!(>::get(&ValidatorIndex(0)).is_some()); - assert!(>::get(&ValidatorIndex(1)).is_some()); - assert!(>::get(&ValidatorIndex(4)).is_some()); - - assert!(>::get(&chain_a).is_some()); - assert!(>::get(&chain_b).is_some()); - assert!(>::get(&chain_a).is_some()); - assert!(>::get(&chain_b).is_some()); - - run_to_block(12, |n| match n { - 12 => Some(SessionChangeNotification { - validators: validator_public_new.clone(), - queued: Vec::new(), - prev_config: default_config(), - new_config: default_config(), - random_seed: Default::default(), - session_index: 6, - }), - _ => None, - }); - - assert_eq!(shared::Pallet::::session_index(), 6); - - assert!(>::get(&ValidatorIndex(0)).is_none()); - assert!(>::get(&ValidatorIndex(1)).is_none()); - assert!(>::get(&ValidatorIndex(4)).is_none()); - - assert!(>::get(&chain_a).is_none()); - assert!(>::get(&chain_b).is_none()); - assert!(>::get(&chain_a).is_none()); - assert!(>::get(&chain_b).is_none()); - - assert!(>::iter().collect::>().is_empty()); - assert!(>::iter().collect::>().is_empty()); - assert!(>::iter().collect::>().is_empty()); - }); - } - - // TODO [now]: test `collect_disputed` -} diff --git a/polkadot/runtime/parachains/src/inclusion/mod.rs b/polkadot/runtime/parachains/src/inclusion/mod.rs new file mode 100644 index 0000000000..9aa7d86e98 --- /dev/null +++ b/polkadot/runtime/parachains/src/inclusion/mod.rs @@ -0,0 +1,1043 @@ +// Copyright 2020 Parity Technologies (UK) Ltd. +// This file is part of Polkadot. + +// Polkadot is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Polkadot is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Polkadot. If not, see . + +//! The inclusion pallet is responsible for inclusion and availability of scheduled parachains +//! and parathreads. +//! +//! It is responsible for carrying candidates from being backable to being backed, and then from backed +//! to included. + +use crate::{ + configuration, disputes, dmp, hrmp, paras, + paras_inherent::{sanitize_bitfields, DisputedBitfield}, + scheduler::CoreAssignment, + shared, ump, +}; +use bitvec::{order::Lsb0 as BitOrderLsb0, vec::BitVec}; +use frame_support::pallet_prelude::*; +use parity_scale_codec::{Decode, Encode}; +use primitives::v1::{ + AvailabilityBitfield, BackedCandidate, CandidateCommitments, CandidateDescriptor, + CandidateHash, CandidateReceipt, CommittedCandidateReceipt, CoreIndex, GroupIndex, Hash, + HeadData, Id as ParaId, SigningContext, UncheckedSignedAvailabilityBitfields, ValidatorId, + ValidatorIndex, ValidityAttestation, +}; +use scale_info::TypeInfo; +use sp_runtime::{ + traits::{One, Saturating}, + DispatchError, +}; +use sp_std::{collections::btree_set::BTreeSet, prelude::*}; + +pub use pallet::*; + +#[cfg(test)] +pub(crate) mod tests; + +/// A bitfield signed by a validator indicating that it is keeping its piece of the erasure-coding +/// for any backed candidates referred to by a `1` bit available. +/// +/// The bitfield's signature should be checked at the point of submission. Afterwards it can be +/// dropped. +#[derive(Encode, Decode, TypeInfo)] +#[cfg_attr(test, derive(Debug))] +pub struct AvailabilityBitfieldRecord { + bitfield: AvailabilityBitfield, // one bit per core. + submitted_at: N, // for accounting, as meaning of bits may change over time. +} + +/// Determines if all checks should be applied or if a subset was already completed +/// in a code path that will be executed afterwards or was already executed before. +#[derive(Encode, Decode, PartialEq, Eq, RuntimeDebug, TypeInfo)] +pub(crate) enum FullCheck { + /// Yes, do a full check, skip nothing. + Yes, + /// Skip a subset of checks that are already completed before. + /// + /// Attention: Should only be used when absolutely sure that the required + /// checks are completed before. + Skip, +} + +/// A backed candidate pending availability. +#[derive(Encode, Decode, PartialEq, TypeInfo)] +#[cfg_attr(test, derive(Debug, Default))] +pub struct CandidatePendingAvailability { + /// The availability core this is assigned to. + core: CoreIndex, + /// The candidate hash. + hash: CandidateHash, + /// The candidate descriptor. + descriptor: CandidateDescriptor, + /// The received availability votes. One bit per validator. + availability_votes: BitVec, + /// The backers of the candidate pending availability. + backers: BitVec, + /// The block number of the relay-parent of the receipt. + relay_parent_number: N, + /// The block number of the relay-chain block this was backed in. + backed_in_number: N, + /// The group index backing this block. + backing_group: GroupIndex, +} + +impl CandidatePendingAvailability { + /// Get the availability votes on the candidate. + pub(crate) fn availability_votes(&self) -> &BitVec { + &self.availability_votes + } + + /// Get the relay-chain block number this was backed in. + pub(crate) fn backed_in_number(&self) -> &N { + &self.backed_in_number + } + + /// Get the core index. + pub(crate) fn core_occupied(&self) -> CoreIndex { + self.core.clone() + } + + /// Get the candidate hash. + pub(crate) fn candidate_hash(&self) -> CandidateHash { + self.hash + } + + /// Get the candidate descriptor. + pub(crate) fn candidate_descriptor(&self) -> &CandidateDescriptor { + &self.descriptor + } + + #[cfg(any(feature = "runtime-benchmarks", test))] + pub(crate) fn new( + core: CoreIndex, + hash: CandidateHash, + descriptor: CandidateDescriptor, + availability_votes: BitVec, + backers: BitVec, + relay_parent_number: N, + backed_in_number: N, + backing_group: GroupIndex, + ) -> Self { + Self { + core, + hash, + descriptor, + availability_votes, + backers, + relay_parent_number, + backed_in_number, + backing_group, + } + } +} + +/// A hook for applying validator rewards +pub trait RewardValidators { + // Reward the validators with the given indices for issuing backing statements. + fn reward_backing(validators: impl IntoIterator); + // Reward the validators with the given indices for issuing availability bitfields. + // Validators are sent to this hook when they have contributed to the availability + // of a candidate by setting a bit in their bitfield. + fn reward_bitfields(validators: impl IntoIterator); +} + +/// Helper return type for `process_candidates`. +#[derive(Encode, Decode, PartialEq, TypeInfo)] +#[cfg_attr(test, derive(Debug))] +pub(crate) struct ProcessedCandidates { + pub(crate) core_indices: Vec, + pub(crate) candidate_receipt_with_backing_validator_indices: + Vec<(CandidateReceipt, Vec<(ValidatorIndex, ValidityAttestation)>)>, +} + +impl Default for ProcessedCandidates { + fn default() -> Self { + Self { + core_indices: Vec::new(), + candidate_receipt_with_backing_validator_indices: Vec::new(), + } + } +} + +#[frame_support::pallet] +pub mod pallet { + use super::*; + + #[pallet::pallet] + #[pallet::generate_store(pub(super) trait Store)] + pub struct Pallet(_); + + #[pallet::config] + pub trait Config: + frame_system::Config + + shared::Config + + paras::Config + + dmp::Config + + ump::Config + + hrmp::Config + + configuration::Config + { + type Event: From> + IsType<::Event>; + type DisputesHandler: disputes::DisputesHandler; + type RewardValidators: RewardValidators; + } + + #[pallet::event] + #[pallet::generate_deposit(pub(super) fn deposit_event)] + pub enum Event { + /// A candidate was backed. `[candidate, head_data]` + CandidateBacked(CandidateReceipt, HeadData, CoreIndex, GroupIndex), + /// A candidate was included. `[candidate, head_data]` + CandidateIncluded(CandidateReceipt, HeadData, CoreIndex, GroupIndex), + /// A candidate timed out. `[candidate, head_data]` + CandidateTimedOut(CandidateReceipt, HeadData, CoreIndex), + } + + #[pallet::error] + pub enum Error { + /// Availability bitfield has unexpected size. + WrongBitfieldSize, + /// Multiple bitfields submitted by same validator or validators out of order by index. + BitfieldDuplicateOrUnordered, + /// Validator index out of bounds. + ValidatorIndexOutOfBounds, + /// Invalid signature + InvalidBitfieldSignature, + /// Candidate submitted but para not scheduled. + UnscheduledCandidate, + /// Candidate scheduled despite pending candidate already existing for the para. + CandidateScheduledBeforeParaFree, + /// Candidate included with the wrong collator. + WrongCollator, + /// Scheduled cores out of order. + ScheduledOutOfOrder, + /// Head data exceeds the configured maximum. + HeadDataTooLarge, + /// Code upgrade prematurely. + PrematureCodeUpgrade, + /// Output code is too large + NewCodeTooLarge, + /// Candidate not in parent context. + CandidateNotInParentContext, + /// Invalid group index in core assignment. + InvalidGroupIndex, + /// Insufficient (non-majority) backing. + InsufficientBacking, + /// Invalid (bad signature, unknown validator, etc.) backing. + InvalidBacking, + /// Collator did not sign PoV. + NotCollatorSigned, + /// The validation data hash does not match expected. + ValidationDataHashMismatch, + /// The downward message queue is not processed correctly. + IncorrectDownwardMessageHandling, + /// At least one upward message sent does not pass the acceptance criteria. + InvalidUpwardMessages, + /// The candidate didn't follow the rules of HRMP watermark advancement. + HrmpWatermarkMishandling, + /// The HRMP messages sent by the candidate is not valid. + InvalidOutboundHrmp, + /// The validation code hash of the candidate is not valid. + InvalidValidationCodeHash, + /// The `para_head` hash in the candidate descriptor doesn't match the hash of the actual para head in the + /// commitments. + ParaHeadMismatch, + /// A bitfield that references a freed core, + /// either intentionally or as part of a concluded + /// invalid dispute. + BitfieldReferencesFreedCore, + } + + /// The latest bitfield for each validator, referred to by their index in the validator set. + #[pallet::storage] + pub(crate) type AvailabilityBitfields = + StorageMap<_, Twox64Concat, ValidatorIndex, AvailabilityBitfieldRecord>; + + /// Candidates pending availability by `ParaId`. + #[pallet::storage] + pub(crate) type PendingAvailability = + StorageMap<_, Twox64Concat, ParaId, CandidatePendingAvailability>; + + /// The commitments of candidates pending availability, by `ParaId`. + #[pallet::storage] + pub(crate) type PendingAvailabilityCommitments = + StorageMap<_, Twox64Concat, ParaId, CandidateCommitments>; + + #[pallet::call] + impl Pallet {} +} + +const LOG_TARGET: &str = "runtime::inclusion"; + +impl Pallet { + /// Block initialization logic, called by initializer. + pub(crate) fn initializer_initialize(_now: T::BlockNumber) -> Weight { + 0 + } + + /// Block finalization logic, called by initializer. + pub(crate) fn initializer_finalize() {} + + /// Handle an incoming session change. + pub(crate) fn initializer_on_new_session( + _notification: &crate::initializer::SessionChangeNotification, + ) { + // unlike most drain methods, drained elements are not cleared on `Drop` of the iterator + // and require consumption. + for _ in >::drain() {} + for _ in >::drain() {} + for _ in >::drain() {} + } + + /// Extract the freed cores based on cores that became available. + /// + /// Updates storage items `PendingAvailability` and `AvailabilityBitfields`. + pub(crate) fn update_pending_availability_and_get_freed_cores( + expected_bits: usize, + validators: &[ValidatorId], + signed_bitfields: UncheckedSignedAvailabilityBitfields, + core_lookup: F, + ) -> Vec<(CoreIndex, CandidateHash)> + where + F: Fn(CoreIndex) -> Option, + { + let mut assigned_paras_record = (0..expected_bits) + .map(|bit_index| core_lookup(CoreIndex::from(bit_index as u32))) + .map(|opt_para_id| { + opt_para_id.map(|para_id| (para_id, PendingAvailability::::get(¶_id))) + }) + .collect::>(); + + let now = >::block_number(); + for (checked_bitfield, validator_index) in + signed_bitfields.into_iter().map(|signed_bitfield| { + // extracting unchecked data, since it's checked in `fn sanitize_bitfields` already. + let validator_idx = signed_bitfield.unchecked_validator_index(); + let checked_bitfield = signed_bitfield.unchecked_into_payload(); + (checked_bitfield, validator_idx) + }) { + for (bit_idx, _) in checked_bitfield.0.iter().enumerate().filter(|(_, is_av)| **is_av) { + let pending_availability = if let Some((_, pending_availability)) = + assigned_paras_record[bit_idx].as_mut() + { + pending_availability + } else { + // For honest validators, this happens in case of unoccupied cores, + // which in turn happens in case of a disputed candidate. + // A malicious one might include arbitrary indices, but they are represented + // by `None` values and will be sorted out in the next if case. + continue + }; + + // defensive check - this is constructed by loading the availability bitfield record, + // which is always `Some` if the core is occupied - that's why we're here. + let validator_index = validator_index.0 as usize; + if let Some(mut bit) = + pending_availability.as_mut().and_then(|candidate_pending_availability| { + candidate_pending_availability.availability_votes.get_mut(validator_index) + }) { + *bit = true; + } + } + + let record = + AvailabilityBitfieldRecord { bitfield: checked_bitfield, submitted_at: now }; + + >::insert(&validator_index, record); + } + + let threshold = availability_threshold(validators.len()); + + let mut freed_cores = Vec::with_capacity(expected_bits); + for (para_id, pending_availability) in assigned_paras_record + .into_iter() + .filter_map(|x| x) + .filter_map(|(id, p)| p.map(|p| (id, p))) + { + if pending_availability.availability_votes.count_ones() >= threshold { + >::remove(¶_id); + let commitments = match PendingAvailabilityCommitments::::take(¶_id) { + Some(commitments) => commitments, + None => { + log::warn!( + target: LOG_TARGET, + "Inclusion::process_bitfields: PendingAvailability and PendingAvailabilityCommitments + are out of sync, did someone mess with the storage?", + ); + continue + }, + }; + + if ON_CHAIN_USE { + let receipt = CommittedCandidateReceipt { + descriptor: pending_availability.descriptor, + commitments, + }; + let _weight = Self::enact_candidate( + pending_availability.relay_parent_number, + receipt, + pending_availability.backers, + pending_availability.availability_votes, + pending_availability.core, + pending_availability.backing_group, + ); + } + + freed_cores.push((pending_availability.core, pending_availability.hash)); + } else { + >::insert(¶_id, &pending_availability); + } + } + + freed_cores + } + + /// Process a set of incoming bitfields. + /// + /// Returns a `Vec` of `CandidateHash`es and their respective `AvailabilityCore`s that became available, + /// and cores free. + pub(crate) fn process_bitfields( + expected_bits: usize, + signed_bitfields: UncheckedSignedAvailabilityBitfields, + disputed_bitfield: DisputedBitfield, + core_lookup: impl Fn(CoreIndex) -> Option, + ) -> Vec<(CoreIndex, CandidateHash)> { + let validators = shared::Pallet::::active_validator_keys(); + let session_index = shared::Pallet::::session_index(); + let parent_hash = frame_system::Pallet::::parent_hash(); + + let checked_bitfields = sanitize_bitfields::( + signed_bitfields, + disputed_bitfield, + expected_bits, + parent_hash, + session_index, + &validators[..], + FullCheck::Yes, + ); + + let freed_cores = Self::update_pending_availability_and_get_freed_cores::<_, true>( + expected_bits, + &validators[..], + checked_bitfields, + core_lookup, + ); + + freed_cores + } + + /// Process candidates that have been backed. Provide the relay storage root, a set of candidates + /// and scheduled cores. + /// + /// Both should be sorted ascending by core index, and the candidates should be a subset of + /// scheduled cores. If these conditions are not met, the execution of the function fails. + pub(crate) fn process_candidates( + parent_storage_root: T::Hash, + candidates: Vec>, + scheduled: Vec, + group_validators: GV, + full_check: FullCheck, + ) -> Result, DispatchError> + where + GV: Fn(GroupIndex) -> Option>, + { + ensure!(candidates.len() <= scheduled.len(), Error::::UnscheduledCandidate); + + if scheduled.is_empty() { + return Ok(ProcessedCandidates::default()) + } + + let validators = shared::Pallet::::active_validator_keys(); + let parent_hash = >::parent_hash(); + + // At the moment we assume (and in fact enforce, below) that the relay-parent is always one + // before of the block where we include a candidate (i.e. this code path). + let now = >::block_number(); + let relay_parent_number = now - One::one(); + let check_ctx = CandidateCheckContext::::new(now, relay_parent_number); + + // Collect candidate receipts with backers. + let mut candidate_receipt_with_backing_validator_indices = + Vec::with_capacity(candidates.len()); + + // Do all checks before writing storage. + let core_indices_and_backers = { + let mut skip = 0; + let mut core_indices_and_backers = Vec::with_capacity(candidates.len()); + let mut last_core = None; + + let mut check_assignment_in_order = |assignment: &CoreAssignment| -> DispatchResult { + ensure!( + last_core.map_or(true, |core| assignment.core > core), + Error::::ScheduledOutOfOrder, + ); + + last_core = Some(assignment.core); + Ok(()) + }; + + let signing_context = + SigningContext { parent_hash, session_index: shared::Pallet::::session_index() }; + + // We combine an outer loop over candidates with an inner loop over the scheduled, + // where each iteration of the outer loop picks up at the position + // in scheduled just after the past iteration left off. + // + // If the candidates appear in the same order as they appear in `scheduled`, + // then they should always be found. If the end of `scheduled` is reached, + // then the candidate was either not scheduled or out-of-order. + // + // In the meantime, we do certain sanity checks on the candidates and on the scheduled + // list. + 'next_backed_candidate: for (candidate_idx, backed_candidate) in + candidates.iter().enumerate() + { + if let FullCheck::Yes = full_check { + check_ctx.verify_backed_candidate( + parent_hash, + candidate_idx, + backed_candidate, + )?; + } + + let para_id = backed_candidate.descriptor().para_id; + let mut backers = bitvec::bitvec![BitOrderLsb0, u8; 0; validators.len()]; + + for (i, assignment) in scheduled[skip..].iter().enumerate() { + check_assignment_in_order(assignment)?; + + if para_id == assignment.para_id { + if let Some(required_collator) = assignment.required_collator() { + ensure!( + required_collator == &backed_candidate.descriptor().collator, + Error::::WrongCollator, + ); + } + + { + // this should never fail because the para is registered + let persisted_validation_data = + match crate::util::make_persisted_validation_data::( + para_id, + relay_parent_number, + parent_storage_root, + ) { + Some(l) => l, + None => { + // We don't want to error out here because it will + // brick the relay-chain. So we return early without + // doing anything. + return Ok(ProcessedCandidates::default()) + }, + }; + + let expected = persisted_validation_data.hash(); + + ensure!( + expected == + backed_candidate.descriptor().persisted_validation_data_hash, + Error::::ValidationDataHashMismatch, + ); + } + + ensure!( + >::get(¶_id).is_none() && + >::get(¶_id).is_none(), + Error::::CandidateScheduledBeforeParaFree, + ); + + // account for already skipped, and then skip this one. + skip = i + skip + 1; + + let group_vals = group_validators(assignment.group_idx) + .ok_or_else(|| Error::::InvalidGroupIndex)?; + + // check the signatures in the backing and that it is a majority. + { + let maybe_amount_validated = primitives::v1::check_candidate_backing( + &backed_candidate, + &signing_context, + group_vals.len(), + |intra_group_vi| { + group_vals + .get(intra_group_vi) + .and_then(|vi| validators.get(vi.0 as usize)) + .map(|v| v.clone()) + }, + ); + + match maybe_amount_validated { + Ok(amount_validated) => ensure!( + amount_validated * 2 > group_vals.len(), + Error::::InsufficientBacking, + ), + Err(()) => { + Err(Error::::InvalidBacking)?; + }, + } + + let mut backer_idx_and_attestation = + Vec::<(ValidatorIndex, ValidityAttestation)>::with_capacity( + backed_candidate.validator_indices.count_ones(), + ); + let candidate_receipt = backed_candidate.receipt(); + + for ((bit_idx, _), attestation) in backed_candidate + .validator_indices + .iter() + .enumerate() + .filter(|(_, signed)| **signed) + .zip(backed_candidate.validity_votes.iter().cloned()) + { + let val_idx = group_vals + .get(bit_idx) + .expect("this query succeeded above; qed"); + backer_idx_and_attestation.push((*val_idx, attestation)); + + backers.set(val_idx.0 as _, true); + } + candidate_receipt_with_backing_validator_indices + .push((candidate_receipt, backer_idx_and_attestation)); + } + + core_indices_and_backers.push(( + assignment.core, + backers, + assignment.group_idx, + )); + continue 'next_backed_candidate + } + } + + // end of loop reached means that the candidate didn't appear in the non-traversed + // section of the `scheduled` slice. either it was not scheduled or didn't appear in + // `candidates` in the correct order. + ensure!(false, Error::::UnscheduledCandidate); + } + + // check remainder of scheduled cores, if any. + for assignment in scheduled[skip..].iter() { + check_assignment_in_order(assignment)?; + } + + core_indices_and_backers + }; + + // one more sweep for actually writing to storage. + let core_indices = + core_indices_and_backers.iter().map(|&(ref c, _, _)| c.clone()).collect(); + for (candidate, (core, backers, group)) in + candidates.into_iter().zip(core_indices_and_backers) + { + let para_id = candidate.descriptor().para_id; + + // initialize all availability votes to 0. + let availability_votes: BitVec = + bitvec::bitvec![BitOrderLsb0, u8; 0; validators.len()]; + + Self::deposit_event(Event::::CandidateBacked( + candidate.candidate.to_plain(), + candidate.candidate.commitments.head_data.clone(), + core, + group, + )); + + let candidate_hash = candidate.candidate.hash(); + + let (descriptor, commitments) = + (candidate.candidate.descriptor, candidate.candidate.commitments); + + >::insert( + ¶_id, + CandidatePendingAvailability { + core, + hash: candidate_hash, + descriptor, + availability_votes, + relay_parent_number, + backers: backers.to_bitvec(), + backed_in_number: check_ctx.now, + backing_group: group, + }, + ); + >::insert(¶_id, commitments); + } + + Ok(ProcessedCandidates:: { + core_indices, + candidate_receipt_with_backing_validator_indices, + }) + } + + /// Run the acceptance criteria checks on the given candidate commitments. + pub(crate) fn check_validation_outputs_for_runtime_api( + para_id: ParaId, + validation_outputs: primitives::v1::CandidateCommitments, + ) -> bool { + // This function is meant to be called from the runtime APIs against the relay-parent, hence + // `relay_parent_number` is equal to `now`. + let now = >::block_number(); + let relay_parent_number = now; + let check_ctx = CandidateCheckContext::::new(now, relay_parent_number); + + if let Err(err) = check_ctx.check_validation_outputs( + para_id, + &validation_outputs.head_data, + &validation_outputs.new_validation_code, + validation_outputs.processed_downward_messages, + &validation_outputs.upward_messages, + T::BlockNumber::from(validation_outputs.hrmp_watermark), + &validation_outputs.horizontal_messages, + ) { + log::debug!( + target: LOG_TARGET, + "Validation outputs checking for parachain `{}` failed: {:?}", + u32::from(para_id), + err, + ); + false + } else { + true + } + } + + fn enact_candidate( + relay_parent_number: T::BlockNumber, + receipt: CommittedCandidateReceipt, + backers: BitVec, + availability_votes: BitVec, + core_index: CoreIndex, + backing_group: GroupIndex, + ) -> Weight { + let plain = receipt.to_plain(); + let commitments = receipt.commitments; + let config = >::config(); + + T::RewardValidators::reward_backing( + backers + .iter() + .enumerate() + .filter(|(_, backed)| **backed) + .map(|(i, _)| ValidatorIndex(i as _)), + ); + + T::RewardValidators::reward_bitfields( + availability_votes + .iter() + .enumerate() + .filter(|(_, voted)| **voted) + .map(|(i, _)| ValidatorIndex(i as _)), + ); + + // initial weight is config read. + let mut weight = T::DbWeight::get().reads_writes(1, 0); + if let Some(new_code) = commitments.new_validation_code { + weight += >::schedule_code_upgrade( + receipt.descriptor.para_id, + new_code, + relay_parent_number, + &config, + ); + } + + // enact the messaging facet of the candidate. + weight += >::prune_dmq( + receipt.descriptor.para_id, + commitments.processed_downward_messages, + ); + weight += >::receive_upward_messages( + receipt.descriptor.para_id, + commitments.upward_messages, + ); + weight += >::prune_hrmp( + receipt.descriptor.para_id, + T::BlockNumber::from(commitments.hrmp_watermark), + ); + weight += >::queue_outbound_hrmp( + receipt.descriptor.para_id, + commitments.horizontal_messages, + ); + + Self::deposit_event(Event::::CandidateIncluded( + plain, + commitments.head_data.clone(), + core_index, + backing_group, + )); + + weight + + >::note_new_head( + receipt.descriptor.para_id, + commitments.head_data, + relay_parent_number, + ) + } + + /// Cleans up all paras pending availability that the predicate returns true for. + /// + /// The predicate accepts the index of the core and the block number the core has been occupied + /// since (i.e. the block number the candidate was backed at in this fork of the relay chain). + /// + /// Returns a vector of cleaned-up core IDs. + pub(crate) fn collect_pending( + pred: impl Fn(CoreIndex, T::BlockNumber) -> bool, + ) -> Vec { + let mut cleaned_up_ids = Vec::new(); + let mut cleaned_up_cores = Vec::new(); + + for (para_id, pending_record) in >::iter() { + if pred(pending_record.core, pending_record.backed_in_number) { + cleaned_up_ids.push(para_id); + cleaned_up_cores.push(pending_record.core); + } + } + + for para_id in cleaned_up_ids { + let pending = >::take(¶_id); + let commitments = >::take(¶_id); + + if let (Some(pending), Some(commitments)) = (pending, commitments) { + // defensive: this should always be true. + let candidate = CandidateReceipt { + descriptor: pending.descriptor, + commitments_hash: commitments.hash(), + }; + + Self::deposit_event(Event::::CandidateTimedOut( + candidate, + commitments.head_data, + pending.core, + )); + } + } + + cleaned_up_cores + } + + /// Cleans up all paras pending availability that are in the given list of disputed candidates. + /// + /// Returns a vector of cleaned-up core IDs. + pub(crate) fn collect_disputed(disputed: &BTreeSet) -> Vec { + let mut cleaned_up_ids = Vec::new(); + let mut cleaned_up_cores = Vec::new(); + + for (para_id, pending_record) in >::iter() { + if disputed.contains(&pending_record.hash) { + cleaned_up_ids.push(para_id); + cleaned_up_cores.push(pending_record.core); + } + } + + for para_id in cleaned_up_ids { + let _ = >::take(¶_id); + let _ = >::take(¶_id); + } + + cleaned_up_cores + } + + /// Forcibly enact the candidate with the given ID as though it had been deemed available + /// by bitfields. + /// + /// Is a no-op if there is no candidate pending availability for this para-id. + /// This should generally not be used but it is useful during execution of Runtime APIs, + /// where the changes to the state are expected to be discarded directly after. + pub(crate) fn force_enact(para: ParaId) { + let pending = >::take(¶); + let commitments = >::take(¶); + + if let (Some(pending), Some(commitments)) = (pending, commitments) { + let candidate = + CommittedCandidateReceipt { descriptor: pending.descriptor, commitments }; + + Self::enact_candidate( + pending.relay_parent_number, + candidate, + pending.backers, + pending.availability_votes, + pending.core, + pending.backing_group, + ); + } + } + + /// Returns the `CommittedCandidateReceipt` pending availability for the para provided, if any. + pub(crate) fn candidate_pending_availability( + para: ParaId, + ) -> Option> { + >::get(¶) + .map(|p| p.descriptor) + .and_then(|d| >::get(¶).map(move |c| (d, c))) + .map(|(d, c)| CommittedCandidateReceipt { descriptor: d, commitments: c }) + } + + /// Returns the metadata around the candidate pending availability for the + /// para provided, if any. + pub(crate) fn pending_availability( + para: ParaId, + ) -> Option> { + >::get(¶) + } +} + +const fn availability_threshold(n_validators: usize) -> usize { + let mut threshold = (n_validators * 2) / 3; + threshold += (n_validators * 2) % 3; + threshold +} + +#[derive(derive_more::From, Debug)] +enum AcceptanceCheckErr { + HeadDataTooLarge, + PrematureCodeUpgrade, + NewCodeTooLarge, + ProcessedDownwardMessages(dmp::ProcessedDownwardMessagesAcceptanceErr), + UpwardMessages(ump::AcceptanceCheckErr), + HrmpWatermark(hrmp::HrmpWatermarkAcceptanceErr), + OutboundHrmp(hrmp::OutboundHrmpAcceptanceErr), +} + +impl AcceptanceCheckErr { + /// Returns the same error so that it can be threaded through a needle of `DispatchError` and + /// ultimately returned from a `Dispatchable`. + fn strip_into_dispatch_err(self) -> Error { + use AcceptanceCheckErr::*; + match self { + HeadDataTooLarge => Error::::HeadDataTooLarge, + PrematureCodeUpgrade => Error::::PrematureCodeUpgrade, + NewCodeTooLarge => Error::::NewCodeTooLarge, + ProcessedDownwardMessages(_) => Error::::IncorrectDownwardMessageHandling, + UpwardMessages(_) => Error::::InvalidUpwardMessages, + HrmpWatermark(_) => Error::::HrmpWatermarkMishandling, + OutboundHrmp(_) => Error::::InvalidOutboundHrmp, + } + } +} + +/// A collection of data required for checking a candidate. +pub(crate) struct CandidateCheckContext { + config: configuration::HostConfiguration, + now: T::BlockNumber, + relay_parent_number: T::BlockNumber, +} + +impl CandidateCheckContext { + pub(crate) fn new(now: T::BlockNumber, relay_parent_number: T::BlockNumber) -> Self { + Self { config: >::config(), now, relay_parent_number } + } + + /// Execute verification of the candidate. + /// + /// Assures: + /// * correct expected relay parent reference + /// * collator signature check passes + /// * code hash of commitments matches current code hash + /// * para head in the descriptor and commitments match + pub(crate) fn verify_backed_candidate( + &self, + parent_hash: ::Hash, + candidate_idx: usize, + backed_candidate: &BackedCandidate<::Hash>, + ) -> Result<(), Error> { + let para_id = backed_candidate.descriptor().para_id; + let now = self.now; + + // we require that the candidate is in the context of the parent block. + ensure!( + backed_candidate.descriptor().relay_parent == parent_hash, + Error::::CandidateNotInParentContext, + ); + ensure!( + backed_candidate.descriptor().check_collator_signature().is_ok(), + Error::::NotCollatorSigned, + ); + + let validation_code_hash = >::validation_code_hash_at(para_id, now, None) + // A candidate for a parachain without current validation code is not scheduled. + .ok_or_else(|| Error::::UnscheduledCandidate)?; + ensure!( + backed_candidate.descriptor().validation_code_hash == validation_code_hash, + Error::::InvalidValidationCodeHash, + ); + + ensure!( + backed_candidate.descriptor().para_head == + backed_candidate.candidate.commitments.head_data.hash(), + Error::::ParaHeadMismatch, + ); + + if let Err(err) = self.check_validation_outputs( + para_id, + &backed_candidate.candidate.commitments.head_data, + &backed_candidate.candidate.commitments.new_validation_code, + backed_candidate.candidate.commitments.processed_downward_messages, + &backed_candidate.candidate.commitments.upward_messages, + T::BlockNumber::from(backed_candidate.candidate.commitments.hrmp_watermark), + &backed_candidate.candidate.commitments.horizontal_messages, + ) { + log::debug!( + target: LOG_TARGET, + "Validation outputs checking during inclusion of a candidate {} for parachain `{}` failed: {:?}", + candidate_idx, + u32::from(para_id), + err, + ); + Err(err.strip_into_dispatch_err::())?; + }; + Ok(()) + } + + /// Check the given outputs after candidate validation on whether it passes the acceptance + /// criteria. + fn check_validation_outputs( + &self, + para_id: ParaId, + head_data: &HeadData, + new_validation_code: &Option, + processed_downward_messages: u32, + upward_messages: &[primitives::v1::UpwardMessage], + hrmp_watermark: T::BlockNumber, + horizontal_messages: &[primitives::v1::OutboundHrmpMessage], + ) -> Result<(), AcceptanceCheckErr> { + ensure!( + head_data.0.len() <= self.config.max_head_data_size as _, + AcceptanceCheckErr::HeadDataTooLarge, + ); + + // if any, the code upgrade attempt is allowed. + if let Some(new_validation_code) = new_validation_code { + let valid_upgrade_attempt = >::last_code_upgrade(para_id, true) + .map_or(true, |last| { + last <= self.relay_parent_number && + self.relay_parent_number.saturating_sub(last) >= + self.config.validation_upgrade_frequency + }); + ensure!(valid_upgrade_attempt, AcceptanceCheckErr::PrematureCodeUpgrade); + ensure!( + new_validation_code.0.len() <= self.config.max_code_size as _, + AcceptanceCheckErr::NewCodeTooLarge, + ); + } + + // check if the candidate passes the messaging acceptance criteria + >::check_processed_downward_messages(para_id, processed_downward_messages)?; + >::check_upward_messages(&self.config, para_id, upward_messages)?; + >::check_hrmp_watermark(para_id, self.relay_parent_number, hrmp_watermark)?; + >::check_outbound_hrmp(&self.config, para_id, horizontal_messages)?; + + Ok(()) + } +} diff --git a/polkadot/runtime/parachains/src/inclusion/tests.rs b/polkadot/runtime/parachains/src/inclusion/tests.rs new file mode 100644 index 0000000000..c09a0edea2 --- /dev/null +++ b/polkadot/runtime/parachains/src/inclusion/tests.rs @@ -0,0 +1,1911 @@ +// Copyright 2021 Parity Technologies (UK) Ltd. +// This file is part of Polkadot. + +// Polkadot is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Polkadot is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Polkadot. If not, see . + +use super::*; +use crate::{ + configuration::HostConfiguration, + initializer::SessionChangeNotification, + mock::{ + new_test_ext, Configuration, MockGenesisConfig, ParaInclusion, Paras, ParasShared, System, + Test, + }, + paras::ParaGenesisArgs, + paras_inherent::DisputedBitfield, + scheduler::AssignmentKind, +}; +use frame_support::assert_noop; +use futures::executor::block_on; +use keyring::Sr25519Keyring; +use primitives::{ + v0::PARACHAIN_KEY_TYPE_ID, + v1::{ + BlockNumber, CandidateCommitments, CandidateDescriptor, CollatorId, + CompactStatement as Statement, Hash, SignedAvailabilityBitfield, SignedStatement, + UncheckedSignedAvailabilityBitfield, ValidationCode, ValidatorId, ValidityAttestation, + }, +}; +use sc_keystore::LocalKeystore; +use sp_keystore::{SyncCryptoStore, SyncCryptoStorePtr}; +use std::sync::Arc; + +fn default_config() -> HostConfiguration { + let mut config = HostConfiguration::default(); + config.parathread_cores = 1; + config.max_code_size = 3; + config +} + +pub(crate) fn genesis_config(paras: Vec<(ParaId, bool)>) -> MockGenesisConfig { + MockGenesisConfig { + paras: paras::GenesisConfig { + paras: paras + .into_iter() + .map(|(id, is_chain)| { + ( + id, + ParaGenesisArgs { + genesis_head: Vec::new().into(), + validation_code: Vec::new().into(), + parachain: is_chain, + }, + ) + }) + .collect(), + ..Default::default() + }, + configuration: configuration::GenesisConfig { + config: default_config(), + ..Default::default() + }, + ..Default::default() + } +} + +#[derive(Debug, Clone, Copy, PartialEq)] +pub(crate) enum BackingKind { + #[allow(unused)] + Unanimous, + Threshold, + Lacking, +} + +pub(crate) fn collator_sign_candidate( + collator: Sr25519Keyring, + candidate: &mut CommittedCandidateReceipt, +) { + candidate.descriptor.collator = collator.public().into(); + + let payload = primitives::v1::collator_signature_payload( + &candidate.descriptor.relay_parent, + &candidate.descriptor.para_id, + &candidate.descriptor.persisted_validation_data_hash, + &candidate.descriptor.pov_hash, + &candidate.descriptor.validation_code_hash, + ); + + candidate.descriptor.signature = collator.sign(&payload[..]).into(); + assert!(candidate.descriptor().check_collator_signature().is_ok()); +} + +pub(crate) async fn back_candidate( + candidate: CommittedCandidateReceipt, + validators: &[Sr25519Keyring], + group: &[ValidatorIndex], + keystore: &SyncCryptoStorePtr, + signing_context: &SigningContext, + kind: BackingKind, +) -> BackedCandidate { + let mut validator_indices = bitvec::bitvec![BitOrderLsb0, u8; 0; group.len()]; + let threshold = (group.len() / 2) + 1; + + let signing = match kind { + BackingKind::Unanimous => group.len(), + BackingKind::Threshold => threshold, + BackingKind::Lacking => threshold.saturating_sub(1), + }; + + let mut validity_votes = Vec::with_capacity(signing); + let candidate_hash = candidate.hash(); + + for (idx_in_group, val_idx) in group.iter().enumerate().take(signing) { + let key: Sr25519Keyring = validators[val_idx.0 as usize]; + *validator_indices.get_mut(idx_in_group).unwrap() = true; + + let signature = SignedStatement::sign( + &keystore, + Statement::Valid(candidate_hash), + signing_context, + *val_idx, + &key.public().into(), + ) + .await + .unwrap() + .unwrap() + .signature() + .clone(); + + validity_votes.push(ValidityAttestation::Explicit(signature).into()); + } + + let backed = BackedCandidate { candidate, validity_votes, validator_indices }; + + let successfully_backed = + primitives::v1::check_candidate_backing(&backed, signing_context, group.len(), |i| { + Some(validators[group[i].0 as usize].public().into()) + }) + .ok() + .unwrap_or(0) * + 2 > group.len(); + + match kind { + BackingKind::Unanimous | BackingKind::Threshold => assert!(successfully_backed), + BackingKind::Lacking => assert!(!successfully_backed), + }; + + backed +} + +pub(crate) fn run_to_block( + to: BlockNumber, + new_session: impl Fn(BlockNumber) -> Option>, +) { + while System::block_number() < to { + let b = System::block_number(); + + ParaInclusion::initializer_finalize(); + Paras::initializer_finalize(); + ParasShared::initializer_finalize(); + + if let Some(notification) = new_session(b + 1) { + ParasShared::initializer_on_new_session( + notification.session_index, + notification.random_seed, + ¬ification.new_config, + notification.validators.clone(), + ); + Paras::initializer_on_new_session(¬ification); + ParaInclusion::initializer_on_new_session(¬ification); + } + + System::on_finalize(b); + + System::on_initialize(b + 1); + System::set_block_number(b + 1); + + ParasShared::initializer_initialize(b + 1); + Paras::initializer_initialize(b + 1); + ParaInclusion::initializer_initialize(b + 1); + } +} + +pub(crate) fn expected_bits() -> usize { + Paras::parachains().len() + Configuration::config().parathread_cores as usize +} + +fn default_bitfield() -> AvailabilityBitfield { + AvailabilityBitfield(bitvec::bitvec![BitOrderLsb0, u8; 0; expected_bits()]) +} + +fn default_availability_votes() -> BitVec { + bitvec::bitvec![BitOrderLsb0, u8; 0; ParasShared::active_validator_keys().len()] +} + +fn default_backing_bitfield() -> BitVec { + bitvec::bitvec![BitOrderLsb0, u8; 0; ParasShared::active_validator_keys().len()] +} + +fn backing_bitfield(v: &[usize]) -> BitVec { + let mut b = default_backing_bitfield(); + for i in v { + b.set(*i, true); + } + b +} + +pub(crate) fn validator_pubkeys(val_ids: &[Sr25519Keyring]) -> Vec { + val_ids.iter().map(|v| v.public().into()).collect() +} + +pub(crate) async fn sign_bitfield( + keystore: &SyncCryptoStorePtr, + key: &Sr25519Keyring, + validator_index: ValidatorIndex, + bitfield: AvailabilityBitfield, + signing_context: &SigningContext, +) -> SignedAvailabilityBitfield { + SignedAvailabilityBitfield::sign( + &keystore, + bitfield, + &signing_context, + validator_index, + &key.public().into(), + ) + .await + .unwrap() + .unwrap() +} + +#[derive(Default)] +pub(crate) struct TestCandidateBuilder { + pub(crate) para_id: ParaId, + pub(crate) head_data: HeadData, + pub(crate) para_head_hash: Option, + pub(crate) pov_hash: Hash, + pub(crate) relay_parent: Hash, + pub(crate) persisted_validation_data_hash: Hash, + pub(crate) new_validation_code: Option, + pub(crate) validation_code: ValidationCode, + pub(crate) hrmp_watermark: BlockNumber, +} + +impl TestCandidateBuilder { + pub(crate) fn build(self) -> CommittedCandidateReceipt { + CommittedCandidateReceipt { + descriptor: CandidateDescriptor { + para_id: self.para_id, + pov_hash: self.pov_hash, + relay_parent: self.relay_parent, + persisted_validation_data_hash: self.persisted_validation_data_hash, + validation_code_hash: self.validation_code.hash(), + para_head: self.para_head_hash.unwrap_or_else(|| self.head_data.hash()), + ..Default::default() + }, + commitments: CandidateCommitments { + head_data: self.head_data, + new_validation_code: self.new_validation_code, + hrmp_watermark: self.hrmp_watermark, + ..Default::default() + }, + } + } +} + +pub(crate) fn make_vdata_hash(para_id: ParaId) -> Option { + let relay_parent_number = >::block_number() - 1; + let persisted_validation_data = crate::util::make_persisted_validation_data::( + para_id, + relay_parent_number, + Default::default(), + )?; + Some(persisted_validation_data.hash()) +} + +#[test] +fn collect_pending_cleans_up_pending() { + let chain_a = ParaId::from(1); + let chain_b = ParaId::from(2); + let thread_a = ParaId::from(3); + + let paras = vec![(chain_a, true), (chain_b, true), (thread_a, false)]; + new_test_ext(genesis_config(paras)).execute_with(|| { + let default_candidate = TestCandidateBuilder::default().build(); + >::insert( + chain_a, + CandidatePendingAvailability { + core: CoreIndex::from(0), + hash: default_candidate.hash(), + descriptor: default_candidate.descriptor.clone(), + availability_votes: default_availability_votes(), + relay_parent_number: 0, + backed_in_number: 0, + backers: default_backing_bitfield(), + backing_group: GroupIndex::from(0), + }, + ); + PendingAvailabilityCommitments::::insert( + chain_a, + default_candidate.commitments.clone(), + ); + + >::insert( + &chain_b, + CandidatePendingAvailability { + core: CoreIndex::from(1), + hash: default_candidate.hash(), + descriptor: default_candidate.descriptor, + availability_votes: default_availability_votes(), + relay_parent_number: 0, + backed_in_number: 0, + backers: default_backing_bitfield(), + backing_group: GroupIndex::from(1), + }, + ); + PendingAvailabilityCommitments::::insert(chain_b, default_candidate.commitments); + + run_to_block(5, |_| None); + + assert!(>::get(&chain_a).is_some()); + assert!(>::get(&chain_b).is_some()); + assert!(>::get(&chain_a).is_some()); + assert!(>::get(&chain_b).is_some()); + + ParaInclusion::collect_pending(|core, _since| core == CoreIndex::from(0)); + + assert!(>::get(&chain_a).is_none()); + assert!(>::get(&chain_b).is_some()); + assert!(>::get(&chain_a).is_none()); + assert!(>::get(&chain_b).is_some()); + }); +} + +#[test] +fn bitfield_checks() { + let chain_a = ParaId::from(1); + let chain_b = ParaId::from(2); + let thread_a = ParaId::from(3); + + let paras = vec![(chain_a, true), (chain_b, true), (thread_a, false)]; + let validators = vec![ + Sr25519Keyring::Alice, + Sr25519Keyring::Bob, + Sr25519Keyring::Charlie, + Sr25519Keyring::Dave, + Sr25519Keyring::Ferdie, + ]; + let keystore: SyncCryptoStorePtr = Arc::new(LocalKeystore::in_memory()); + for validator in validators.iter() { + SyncCryptoStore::sr25519_generate_new( + &*keystore, + PARACHAIN_KEY_TYPE_ID, + Some(&validator.to_seed()), + ) + .unwrap(); + } + let validator_public = validator_pubkeys(&validators); + + new_test_ext(genesis_config(paras.clone())).execute_with(|| { + shared::Pallet::::set_active_validators_ascending(validator_public.clone()); + shared::Pallet::::set_session_index(5); + + let signing_context = + SigningContext { parent_hash: System::parent_hash(), session_index: 5 }; + + let core_lookup = |core| match core { + core if core == CoreIndex::from(0) => Some(chain_a), + core if core == CoreIndex::from(1) => Some(chain_b), + core if core == CoreIndex::from(2) => Some(thread_a), + core if core == CoreIndex::from(3) => None, // for the expected_cores() + 1 test below. + _ => panic!("out of bounds for testing"), + }; + + // mark all candidates as pending availability + let set_pending_av = || { + for (p_id, _) in paras { + PendingAvailability::::insert( + p_id, + CandidatePendingAvailability { + availability_votes: default_availability_votes(), + ..Default::default() + }, + ) + } + }; + + // too many bits in bitfield + { + let mut bare_bitfield = default_bitfield(); + bare_bitfield.0.push(false); + let signed = block_on(sign_bitfield( + &keystore, + &validators[0], + ValidatorIndex(0), + bare_bitfield, + &signing_context, + )); + + assert_eq!( + ParaInclusion::process_bitfields( + expected_bits(), + vec![signed.into()], + DisputedBitfield::zeros(expected_bits()), + &core_lookup, + ), + vec![] + ); + } + + // not enough bits + { + let bare_bitfield = default_bitfield(); + let signed = block_on(sign_bitfield( + &keystore, + &validators[0], + ValidatorIndex(0), + bare_bitfield, + &signing_context, + )); + + assert_eq!( + ParaInclusion::process_bitfields( + expected_bits() + 1, + vec![signed.into()], + DisputedBitfield::zeros(expected_bits()), + &core_lookup, + ), + vec![] + ); + } + + // duplicate. + { + set_pending_av.clone()(); + let back_core_0_bitfield = { + let mut b = default_bitfield(); + b.0.set(0, true); + b + }; + let signed: UncheckedSignedAvailabilityBitfield = block_on(sign_bitfield( + &keystore, + &validators[0], + ValidatorIndex(0), + back_core_0_bitfield, + &signing_context, + )) + .into(); + + assert_eq!( + >::get(chain_a) + .unwrap() + .availability_votes + .count_ones(), + 0 + ); + + // the threshold to free a core is 4 availability votes, but we only expect 1 valid + // valid bitfield. + assert!(ParaInclusion::process_bitfields( + expected_bits(), + vec![signed.clone(), signed], + DisputedBitfield::zeros(expected_bits()), + &core_lookup, + ) + .is_empty()); + + assert_eq!( + >::get(chain_a) + .unwrap() + .availability_votes + .count_ones(), + 1 + ); + + // clean up + PendingAvailability::::remove_all(None); + } + + // out of order. + { + set_pending_av.clone()(); + let back_core_0_bitfield = { + let mut b = default_bitfield(); + b.0.set(0, true); + b + }; + let signed_0 = block_on(sign_bitfield( + &keystore, + &validators[0], + ValidatorIndex(0), + back_core_0_bitfield.clone(), + &signing_context, + )) + .into(); + + let signed_1 = block_on(sign_bitfield( + &keystore, + &validators[1], + ValidatorIndex(1), + back_core_0_bitfield, + &signing_context, + )) + .into(); + + assert_eq!( + >::get(chain_a) + .unwrap() + .availability_votes + .count_ones(), + 0 + ); + + // the threshold to free a core is 4 availability votes, but we only expect 1 valid + // valid bitfield because `signed_0` will get skipped for being out of order. + assert!(ParaInclusion::process_bitfields( + expected_bits(), + vec![signed_1, signed_0], + DisputedBitfield::zeros(expected_bits()), + &core_lookup, + ) + .is_empty()); + + assert_eq!( + >::get(chain_a) + .unwrap() + .availability_votes + .count_ones(), + 1 + ); + + PendingAvailability::::remove_all(None); + } + + // non-pending bit set. + { + let mut bare_bitfield = default_bitfield(); + *bare_bitfield.0.get_mut(0).unwrap() = true; + let signed = block_on(sign_bitfield( + &keystore, + &validators[0], + ValidatorIndex(0), + bare_bitfield, + &signing_context, + )); + + assert!(ParaInclusion::process_bitfields( + expected_bits(), + vec![signed.into()], + DisputedBitfield::zeros(expected_bits()), + &core_lookup, + ) + .is_empty()); + } + + // empty bitfield signed: always ok, but kind of useless. + { + let bare_bitfield = default_bitfield(); + let signed = block_on(sign_bitfield( + &keystore, + &validators[0], + ValidatorIndex(0), + bare_bitfield, + &signing_context, + )); + + assert!(ParaInclusion::process_bitfields( + expected_bits(), + vec![signed.into()], + DisputedBitfield::zeros(expected_bits()), + &core_lookup, + ) + .is_empty()); + } + + // bitfield signed with pending bit signed. + { + let mut bare_bitfield = default_bitfield(); + + assert_eq!(core_lookup(CoreIndex::from(0)), Some(chain_a)); + + let default_candidate = TestCandidateBuilder::default().build(); + >::insert( + chain_a, + CandidatePendingAvailability { + core: CoreIndex::from(0), + hash: default_candidate.hash(), + descriptor: default_candidate.descriptor, + availability_votes: default_availability_votes(), + relay_parent_number: 0, + backed_in_number: 0, + backers: default_backing_bitfield(), + backing_group: GroupIndex::from(0), + }, + ); + PendingAvailabilityCommitments::::insert(chain_a, default_candidate.commitments); + + *bare_bitfield.0.get_mut(0).unwrap() = true; + let signed = block_on(sign_bitfield( + &keystore, + &validators[0], + ValidatorIndex(0), + bare_bitfield, + &signing_context, + )); + + assert!(ParaInclusion::process_bitfields( + expected_bits(), + vec![signed.into()], + DisputedBitfield::zeros(expected_bits()), + &core_lookup, + ) + .is_empty()); + + >::remove(chain_a); + PendingAvailabilityCommitments::::remove(chain_a); + } + + // bitfield signed with pending bit signed, but no commitments. + { + let mut bare_bitfield = default_bitfield(); + + assert_eq!(core_lookup(CoreIndex::from(0)), Some(chain_a)); + + let default_candidate = TestCandidateBuilder::default().build(); + >::insert( + chain_a, + CandidatePendingAvailability { + core: CoreIndex::from(0), + hash: default_candidate.hash(), + descriptor: default_candidate.descriptor, + availability_votes: default_availability_votes(), + relay_parent_number: 0, + backed_in_number: 0, + backers: default_backing_bitfield(), + backing_group: GroupIndex::from(0), + }, + ); + + *bare_bitfield.0.get_mut(0).unwrap() = true; + let signed = block_on(sign_bitfield( + &keystore, + &validators[0], + ValidatorIndex(0), + bare_bitfield, + &signing_context, + )); + + // no core is freed + assert!(ParaInclusion::process_bitfields( + expected_bits(), + vec![signed.into()], + DisputedBitfield::zeros(expected_bits()), + &core_lookup, + ) + .is_empty()); + } + }); +} + +#[test] +fn supermajority_bitfields_trigger_availability() { + let chain_a = ParaId::from(1); + let chain_b = ParaId::from(2); + let thread_a = ParaId::from(3); + + let paras = vec![(chain_a, true), (chain_b, true), (thread_a, false)]; + let validators = vec![ + Sr25519Keyring::Alice, + Sr25519Keyring::Bob, + Sr25519Keyring::Charlie, + Sr25519Keyring::Dave, + Sr25519Keyring::Ferdie, + ]; + let keystore: SyncCryptoStorePtr = Arc::new(LocalKeystore::in_memory()); + for validator in validators.iter() { + SyncCryptoStore::sr25519_generate_new( + &*keystore, + PARACHAIN_KEY_TYPE_ID, + Some(&validator.to_seed()), + ) + .unwrap(); + } + let validator_public = validator_pubkeys(&validators); + + new_test_ext(genesis_config(paras)).execute_with(|| { + shared::Pallet::::set_active_validators_ascending(validator_public.clone()); + shared::Pallet::::set_session_index(5); + + let signing_context = + SigningContext { parent_hash: System::parent_hash(), session_index: 5 }; + + let core_lookup = |core| match core { + core if core == CoreIndex::from(0) => Some(chain_a), + core if core == CoreIndex::from(1) => Some(chain_b), + core if core == CoreIndex::from(2) => Some(thread_a), + _ => panic!("Core out of bounds for 2 parachains and 1 parathread core."), + }; + + let candidate_a = TestCandidateBuilder { + para_id: chain_a, + head_data: vec![1, 2, 3, 4].into(), + ..Default::default() + } + .build(); + + >::insert( + chain_a, + CandidatePendingAvailability { + core: CoreIndex::from(0), + hash: candidate_a.hash(), + descriptor: candidate_a.clone().descriptor, + availability_votes: default_availability_votes(), + relay_parent_number: 0, + backed_in_number: 0, + backers: backing_bitfield(&[3, 4]), + backing_group: GroupIndex::from(0), + }, + ); + PendingAvailabilityCommitments::::insert(chain_a, candidate_a.clone().commitments); + + let candidate_b = TestCandidateBuilder { + para_id: chain_b, + head_data: vec![5, 6, 7, 8].into(), + ..Default::default() + } + .build(); + + >::insert( + chain_b, + CandidatePendingAvailability { + core: CoreIndex::from(1), + hash: candidate_b.hash(), + descriptor: candidate_b.descriptor, + availability_votes: default_availability_votes(), + relay_parent_number: 0, + backed_in_number: 0, + backers: backing_bitfield(&[0, 2]), + backing_group: GroupIndex::from(1), + }, + ); + PendingAvailabilityCommitments::::insert(chain_b, candidate_b.commitments); + + // this bitfield signals that a and b are available. + let a_and_b_available = { + let mut bare_bitfield = default_bitfield(); + *bare_bitfield.0.get_mut(0).unwrap() = true; + *bare_bitfield.0.get_mut(1).unwrap() = true; + + bare_bitfield + }; + + // this bitfield signals that only a is available. + let a_available = { + let mut bare_bitfield = default_bitfield(); + *bare_bitfield.0.get_mut(0).unwrap() = true; + + bare_bitfield + }; + + let threshold = availability_threshold(validators.len()); + + // 4 of 5 first value >= 2/3 + assert_eq!(threshold, 4); + + let signed_bitfields = validators + .iter() + .enumerate() + .filter_map(|(i, key)| { + let to_sign = if i < 3 { + a_and_b_available.clone() + } else if i < 4 { + a_available.clone() + } else { + // sign nothing. + return None + }; + + Some( + block_on(sign_bitfield( + &keystore, + key, + ValidatorIndex(i as _), + to_sign, + &signing_context, + )) + .into(), + ) + }) + .collect(); + + // only chain A's core is freed. + assert_eq!( + ParaInclusion::process_bitfields( + expected_bits(), + signed_bitfields, + DisputedBitfield::zeros(expected_bits()), + &core_lookup, + ), + vec![(CoreIndex(0), candidate_a.hash())] + ); + + // chain A had 4 signing off, which is >= threshold. + // chain B has 3 signing off, which is < threshold. + assert!(>::get(&chain_a).is_none()); + assert!(>::get(&chain_a).is_none()); + assert!(>::get(&chain_b).is_some()); + assert_eq!(>::get(&chain_b).unwrap().availability_votes, { + // check that votes from first 3 were tracked. + + let mut votes = default_availability_votes(); + *votes.get_mut(0).unwrap() = true; + *votes.get_mut(1).unwrap() = true; + *votes.get_mut(2).unwrap() = true; + + votes + }); + + // and check that chain head was enacted. + assert_eq!(Paras::para_head(&chain_a), Some(vec![1, 2, 3, 4].into())); + + // Check that rewards are applied. + { + let rewards = crate::mock::availability_rewards(); + + assert_eq!(rewards.len(), 4); + assert_eq!(rewards.get(&ValidatorIndex(0)).unwrap(), &1); + assert_eq!(rewards.get(&ValidatorIndex(1)).unwrap(), &1); + assert_eq!(rewards.get(&ValidatorIndex(2)).unwrap(), &1); + assert_eq!(rewards.get(&ValidatorIndex(3)).unwrap(), &1); + } + + { + let rewards = crate::mock::backing_rewards(); + + assert_eq!(rewards.len(), 2); + assert_eq!(rewards.get(&ValidatorIndex(3)).unwrap(), &1); + assert_eq!(rewards.get(&ValidatorIndex(4)).unwrap(), &1); + } + }); +} + +#[test] +fn candidate_checks() { + let chain_a = ParaId::from(1); + let chain_b = ParaId::from(2); + let thread_a = ParaId::from(3); + + // The block number of the relay-parent for testing. + const RELAY_PARENT_NUM: BlockNumber = 4; + + let paras = vec![(chain_a, true), (chain_b, true), (thread_a, false)]; + let validators = vec![ + Sr25519Keyring::Alice, + Sr25519Keyring::Bob, + Sr25519Keyring::Charlie, + Sr25519Keyring::Dave, + Sr25519Keyring::Ferdie, + ]; + let keystore: SyncCryptoStorePtr = Arc::new(LocalKeystore::in_memory()); + for validator in validators.iter() { + SyncCryptoStore::sr25519_generate_new( + &*keystore, + PARACHAIN_KEY_TYPE_ID, + Some(&validator.to_seed()), + ) + .unwrap(); + } + let validator_public = validator_pubkeys(&validators); + + new_test_ext(genesis_config(paras)).execute_with(|| { + shared::Pallet::::set_active_validators_ascending(validator_public.clone()); + shared::Pallet::::set_session_index(5); + + run_to_block(5, |_| None); + + let signing_context = + SigningContext { parent_hash: System::parent_hash(), session_index: 5 }; + + let group_validators = |group_index: GroupIndex| { + match group_index { + group_index if group_index == GroupIndex::from(0) => Some(vec![0, 1]), + group_index if group_index == GroupIndex::from(1) => Some(vec![2, 3]), + group_index if group_index == GroupIndex::from(2) => Some(vec![4]), + _ => panic!("Group index out of bounds for 2 parachains and 1 parathread core"), + } + .map(|m| m.into_iter().map(ValidatorIndex).collect::>()) + }; + + let thread_collator: CollatorId = Sr25519Keyring::Two.public().into(); + + let chain_a_assignment = CoreAssignment { + core: CoreIndex::from(0), + para_id: chain_a, + kind: AssignmentKind::Parachain, + group_idx: GroupIndex::from(0), + }; + + let chain_b_assignment = CoreAssignment { + core: CoreIndex::from(1), + para_id: chain_b, + kind: AssignmentKind::Parachain, + group_idx: GroupIndex::from(1), + }; + + let thread_a_assignment = CoreAssignment { + core: CoreIndex::from(2), + para_id: thread_a, + kind: AssignmentKind::Parathread(thread_collator.clone(), 0), + group_idx: GroupIndex::from(2), + }; + + // unscheduled candidate. + { + let mut candidate = TestCandidateBuilder { + para_id: chain_a, + relay_parent: System::parent_hash(), + pov_hash: Hash::repeat_byte(1), + persisted_validation_data_hash: make_vdata_hash(chain_a).unwrap(), + hrmp_watermark: RELAY_PARENT_NUM, + ..Default::default() + } + .build(); + collator_sign_candidate(Sr25519Keyring::One, &mut candidate); + + let backed = block_on(back_candidate( + candidate, + &validators, + group_validators(GroupIndex::from(0)).unwrap().as_ref(), + &keystore, + &signing_context, + BackingKind::Threshold, + )); + + assert_noop!( + ParaInclusion::process_candidates( + Default::default(), + vec![backed], + vec![chain_b_assignment.clone()], + &group_validators, + FullCheck::Yes, + ), + Error::::UnscheduledCandidate + ); + } + + // candidates out of order. + { + let mut candidate_a = TestCandidateBuilder { + para_id: chain_a, + relay_parent: System::parent_hash(), + pov_hash: Hash::repeat_byte(1), + persisted_validation_data_hash: make_vdata_hash(chain_a).unwrap(), + hrmp_watermark: RELAY_PARENT_NUM, + ..Default::default() + } + .build(); + let mut candidate_b = TestCandidateBuilder { + para_id: chain_b, + relay_parent: System::parent_hash(), + pov_hash: Hash::repeat_byte(2), + persisted_validation_data_hash: make_vdata_hash(chain_b).unwrap(), + hrmp_watermark: RELAY_PARENT_NUM, + ..Default::default() + } + .build(); + + collator_sign_candidate(Sr25519Keyring::One, &mut candidate_a); + + collator_sign_candidate(Sr25519Keyring::Two, &mut candidate_b); + + let backed_a = block_on(back_candidate( + candidate_a, + &validators, + group_validators(GroupIndex::from(0)).unwrap().as_ref(), + &keystore, + &signing_context, + BackingKind::Threshold, + )); + + let backed_b = block_on(back_candidate( + candidate_b, + &validators, + group_validators(GroupIndex::from(1)).unwrap().as_ref(), + &keystore, + &signing_context, + BackingKind::Threshold, + )); + + // out-of-order manifests as unscheduled. + assert_noop!( + ParaInclusion::process_candidates( + Default::default(), + vec![backed_b, backed_a], + vec![chain_a_assignment.clone(), chain_b_assignment.clone()], + &group_validators, + FullCheck::Yes, + ), + Error::::UnscheduledCandidate + ); + } + + // candidate not backed. + { + let mut candidate = TestCandidateBuilder { + para_id: chain_a, + relay_parent: System::parent_hash(), + pov_hash: Hash::repeat_byte(1), + persisted_validation_data_hash: make_vdata_hash(chain_a).unwrap(), + hrmp_watermark: RELAY_PARENT_NUM, + ..Default::default() + } + .build(); + collator_sign_candidate(Sr25519Keyring::One, &mut candidate); + + let backed = block_on(back_candidate( + candidate, + &validators, + group_validators(GroupIndex::from(0)).unwrap().as_ref(), + &keystore, + &signing_context, + BackingKind::Lacking, + )); + + assert_noop!( + ParaInclusion::process_candidates( + Default::default(), + vec![backed], + vec![chain_a_assignment.clone()], + &group_validators, + FullCheck::Yes, + ), + Error::::InsufficientBacking + ); + } + + // candidate not in parent context. + { + let wrong_parent_hash = Hash::repeat_byte(222); + assert!(System::parent_hash() != wrong_parent_hash); + + let mut candidate = TestCandidateBuilder { + para_id: chain_a, + relay_parent: wrong_parent_hash, + pov_hash: Hash::repeat_byte(1), + persisted_validation_data_hash: make_vdata_hash(chain_a).unwrap(), + ..Default::default() + } + .build(); + collator_sign_candidate(Sr25519Keyring::One, &mut candidate); + + let backed = block_on(back_candidate( + candidate, + &validators, + group_validators(GroupIndex::from(0)).unwrap().as_ref(), + &keystore, + &signing_context, + BackingKind::Threshold, + )); + + assert_noop!( + ParaInclusion::process_candidates( + Default::default(), + vec![backed], + vec![chain_a_assignment.clone()], + &group_validators, + FullCheck::Yes, + ), + Error::::CandidateNotInParentContext + ); + } + + // candidate has wrong collator. + { + let mut candidate = TestCandidateBuilder { + para_id: thread_a, + relay_parent: System::parent_hash(), + pov_hash: Hash::repeat_byte(1), + persisted_validation_data_hash: make_vdata_hash(thread_a).unwrap(), + hrmp_watermark: RELAY_PARENT_NUM, + ..Default::default() + } + .build(); + + assert!(CollatorId::from(Sr25519Keyring::One.public()) != thread_collator); + collator_sign_candidate(Sr25519Keyring::One, &mut candidate); + + let backed = block_on(back_candidate( + candidate, + &validators, + group_validators(GroupIndex::from(2)).unwrap().as_ref(), + &keystore, + &signing_context, + BackingKind::Threshold, + )); + + assert_noop!( + ParaInclusion::process_candidates( + Default::default(), + vec![backed], + vec![ + chain_a_assignment.clone(), + chain_b_assignment.clone(), + thread_a_assignment.clone(), + ], + &group_validators, + FullCheck::Yes, + ), + Error::::WrongCollator, + ); + } + + // candidate not well-signed by collator. + { + let mut candidate = TestCandidateBuilder { + para_id: thread_a, + relay_parent: System::parent_hash(), + pov_hash: Hash::repeat_byte(1), + persisted_validation_data_hash: make_vdata_hash(thread_a).unwrap(), + hrmp_watermark: RELAY_PARENT_NUM, + ..Default::default() + } + .build(); + + assert_eq!(CollatorId::from(Sr25519Keyring::Two.public()), thread_collator); + collator_sign_candidate(Sr25519Keyring::Two, &mut candidate); + + // change the candidate after signing. + candidate.descriptor.pov_hash = Hash::repeat_byte(2); + + let backed = block_on(back_candidate( + candidate, + &validators, + group_validators(GroupIndex::from(2)).unwrap().as_ref(), + &keystore, + &signing_context, + BackingKind::Threshold, + )); + + assert_noop!( + ParaInclusion::process_candidates( + Default::default(), + vec![backed], + vec![thread_a_assignment.clone()], + &group_validators, + FullCheck::Yes, + ), + Error::::NotCollatorSigned + ); + } + + // para occupied - reject. + { + let mut candidate = TestCandidateBuilder { + para_id: chain_a, + relay_parent: System::parent_hash(), + pov_hash: Hash::repeat_byte(1), + persisted_validation_data_hash: make_vdata_hash(chain_a).unwrap(), + hrmp_watermark: RELAY_PARENT_NUM, + ..Default::default() + } + .build(); + + collator_sign_candidate(Sr25519Keyring::One, &mut candidate); + + let backed = block_on(back_candidate( + candidate, + &validators, + group_validators(GroupIndex::from(0)).unwrap().as_ref(), + &keystore, + &signing_context, + BackingKind::Threshold, + )); + + let candidate = TestCandidateBuilder::default().build(); + >::insert( + &chain_a, + CandidatePendingAvailability { + core: CoreIndex::from(0), + hash: candidate.hash(), + descriptor: candidate.descriptor, + availability_votes: default_availability_votes(), + relay_parent_number: 3, + backed_in_number: 4, + backers: default_backing_bitfield(), + backing_group: GroupIndex::from(0), + }, + ); + >::insert(&chain_a, candidate.commitments); + + assert_noop!( + ParaInclusion::process_candidates( + Default::default(), + vec![backed], + vec![chain_a_assignment.clone()], + &group_validators, + FullCheck::Yes, + ), + Error::::CandidateScheduledBeforeParaFree + ); + + >::remove(&chain_a); + >::remove(&chain_a); + } + + // messed up commitments storage - do not panic - reject. + { + let mut candidate = TestCandidateBuilder { + para_id: chain_a, + relay_parent: System::parent_hash(), + pov_hash: Hash::repeat_byte(1), + persisted_validation_data_hash: make_vdata_hash(chain_a).unwrap(), + hrmp_watermark: RELAY_PARENT_NUM, + ..Default::default() + } + .build(); + + collator_sign_candidate(Sr25519Keyring::One, &mut candidate); + + // this is not supposed to happen + >::insert(&chain_a, candidate.commitments.clone()); + + let backed = block_on(back_candidate( + candidate, + &validators, + group_validators(GroupIndex::from(0)).unwrap().as_ref(), + &keystore, + &signing_context, + BackingKind::Threshold, + )); + + assert_noop!( + ParaInclusion::process_candidates( + Default::default(), + vec![backed], + vec![chain_a_assignment.clone()], + &group_validators, + FullCheck::Yes, + ), + Error::::CandidateScheduledBeforeParaFree + ); + + >::remove(&chain_a); + } + + // interfering code upgrade - reject + { + let mut candidate = TestCandidateBuilder { + para_id: chain_a, + relay_parent: System::parent_hash(), + pov_hash: Hash::repeat_byte(1), + new_validation_code: Some(vec![5, 6, 7, 8].into()), + persisted_validation_data_hash: make_vdata_hash(chain_a).unwrap(), + hrmp_watermark: RELAY_PARENT_NUM, + ..Default::default() + } + .build(); + + collator_sign_candidate(Sr25519Keyring::One, &mut candidate); + + let backed = block_on(back_candidate( + candidate, + &validators, + group_validators(GroupIndex::from(0)).unwrap().as_ref(), + &keystore, + &signing_context, + BackingKind::Threshold, + )); + + { + let cfg = Configuration::config(); + let expected_at = 10 + cfg.validation_upgrade_delay; + assert_eq!(expected_at, 10); + Paras::schedule_code_upgrade(chain_a, vec![1, 2, 3, 4].into(), expected_at, &cfg); + + assert_eq!(Paras::last_code_upgrade(chain_a, true), Some(expected_at)); + } + + assert_noop!( + ParaInclusion::process_candidates( + Default::default(), + vec![backed], + vec![chain_a_assignment.clone()], + &group_validators, + FullCheck::Yes, + ), + Error::::PrematureCodeUpgrade + ); + } + + // Bad validation data hash - reject + { + let mut candidate = TestCandidateBuilder { + para_id: chain_a, + relay_parent: System::parent_hash(), + pov_hash: Hash::repeat_byte(1), + persisted_validation_data_hash: [42u8; 32].into(), + hrmp_watermark: RELAY_PARENT_NUM, + ..Default::default() + } + .build(); + + collator_sign_candidate(Sr25519Keyring::One, &mut candidate); + + let backed = block_on(back_candidate( + candidate, + &validators, + group_validators(GroupIndex::from(0)).unwrap().as_ref(), + &keystore, + &signing_context, + BackingKind::Threshold, + )); + + assert_eq!( + ParaInclusion::process_candidates( + Default::default(), + vec![backed], + vec![chain_a_assignment.clone()], + &group_validators, + FullCheck::Yes, + ), + Err(Error::::ValidationDataHashMismatch.into()), + ); + } + + // bad validation code hash + { + let mut candidate = TestCandidateBuilder { + para_id: chain_a, + relay_parent: System::parent_hash(), + pov_hash: Hash::repeat_byte(1), + persisted_validation_data_hash: make_vdata_hash(chain_a).unwrap(), + hrmp_watermark: RELAY_PARENT_NUM, + validation_code: ValidationCode(vec![1]), + ..Default::default() + } + .build(); + + collator_sign_candidate(Sr25519Keyring::One, &mut candidate); + + let backed = block_on(back_candidate( + candidate, + &validators, + group_validators(GroupIndex::from(0)).unwrap().as_ref(), + &keystore, + &signing_context, + BackingKind::Threshold, + )); + + assert_noop!( + ParaInclusion::process_candidates( + Default::default(), + vec![backed], + vec![chain_a_assignment.clone()], + &group_validators, + FullCheck::Yes, + ), + Error::::InvalidValidationCodeHash + ); + } + + // Para head hash in descriptor doesn't match head data + { + let mut candidate = TestCandidateBuilder { + para_id: chain_a, + relay_parent: System::parent_hash(), + pov_hash: Hash::repeat_byte(1), + persisted_validation_data_hash: make_vdata_hash(chain_a).unwrap(), + hrmp_watermark: RELAY_PARENT_NUM, + para_head_hash: Some(Hash::random()), + ..Default::default() + } + .build(); + + collator_sign_candidate(Sr25519Keyring::One, &mut candidate); + + let backed = block_on(back_candidate( + candidate, + &validators, + group_validators(GroupIndex::from(0)).unwrap().as_ref(), + &keystore, + &signing_context, + BackingKind::Threshold, + )); + + assert_noop!( + ParaInclusion::process_candidates( + Default::default(), + vec![backed], + vec![chain_a_assignment.clone()], + &group_validators, + FullCheck::Yes, + ), + Error::::ParaHeadMismatch + ); + } + }); +} + +#[test] +fn backing_works() { + let chain_a = ParaId::from(1); + let chain_b = ParaId::from(2); + let thread_a = ParaId::from(3); + + // The block number of the relay-parent for testing. + const RELAY_PARENT_NUM: BlockNumber = 4; + + let paras = vec![(chain_a, true), (chain_b, true), (thread_a, false)]; + let validators = vec![ + Sr25519Keyring::Alice, + Sr25519Keyring::Bob, + Sr25519Keyring::Charlie, + Sr25519Keyring::Dave, + Sr25519Keyring::Ferdie, + ]; + let keystore: SyncCryptoStorePtr = Arc::new(LocalKeystore::in_memory()); + for validator in validators.iter() { + SyncCryptoStore::sr25519_generate_new( + &*keystore, + PARACHAIN_KEY_TYPE_ID, + Some(&validator.to_seed()), + ) + .unwrap(); + } + let validator_public = validator_pubkeys(&validators); + + new_test_ext(genesis_config(paras)).execute_with(|| { + shared::Pallet::::set_active_validators_ascending(validator_public.clone()); + shared::Pallet::::set_session_index(5); + + run_to_block(5, |_| None); + + let signing_context = + SigningContext { parent_hash: System::parent_hash(), session_index: 5 }; + + let group_validators = |group_index: GroupIndex| { + match group_index { + group_index if group_index == GroupIndex::from(0) => Some(vec![0, 1]), + group_index if group_index == GroupIndex::from(1) => Some(vec![2, 3]), + group_index if group_index == GroupIndex::from(2) => Some(vec![4]), + _ => panic!("Group index out of bounds for 2 parachains and 1 parathread core"), + } + .map(|vs| vs.into_iter().map(ValidatorIndex).collect::>()) + }; + + let thread_collator: CollatorId = Sr25519Keyring::Two.public().into(); + + let chain_a_assignment = CoreAssignment { + core: CoreIndex::from(0), + para_id: chain_a, + kind: AssignmentKind::Parachain, + group_idx: GroupIndex::from(0), + }; + + let chain_b_assignment = CoreAssignment { + core: CoreIndex::from(1), + para_id: chain_b, + kind: AssignmentKind::Parachain, + group_idx: GroupIndex::from(1), + }; + + let thread_a_assignment = CoreAssignment { + core: CoreIndex::from(2), + para_id: thread_a, + kind: AssignmentKind::Parathread(thread_collator.clone(), 0), + group_idx: GroupIndex::from(2), + }; + + let mut candidate_a = TestCandidateBuilder { + para_id: chain_a, + relay_parent: System::parent_hash(), + pov_hash: Hash::repeat_byte(1), + persisted_validation_data_hash: make_vdata_hash(chain_a).unwrap(), + hrmp_watermark: RELAY_PARENT_NUM, + ..Default::default() + } + .build(); + collator_sign_candidate(Sr25519Keyring::One, &mut candidate_a); + + let mut candidate_b = TestCandidateBuilder { + para_id: chain_b, + relay_parent: System::parent_hash(), + pov_hash: Hash::repeat_byte(2), + persisted_validation_data_hash: make_vdata_hash(chain_b).unwrap(), + hrmp_watermark: RELAY_PARENT_NUM, + ..Default::default() + } + .build(); + collator_sign_candidate(Sr25519Keyring::One, &mut candidate_b); + + let mut candidate_c = TestCandidateBuilder { + para_id: thread_a, + relay_parent: System::parent_hash(), + pov_hash: Hash::repeat_byte(3), + persisted_validation_data_hash: make_vdata_hash(thread_a).unwrap(), + hrmp_watermark: RELAY_PARENT_NUM, + ..Default::default() + } + .build(); + collator_sign_candidate(Sr25519Keyring::Two, &mut candidate_c); + + let backed_a = block_on(back_candidate( + candidate_a.clone(), + &validators, + group_validators(GroupIndex::from(0)).unwrap().as_ref(), + &keystore, + &signing_context, + BackingKind::Threshold, + )); + + let backed_b = block_on(back_candidate( + candidate_b.clone(), + &validators, + group_validators(GroupIndex::from(1)).unwrap().as_ref(), + &keystore, + &signing_context, + BackingKind::Threshold, + )); + + let backed_c = block_on(back_candidate( + candidate_c.clone(), + &validators, + group_validators(GroupIndex::from(2)).unwrap().as_ref(), + &keystore, + &signing_context, + BackingKind::Threshold, + )); + + let backed_candidates = vec![backed_a, backed_b, backed_c]; + let get_backing_group_idx = { + // the order defines the group implicitly for this test case + let backed_candidates_with_groups = backed_candidates + .iter() + .enumerate() + .map(|(idx, backed_candidate)| (backed_candidate.hash(), GroupIndex(idx as _))) + .collect::>(); + + move |candidate_hash_x: CandidateHash| -> Option { + backed_candidates_with_groups.iter().find_map(|(candidate_hash, grp)| { + if *candidate_hash == candidate_hash_x { + Some(*grp) + } else { + None + } + }) + } + }; + + let ProcessedCandidates { + core_indices: occupied_cores, + candidate_receipt_with_backing_validator_indices, + } = ParaInclusion::process_candidates( + Default::default(), + backed_candidates.clone(), + vec![ + chain_a_assignment.clone(), + chain_b_assignment.clone(), + thread_a_assignment.clone(), + ], + &group_validators, + FullCheck::Yes, + ) + .expect("candidates scheduled, in order, and backed"); + + assert_eq!( + occupied_cores, + vec![CoreIndex::from(0), CoreIndex::from(1), CoreIndex::from(2)] + ); + + // Transform the votes into the setup we expect + let expected = { + let mut intermediate = std::collections::HashMap::< + CandidateHash, + (CandidateReceipt, Vec<(ValidatorIndex, ValidityAttestation)>), + >::new(); + backed_candidates.into_iter().for_each(|backed_candidate| { + let candidate_receipt_with_backers = intermediate + .entry(backed_candidate.hash()) + .or_insert_with(|| (backed_candidate.receipt(), Vec::new())); + + assert_eq!( + backed_candidate.validity_votes.len(), + backed_candidate.validator_indices.count_ones() + ); + candidate_receipt_with_backers.1.extend( + backed_candidate + .validator_indices + .iter() + .enumerate() + .filter(|(_, signed)| **signed) + .zip(backed_candidate.validity_votes.iter().cloned()) + .filter_map(|((validator_index_within_group, _), attestation)| { + let grp_idx = get_backing_group_idx(backed_candidate.hash()).unwrap(); + group_validators(grp_idx).map(|validator_indices| { + (validator_indices[validator_index_within_group], attestation) + }) + }), + ); + }); + intermediate.into_values().collect::>() + }; + + // sort, since we use a hashmap above + let assure_candidate_sorting = |mut candidate_receipts_with_backers: Vec<( + CandidateReceipt, + Vec<(ValidatorIndex, ValidityAttestation)>, + )>| { + candidate_receipts_with_backers.sort_by(|(cr1, _), (cr2, _)| { + cr1.descriptor().para_id.cmp(&cr2.descriptor().para_id) + }); + candidate_receipts_with_backers + }; + assert_eq!( + assure_candidate_sorting(expected), + assure_candidate_sorting(candidate_receipt_with_backing_validator_indices) + ); + + assert_eq!( + >::get(&chain_a), + Some(CandidatePendingAvailability { + core: CoreIndex::from(0), + hash: candidate_a.hash(), + descriptor: candidate_a.descriptor, + availability_votes: default_availability_votes(), + relay_parent_number: System::block_number() - 1, + backed_in_number: System::block_number(), + backers: backing_bitfield(&[0, 1]), + backing_group: GroupIndex::from(0), + }) + ); + assert_eq!( + >::get(&chain_a), + Some(candidate_a.commitments), + ); + + assert_eq!( + >::get(&chain_b), + Some(CandidatePendingAvailability { + core: CoreIndex::from(1), + hash: candidate_b.hash(), + descriptor: candidate_b.descriptor, + availability_votes: default_availability_votes(), + relay_parent_number: System::block_number() - 1, + backed_in_number: System::block_number(), + backers: backing_bitfield(&[2, 3]), + backing_group: GroupIndex::from(1), + }) + ); + assert_eq!( + >::get(&chain_b), + Some(candidate_b.commitments), + ); + + assert_eq!( + >::get(&thread_a), + Some(CandidatePendingAvailability { + core: CoreIndex::from(2), + hash: candidate_c.hash(), + descriptor: candidate_c.descriptor, + availability_votes: default_availability_votes(), + relay_parent_number: System::block_number() - 1, + backed_in_number: System::block_number(), + backers: backing_bitfield(&[4]), + backing_group: GroupIndex::from(2), + }) + ); + assert_eq!( + >::get(&thread_a), + Some(candidate_c.commitments), + ); + }); +} + +#[test] +fn can_include_candidate_with_ok_code_upgrade() { + let chain_a = ParaId::from(1); + + // The block number of the relay-parent for testing. + const RELAY_PARENT_NUM: BlockNumber = 4; + + let paras = vec![(chain_a, true)]; + let validators = vec![ + Sr25519Keyring::Alice, + Sr25519Keyring::Bob, + Sr25519Keyring::Charlie, + Sr25519Keyring::Dave, + Sr25519Keyring::Ferdie, + ]; + let keystore: SyncCryptoStorePtr = Arc::new(LocalKeystore::in_memory()); + for validator in validators.iter() { + SyncCryptoStore::sr25519_generate_new( + &*keystore, + PARACHAIN_KEY_TYPE_ID, + Some(&validator.to_seed()), + ) + .unwrap(); + } + let validator_public = validator_pubkeys(&validators); + + new_test_ext(genesis_config(paras)).execute_with(|| { + shared::Pallet::::set_active_validators_ascending(validator_public.clone()); + shared::Pallet::::set_session_index(5); + + run_to_block(5, |_| None); + + let signing_context = + SigningContext { parent_hash: System::parent_hash(), session_index: 5 }; + + let group_validators = |group_index: GroupIndex| { + match group_index { + group_index if group_index == GroupIndex::from(0) => Some(vec![0, 1, 2, 3, 4]), + _ => panic!("Group index out of bounds for 1 parachain"), + } + .map(|vs| vs.into_iter().map(ValidatorIndex).collect::>()) + }; + + let chain_a_assignment = CoreAssignment { + core: CoreIndex::from(0), + para_id: chain_a, + kind: AssignmentKind::Parachain, + group_idx: GroupIndex::from(0), + }; + + let mut candidate_a = TestCandidateBuilder { + para_id: chain_a, + relay_parent: System::parent_hash(), + pov_hash: Hash::repeat_byte(1), + persisted_validation_data_hash: make_vdata_hash(chain_a).unwrap(), + new_validation_code: Some(vec![1, 2, 3].into()), + hrmp_watermark: RELAY_PARENT_NUM, + ..Default::default() + } + .build(); + collator_sign_candidate(Sr25519Keyring::One, &mut candidate_a); + + let backed_a = block_on(back_candidate( + candidate_a.clone(), + &validators, + group_validators(GroupIndex::from(0)).unwrap().as_ref(), + &keystore, + &signing_context, + BackingKind::Threshold, + )); + + let ProcessedCandidates { core_indices: occupied_cores, .. } = + ParaInclusion::process_candidates( + Default::default(), + vec![backed_a], + vec![chain_a_assignment.clone()], + &group_validators, + FullCheck::Yes, + ) + .expect("candidates scheduled, in order, and backed"); + + assert_eq!(occupied_cores, vec![CoreIndex::from(0)]); + + assert_eq!( + >::get(&chain_a), + Some(CandidatePendingAvailability { + core: CoreIndex::from(0), + hash: candidate_a.hash(), + descriptor: candidate_a.descriptor, + availability_votes: default_availability_votes(), + relay_parent_number: System::block_number() - 1, + backed_in_number: System::block_number(), + backers: backing_bitfield(&[0, 1, 2]), + backing_group: GroupIndex::from(0), + }) + ); + assert_eq!( + >::get(&chain_a), + Some(candidate_a.commitments), + ); + }); +} + +#[test] +fn session_change_wipes() { + let chain_a = ParaId::from(1); + let chain_b = ParaId::from(2); + let thread_a = ParaId::from(3); + + let paras = vec![(chain_a, true), (chain_b, true), (thread_a, false)]; + let validators = vec![ + Sr25519Keyring::Alice, + Sr25519Keyring::Bob, + Sr25519Keyring::Charlie, + Sr25519Keyring::Dave, + Sr25519Keyring::Ferdie, + ]; + let keystore: SyncCryptoStorePtr = Arc::new(LocalKeystore::in_memory()); + for validator in validators.iter() { + SyncCryptoStore::sr25519_generate_new( + &*keystore, + PARACHAIN_KEY_TYPE_ID, + Some(&validator.to_seed()), + ) + .unwrap(); + } + let validator_public = validator_pubkeys(&validators); + + new_test_ext(genesis_config(paras)).execute_with(|| { + shared::Pallet::::set_active_validators_ascending(validator_public.clone()); + shared::Pallet::::set_session_index(5); + + let validators_new = + vec![Sr25519Keyring::Alice, Sr25519Keyring::Bob, Sr25519Keyring::Charlie]; + + let validator_public_new = validator_pubkeys(&validators_new); + + run_to_block(10, |_| None); + + >::insert( + &ValidatorIndex(0), + AvailabilityBitfieldRecord { bitfield: default_bitfield(), submitted_at: 9 }, + ); + + >::insert( + &ValidatorIndex(1), + AvailabilityBitfieldRecord { bitfield: default_bitfield(), submitted_at: 9 }, + ); + + >::insert( + &ValidatorIndex(4), + AvailabilityBitfieldRecord { bitfield: default_bitfield(), submitted_at: 9 }, + ); + + let candidate = TestCandidateBuilder::default().build(); + >::insert( + &chain_a, + CandidatePendingAvailability { + core: CoreIndex::from(0), + hash: candidate.hash(), + descriptor: candidate.descriptor.clone(), + availability_votes: default_availability_votes(), + relay_parent_number: 5, + backed_in_number: 6, + backers: default_backing_bitfield(), + backing_group: GroupIndex::from(0), + }, + ); + >::insert(&chain_a, candidate.commitments.clone()); + + >::insert( + &chain_b, + CandidatePendingAvailability { + core: CoreIndex::from(1), + hash: candidate.hash(), + descriptor: candidate.descriptor, + availability_votes: default_availability_votes(), + relay_parent_number: 6, + backed_in_number: 7, + backers: default_backing_bitfield(), + backing_group: GroupIndex::from(1), + }, + ); + >::insert(&chain_b, candidate.commitments); + + run_to_block(11, |_| None); + + assert_eq!(shared::Pallet::::session_index(), 5); + + assert!(>::get(&ValidatorIndex(0)).is_some()); + assert!(>::get(&ValidatorIndex(1)).is_some()); + assert!(>::get(&ValidatorIndex(4)).is_some()); + + assert!(>::get(&chain_a).is_some()); + assert!(>::get(&chain_b).is_some()); + assert!(>::get(&chain_a).is_some()); + assert!(>::get(&chain_b).is_some()); + + run_to_block(12, |n| match n { + 12 => Some(SessionChangeNotification { + validators: validator_public_new.clone(), + queued: Vec::new(), + prev_config: default_config(), + new_config: default_config(), + random_seed: Default::default(), + session_index: 6, + }), + _ => None, + }); + + assert_eq!(shared::Pallet::::session_index(), 6); + + assert!(>::get(&ValidatorIndex(0)).is_none()); + assert!(>::get(&ValidatorIndex(1)).is_none()); + assert!(>::get(&ValidatorIndex(4)).is_none()); + + assert!(>::get(&chain_a).is_none()); + assert!(>::get(&chain_b).is_none()); + assert!(>::get(&chain_a).is_none()); + assert!(>::get(&chain_b).is_none()); + + assert!(>::iter().collect::>().is_empty()); + assert!(>::iter().collect::>().is_empty()); + assert!(>::iter().collect::>().is_empty()); + }); +} + +// TODO [now]: test `collect_disputed` diff --git a/polkadot/runtime/parachains/src/paras_inherent.rs b/polkadot/runtime/parachains/src/paras_inherent.rs deleted file mode 100644 index 15abaca247..0000000000 --- a/polkadot/runtime/parachains/src/paras_inherent.rs +++ /dev/null @@ -1,2244 +0,0 @@ -// Copyright 2020 Parity Technologies (UK) Ltd. -// This file is part of Polkadot. - -// Polkadot is free software: you can redistribute it and/or modify -// it under the terms of the GNU General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. - -// Polkadot is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU General Public License for more details. - -// You should have received a copy of the GNU General Public License -// along with Polkadot. If not, see . - -//! Provides glue code over the scheduler and inclusion modules, and accepting -//! one inherent per block that can include new para candidates and bitfields. -//! -//! Unlike other modules in this crate, it does not need to be initialized by the initializer, -//! as it has no initialization logic and its finalization logic depends only on the details of -//! this module. - -use crate::{ - disputes::DisputesHandler, - inclusion, - inclusion::{CandidateCheckContext, FullCheck}, - initializer, - scheduler::{self, CoreAssignment, FreedReason}, - shared, ump, -}; -use bitvec::prelude::BitVec; -use frame_support::{ - inherent::{InherentData, InherentIdentifier, MakeFatalError, ProvideInherent}, - pallet_prelude::*, - traits::Randomness, -}; -use frame_system::pallet_prelude::*; -use pallet_babe::{self, CurrentBlockRandomness}; -use primitives::v1::{ - BackedCandidate, CandidateHash, CoreIndex, DisputeStatementSet, - InherentData as ParachainsInherentData, MultiDisputeStatementSet, ScrapedOnChainVotes, - SessionIndex, SigningContext, UncheckedSignedAvailabilityBitfield, - UncheckedSignedAvailabilityBitfields, ValidatorId, ValidatorIndex, - PARACHAINS_INHERENT_IDENTIFIER, -}; -use rand::{seq::SliceRandom, SeedableRng}; - -use scale_info::TypeInfo; -use sp_runtime::traits::{Header as HeaderT, One}; -use sp_std::{ - cmp::Ordering, - collections::{btree_map::BTreeMap, btree_set::BTreeSet}, - prelude::*, - vec::Vec, -}; - -#[cfg(feature = "runtime-benchmarks")] -mod benchmarking; - -const LOG_TARGET: &str = "runtime::inclusion-inherent"; - -pub trait WeightInfo { - /// Variant over `v`, the count of dispute statements in a dispute statement set. This gives the - /// weight of a single dispute statement set. - fn enter_variable_disputes(v: u32) -> Weight; - /// The weight of one bitfield. - fn enter_bitfields() -> Weight; - /// Variant over `v`, the count of validity votes for a backed candidate. This gives the weight - /// of a single backed candidate. - fn enter_backed_candidates_variable(v: u32) -> Weight; - /// The weight of a single backed candidate with a code upgrade. - fn enter_backed_candidate_code_upgrade() -> Weight; -} - -pub struct TestWeightInfo; -// `WeightInfo` impl for unit and integration tests. Based off of the `max_block` weight for the -// mock. -#[cfg(not(feature = "runtime-benchmarks"))] -impl WeightInfo for TestWeightInfo { - fn enter_variable_disputes(v: u32) -> Weight { - // MAX Block Weight should fit 4 disputes - 80_000 * v as Weight + 80_000 - } - fn enter_bitfields() -> Weight { - // MAX Block Weight should fit 4 backed candidates - 40_000 as Weight - } - fn enter_backed_candidates_variable(v: u32) -> Weight { - // MAX Block Weight should fit 4 backed candidates - 40_000 * v as Weight + 40_000 - } - fn enter_backed_candidate_code_upgrade() -> Weight { - 0 - } -} -// To simplify benchmarks running as tests, we set all the weights to 0. `enter` will exit early -// when if the data causes it to be over weight, but we don't want that to block a benchmark from -// running as a test. -#[cfg(feature = "runtime-benchmarks")] -impl WeightInfo for TestWeightInfo { - fn enter_variable_disputes(_v: u32) -> Weight { - 0 - } - fn enter_bitfields() -> Weight { - 0 - } - fn enter_backed_candidates_variable(_v: u32) -> Weight { - 0 - } - fn enter_backed_candidate_code_upgrade() -> Weight { - 0 - } -} - -fn paras_inherent_total_weight( - backed_candidates: &[BackedCandidate<::Hash>], - bitfields: &[UncheckedSignedAvailabilityBitfield], - disputes: &[DisputeStatementSet], -) -> Weight { - backed_candidates_weight::(backed_candidates) - .saturating_add(signed_bitfields_weight::(bitfields.len())) - .saturating_add(dispute_statements_weight::(disputes)) -} - -fn dispute_statements_weight(disputes: &[DisputeStatementSet]) -> Weight { - disputes - .iter() - .map(|d| { - <::WeightInfo as WeightInfo>::enter_variable_disputes( - d.statements.len() as u32 - ) - }) - .fold(0, |acc, x| acc.saturating_add(x)) -} - -fn signed_bitfields_weight(bitfields_len: usize) -> Weight { - <::WeightInfo as WeightInfo>::enter_bitfields() - .saturating_mul(bitfields_len as Weight) -} - -fn backed_candidate_weight( - candidate: &BackedCandidate, -) -> Weight { - if candidate.candidate.commitments.new_validation_code.is_some() { - <::WeightInfo as WeightInfo>::enter_backed_candidate_code_upgrade() - } else { - <::WeightInfo as WeightInfo>::enter_backed_candidates_variable( - candidate.validity_votes.len() as u32, - ) - } -} - -fn backed_candidates_weight( - candidates: &[BackedCandidate], -) -> Weight { - candidates - .iter() - .map(|c| backed_candidate_weight::(c)) - .fold(0, |acc, x| acc.saturating_add(x)) -} - -/// A helper trait to allow calling retain while getting access -/// to the index of the item in the `vec`. -trait IndexedRetain { - /// Retains only the elements specified by the predicate. - /// - /// In other words, remove all elements `e` residing at - /// index `i` such that `f(i, &e)` returns `false`. This method - /// operates in place, visiting each element exactly once in the - /// original order, and preserves the order of the retained elements. - fn indexed_retain(&mut self, f: impl FnMut(usize, &T) -> bool); -} - -impl IndexedRetain for Vec { - fn indexed_retain(&mut self, mut f: impl FnMut(usize, &T) -> bool) { - let mut idx = 0_usize; - self.retain(move |item| { - let ret = f(idx, item); - idx += 1_usize; - ret - }) - } -} - -/// A bitfield concerning concluded disputes for candidates -/// associated to the core index equivalent to the bit position. -#[derive(Default, PartialEq, Eq, Clone, Encode, Decode, RuntimeDebug, TypeInfo)] -pub(crate) struct DisputedBitfield(pub(crate) BitVec); - -impl From> for DisputedBitfield { - fn from(inner: BitVec) -> Self { - Self(inner) - } -} - -#[cfg(test)] -impl DisputedBitfield { - /// Create a new bitfield, where each bit is set to `false`. - pub fn zeros(n: usize) -> Self { - Self::from(BitVec::::repeat(false, n)) - } -} - -pub use pallet::*; - -#[frame_support::pallet] -pub mod pallet { - use super::*; - - #[pallet::pallet] - #[pallet::generate_store(pub(super) trait Store)] - pub struct Pallet(_); - - #[pallet::config] - #[pallet::disable_frame_system_supertrait_check] - pub trait Config: - inclusion::Config + scheduler::Config + initializer::Config + pallet_babe::Config - { - /// Weight information for extrinsics in this pallet. - type WeightInfo: WeightInfo; - } - - #[pallet::error] - pub enum Error { - /// Inclusion inherent called more than once per block. - TooManyInclusionInherents, - /// The hash of the submitted parent header doesn't correspond to the saved block hash of - /// the parent. - InvalidParentHeader, - /// Disputed candidate that was concluded invalid. - CandidateConcludedInvalid, - /// The data given to the inherent will result in an overweight block. - InherentOverweight, - } - - /// Whether the paras inherent was included within this block. - /// - /// The `Option<()>` is effectively a `bool`, but it never hits storage in the `None` variant - /// due to the guarantees of FRAME's storage APIs. - /// - /// If this is `None` at the end of the block, we panic and render the block invalid. - #[pallet::storage] - pub(crate) type Included = StorageValue<_, ()>; - - /// Scraped on chain data for extracting resolved disputes as well as backing votes. - #[pallet::storage] - #[pallet::getter(fn on_chain_votes)] - pub(crate) type OnChainVotes = StorageValue<_, ScrapedOnChainVotes>; - - #[pallet::hooks] - impl Hooks> for Pallet { - fn on_initialize(_: T::BlockNumber) -> Weight { - T::DbWeight::get().reads_writes(1, 1) // in on_finalize. - } - - fn on_finalize(_: T::BlockNumber) { - if Included::::take().is_none() { - panic!("Bitfields and heads must be included every block"); - } - } - } - - #[pallet::inherent] - impl ProvideInherent for Pallet { - type Call = Call; - type Error = MakeFatalError<()>; - const INHERENT_IDENTIFIER: InherentIdentifier = PARACHAINS_INHERENT_IDENTIFIER; - - fn create_inherent(data: &InherentData) -> Option { - let inherent_data = Self::create_inherent_inner(data)?; - // Sanity check: session changes can invalidate an inherent, - // and we _really_ don't want that to happen. - // See - - // Calling `Self::enter` here is a safe-guard, to avoid any discrepancy between on-chain checks - // (`enter`) and the off-chain checks by the block author (this function). Once we are confident - // in all the logic in this module this check should be removed to optimize performance. - - let inherent_data = match Self::enter_inner(inherent_data.clone(), FullCheck::Skip) { - Ok(_) => inherent_data, - Err(err) => { - log::error!( - target: LOG_TARGET, - "dropping paras inherent data because they produced \ - an invalid paras inherent: {:?}", - err.error, - ); - - ParachainsInherentData { - bitfields: Vec::new(), - backed_candidates: Vec::new(), - disputes: Vec::new(), - parent_header: inherent_data.parent_header, - } - }, - }; - - Some(Call::enter { data: inherent_data }) - } - - fn is_inherent(call: &Self::Call) -> bool { - matches!(call, Call::enter { .. }) - } - } - - /// Collect all freed cores based on storage data. (i.e. append cores freed from timeouts to - /// the given `freed_concluded`). - /// - /// The parameter `freed_concluded` contains all core indicies that became - /// free due to candidate that became available. - pub(crate) fn collect_all_freed_cores( - freed_concluded: I, - ) -> BTreeMap - where - I: core::iter::IntoIterator, - T: Config, - { - // Handle timeouts for any availability core work. - let availability_pred = >::availability_timeout_predicate(); - let freed_timeout = if let Some(pred) = availability_pred { - >::collect_pending(pred) - } else { - Vec::new() - }; - - // Schedule paras again, given freed cores, and reasons for freeing. - let freed = freed_concluded - .into_iter() - .map(|(c, _hash)| (c, FreedReason::Concluded)) - .chain(freed_timeout.into_iter().map(|c| (c, FreedReason::TimedOut))) - .collect::>(); - freed - } - - #[pallet::call] - impl Pallet { - /// Enter the paras inherent. This will process bitfields and backed candidates. - #[pallet::weight(( - paras_inherent_total_weight::( - data.backed_candidates.as_slice(), - data.bitfields.as_slice(), - data.disputes.as_slice(), - ), - DispatchClass::Mandatory, - ))] - pub fn enter( - origin: OriginFor, - data: ParachainsInherentData, - ) -> DispatchResultWithPostInfo { - ensure_none(origin)?; - - ensure!(!Included::::exists(), Error::::TooManyInclusionInherents); - Included::::set(Some(())); - - Self::enter_inner(data, FullCheck::Yes) - } - } -} - -impl Pallet { - pub(crate) fn enter_inner( - data: ParachainsInherentData, - full_check: FullCheck, - ) -> DispatchResultWithPostInfo { - let ParachainsInherentData { - bitfields: mut signed_bitfields, - mut backed_candidates, - parent_header, - mut disputes, - } = data; - - log::debug!( - target: LOG_TARGET, - "[enter] bitfields.len(): {}, backed_candidates.len(): {}, disputes.len() {}", - signed_bitfields.len(), - backed_candidates.len(), - disputes.len() - ); - - // Check that the submitted parent header indeed corresponds to the previous block hash. - let parent_hash = >::parent_hash(); - ensure!( - parent_header.hash().as_ref() == parent_hash.as_ref(), - Error::::InvalidParentHeader, - ); - - let now = >::block_number(); - - let mut candidate_weight = backed_candidates_weight::(&backed_candidates); - let mut bitfields_weight = signed_bitfields_weight::(signed_bitfields.len()); - let disputes_weight = dispute_statements_weight::(&disputes); - - let max_block_weight = ::BlockWeights::get().max_block; - - // Potentially trim inherent data to ensure processing will be within weight limits - let total_weight = { - if candidate_weight - .saturating_add(bitfields_weight) - .saturating_add(disputes_weight) > - max_block_weight - { - // if the total weight is over the max block weight, first try clearing backed - // candidates and bitfields. - backed_candidates.clear(); - candidate_weight = 0; - signed_bitfields.clear(); - bitfields_weight = 0; - } - - if disputes_weight > max_block_weight { - // if disputes are by themselves overweight already, trim the disputes. - debug_assert!(candidate_weight == 0 && bitfields_weight == 0); - - let entropy = compute_entropy::(parent_hash); - let mut rng = rand_chacha::ChaChaRng::from_seed(entropy.into()); - - let remaining_weight = - limit_disputes::(&mut disputes, max_block_weight, &mut rng); - max_block_weight.saturating_sub(remaining_weight) - } else { - candidate_weight - .saturating_add(bitfields_weight) - .saturating_add(disputes_weight) - } - }; - - let expected_bits = >::availability_cores().len(); - - // Handle disputes logic. - let current_session = >::session_index(); - let disputed_bitfield = { - let new_current_dispute_sets: Vec<_> = disputes - .iter() - .filter(|s| s.session == current_session) - .map(|s| (s.session, s.candidate_hash)) - .collect(); - - // Note that `provide_multi_dispute_data` will iterate, verify, and import each - // dispute; so the input here must be reasonably bounded. - let _ = T::DisputesHandler::provide_multi_dispute_data(disputes.clone())?; - if T::DisputesHandler::is_frozen() { - // The relay chain we are currently on is invalid. Proceed no further on parachains. - return Ok(Some(dispute_statements_weight::(&disputes)).into()) - } - - let mut freed_disputed = if !new_current_dispute_sets.is_empty() { - let concluded_invalid_disputes = new_current_dispute_sets - .iter() - .filter(|(session, candidate)| { - T::DisputesHandler::concluded_invalid(*session, *candidate) - }) - .map(|(_, candidate)| *candidate) - .collect::>(); - - let freed_disputed = - >::collect_disputed(&concluded_invalid_disputes) - .into_iter() - .map(|core| (core, FreedReason::Concluded)) - .collect(); - freed_disputed - } else { - Vec::new() - }; - - // Create a bit index from the set of core indices where each index corresponds to - // a core index that was freed due to a dispute. - let disputed_bitfield = create_disputed_bitfield( - expected_bits, - freed_disputed.iter().map(|(core_index, _)| core_index), - ); - - if !freed_disputed.is_empty() { - // unstable sort is fine, because core indices are unique - // i.e. the same candidate can't occupy 2 cores at once. - freed_disputed.sort_unstable_by_key(|pair| pair.0); // sort by core index - >::free_cores(freed_disputed); - } - - disputed_bitfield - }; - - // Process new availability bitfields, yielding any availability cores whose - // work has now concluded. - let freed_concluded = >::process_bitfields( - expected_bits, - signed_bitfields, - disputed_bitfield, - >::core_para, - ); - - // Inform the disputes module of all included candidates. - for (_, candidate_hash) in &freed_concluded { - T::DisputesHandler::note_included(current_session, *candidate_hash, now); - } - - let freed = collect_all_freed_cores::(freed_concluded.iter().cloned()); - - >::clear(); - >::schedule(freed, now); - - let scheduled = >::scheduled(); - let backed_candidates = sanitize_backed_candidates::( - parent_hash, - backed_candidates, - move |_candidate_index: usize, backed_candidate: &BackedCandidate| -> bool { - ::DisputesHandler::concluded_invalid(current_session, backed_candidate.hash()) - // `fn process_candidates` does the verification checks - }, - &scheduled[..], - ); - - // Process backed candidates according to scheduled cores. - let parent_storage_root = parent_header.state_root().clone(); - let inclusion::ProcessedCandidates::<::Hash> { - core_indices: occupied, - candidate_receipt_with_backing_validator_indices, - } = >::process_candidates( - parent_storage_root, - backed_candidates, - scheduled, - >::group_validators, - full_check, - )?; - - // The number of disputes included in a block is - // limited by the weight as well as the number of candidate blocks. - OnChainVotes::::put(ScrapedOnChainVotes::<::Hash> { - session: current_session, - backing_validators_per_candidate: candidate_receipt_with_backing_validator_indices, - disputes, - }); - - // Note which of the scheduled cores were actually occupied by a backed candidate. - >::occupied(&occupied); - - // Give some time slice to dispatch pending upward messages. - // this is max config.ump_service_total_weight - let _ump_weight = >::process_pending_upward_messages(); - - Ok(Some(total_weight).into()) - } -} - -impl Pallet { - /// Create the `ParachainsInherentData` that gets passed to [`Self::enter`] in [`Self::create_inherent`]. - /// This code is pulled out of [`Self::create_inherent`] so it can be unit tested. - fn create_inherent_inner(data: &InherentData) -> Option> { - let ParachainsInherentData:: { - bitfields, - backed_candidates, - mut disputes, - parent_header, - } = match data.get_data(&Self::INHERENT_IDENTIFIER) { - Ok(Some(d)) => d, - Ok(None) => return None, - Err(_) => { - log::warn!(target: LOG_TARGET, "ParachainsInherentData failed to decode"); - return None - }, - }; - - log::debug!( - target: LOG_TARGET, - "[create_inherent_inner] bitfields.len(): {}, backed_candidates.len(): {}, disputes.len() {}", - bitfields.len(), - backed_candidates.len(), - disputes.len() - ); - - let parent_hash = >::parent_hash(); - - if parent_hash != parent_header.hash() { - log::warn!( - target: LOG_TARGET, - "ParachainsInherentData references a different parent header hash than frame" - ); - return None - } - - let current_session = >::session_index(); - let expected_bits = >::availability_cores().len(); - let validator_public = shared::Pallet::::active_validator_keys(); - - T::DisputesHandler::filter_multi_dispute_data(&mut disputes); - - let (mut backed_candidates, mut bitfields) = - frame_support::storage::with_transaction(|| { - // we don't care about fresh or not disputes - // this writes them to storage, so let's query it via those means - // if this fails for whatever reason, that's ok - let _ = - T::DisputesHandler::provide_multi_dispute_data(disputes.clone()).map_err(|e| { - log::warn!( - target: LOG_TARGET, - "MultiDisputesData failed to update: {:?}", - e - ); - e - }); - - // Contains the disputes that are concluded in the current session only, - // since these are the only ones that are relevant for the occupied cores - // and lightens the load on `collect_disputed` significantly. - // Cores can't be occupied with candidates of the previous sessions, and only - // things with new votes can have just concluded. We only need to collect - // cores with disputes that conclude just now, because disputes that - // concluded longer ago have already had any corresponding cores cleaned up. - let current_concluded_invalid_disputes = disputes - .iter() - .filter(|dss| dss.session == current_session) - .map(|dss| (dss.session, dss.candidate_hash)) - .filter(|(session, candidate)| { - ::DisputesHandler::concluded_invalid(*session, *candidate) - }) - .map(|(_session, candidate)| candidate) - .collect::>(); - - // All concluded invalid disputes, that are relevant for the set of candidates - // the inherent provided. - let concluded_invalid_disputes = backed_candidates - .iter() - .map(|backed_candidate| backed_candidate.hash()) - .filter(|candidate| { - ::DisputesHandler::concluded_invalid(current_session, *candidate) - }) - .collect::>(); - - let mut freed_disputed: Vec<_> = - >::collect_disputed(¤t_concluded_invalid_disputes) - .into_iter() - .map(|core| (core, FreedReason::Concluded)) - .collect(); - - let disputed_bitfield = - create_disputed_bitfield(expected_bits, freed_disputed.iter().map(|(x, _)| x)); - - if !freed_disputed.is_empty() { - // unstable sort is fine, because core indices are unique - // i.e. the same candidate can't occupy 2 cores at once. - freed_disputed.sort_unstable_by_key(|pair| pair.0); // sort by core index - >::free_cores(freed_disputed.clone()); - } - - // The following 3 calls are equiv to a call to `process_bitfields` - // but we can retain access to `bitfields`. - let bitfields = sanitize_bitfields::( - bitfields, - disputed_bitfield, - expected_bits, - parent_hash, - current_session, - &validator_public[..], - FullCheck::Skip, - ); - - let freed_concluded = - >::update_pending_availability_and_get_freed_cores::< - _, - false, - >( - expected_bits, - &validator_public[..], - bitfields.clone(), - >::core_para, - ); - - let freed = collect_all_freed_cores::(freed_concluded.iter().cloned()); - - >::clear(); - let now = >::block_number(); - >::schedule(freed, now); - - let scheduled = >::scheduled(); - - let relay_parent_number = now - One::one(); - - let check_ctx = CandidateCheckContext::::new(now, relay_parent_number); - let backed_candidates = sanitize_backed_candidates::( - parent_hash, - backed_candidates, - move |candidate_idx: usize, - backed_candidate: &BackedCandidate<::Hash>| - -> bool { - // never include a concluded-invalid candidate - concluded_invalid_disputes.contains(&backed_candidate.hash()) || - // Instead of checking the candidates with code upgrades twice - // move the checking up here and skip it in the training wheels fallback. - // That way we avoid possible duplicate checks while assuring all - // backed candidates fine to pass on. - check_ctx - .verify_backed_candidate(parent_hash, candidate_idx, backed_candidate) - .is_err() - }, - &scheduled[..], - ); - - frame_support::storage::TransactionOutcome::Rollback(( - // filtered backed candidates - backed_candidates, - // filtered bitfields - bitfields, - )) - }); - - let entropy = compute_entropy::(parent_hash); - let mut rng = rand_chacha::ChaChaRng::from_seed(entropy.into()); - - // Assure the maximum block weight is adhered. - let max_block_weight = ::BlockWeights::get().max_block; - let _consumed_weight = apply_weight_limit::( - &mut backed_candidates, - &mut bitfields, - &mut disputes, - max_block_weight, - &mut rng, - ); - - Some(ParachainsInherentData:: { - bitfields, - backed_candidates, - disputes, - parent_header, - }) - } -} - -/// Derive a bitfield from dispute -pub(super) fn create_disputed_bitfield<'a, I>( - expected_bits: usize, - freed_cores: I, -) -> DisputedBitfield -where - I: 'a + IntoIterator, -{ - let mut bitvec = BitVec::repeat(false, expected_bits); - for core_idx in freed_cores { - let core_idx = core_idx.0 as usize; - if core_idx < expected_bits { - bitvec.set(core_idx, true); - } - } - DisputedBitfield::from(bitvec) -} - -/// Select a random subset, with preference for certain indices. -/// -/// Adds random items to the set until all candidates -/// are tried or the remaining weight is depleted. -/// -/// Returns the weight of all selected items from `selectables` -/// as well as their indices in ascending order. -fn random_sel Weight>( - rng: &mut rand_chacha::ChaChaRng, - selectables: Vec, - mut preferred_indices: Vec, - weight_fn: F, - weight_limit: Weight, -) -> (Weight, Vec) { - if selectables.is_empty() { - return (0 as Weight, Vec::new()) - } - // all indices that are not part of the preferred set - let mut indices = (0..selectables.len()) - .into_iter() - .filter(|idx| !preferred_indices.contains(idx)) - .collect::>(); - let mut picked_indices = Vec::with_capacity(selectables.len().saturating_sub(1)); - - let mut weight_acc = 0 as Weight; - - preferred_indices.shuffle(rng); - for preferred_idx in preferred_indices { - // preferred indices originate from outside - if let Some(item) = selectables.get(preferred_idx) { - let updated = weight_acc.saturating_add(weight_fn(item)); - if updated > weight_limit { - continue - } - weight_acc = updated; - picked_indices.push(preferred_idx); - } - } - - indices.shuffle(rng); - for idx in indices { - let item = &selectables[idx]; - let updated = weight_acc.saturating_add(weight_fn(item)); - - if updated > weight_limit { - continue - } - weight_acc = updated; - - picked_indices.push(idx); - } - - // sorting indices, so the ordering is retained - // unstable sorting is fine, since there are no duplicates - picked_indices.sort_unstable(); - (weight_acc, picked_indices) -} - -/// Considers an upper threshold that the inherent data must not exceed. -/// -/// If there is sufficient space, all disputes, all bitfields and all candidates -/// will be included. -/// -/// Otherwise tries to include all disputes, and then tries to fill the remaining space with bitfields and then candidates. -/// -/// The selection process is random. For candidates, there is an exception for code upgrades as they are preferred. -/// And for disputes, local and older disputes are preferred (see `limit_disputes`). -/// for backed candidates, since with a increasing number of parachains their chances of -/// inclusion become slim. All backed candidates are checked beforehands in `fn create_inherent_inner` -/// which guarantees sanity. -fn apply_weight_limit( - candidates: &mut Vec::Hash>>, - bitfields: &mut UncheckedSignedAvailabilityBitfields, - disputes: &mut MultiDisputeStatementSet, - max_block_weight: Weight, - rng: &mut rand_chacha::ChaChaRng, -) -> Weight { - // include as many disputes as possible, always - let remaining_weight = limit_disputes::(disputes, max_block_weight, rng); - - let total_candidates_weight = backed_candidates_weight::(candidates.as_slice()); - - let total_bitfields_weight = signed_bitfields_weight::(bitfields.len()); - - let total = total_bitfields_weight.saturating_add(total_candidates_weight); - - // candidates + bitfields fit into the block - if remaining_weight >= total { - return total - } - - // Prefer code upgrades, they tend to be large and hence stand no chance to be picked - // late while maintaining the weight bounds - let preferred_indices = candidates - .iter() - .enumerate() - .filter_map(|(idx, candidate)| { - candidate.candidate.commitments.new_validation_code.as_ref().map(|_code| idx) - }) - .collect::>(); - - // There is weight remaining to be consumed by a subset of candidates - // which are going to be picked now. - if let Some(remaining_weight) = remaining_weight.checked_sub(total_bitfields_weight) { - let (acc_candidate_weight, indices) = - random_sel::::Hash>, _>( - rng, - candidates.clone(), - preferred_indices, - |c| backed_candidate_weight::(c), - remaining_weight, - ); - candidates.indexed_retain(|idx, _backed_candidate| indices.binary_search(&idx).is_ok()); - // pick all bitfields, and - // fill the remaining space with candidates - let total = acc_candidate_weight.saturating_add(total_bitfields_weight); - return total - } - - candidates.clear(); - - // insufficient space for even the bitfields alone, so only try to fit as many of those - // into the block and skip the candidates entirely - let (total, indices) = random_sel::( - rng, - bitfields.clone(), - vec![], - |_| <::WeightInfo as WeightInfo>::enter_bitfields(), - remaining_weight, - ); - - bitfields.indexed_retain(|idx, _bitfield| indices.binary_search(&idx).is_ok()); - - total -} - -/// Filter bitfields based on freed core indices, validity, and other sanity checks. -/// -/// Do sanity checks on the bitfields: -/// -/// 1. no more than one bitfield per validator -/// 2. bitfields are ascending by validator index. -/// 3. each bitfield has exactly `expected_bits` -/// 4. signature is valid -/// 5. remove any disputed core indices -/// -/// If any of those is not passed, the bitfield is dropped. -/// -/// While this function technically returns a set of unchecked bitfields, -/// they were actually checked and filtered to allow using it in both -/// cases, as `filtering` and `checking` stage. -/// -/// `full_check` determines if validator signatures are checked. If `::Yes`, -/// bitfields that have an invalid signature will be filtered out. -pub(crate) fn sanitize_bitfields( - unchecked_bitfields: UncheckedSignedAvailabilityBitfields, - disputed_bitfield: DisputedBitfield, - expected_bits: usize, - parent_hash: T::Hash, - session_index: SessionIndex, - validators: &[ValidatorId], - full_check: FullCheck, -) -> UncheckedSignedAvailabilityBitfields { - let mut bitfields = Vec::with_capacity(unchecked_bitfields.len()); - - let mut last_index: Option = None; - - if disputed_bitfield.0.len() != expected_bits { - // This is a system logic error that should never occur, but we want to handle it gracefully - // so we just drop all bitfields - log::error!(target: LOG_TARGET, "BUG: disputed_bitfield != expected_bits"); - return vec![] - } - - let all_zeros = BitVec::::repeat(false, expected_bits); - let signing_context = SigningContext { parent_hash, session_index }; - for unchecked_bitfield in unchecked_bitfields { - // Find and skip invalid bitfields. - if unchecked_bitfield.unchecked_payload().0.len() != expected_bits { - log::trace!( - target: LOG_TARGET, - "[{:?}] bad bitfield length: {} != {:?}", - full_check, - unchecked_bitfield.unchecked_payload().0.len(), - expected_bits, - ); - continue - } - - if unchecked_bitfield.unchecked_payload().0.clone() & disputed_bitfield.0.clone() != - all_zeros - { - log::trace!( - target: LOG_TARGET, - "[{:?}] bitfield contains disputed cores: {:?}", - full_check, - unchecked_bitfield.unchecked_payload().0.clone() & disputed_bitfield.0.clone() - ); - continue - } - - let validator_index = unchecked_bitfield.unchecked_validator_index(); - - if !last_index.map_or(true, |last_index: ValidatorIndex| last_index < validator_index) { - log::trace!( - target: LOG_TARGET, - "[{:?}] bitfield validator index is not greater than last: !({:?} < {})", - full_check, - last_index.as_ref().map(|x| x.0), - validator_index.0 - ); - continue - } - - if unchecked_bitfield.unchecked_validator_index().0 as usize >= validators.len() { - log::trace!( - target: LOG_TARGET, - "[{:?}] bitfield validator index is out of bounds: {} >= {}", - full_check, - validator_index.0, - validators.len(), - ); - continue - } - - let validator_public = &validators[validator_index.0 as usize]; - - if let FullCheck::Yes = full_check { - if let Ok(signed_bitfield) = - unchecked_bitfield.try_into_checked(&signing_context, validator_public) - { - bitfields.push(signed_bitfield.into_unchecked()); - } else { - log::warn!(target: LOG_TARGET, "Invalid bitfield signature"); - }; - } else { - bitfields.push(unchecked_bitfield); - } - - last_index = Some(validator_index); - } - bitfields -} - -/// Filter out any candidates that have a concluded invalid dispute. -/// -/// `scheduled` follows the same naming scheme as provided in the -/// guide: Currently `free` but might become `occupied`. -/// For the filtering here the relevant part is only the current `free` -/// state. -/// -/// `candidate_has_concluded_invalid_dispute` must return `true` if the candidate -/// is disputed, false otherwise -fn sanitize_backed_candidates< - T: crate::inclusion::Config, - F: FnMut(usize, &BackedCandidate) -> bool, ->( - relay_parent: T::Hash, - mut backed_candidates: Vec>, - mut candidate_has_concluded_invalid_dispute_or_is_invalid: F, - scheduled: &[CoreAssignment], -) -> Vec> { - // Remove any candidates that were concluded invalid. - backed_candidates.indexed_retain(move |idx, backed_candidate| { - !candidate_has_concluded_invalid_dispute_or_is_invalid(idx, backed_candidate) - }); - - // Assure the backed candidate's `ParaId`'s core is free. - // This holds under the assumption that `Scheduler::schedule` is called _before_. - // Also checks the candidate references the correct relay parent. - let scheduled_paras_set = scheduled - .into_iter() - .map(|core_assignment| core_assignment.para_id) - .collect::>(); - backed_candidates.retain(|backed_candidate| { - let desc = backed_candidate.descriptor(); - desc.relay_parent == relay_parent && scheduled_paras_set.contains(&desc.para_id) - }); - - backed_candidates -} - -/// Derive entropy from babe provided per block randomness. -/// -/// In the odd case none is available, uses the `parent_hash` and -/// a const value, while emitting a warning. -fn compute_entropy(parent_hash: T::Hash) -> [u8; 32] { - const CANDIDATE_SEED_SUBJECT: [u8; 32] = *b"candidate-seed-selection-subject"; - let vrf_random = CurrentBlockRandomness::::random(&CANDIDATE_SEED_SUBJECT[..]).0; - let mut entropy: [u8; 32] = CANDIDATE_SEED_SUBJECT.clone(); - if let Some(vrf_random) = vrf_random { - entropy.as_mut().copy_from_slice(vrf_random.as_ref()); - } else { - // in case there is no vrf randomness present, we utilize the relay parent - // as seed, it's better than a static value. - log::warn!(target: LOG_TARGET, "CurrentBlockRandomness did not provide entropy"); - entropy.as_mut().copy_from_slice(parent_hash.as_ref()); - } - entropy -} - -/// Limit disputes in place. -/// -/// Returns the unused weight of `remaining_weight`. -fn limit_disputes( - disputes: &mut MultiDisputeStatementSet, - remaining_weight: Weight, - rng: &mut rand_chacha::ChaChaRng, -) -> Weight { - let mut remaining_weight = remaining_weight; - let disputes_weight = dispute_statements_weight::(&disputes); - if disputes_weight > remaining_weight { - // Sort the dispute statements according to the following prioritization: - // 1. Prioritize local disputes over remote disputes. - // 2. Prioritize older disputes over newer disputes. - disputes.sort_by(|a, b| { - let a_local_block = T::DisputesHandler::included_state(a.session, a.candidate_hash); - let b_local_block = T::DisputesHandler::included_state(b.session, b.candidate_hash); - match (a_local_block, b_local_block) { - // Prioritize local disputes over remote disputes. - (None, Some(_)) => Ordering::Greater, - (Some(_), None) => Ordering::Less, - // For local disputes, prioritize those that occur at an earlier height. - (Some(a_height), Some(b_height)) => a_height.cmp(&b_height), - // Prioritize earlier remote disputes using session as rough proxy. - (None, None) => a.session.cmp(&b.session), - } - }); - - // Since the disputes array is sorted, we may use binary search to find the beginning of - // remote disputes - let idx = disputes - .binary_search_by(|probe| { - if T::DisputesHandler::included_state(probe.session, probe.candidate_hash).is_some() - { - Ordering::Greater - } else { - Ordering::Less - } - }) - // The above predicate will never find an item and therefore we are guaranteed to obtain - // an error, which we can safely unwrap. QED. - .unwrap_err(); - - // Due to the binary search predicate above, the index computed will constitute the beginning - // of the remote disputes sub-array - let remote_disputes = disputes.split_off(idx); - - // Select disputes in-order until the remaining weight is attained - disputes.retain(|d| { - let dispute_weight = <::WeightInfo as WeightInfo>::enter_variable_disputes( - d.statements.len() as u32, - ); - if remaining_weight >= dispute_weight { - remaining_weight -= dispute_weight; - true - } else { - false - } - }); - - // Compute the statements length of all remote disputes - let d = remote_disputes.iter().map(|d| d.statements.len() as u32).collect::>(); - - // Select remote disputes at random until the block is full - let (acc_remote_disputes_weight, indices) = random_sel::( - rng, - d, - vec![], - |v| <::WeightInfo as WeightInfo>::enter_variable_disputes(*v), - remaining_weight, - ); - - // Collect all remote disputes - let mut remote_disputes = - indices.into_iter().map(|idx| disputes[idx].clone()).collect::>(); - - // Construct the full list of selected disputes - disputes.append(&mut remote_disputes); - - // Update the remaining weight - remaining_weight = remaining_weight.saturating_sub(acc_remote_disputes_weight); - } - - remaining_weight -} - -#[cfg(test)] -mod tests { - use super::*; - - // In order to facilitate benchmarks as tests we have a benchmark feature gated `WeightInfo` impl - // that uses 0 for all the weights. Because all the weights are 0, the tests that rely on - // weights for limiting data will fail, so we don't run them when using the benchmark feature. - #[cfg(not(feature = "runtime-benchmarks"))] - mod enter { - use super::*; - use crate::{ - builder::{Bench, BenchBuilder}, - mock::{new_test_ext, MockGenesisConfig, Test}, - }; - use frame_support::assert_ok; - use sp_std::collections::btree_map::BTreeMap; - - struct TestConfig { - dispute_statements: BTreeMap, - dispute_sessions: Vec, - backed_and_concluding: BTreeMap, - num_validators_per_core: u32, - includes_code_upgrade: Option, - } - - fn make_inherent_data( - TestConfig { - dispute_statements, - dispute_sessions, - backed_and_concluding, - num_validators_per_core, - includes_code_upgrade, - }: TestConfig, - ) -> Bench { - BenchBuilder::::new() - .set_max_validators((dispute_sessions.len() as u32) * num_validators_per_core) - .set_max_validators_per_core(num_validators_per_core) - .set_dispute_statements(dispute_statements) - .build(backed_and_concluding, dispute_sessions.as_slice(), includes_code_upgrade) - } - - #[test] - // Validate that if we create 2 backed candidates which are assigned to 2 cores that will be freed via - // becoming fully available, the backed candidates will not be filtered out in `create_inherent` and - // will not cause `enter` to early. - fn include_backed_candidates() { - new_test_ext(MockGenesisConfig::default()).execute_with(|| { - let dispute_statements = BTreeMap::new(); - - let mut backed_and_concluding = BTreeMap::new(); - backed_and_concluding.insert(0, 1); - backed_and_concluding.insert(1, 1); - - let scenario = make_inherent_data(TestConfig { - dispute_statements, - dispute_sessions: vec![0, 0], - backed_and_concluding, - num_validators_per_core: 1, - includes_code_upgrade: None, - }); - - // We expect the scenario to have cores 0 & 1 with pending availability. The backed - // candidates are also created for cores 0 & 1, so once the pending available - // become fully available those cores are marked as free and scheduled for the backed - // candidates. - let expected_para_inherent_data = scenario.data.clone(); - - // Check the para inherent data is as expected: - // * 1 bitfield per validator (2 validators) - assert_eq!(expected_para_inherent_data.bitfields.len(), 2); - // * 1 backed candidate per core (2 cores) - assert_eq!(expected_para_inherent_data.backed_candidates.len(), 2); - // * 0 disputes. - assert_eq!(expected_para_inherent_data.disputes.len(), 0); - let mut inherent_data = InherentData::new(); - inherent_data - .put_data(PARACHAINS_INHERENT_IDENTIFIER, &expected_para_inherent_data) - .unwrap(); - - // The current schedule is empty prior to calling `create_inherent_enter`. - assert_eq!(>::scheduled(), vec![]); - - // Nothing is filtered out (including the backed candidates.) - assert_eq!( - Pallet::::create_inherent_inner(&inherent_data.clone()).unwrap(), - expected_para_inherent_data - ); - - // The schedule is still empty prior to calling `enter`. (`create_inherent_inner` should not - // alter storage, but just double checking for sanity). - assert_eq!(>::scheduled(), vec![]); - - assert_eq!(Pallet::::on_chain_votes(), None); - // Call enter with our 2 backed candidates - assert_ok!(Pallet::::enter( - frame_system::RawOrigin::None.into(), - expected_para_inherent_data - )); - assert_eq!( - // The length of this vec is equal to the number of candidates, so we know our 2 - // backed candidates did not get filtered out - Pallet::::on_chain_votes() - .unwrap() - .backing_validators_per_candidate - .len(), - 2 - ); - }); - } - - #[test] - // Ensure that disputes are filtered out if the session is in the future. - fn filter_multi_dispute_data() { - new_test_ext(MockGenesisConfig::default()).execute_with(|| { - // Create the inherent data for this block - let dispute_statements = BTreeMap::new(); - - let backed_and_concluding = BTreeMap::new(); - - let scenario = make_inherent_data(TestConfig { - dispute_statements, - dispute_sessions: vec![ - 1, 2, 3, /* Session 3 too new, will get filtered out */ - ], - backed_and_concluding, - num_validators_per_core: 5, - includes_code_upgrade: None, - }); - - let expected_para_inherent_data = scenario.data.clone(); - - // Check the para inherent data is as expected: - // * 1 bitfield per validator (5 validators per core, 3 disputes => 3 cores, 15 validators) - assert_eq!(expected_para_inherent_data.bitfields.len(), 15); - // * 0 backed candidate per core - assert_eq!(expected_para_inherent_data.backed_candidates.len(), 0); - // * 3 disputes. - assert_eq!(expected_para_inherent_data.disputes.len(), 3); - let mut inherent_data = InherentData::new(); - inherent_data - .put_data(PARACHAINS_INHERENT_IDENTIFIER, &expected_para_inherent_data) - .unwrap(); - - // The current schedule is empty prior to calling `create_inherent_enter`. - assert_eq!(>::scheduled(), vec![]); - - let multi_dispute_inherent_data = - Pallet::::create_inherent_inner(&inherent_data.clone()).unwrap(); - // Dispute for session that lies too far in the future should be filtered out - assert!(multi_dispute_inherent_data != expected_para_inherent_data); - - assert_eq!(multi_dispute_inherent_data.disputes.len(), 2); - - // Assert that the first 2 disputes are included - assert_eq!( - &multi_dispute_inherent_data.disputes[..2], - &expected_para_inherent_data.disputes[..2], - ); - - // The schedule is still empty prior to calling `enter`. (`create_inherent_inner` should not - // alter storage, but just double checking for sanity). - assert_eq!(>::scheduled(), vec![]); - - assert_eq!(Pallet::::on_chain_votes(), None); - // Call enter with our 2 disputes - assert_ok!(Pallet::::enter( - frame_system::RawOrigin::None.into(), - multi_dispute_inherent_data, - )); - - assert_eq!( - // The length of this vec is equal to the number of candidates, so we know there - // where no backed candidates included - Pallet::::on_chain_votes() - .unwrap() - .backing_validators_per_candidate - .len(), - 0 - ); - }); - } - - #[test] - // Ensure that when dispute data establishes an over weight block that we adequately - // filter out disputes according to our prioritization rule - fn limit_dispute_data() { - new_test_ext(MockGenesisConfig::default()).execute_with(|| { - // Create the inherent data for this block - let dispute_statements = BTreeMap::new(); - // No backed and concluding cores, so all cores will be fileld with disputesw - let backed_and_concluding = BTreeMap::new(); - - let scenario = make_inherent_data(TestConfig { - dispute_statements, - dispute_sessions: vec![2, 2, 1], // 3 cores, all disputes - backed_and_concluding, - num_validators_per_core: 6, - includes_code_upgrade: None, - }); - - let expected_para_inherent_data = scenario.data.clone(); - - // Check the para inherent data is as expected: - // * 1 bitfield per validator (6 validators per core, 3 disputes => 18 validators) - assert_eq!(expected_para_inherent_data.bitfields.len(), 18); - // * 0 backed candidate per core - assert_eq!(expected_para_inherent_data.backed_candidates.len(), 0); - // * 3 disputes. - assert_eq!(expected_para_inherent_data.disputes.len(), 3); - let mut inherent_data = InherentData::new(); - inherent_data - .put_data(PARACHAINS_INHERENT_IDENTIFIER, &expected_para_inherent_data) - .unwrap(); - - // The current schedule is empty prior to calling `create_inherent_enter`. - assert_eq!(>::scheduled(), vec![]); - - let limit_inherent_data = - Pallet::::create_inherent_inner(&inherent_data.clone()).unwrap(); - // Expect that inherent data is filtered to include only 2 disputes - assert!(limit_inherent_data != expected_para_inherent_data); - - // Ensure that the included disputes are sorted by session - assert_eq!(limit_inherent_data.disputes.len(), 2); - assert_eq!(limit_inherent_data.disputes[0].session, 1); - assert_eq!(limit_inherent_data.disputes[1].session, 2); - - // The schedule is still empty prior to calling `enter`. (`create_inherent_inner` should not - // alter storage, but just double checking for sanity). - assert_eq!(>::scheduled(), vec![]); - - assert_eq!(Pallet::::on_chain_votes(), None); - // Call enter with our 2 disputes - assert_ok!(Pallet::::enter( - frame_system::RawOrigin::None.into(), - limit_inherent_data, - )); - - assert_eq!( - // Ensure that our inherent data did not included backed candidates as expected - Pallet::::on_chain_votes() - .unwrap() - .backing_validators_per_candidate - .len(), - 0 - ); - }); - } - - #[test] - // Ensure that when dispute data establishes an over weight block that we abort - // due to an over weight block - fn limit_dispute_data_overweight() { - new_test_ext(MockGenesisConfig::default()).execute_with(|| { - // Create the inherent data for this block - let dispute_statements = BTreeMap::new(); - // No backed and concluding cores, so all cores will be fileld with disputesw - let backed_and_concluding = BTreeMap::new(); - - let scenario = make_inherent_data(TestConfig { - dispute_statements, - dispute_sessions: vec![2, 2, 1], // 3 cores, all disputes - backed_and_concluding, - num_validators_per_core: 6, - includes_code_upgrade: None, - }); - - let expected_para_inherent_data = scenario.data.clone(); - - // Check the para inherent data is as expected: - // * 1 bitfield per validator (6 validators per core, 3 disputes => 18 validators) - assert_eq!(expected_para_inherent_data.bitfields.len(), 18); - // * 0 backed candidate per core - assert_eq!(expected_para_inherent_data.backed_candidates.len(), 0); - // * 3 disputes. - assert_eq!(expected_para_inherent_data.disputes.len(), 3); - let mut inherent_data = InherentData::new(); - inherent_data - .put_data(PARACHAINS_INHERENT_IDENTIFIER, &expected_para_inherent_data) - .unwrap(); - - // The current schedule is empty prior to calling `create_inherent_enter`. - assert_eq!(>::scheduled(), vec![]); - - assert_ok!(Pallet::::enter( - frame_system::RawOrigin::None.into(), - expected_para_inherent_data, - )); - }); - } - - #[test] - // Ensure that when a block is over weight due to disputes, but there is still sufficient - // block weight to include a number of signed bitfields, the inherent data is filtered - // as expected - fn limit_dispute_data_ignore_backed_candidates() { - new_test_ext(MockGenesisConfig::default()).execute_with(|| { - // Create the inherent data for this block - let dispute_statements = BTreeMap::new(); - - let mut backed_and_concluding = BTreeMap::new(); - // 2 backed candidates shall be scheduled - backed_and_concluding.insert(0, 2); - backed_and_concluding.insert(1, 2); - - let scenario = make_inherent_data(TestConfig { - dispute_statements, - // 2 backed candidates + 3 disputes (at sessions 2, 1 and 1) - dispute_sessions: vec![0, 0, 2, 2, 1], - backed_and_concluding, - num_validators_per_core: 4, - includes_code_upgrade: None, - }); - - let expected_para_inherent_data = scenario.data.clone(); - - // Check the para inherent data is as expected: - // * 1 bitfield per validator (4 validators per core, 2 backed candidates, 3 disputes => 4*5 = 20) - assert_eq!(expected_para_inherent_data.bitfields.len(), 20); - // * 2 backed candidates - assert_eq!(expected_para_inherent_data.backed_candidates.len(), 2); - // * 3 disputes. - assert_eq!(expected_para_inherent_data.disputes.len(), 3); - let mut inherent_data = InherentData::new(); - inherent_data - .put_data(PARACHAINS_INHERENT_IDENTIFIER, &expected_para_inherent_data) - .unwrap(); - - // The current schedule is empty prior to calling `create_inherent_enter`. - assert_eq!(>::scheduled(), vec![]); - - // Nothing is filtered out (including the backed candidates.) - let limit_inherent_data = - Pallet::::create_inherent_inner(&inherent_data.clone()).unwrap(); - assert!(limit_inherent_data != expected_para_inherent_data); - - // Three disputes is over weight (see previous test), so we expect to only see 2 disputes - assert_eq!(limit_inherent_data.disputes.len(), 2); - // Ensure disputes are filtered as expected - assert_eq!(limit_inherent_data.disputes[0].session, 1); - assert_eq!(limit_inherent_data.disputes[1].session, 2); - // Ensure all bitfields are included as these are still not over weight - assert_eq!( - limit_inherent_data.bitfields.len(), - expected_para_inherent_data.bitfields.len() - ); - // Ensure that all backed candidates are filtered out as either would make the block over weight - assert_eq!(limit_inherent_data.backed_candidates.len(), 0); - - // The schedule is still empty prior to calling `enter`. (`create_inherent_inner` should not - // alter storage, but just double checking for sanity). - assert_eq!(>::scheduled(), vec![]); - - assert_eq!(Pallet::::on_chain_votes(), None); - // Call enter with our 2 disputes - assert_ok!(Pallet::::enter( - frame_system::RawOrigin::None.into(), - limit_inherent_data, - )); - - assert_eq!( - // The length of this vec is equal to the number of candidates, so we know - // all of our candidates got filtered out - Pallet::::on_chain_votes() - .unwrap() - .backing_validators_per_candidate - .len(), - 0, - ); - }); - } - - #[test] - // Ensure that we abort if we encounter an over weight block for disputes + bitfields - fn limit_dispute_data_ignore_backed_candidates_overweight() { - new_test_ext(MockGenesisConfig::default()).execute_with(|| { - // Create the inherent data for this block - let dispute_statements = BTreeMap::new(); - - let mut backed_and_concluding = BTreeMap::new(); - // 2 backed candidates shall be scheduled - backed_and_concluding.insert(0, 2); - backed_and_concluding.insert(1, 2); - - let scenario = make_inherent_data(TestConfig { - dispute_statements, - // 2 backed candidates + 3 disputes (at sessions 2, 1 and 1) - dispute_sessions: vec![0, 0, 2, 2, 1], - backed_and_concluding, - num_validators_per_core: 4, - includes_code_upgrade: None, - }); - - let expected_para_inherent_data = scenario.data.clone(); - - // Check the para inherent data is as expected: - // * 1 bitfield per validator (4 validators per core, 2 backed candidates, 3 disputes => 4*5 = 20) - assert_eq!(expected_para_inherent_data.bitfields.len(), 20); - // * 2 backed candidates - assert_eq!(expected_para_inherent_data.backed_candidates.len(), 2); - // * 3 disputes. - assert_eq!(expected_para_inherent_data.disputes.len(), 3); - let mut inherent_data = InherentData::new(); - inherent_data - .put_data(PARACHAINS_INHERENT_IDENTIFIER, &expected_para_inherent_data) - .unwrap(); - - // The current schedule is empty prior to calling `create_inherent_enter`. - assert_eq!(>::scheduled(), vec![]); - - // Ensure that calling enter with 3 disputes and 2 candidates is over weight - assert_ok!(Pallet::::enter( - frame_system::RawOrigin::None.into(), - expected_para_inherent_data, - )); - - assert_eq!( - // The length of this vec is equal to the number of candidates, so we know - // all of our candidates got filtered out - Pallet::::on_chain_votes() - .unwrap() - .backing_validators_per_candidate - .len(), - 0, - ); - }); - } - - #[test] - // Ensure that when a block is over weight due to disputes and bitfields, the bitfields are - // filtered to accommodate the block size and no backed candidates are included. - fn limit_bitfields() { - new_test_ext(MockGenesisConfig::default()).execute_with(|| { - // Create the inherent data for this block - let mut dispute_statements = BTreeMap::new(); - // Cap the number of statements per dispute to 20 in order to ensure we have enough - // space in the block for some (but not all) bitfields - dispute_statements.insert(2, 20); - dispute_statements.insert(3, 20); - dispute_statements.insert(4, 20); - - let mut backed_and_concluding = BTreeMap::new(); - // Schedule 2 backed candidates - backed_and_concluding.insert(0, 2); - backed_and_concluding.insert(1, 2); - - let scenario = make_inherent_data(TestConfig { - dispute_statements, - // 2 backed candidates + 3 disputes (at sessions 2, 1 and 1) - dispute_sessions: vec![0, 0, 2, 2, 1], - backed_and_concluding, - num_validators_per_core: 5, - includes_code_upgrade: None, - }); - - let expected_para_inherent_data = scenario.data.clone(); - - // Check the para inherent data is as expected: - // * 1 bitfield per validator (5 validators per core, 2 backed candidates, 3 disputes => 4*5 = 20), - assert_eq!(expected_para_inherent_data.bitfields.len(), 25); - // * 2 backed candidates, - assert_eq!(expected_para_inherent_data.backed_candidates.len(), 2); - // * 3 disputes. - assert_eq!(expected_para_inherent_data.disputes.len(), 3); - let mut inherent_data = InherentData::new(); - inherent_data - .put_data(PARACHAINS_INHERENT_IDENTIFIER, &expected_para_inherent_data) - .unwrap(); - - // The current schedule is empty prior to calling `create_inherent_enter`. - assert_eq!(>::scheduled(), vec![]); - - // Nothing is filtered out (including the backed candidates.) - let limit_inherent_data = - Pallet::::create_inherent_inner(&inherent_data.clone()).unwrap(); - assert!(limit_inherent_data != expected_para_inherent_data); - - // Three disputes is over weight (see previous test), so we expect to only see 2 disputes - assert_eq!(limit_inherent_data.disputes.len(), 2); - // Ensure disputes are filtered as expected - assert_eq!(limit_inherent_data.disputes[0].session, 1); - assert_eq!(limit_inherent_data.disputes[1].session, 2); - // Ensure all bitfields are included as these are still not over weight - assert_eq!(limit_inherent_data.bitfields.len(), 20,); - // Ensure that all backed candidates are filtered out as either would make the block over weight - assert_eq!(limit_inherent_data.backed_candidates.len(), 0); - - // The schedule is still empty prior to calling `enter`. (`create_inherent_inner` should not - // alter storage, but just double checking for sanity). - assert_eq!(>::scheduled(), vec![]); - - assert_eq!(Pallet::::on_chain_votes(), None); - // Call enter with our 2 disputes - assert_ok!(Pallet::::enter( - frame_system::RawOrigin::None.into(), - limit_inherent_data, - )); - - assert_eq!( - // The length of this vec is equal to the number of candidates, so we know - // all of our candidates got filtered out - Pallet::::on_chain_votes() - .unwrap() - .backing_validators_per_candidate - .len(), - 0, - ); - }); - } - - #[test] - // Ensure that when a block is over weight due to disputes and bitfields, we abort - fn limit_bitfields_overweight() { - new_test_ext(MockGenesisConfig::default()).execute_with(|| { - // Create the inherent data for this block - let mut dispute_statements = BTreeMap::new(); - // Control the number of statements per dispute to ensure we have enough space - // in the block for some (but not all) bitfields - dispute_statements.insert(2, 20); - dispute_statements.insert(3, 20); - dispute_statements.insert(4, 20); - - let mut backed_and_concluding = BTreeMap::new(); - // 2 backed candidates shall be scheduled - backed_and_concluding.insert(0, 2); - backed_and_concluding.insert(1, 2); - - let scenario = make_inherent_data(TestConfig { - dispute_statements, - // 2 backed candidates + 3 disputes (at sessions 2, 1 and 1) - dispute_sessions: vec![0, 0, 2, 2, 1], - backed_and_concluding, - num_validators_per_core: 5, - includes_code_upgrade: None, - }); - - let expected_para_inherent_data = scenario.data.clone(); - - // Check the para inherent data is as expected: - // * 1 bitfield per validator (5 validators per core, 2 backed candidates, 3 disputes => 5*5 = 25) - assert_eq!(expected_para_inherent_data.bitfields.len(), 25); - // * 2 backed candidates - assert_eq!(expected_para_inherent_data.backed_candidates.len(), 2); - // * 3 disputes. - assert_eq!(expected_para_inherent_data.disputes.len(), 3); - let mut inherent_data = InherentData::new(); - inherent_data - .put_data(PARACHAINS_INHERENT_IDENTIFIER, &expected_para_inherent_data) - .unwrap(); - - // The current schedule is empty prior to calling `create_inherent_enter`. - assert_eq!(>::scheduled(), vec![]); - - assert_ok!(Pallet::::enter( - frame_system::RawOrigin::None.into(), - expected_para_inherent_data, - )); - - assert_eq!( - // The length of this vec is equal to the number of candidates, so we know - // all of our candidates got filtered out - Pallet::::on_chain_votes() - .unwrap() - .backing_validators_per_candidate - .len(), - 0, - ); - }); - } - - #[test] - // Ensure that when a block is over weight due to disputes and bitfields, we abort - fn limit_candidates_over_weight_1() { - new_test_ext(MockGenesisConfig::default()).execute_with(|| { - // Create the inherent data for this block - let mut dispute_statements = BTreeMap::new(); - // Control the number of statements per dispute to ensure we have enough space - // in the block for some (but not all) bitfields - dispute_statements.insert(2, 17); - dispute_statements.insert(3, 17); - dispute_statements.insert(4, 17); - - let mut backed_and_concluding = BTreeMap::new(); - // 2 backed candidates shall be scheduled - backed_and_concluding.insert(0, 16); - backed_and_concluding.insert(1, 25); - - let scenario = make_inherent_data(TestConfig { - dispute_statements, - dispute_sessions: vec![0, 0, 2, 2, 1], // 2 backed candidates, 3 disputes at sessions 2, 1 and 1 respectively - backed_and_concluding, - num_validators_per_core: 5, - includes_code_upgrade: None, - }); - - let expected_para_inherent_data = scenario.data.clone(); - - // Check the para inherent data is as expected: - // * 1 bitfield per validator (5 validators per core, 2 backed candidates, 3 disputes => 5*5 = 25) - assert_eq!(expected_para_inherent_data.bitfields.len(), 25); - // * 2 backed candidates - assert_eq!(expected_para_inherent_data.backed_candidates.len(), 2); - // * 3 disputes. - assert_eq!(expected_para_inherent_data.disputes.len(), 3); - let mut inherent_data = InherentData::new(); - inherent_data - .put_data(PARACHAINS_INHERENT_IDENTIFIER, &expected_para_inherent_data) - .unwrap(); - - let limit_inherent_data = - Pallet::::create_inherent_inner(&inherent_data.clone()).unwrap(); - // Expect that inherent data is filtered to include only 1 backed candidate and 2 disputes - assert!(limit_inherent_data != expected_para_inherent_data); - - // * 1 bitfields - assert_eq!(limit_inherent_data.bitfields.len(), 25); - // * 2 backed candidates - assert_eq!(limit_inherent_data.backed_candidates.len(), 1); - // * 3 disputes. - assert_eq!(limit_inherent_data.disputes.len(), 2); - - // The current schedule is empty prior to calling `create_inherent_enter`. - assert_eq!(>::scheduled(), vec![]); - - assert_ok!(Pallet::::enter( - frame_system::RawOrigin::None.into(), - limit_inherent_data, - )); - - assert_eq!( - // The length of this vec is equal to the number of candidates, so we know our 2 - // backed candidates did not get filtered out - Pallet::::on_chain_votes() - .unwrap() - .backing_validators_per_candidate - .len(), - 1 - ); - }); - } - - #[test] - // Ensure that when a block is over weight due to disputes and bitfields, we abort - fn limit_candidates_over_weight_0() { - new_test_ext(MockGenesisConfig::default()).execute_with(|| { - // Create the inherent data for this block - let mut dispute_statements = BTreeMap::new(); - // Control the number of statements per dispute to ensure we have enough space - // in the block for some (but not all) bitfields - dispute_statements.insert(2, 17); - dispute_statements.insert(3, 17); - dispute_statements.insert(4, 17); - - let mut backed_and_concluding = BTreeMap::new(); - // 2 backed candidates shall be scheduled - backed_and_concluding.insert(0, 16); - backed_and_concluding.insert(1, 25); - - let scenario = make_inherent_data(TestConfig { - dispute_statements, - dispute_sessions: vec![0, 0, 2, 2, 1], // 2 backed candidates, 3 disputes at sessions 2, 1 and 1 respectively - backed_and_concluding, - num_validators_per_core: 5, - includes_code_upgrade: None, - }); - - let expected_para_inherent_data = scenario.data.clone(); - - // Check the para inherent data is as expected: - // * 1 bitfield per validator (5 validators per core, 2 backed candidates, 3 disputes => 5*5 = 25) - assert_eq!(expected_para_inherent_data.bitfields.len(), 25); - // * 2 backed candidates - assert_eq!(expected_para_inherent_data.backed_candidates.len(), 2); - // * 3 disputes. - assert_eq!(expected_para_inherent_data.disputes.len(), 3); - - assert_ok!(Pallet::::enter( - frame_system::RawOrigin::None.into(), - expected_para_inherent_data, - )); - - assert_eq!( - // The length of this vec is equal to the number of candidates, so we know our 2 - // backed candidates did not get filtered out - Pallet::::on_chain_votes() - .unwrap() - .backing_validators_per_candidate - .len(), - 0 - ); - }); - } - } - - fn default_header() -> primitives::v1::Header { - primitives::v1::Header { - parent_hash: Default::default(), - number: 0, - state_root: Default::default(), - extrinsics_root: Default::default(), - digest: Default::default(), - } - } - - mod sanitizers { - use super::*; - - use crate::inclusion::tests::{ - back_candidate, collator_sign_candidate, BackingKind, TestCandidateBuilder, - }; - use bitvec::order::Lsb0; - use primitives::v1::{ - AvailabilityBitfield, GroupIndex, Hash, Id as ParaId, SignedAvailabilityBitfield, - ValidatorIndex, - }; - - use crate::mock::Test; - use futures::executor::block_on; - use keyring::Sr25519Keyring; - use primitives::v0::PARACHAIN_KEY_TYPE_ID; - use sc_keystore::LocalKeystore; - use sp_keystore::{SyncCryptoStore, SyncCryptoStorePtr}; - use std::sync::Arc; - - fn validator_pubkeys(val_ids: &[keyring::Sr25519Keyring]) -> Vec { - val_ids.iter().map(|v| v.public().into()).collect() - } - - #[test] - fn bitfields() { - let header = default_header(); - let parent_hash = header.hash(); - // 2 cores means two bits - let expected_bits = 2; - let session_index = SessionIndex::from(0_u32); - - let crypto_store = LocalKeystore::in_memory(); - let crypto_store = Arc::new(crypto_store) as SyncCryptoStorePtr; - let signing_context = SigningContext { parent_hash, session_index }; - - let validators = vec![ - keyring::Sr25519Keyring::Alice, - keyring::Sr25519Keyring::Bob, - keyring::Sr25519Keyring::Charlie, - keyring::Sr25519Keyring::Dave, - ]; - for validator in validators.iter() { - SyncCryptoStore::sr25519_generate_new( - &*crypto_store, - PARACHAIN_KEY_TYPE_ID, - Some(&validator.to_seed()), - ) - .unwrap(); - } - let validator_public = validator_pubkeys(&validators); - - let unchecked_bitfields = [ - BitVec::::repeat(true, expected_bits), - BitVec::::repeat(true, expected_bits), - { - let mut bv = BitVec::::repeat(false, expected_bits); - bv.set(expected_bits - 1, true); - bv - }, - ] - .iter() - .enumerate() - .map(|(vi, ab)| { - let validator_index = ValidatorIndex::from(vi as u32); - block_on(SignedAvailabilityBitfield::sign( - &crypto_store, - AvailabilityBitfield::from(ab.clone()), - &signing_context, - validator_index, - &validator_public[vi], - )) - .unwrap() - .unwrap() - .into_unchecked() - }) - .collect::>(); - - let disputed_bitfield = DisputedBitfield::zeros(expected_bits); - - { - assert_eq!( - sanitize_bitfields::( - unchecked_bitfields.clone(), - disputed_bitfield.clone(), - expected_bits, - parent_hash, - session_index, - &validator_public[..], - FullCheck::Skip, - ), - unchecked_bitfields.clone() - ); - assert_eq!( - sanitize_bitfields::( - unchecked_bitfields.clone(), - disputed_bitfield.clone(), - expected_bits, - parent_hash, - session_index, - &validator_public[..], - FullCheck::Yes - ), - unchecked_bitfields.clone() - ); - } - - // disputed bitfield is non-zero - { - let mut disputed_bitfield = DisputedBitfield::zeros(expected_bits); - // pretend the first core was freed by either a malicious validator - // or by resolved dispute - disputed_bitfield.0.set(0, true); - - assert_eq!( - sanitize_bitfields::( - unchecked_bitfields.clone(), - disputed_bitfield.clone(), - expected_bits, - parent_hash, - session_index, - &validator_public[..], - FullCheck::Yes - ) - .len(), - 1 - ); - assert_eq!( - sanitize_bitfields::( - unchecked_bitfields.clone(), - disputed_bitfield.clone(), - expected_bits, - parent_hash, - session_index, - &validator_public[..], - FullCheck::Skip - ) - .len(), - 1 - ); - } - - // bitfield size mismatch - { - assert!(sanitize_bitfields::( - unchecked_bitfields.clone(), - disputed_bitfield.clone(), - expected_bits + 1, - parent_hash, - session_index, - &validator_public[..], - FullCheck::Yes - ) - .is_empty()); - assert!(sanitize_bitfields::( - unchecked_bitfields.clone(), - disputed_bitfield.clone(), - expected_bits + 1, - parent_hash, - session_index, - &validator_public[..], - FullCheck::Skip - ) - .is_empty()); - } - - // remove the last validator - { - let shortened = validator_public.len() - 2; - assert_eq!( - &sanitize_bitfields::( - unchecked_bitfields.clone(), - disputed_bitfield.clone(), - expected_bits, - parent_hash, - session_index, - &validator_public[..shortened], - FullCheck::Yes, - )[..], - &unchecked_bitfields[..shortened] - ); - assert_eq!( - &sanitize_bitfields::( - unchecked_bitfields.clone(), - disputed_bitfield.clone(), - expected_bits, - parent_hash, - session_index, - &validator_public[..shortened], - FullCheck::Skip, - )[..], - &unchecked_bitfields[..shortened] - ); - } - - // switch ordering of bitfields - { - let mut unchecked_bitfields = unchecked_bitfields.clone(); - let x = unchecked_bitfields.swap_remove(0); - unchecked_bitfields.push(x); - assert_eq!( - &sanitize_bitfields::( - unchecked_bitfields.clone(), - disputed_bitfield.clone(), - expected_bits, - parent_hash, - session_index, - &validator_public[..], - FullCheck::Yes - )[..], - &unchecked_bitfields[..(unchecked_bitfields.len() - 2)] - ); - assert_eq!( - &sanitize_bitfields::( - unchecked_bitfields.clone(), - disputed_bitfield.clone(), - expected_bits, - parent_hash, - session_index, - &validator_public[..], - FullCheck::Skip - )[..], - &unchecked_bitfields[..(unchecked_bitfields.len() - 2)] - ); - } - - // check the validators signature - { - use primitives::v1::ValidatorSignature; - let mut unchecked_bitfields = unchecked_bitfields.clone(); - - // insert a bad signature for the last bitfield - let last_bit_idx = unchecked_bitfields.len() - 1; - unchecked_bitfields - .get_mut(last_bit_idx) - .and_then(|u| Some(u.set_signature(ValidatorSignature::default()))) - .expect("we are accessing a valid index"); - assert_eq!( - &sanitize_bitfields::( - unchecked_bitfields.clone(), - disputed_bitfield.clone(), - expected_bits, - parent_hash, - session_index, - &validator_public[..], - FullCheck::Yes - )[..], - &unchecked_bitfields[..last_bit_idx] - ); - assert_eq!( - &sanitize_bitfields::( - unchecked_bitfields.clone(), - disputed_bitfield.clone(), - expected_bits, - parent_hash, - session_index, - &validator_public[..], - FullCheck::Skip - )[..], - &unchecked_bitfields[..] - ); - } - } - - #[test] - fn candidates() { - const RELAY_PARENT_NUM: u32 = 3; - - let header = default_header(); - let relay_parent = header.hash(); - let session_index = SessionIndex::from(0_u32); - - let keystore = LocalKeystore::in_memory(); - let keystore = Arc::new(keystore) as SyncCryptoStorePtr; - let signing_context = SigningContext { parent_hash: relay_parent, session_index }; - - let validators = vec![ - keyring::Sr25519Keyring::Alice, - keyring::Sr25519Keyring::Bob, - keyring::Sr25519Keyring::Charlie, - keyring::Sr25519Keyring::Dave, - ]; - for validator in validators.iter() { - SyncCryptoStore::sr25519_generate_new( - &*keystore, - PARACHAIN_KEY_TYPE_ID, - Some(&validator.to_seed()), - ) - .unwrap(); - } - - let has_concluded_invalid = - |_idx: usize, _backed_candidate: &BackedCandidate| -> bool { false }; - - let scheduled = (0_usize..2) - .into_iter() - .map(|idx| { - let ca = CoreAssignment { - kind: scheduler::AssignmentKind::Parachain, - group_idx: GroupIndex::from(idx as u32), - para_id: ParaId::from(1_u32 + idx as u32), - core: CoreIndex::from(idx as u32), - }; - ca - }) - .collect::>(); - let scheduled = &scheduled[..]; - - let group_validators = |group_index: GroupIndex| { - match group_index { - group_index if group_index == GroupIndex::from(0) => Some(vec![0, 1]), - group_index if group_index == GroupIndex::from(1) => Some(vec![2, 3]), - _ => panic!("Group index out of bounds for 2 parachains and 1 parathread core"), - } - .map(|m| m.into_iter().map(ValidatorIndex).collect::>()) - }; - - let backed_candidates = (0_usize..2) - .into_iter() - .map(|idx0| { - let idx1 = idx0 + 1; - let mut candidate = TestCandidateBuilder { - para_id: ParaId::from(idx1), - relay_parent, - pov_hash: Hash::repeat_byte(idx1 as u8), - persisted_validation_data_hash: [42u8; 32].into(), - hrmp_watermark: RELAY_PARENT_NUM, - ..Default::default() - } - .build(); - - collator_sign_candidate(Sr25519Keyring::One, &mut candidate); - - let backed = block_on(back_candidate( - candidate, - &validators, - group_validators(GroupIndex::from(idx0 as u32)).unwrap().as_ref(), - &keystore, - &signing_context, - BackingKind::Threshold, - )); - backed - }) - .collect::>(); - - // happy path - assert_eq!( - sanitize_backed_candidates::( - relay_parent, - backed_candidates.clone(), - has_concluded_invalid, - scheduled - ), - backed_candidates - ); - - // nothing is scheduled, so no paraids match, thus all backed candidates are skipped - { - let scheduled = &[][..]; - assert!(sanitize_backed_candidates::( - relay_parent, - backed_candidates.clone(), - has_concluded_invalid, - scheduled - ) - .is_empty()); - } - - // relay parent mismatch - { - let relay_parent = Hash::repeat_byte(0xFA); - assert!(sanitize_backed_candidates::( - relay_parent, - backed_candidates.clone(), - has_concluded_invalid, - scheduled - ) - .is_empty()); - } - - // candidates that have concluded as invalid are filtered out - { - // mark every second one as concluded invalid - let set = { - let mut set = std::collections::HashSet::new(); - for (idx, backed_candidate) in backed_candidates.iter().enumerate() { - if idx & 0x01 == 0 { - set.insert(backed_candidate.hash().clone()); - } - } - set - }; - let has_concluded_invalid = - |_idx: usize, candidate: &BackedCandidate| set.contains(&candidate.hash()); - assert_eq!( - sanitize_backed_candidates::( - relay_parent, - backed_candidates.clone(), - has_concluded_invalid, - scheduled - ) - .len(), - backed_candidates.len() / 2 - ); - } - } - } -} diff --git a/polkadot/runtime/parachains/src/paras_inherent/misc.rs b/polkadot/runtime/parachains/src/paras_inherent/misc.rs new file mode 100644 index 0000000000..51b1253e94 --- /dev/null +++ b/polkadot/runtime/parachains/src/paras_inherent/misc.rs @@ -0,0 +1,40 @@ +// Copyright 2020 Parity Technologies (UK) Ltd. +// This file is part of Polkadot. + +// Polkadot is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Polkadot is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Polkadot. If not, see . + +use sp_std::vec::Vec; + +/// A helper trait to allow calling retain while getting access +/// to the index of the item in the `vec`. +pub trait IndexedRetain { + /// Retains only the elements specified by the predicate. + /// + /// In other words, remove all elements `e` residing at + /// index `i` such that `f(i, &e)` returns `false`. This method + /// operates in place, visiting each element exactly once in the + /// original order, and preserves the order of the retained elements. + fn indexed_retain(&mut self, f: impl FnMut(usize, &T) -> bool); +} + +impl IndexedRetain for Vec { + fn indexed_retain(&mut self, mut f: impl FnMut(usize, &T) -> bool) { + let mut idx = 0_usize; + self.retain(move |item| { + let ret = f(idx, item); + idx += 1_usize; + ret + }) + } +} diff --git a/polkadot/runtime/parachains/src/paras_inherent/mod.rs b/polkadot/runtime/parachains/src/paras_inherent/mod.rs new file mode 100644 index 0000000000..c0740826d7 --- /dev/null +++ b/polkadot/runtime/parachains/src/paras_inherent/mod.rs @@ -0,0 +1,1022 @@ +// Copyright 2020 Parity Technologies (UK) Ltd. +// This file is part of Polkadot. + +// Polkadot is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Polkadot is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Polkadot. If not, see . + +//! Provides glue code over the scheduler and inclusion modules, and accepting +//! one inherent per block that can include new para candidates and bitfields. +//! +//! Unlike other modules in this crate, it does not need to be initialized by the initializer, +//! as it has no initialization logic and its finalization logic depends only on the details of +//! this module. + +use crate::{ + disputes::DisputesHandler, + inclusion, + inclusion::{CandidateCheckContext, FullCheck}, + initializer, + scheduler::{self, CoreAssignment, FreedReason}, + shared, ump, +}; +use bitvec::prelude::BitVec; +use frame_support::{ + inherent::{InherentData, InherentIdentifier, MakeFatalError, ProvideInherent}, + pallet_prelude::*, + traits::Randomness, +}; +use frame_system::pallet_prelude::*; +use pallet_babe::{self, CurrentBlockRandomness}; +use primitives::v1::{ + BackedCandidate, CandidateHash, CoreIndex, DisputeStatementSet, + InherentData as ParachainsInherentData, MultiDisputeStatementSet, ScrapedOnChainVotes, + SessionIndex, SigningContext, UncheckedSignedAvailabilityBitfield, + UncheckedSignedAvailabilityBitfields, ValidatorId, ValidatorIndex, + PARACHAINS_INHERENT_IDENTIFIER, +}; +use rand::{seq::SliceRandom, SeedableRng}; + +use scale_info::TypeInfo; +use sp_runtime::traits::{Header as HeaderT, One}; +use sp_std::{ + cmp::Ordering, + collections::{btree_map::BTreeMap, btree_set::BTreeSet}, + prelude::*, + vec::Vec, +}; + +mod misc; +mod weights; + +pub use self::{ + misc::IndexedRetain, + weights::{ + backed_candidate_weight, backed_candidates_weight, dispute_statements_weight, + paras_inherent_total_weight, signed_bitfields_weight, TestWeightInfo, WeightInfo, + }, +}; + +#[cfg(feature = "runtime-benchmarks")] +mod benchmarking; + +#[cfg(test)] +mod tests; + +const LOG_TARGET: &str = "runtime::inclusion-inherent"; + +/// A bitfield concerning concluded disputes for candidates +/// associated to the core index equivalent to the bit position. +#[derive(Default, PartialEq, Eq, Clone, Encode, Decode, RuntimeDebug, TypeInfo)] +pub(crate) struct DisputedBitfield(pub(crate) BitVec); + +impl From> for DisputedBitfield { + fn from(inner: BitVec) -> Self { + Self(inner) + } +} + +#[cfg(test)] +impl DisputedBitfield { + /// Create a new bitfield, where each bit is set to `false`. + pub fn zeros(n: usize) -> Self { + Self::from(BitVec::::repeat(false, n)) + } +} + +pub use pallet::*; + +#[frame_support::pallet] +pub mod pallet { + use super::*; + + #[pallet::pallet] + #[pallet::generate_store(pub(super) trait Store)] + pub struct Pallet(_); + + #[pallet::config] + #[pallet::disable_frame_system_supertrait_check] + pub trait Config: + inclusion::Config + scheduler::Config + initializer::Config + pallet_babe::Config + { + /// Weight information for extrinsics in this pallet. + type WeightInfo: WeightInfo; + } + + #[pallet::error] + pub enum Error { + /// Inclusion inherent called more than once per block. + TooManyInclusionInherents, + /// The hash of the submitted parent header doesn't correspond to the saved block hash of + /// the parent. + InvalidParentHeader, + /// Disputed candidate that was concluded invalid. + CandidateConcludedInvalid, + /// The data given to the inherent will result in an overweight block. + InherentOverweight, + } + + /// Whether the paras inherent was included within this block. + /// + /// The `Option<()>` is effectively a `bool`, but it never hits storage in the `None` variant + /// due to the guarantees of FRAME's storage APIs. + /// + /// If this is `None` at the end of the block, we panic and render the block invalid. + #[pallet::storage] + pub(crate) type Included = StorageValue<_, ()>; + + /// Scraped on chain data for extracting resolved disputes as well as backing votes. + #[pallet::storage] + #[pallet::getter(fn on_chain_votes)] + pub(crate) type OnChainVotes = StorageValue<_, ScrapedOnChainVotes>; + + #[pallet::hooks] + impl Hooks> for Pallet { + fn on_initialize(_: T::BlockNumber) -> Weight { + T::DbWeight::get().reads_writes(1, 1) // in on_finalize. + } + + fn on_finalize(_: T::BlockNumber) { + if Included::::take().is_none() { + panic!("Bitfields and heads must be included every block"); + } + } + } + + #[pallet::inherent] + impl ProvideInherent for Pallet { + type Call = Call; + type Error = MakeFatalError<()>; + const INHERENT_IDENTIFIER: InherentIdentifier = PARACHAINS_INHERENT_IDENTIFIER; + + fn create_inherent(data: &InherentData) -> Option { + let inherent_data = Self::create_inherent_inner(data)?; + // Sanity check: session changes can invalidate an inherent, + // and we _really_ don't want that to happen. + // See + + // Calling `Self::enter` here is a safe-guard, to avoid any discrepancy between on-chain checks + // (`enter`) and the off-chain checks by the block author (this function). Once we are confident + // in all the logic in this module this check should be removed to optimize performance. + + let inherent_data = match Self::enter_inner(inherent_data.clone(), FullCheck::Skip) { + Ok(_) => inherent_data, + Err(err) => { + log::error!( + target: LOG_TARGET, + "dropping paras inherent data because they produced \ + an invalid paras inherent: {:?}", + err.error, + ); + + ParachainsInherentData { + bitfields: Vec::new(), + backed_candidates: Vec::new(), + disputes: Vec::new(), + parent_header: inherent_data.parent_header, + } + }, + }; + + Some(Call::enter { data: inherent_data }) + } + + fn is_inherent(call: &Self::Call) -> bool { + matches!(call, Call::enter { .. }) + } + } + + /// Collect all freed cores based on storage data. (i.e. append cores freed from timeouts to + /// the given `freed_concluded`). + /// + /// The parameter `freed_concluded` contains all core indicies that became + /// free due to candidate that became available. + pub(crate) fn collect_all_freed_cores( + freed_concluded: I, + ) -> BTreeMap + where + I: core::iter::IntoIterator, + T: Config, + { + // Handle timeouts for any availability core work. + let availability_pred = >::availability_timeout_predicate(); + let freed_timeout = if let Some(pred) = availability_pred { + >::collect_pending(pred) + } else { + Vec::new() + }; + + // Schedule paras again, given freed cores, and reasons for freeing. + let freed = freed_concluded + .into_iter() + .map(|(c, _hash)| (c, FreedReason::Concluded)) + .chain(freed_timeout.into_iter().map(|c| (c, FreedReason::TimedOut))) + .collect::>(); + freed + } + + #[pallet::call] + impl Pallet { + /// Enter the paras inherent. This will process bitfields and backed candidates. + #[pallet::weight(( + paras_inherent_total_weight::( + data.backed_candidates.as_slice(), + data.bitfields.as_slice(), + data.disputes.as_slice(), + ), + DispatchClass::Mandatory, + ))] + pub fn enter( + origin: OriginFor, + data: ParachainsInherentData, + ) -> DispatchResultWithPostInfo { + ensure_none(origin)?; + + ensure!(!Included::::exists(), Error::::TooManyInclusionInherents); + Included::::set(Some(())); + + Self::enter_inner(data, FullCheck::Yes) + } + } +} + +impl Pallet { + pub(crate) fn enter_inner( + data: ParachainsInherentData, + full_check: FullCheck, + ) -> DispatchResultWithPostInfo { + let ParachainsInherentData { + bitfields: mut signed_bitfields, + mut backed_candidates, + parent_header, + mut disputes, + } = data; + + log::debug!( + target: LOG_TARGET, + "[enter] bitfields.len(): {}, backed_candidates.len(): {}, disputes.len() {}", + signed_bitfields.len(), + backed_candidates.len(), + disputes.len() + ); + + // Check that the submitted parent header indeed corresponds to the previous block hash. + let parent_hash = >::parent_hash(); + ensure!( + parent_header.hash().as_ref() == parent_hash.as_ref(), + Error::::InvalidParentHeader, + ); + + let now = >::block_number(); + + let mut candidate_weight = backed_candidates_weight::(&backed_candidates); + let mut bitfields_weight = signed_bitfields_weight::(signed_bitfields.len()); + let disputes_weight = dispute_statements_weight::(&disputes); + + let max_block_weight = ::BlockWeights::get().max_block; + + // Potentially trim inherent data to ensure processing will be within weight limits + let total_weight = { + if candidate_weight + .saturating_add(bitfields_weight) + .saturating_add(disputes_weight) > + max_block_weight + { + // if the total weight is over the max block weight, first try clearing backed + // candidates and bitfields. + backed_candidates.clear(); + candidate_weight = 0; + signed_bitfields.clear(); + bitfields_weight = 0; + } + + if disputes_weight > max_block_weight { + // if disputes are by themselves overweight already, trim the disputes. + debug_assert!(candidate_weight == 0 && bitfields_weight == 0); + + let entropy = compute_entropy::(parent_hash); + let mut rng = rand_chacha::ChaChaRng::from_seed(entropy.into()); + + let remaining_weight = + limit_disputes::(&mut disputes, max_block_weight, &mut rng); + max_block_weight.saturating_sub(remaining_weight) + } else { + candidate_weight + .saturating_add(bitfields_weight) + .saturating_add(disputes_weight) + } + }; + + let expected_bits = >::availability_cores().len(); + + // Handle disputes logic. + let current_session = >::session_index(); + let disputed_bitfield = { + let new_current_dispute_sets: Vec<_> = disputes + .iter() + .filter(|s| s.session == current_session) + .map(|s| (s.session, s.candidate_hash)) + .collect(); + + // Note that `provide_multi_dispute_data` will iterate, verify, and import each + // dispute; so the input here must be reasonably bounded. + let _ = T::DisputesHandler::provide_multi_dispute_data(disputes.clone())?; + if T::DisputesHandler::is_frozen() { + // The relay chain we are currently on is invalid. Proceed no further on parachains. + return Ok(Some(dispute_statements_weight::(&disputes)).into()) + } + + let mut freed_disputed = if !new_current_dispute_sets.is_empty() { + let concluded_invalid_disputes = new_current_dispute_sets + .iter() + .filter(|(session, candidate)| { + T::DisputesHandler::concluded_invalid(*session, *candidate) + }) + .map(|(_, candidate)| *candidate) + .collect::>(); + + let freed_disputed = + >::collect_disputed(&concluded_invalid_disputes) + .into_iter() + .map(|core| (core, FreedReason::Concluded)) + .collect(); + freed_disputed + } else { + Vec::new() + }; + + // Create a bit index from the set of core indices where each index corresponds to + // a core index that was freed due to a dispute. + let disputed_bitfield = create_disputed_bitfield( + expected_bits, + freed_disputed.iter().map(|(core_index, _)| core_index), + ); + + if !freed_disputed.is_empty() { + // unstable sort is fine, because core indices are unique + // i.e. the same candidate can't occupy 2 cores at once. + freed_disputed.sort_unstable_by_key(|pair| pair.0); // sort by core index + >::free_cores(freed_disputed); + } + + disputed_bitfield + }; + + // Process new availability bitfields, yielding any availability cores whose + // work has now concluded. + let freed_concluded = >::process_bitfields( + expected_bits, + signed_bitfields, + disputed_bitfield, + >::core_para, + ); + + // Inform the disputes module of all included candidates. + for (_, candidate_hash) in &freed_concluded { + T::DisputesHandler::note_included(current_session, *candidate_hash, now); + } + + let freed = collect_all_freed_cores::(freed_concluded.iter().cloned()); + + >::clear(); + >::schedule(freed, now); + + let scheduled = >::scheduled(); + let backed_candidates = sanitize_backed_candidates::( + parent_hash, + backed_candidates, + move |_candidate_index: usize, backed_candidate: &BackedCandidate| -> bool { + ::DisputesHandler::concluded_invalid(current_session, backed_candidate.hash()) + // `fn process_candidates` does the verification checks + }, + &scheduled[..], + ); + + // Process backed candidates according to scheduled cores. + let parent_storage_root = parent_header.state_root().clone(); + let inclusion::ProcessedCandidates::<::Hash> { + core_indices: occupied, + candidate_receipt_with_backing_validator_indices, + } = >::process_candidates( + parent_storage_root, + backed_candidates, + scheduled, + >::group_validators, + full_check, + )?; + + // The number of disputes included in a block is + // limited by the weight as well as the number of candidate blocks. + OnChainVotes::::put(ScrapedOnChainVotes::<::Hash> { + session: current_session, + backing_validators_per_candidate: candidate_receipt_with_backing_validator_indices, + disputes, + }); + + // Note which of the scheduled cores were actually occupied by a backed candidate. + >::occupied(&occupied); + + // Give some time slice to dispatch pending upward messages. + // this is max config.ump_service_total_weight + let _ump_weight = >::process_pending_upward_messages(); + + Ok(Some(total_weight).into()) + } +} + +impl Pallet { + /// Create the `ParachainsInherentData` that gets passed to [`Self::enter`] in [`Self::create_inherent`]. + /// This code is pulled out of [`Self::create_inherent`] so it can be unit tested. + fn create_inherent_inner(data: &InherentData) -> Option> { + let ParachainsInherentData:: { + bitfields, + backed_candidates, + mut disputes, + parent_header, + } = match data.get_data(&Self::INHERENT_IDENTIFIER) { + Ok(Some(d)) => d, + Ok(None) => return None, + Err(_) => { + log::warn!(target: LOG_TARGET, "ParachainsInherentData failed to decode"); + return None + }, + }; + + log::debug!( + target: LOG_TARGET, + "[create_inherent_inner] bitfields.len(): {}, backed_candidates.len(): {}, disputes.len() {}", + bitfields.len(), + backed_candidates.len(), + disputes.len() + ); + + let parent_hash = >::parent_hash(); + + if parent_hash != parent_header.hash() { + log::warn!( + target: LOG_TARGET, + "ParachainsInherentData references a different parent header hash than frame" + ); + return None + } + + let current_session = >::session_index(); + let expected_bits = >::availability_cores().len(); + let validator_public = shared::Pallet::::active_validator_keys(); + + T::DisputesHandler::filter_multi_dispute_data(&mut disputes); + + let (mut backed_candidates, mut bitfields) = + frame_support::storage::with_transaction(|| { + // we don't care about fresh or not disputes + // this writes them to storage, so let's query it via those means + // if this fails for whatever reason, that's ok + let _ = + T::DisputesHandler::provide_multi_dispute_data(disputes.clone()).map_err(|e| { + log::warn!( + target: LOG_TARGET, + "MultiDisputesData failed to update: {:?}", + e + ); + e + }); + + // Contains the disputes that are concluded in the current session only, + // since these are the only ones that are relevant for the occupied cores + // and lightens the load on `collect_disputed` significantly. + // Cores can't be occupied with candidates of the previous sessions, and only + // things with new votes can have just concluded. We only need to collect + // cores with disputes that conclude just now, because disputes that + // concluded longer ago have already had any corresponding cores cleaned up. + let current_concluded_invalid_disputes = disputes + .iter() + .filter(|dss| dss.session == current_session) + .map(|dss| (dss.session, dss.candidate_hash)) + .filter(|(session, candidate)| { + ::DisputesHandler::concluded_invalid(*session, *candidate) + }) + .map(|(_session, candidate)| candidate) + .collect::>(); + + // All concluded invalid disputes, that are relevant for the set of candidates + // the inherent provided. + let concluded_invalid_disputes = backed_candidates + .iter() + .map(|backed_candidate| backed_candidate.hash()) + .filter(|candidate| { + ::DisputesHandler::concluded_invalid(current_session, *candidate) + }) + .collect::>(); + + let mut freed_disputed: Vec<_> = + >::collect_disputed(¤t_concluded_invalid_disputes) + .into_iter() + .map(|core| (core, FreedReason::Concluded)) + .collect(); + + let disputed_bitfield = + create_disputed_bitfield(expected_bits, freed_disputed.iter().map(|(x, _)| x)); + + if !freed_disputed.is_empty() { + // unstable sort is fine, because core indices are unique + // i.e. the same candidate can't occupy 2 cores at once. + freed_disputed.sort_unstable_by_key(|pair| pair.0); // sort by core index + >::free_cores(freed_disputed.clone()); + } + + // The following 3 calls are equiv to a call to `process_bitfields` + // but we can retain access to `bitfields`. + let bitfields = sanitize_bitfields::( + bitfields, + disputed_bitfield, + expected_bits, + parent_hash, + current_session, + &validator_public[..], + FullCheck::Skip, + ); + + let freed_concluded = + >::update_pending_availability_and_get_freed_cores::< + _, + false, + >( + expected_bits, + &validator_public[..], + bitfields.clone(), + >::core_para, + ); + + let freed = collect_all_freed_cores::(freed_concluded.iter().cloned()); + + >::clear(); + let now = >::block_number(); + >::schedule(freed, now); + + let scheduled = >::scheduled(); + + let relay_parent_number = now - One::one(); + + let check_ctx = CandidateCheckContext::::new(now, relay_parent_number); + let backed_candidates = sanitize_backed_candidates::( + parent_hash, + backed_candidates, + move |candidate_idx: usize, + backed_candidate: &BackedCandidate<::Hash>| + -> bool { + // never include a concluded-invalid candidate + concluded_invalid_disputes.contains(&backed_candidate.hash()) || + // Instead of checking the candidates with code upgrades twice + // move the checking up here and skip it in the training wheels fallback. + // That way we avoid possible duplicate checks while assuring all + // backed candidates fine to pass on. + check_ctx + .verify_backed_candidate(parent_hash, candidate_idx, backed_candidate) + .is_err() + }, + &scheduled[..], + ); + + frame_support::storage::TransactionOutcome::Rollback(( + // filtered backed candidates + backed_candidates, + // filtered bitfields + bitfields, + )) + }); + + let entropy = compute_entropy::(parent_hash); + let mut rng = rand_chacha::ChaChaRng::from_seed(entropy.into()); + + // Assure the maximum block weight is adhered. + let max_block_weight = ::BlockWeights::get().max_block; + let _consumed_weight = apply_weight_limit::( + &mut backed_candidates, + &mut bitfields, + &mut disputes, + max_block_weight, + &mut rng, + ); + + Some(ParachainsInherentData:: { + bitfields, + backed_candidates, + disputes, + parent_header, + }) + } +} + +/// Derive a bitfield from dispute +pub(super) fn create_disputed_bitfield<'a, I>( + expected_bits: usize, + freed_cores: I, +) -> DisputedBitfield +where + I: 'a + IntoIterator, +{ + let mut bitvec = BitVec::repeat(false, expected_bits); + for core_idx in freed_cores { + let core_idx = core_idx.0 as usize; + if core_idx < expected_bits { + bitvec.set(core_idx, true); + } + } + DisputedBitfield::from(bitvec) +} + +/// Select a random subset, with preference for certain indices. +/// +/// Adds random items to the set until all candidates +/// are tried or the remaining weight is depleted. +/// +/// Returns the weight of all selected items from `selectables` +/// as well as their indices in ascending order. +fn random_sel Weight>( + rng: &mut rand_chacha::ChaChaRng, + selectables: Vec, + mut preferred_indices: Vec, + weight_fn: F, + weight_limit: Weight, +) -> (Weight, Vec) { + if selectables.is_empty() { + return (0 as Weight, Vec::new()) + } + // all indices that are not part of the preferred set + let mut indices = (0..selectables.len()) + .into_iter() + .filter(|idx| !preferred_indices.contains(idx)) + .collect::>(); + let mut picked_indices = Vec::with_capacity(selectables.len().saturating_sub(1)); + + let mut weight_acc = 0 as Weight; + + preferred_indices.shuffle(rng); + for preferred_idx in preferred_indices { + // preferred indices originate from outside + if let Some(item) = selectables.get(preferred_idx) { + let updated = weight_acc.saturating_add(weight_fn(item)); + if updated > weight_limit { + continue + } + weight_acc = updated; + picked_indices.push(preferred_idx); + } + } + + indices.shuffle(rng); + for idx in indices { + let item = &selectables[idx]; + let updated = weight_acc.saturating_add(weight_fn(item)); + + if updated > weight_limit { + continue + } + weight_acc = updated; + + picked_indices.push(idx); + } + + // sorting indices, so the ordering is retained + // unstable sorting is fine, since there are no duplicates + picked_indices.sort_unstable(); + (weight_acc, picked_indices) +} + +/// Considers an upper threshold that the inherent data must not exceed. +/// +/// If there is sufficient space, all disputes, all bitfields and all candidates +/// will be included. +/// +/// Otherwise tries to include all disputes, and then tries to fill the remaining space with bitfields and then candidates. +/// +/// The selection process is random. For candidates, there is an exception for code upgrades as they are preferred. +/// And for disputes, local and older disputes are preferred (see `limit_disputes`). +/// for backed candidates, since with a increasing number of parachains their chances of +/// inclusion become slim. All backed candidates are checked beforehands in `fn create_inherent_inner` +/// which guarantees sanity. +fn apply_weight_limit( + candidates: &mut Vec::Hash>>, + bitfields: &mut UncheckedSignedAvailabilityBitfields, + disputes: &mut MultiDisputeStatementSet, + max_block_weight: Weight, + rng: &mut rand_chacha::ChaChaRng, +) -> Weight { + // include as many disputes as possible, always + let remaining_weight = limit_disputes::(disputes, max_block_weight, rng); + + let total_candidates_weight = backed_candidates_weight::(candidates.as_slice()); + + let total_bitfields_weight = signed_bitfields_weight::(bitfields.len()); + + let total = total_bitfields_weight.saturating_add(total_candidates_weight); + + // candidates + bitfields fit into the block + if remaining_weight >= total { + return total + } + + // Prefer code upgrades, they tend to be large and hence stand no chance to be picked + // late while maintaining the weight bounds + let preferred_indices = candidates + .iter() + .enumerate() + .filter_map(|(idx, candidate)| { + candidate.candidate.commitments.new_validation_code.as_ref().map(|_code| idx) + }) + .collect::>(); + + // There is weight remaining to be consumed by a subset of candidates + // which are going to be picked now. + if let Some(remaining_weight) = remaining_weight.checked_sub(total_bitfields_weight) { + let (acc_candidate_weight, indices) = + random_sel::::Hash>, _>( + rng, + candidates.clone(), + preferred_indices, + |c| backed_candidate_weight::(c), + remaining_weight, + ); + candidates.indexed_retain(|idx, _backed_candidate| indices.binary_search(&idx).is_ok()); + // pick all bitfields, and + // fill the remaining space with candidates + let total = acc_candidate_weight.saturating_add(total_bitfields_weight); + return total + } + + candidates.clear(); + + // insufficient space for even the bitfields alone, so only try to fit as many of those + // into the block and skip the candidates entirely + let (total, indices) = random_sel::( + rng, + bitfields.clone(), + vec![], + |_| <::WeightInfo as WeightInfo>::enter_bitfields(), + remaining_weight, + ); + + bitfields.indexed_retain(|idx, _bitfield| indices.binary_search(&idx).is_ok()); + + total +} + +/// Filter bitfields based on freed core indices, validity, and other sanity checks. +/// +/// Do sanity checks on the bitfields: +/// +/// 1. no more than one bitfield per validator +/// 2. bitfields are ascending by validator index. +/// 3. each bitfield has exactly `expected_bits` +/// 4. signature is valid +/// 5. remove any disputed core indices +/// +/// If any of those is not passed, the bitfield is dropped. +/// +/// While this function technically returns a set of unchecked bitfields, +/// they were actually checked and filtered to allow using it in both +/// cases, as `filtering` and `checking` stage. +/// +/// `full_check` determines if validator signatures are checked. If `::Yes`, +/// bitfields that have an invalid signature will be filtered out. +pub(crate) fn sanitize_bitfields( + unchecked_bitfields: UncheckedSignedAvailabilityBitfields, + disputed_bitfield: DisputedBitfield, + expected_bits: usize, + parent_hash: T::Hash, + session_index: SessionIndex, + validators: &[ValidatorId], + full_check: FullCheck, +) -> UncheckedSignedAvailabilityBitfields { + let mut bitfields = Vec::with_capacity(unchecked_bitfields.len()); + + let mut last_index: Option = None; + + if disputed_bitfield.0.len() != expected_bits { + // This is a system logic error that should never occur, but we want to handle it gracefully + // so we just drop all bitfields + log::error!(target: LOG_TARGET, "BUG: disputed_bitfield != expected_bits"); + return vec![] + } + + let all_zeros = BitVec::::repeat(false, expected_bits); + let signing_context = SigningContext { parent_hash, session_index }; + for unchecked_bitfield in unchecked_bitfields { + // Find and skip invalid bitfields. + if unchecked_bitfield.unchecked_payload().0.len() != expected_bits { + log::trace!( + target: LOG_TARGET, + "[{:?}] bad bitfield length: {} != {:?}", + full_check, + unchecked_bitfield.unchecked_payload().0.len(), + expected_bits, + ); + continue + } + + if unchecked_bitfield.unchecked_payload().0.clone() & disputed_bitfield.0.clone() != + all_zeros + { + log::trace!( + target: LOG_TARGET, + "[{:?}] bitfield contains disputed cores: {:?}", + full_check, + unchecked_bitfield.unchecked_payload().0.clone() & disputed_bitfield.0.clone() + ); + continue + } + + let validator_index = unchecked_bitfield.unchecked_validator_index(); + + if !last_index.map_or(true, |last_index: ValidatorIndex| last_index < validator_index) { + log::trace!( + target: LOG_TARGET, + "[{:?}] bitfield validator index is not greater than last: !({:?} < {})", + full_check, + last_index.as_ref().map(|x| x.0), + validator_index.0 + ); + continue + } + + if unchecked_bitfield.unchecked_validator_index().0 as usize >= validators.len() { + log::trace!( + target: LOG_TARGET, + "[{:?}] bitfield validator index is out of bounds: {} >= {}", + full_check, + validator_index.0, + validators.len(), + ); + continue + } + + let validator_public = &validators[validator_index.0 as usize]; + + if let FullCheck::Yes = full_check { + if let Ok(signed_bitfield) = + unchecked_bitfield.try_into_checked(&signing_context, validator_public) + { + bitfields.push(signed_bitfield.into_unchecked()); + } else { + log::warn!(target: LOG_TARGET, "Invalid bitfield signature"); + }; + } else { + bitfields.push(unchecked_bitfield); + } + + last_index = Some(validator_index); + } + bitfields +} + +/// Filter out any candidates that have a concluded invalid dispute. +/// +/// `scheduled` follows the same naming scheme as provided in the +/// guide: Currently `free` but might become `occupied`. +/// For the filtering here the relevant part is only the current `free` +/// state. +/// +/// `candidate_has_concluded_invalid_dispute` must return `true` if the candidate +/// is disputed, false otherwise +fn sanitize_backed_candidates< + T: crate::inclusion::Config, + F: FnMut(usize, &BackedCandidate) -> bool, +>( + relay_parent: T::Hash, + mut backed_candidates: Vec>, + mut candidate_has_concluded_invalid_dispute_or_is_invalid: F, + scheduled: &[CoreAssignment], +) -> Vec> { + // Remove any candidates that were concluded invalid. + backed_candidates.indexed_retain(move |idx, backed_candidate| { + !candidate_has_concluded_invalid_dispute_or_is_invalid(idx, backed_candidate) + }); + + // Assure the backed candidate's `ParaId`'s core is free. + // This holds under the assumption that `Scheduler::schedule` is called _before_. + // Also checks the candidate references the correct relay parent. + let scheduled_paras_set = scheduled + .into_iter() + .map(|core_assignment| core_assignment.para_id) + .collect::>(); + backed_candidates.retain(|backed_candidate| { + let desc = backed_candidate.descriptor(); + desc.relay_parent == relay_parent && scheduled_paras_set.contains(&desc.para_id) + }); + + backed_candidates +} + +/// Derive entropy from babe provided per block randomness. +/// +/// In the odd case none is available, uses the `parent_hash` and +/// a const value, while emitting a warning. +fn compute_entropy(parent_hash: T::Hash) -> [u8; 32] { + const CANDIDATE_SEED_SUBJECT: [u8; 32] = *b"candidate-seed-selection-subject"; + let vrf_random = CurrentBlockRandomness::::random(&CANDIDATE_SEED_SUBJECT[..]).0; + let mut entropy: [u8; 32] = CANDIDATE_SEED_SUBJECT.clone(); + if let Some(vrf_random) = vrf_random { + entropy.as_mut().copy_from_slice(vrf_random.as_ref()); + } else { + // in case there is no vrf randomness present, we utilize the relay parent + // as seed, it's better than a static value. + log::warn!(target: LOG_TARGET, "CurrentBlockRandomness did not provide entropy"); + entropy.as_mut().copy_from_slice(parent_hash.as_ref()); + } + entropy +} + +/// Limit disputes in place. +/// +/// Returns the unused weight of `remaining_weight`. +fn limit_disputes( + disputes: &mut MultiDisputeStatementSet, + remaining_weight: Weight, + rng: &mut rand_chacha::ChaChaRng, +) -> Weight { + let mut remaining_weight = remaining_weight; + let disputes_weight = dispute_statements_weight::(&disputes); + if disputes_weight > remaining_weight { + // Sort the dispute statements according to the following prioritization: + // 1. Prioritize local disputes over remote disputes. + // 2. Prioritize older disputes over newer disputes. + disputes.sort_by(|a, b| { + let a_local_block = T::DisputesHandler::included_state(a.session, a.candidate_hash); + let b_local_block = T::DisputesHandler::included_state(b.session, b.candidate_hash); + match (a_local_block, b_local_block) { + // Prioritize local disputes over remote disputes. + (None, Some(_)) => Ordering::Greater, + (Some(_), None) => Ordering::Less, + // For local disputes, prioritize those that occur at an earlier height. + (Some(a_height), Some(b_height)) => a_height.cmp(&b_height), + // Prioritize earlier remote disputes using session as rough proxy. + (None, None) => a.session.cmp(&b.session), + } + }); + + // Since the disputes array is sorted, we may use binary search to find the beginning of + // remote disputes + let idx = disputes + .binary_search_by(|probe| { + if T::DisputesHandler::included_state(probe.session, probe.candidate_hash).is_some() + { + Ordering::Greater + } else { + Ordering::Less + } + }) + // The above predicate will never find an item and therefore we are guaranteed to obtain + // an error, which we can safely unwrap. QED. + .unwrap_err(); + + // Due to the binary search predicate above, the index computed will constitute the beginning + // of the remote disputes sub-array + let remote_disputes = disputes.split_off(idx); + + // Select disputes in-order until the remaining weight is attained + disputes.retain(|d| { + let dispute_weight = <::WeightInfo as WeightInfo>::enter_variable_disputes( + d.statements.len() as u32, + ); + if remaining_weight >= dispute_weight { + remaining_weight -= dispute_weight; + true + } else { + false + } + }); + + // Compute the statements length of all remote disputes + let d = remote_disputes.iter().map(|d| d.statements.len() as u32).collect::>(); + + // Select remote disputes at random until the block is full + let (acc_remote_disputes_weight, indices) = random_sel::( + rng, + d, + vec![], + |v| <::WeightInfo as WeightInfo>::enter_variable_disputes(*v), + remaining_weight, + ); + + // Collect all remote disputes + let mut remote_disputes = + indices.into_iter().map(|idx| disputes[idx].clone()).collect::>(); + + // Construct the full list of selected disputes + disputes.append(&mut remote_disputes); + + // Update the remaining weight + remaining_weight = remaining_weight.saturating_sub(acc_remote_disputes_weight); + } + + remaining_weight +} diff --git a/polkadot/runtime/parachains/src/paras_inherent/tests.rs b/polkadot/runtime/parachains/src/paras_inherent/tests.rs new file mode 100644 index 0000000000..7eeacb5c16 --- /dev/null +++ b/polkadot/runtime/parachains/src/paras_inherent/tests.rs @@ -0,0 +1,1096 @@ +// Copyright 2021 Parity Technologies (UK) Ltd. +// This file is part of Polkadot. + +// Polkadot is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Polkadot is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Polkadot. If not, see . + +use super::*; + +// In order to facilitate benchmarks as tests we have a benchmark feature gated `WeightInfo` impl +// that uses 0 for all the weights. Because all the weights are 0, the tests that rely on +// weights for limiting data will fail, so we don't run them when using the benchmark feature. +#[cfg(not(feature = "runtime-benchmarks"))] +mod enter { + use super::*; + use crate::{ + builder::{Bench, BenchBuilder}, + mock::{new_test_ext, MockGenesisConfig, Test}, + }; + use frame_support::assert_ok; + use sp_std::collections::btree_map::BTreeMap; + + struct TestConfig { + dispute_statements: BTreeMap, + dispute_sessions: Vec, + backed_and_concluding: BTreeMap, + num_validators_per_core: u32, + includes_code_upgrade: Option, + } + + fn make_inherent_data( + TestConfig { + dispute_statements, + dispute_sessions, + backed_and_concluding, + num_validators_per_core, + includes_code_upgrade, + }: TestConfig, + ) -> Bench { + BenchBuilder::::new() + .set_max_validators((dispute_sessions.len() as u32) * num_validators_per_core) + .set_max_validators_per_core(num_validators_per_core) + .set_dispute_statements(dispute_statements) + .build(backed_and_concluding, dispute_sessions.as_slice(), includes_code_upgrade) + } + + #[test] + // Validate that if we create 2 backed candidates which are assigned to 2 cores that will be freed via + // becoming fully available, the backed candidates will not be filtered out in `create_inherent` and + // will not cause `enter` to early. + fn include_backed_candidates() { + new_test_ext(MockGenesisConfig::default()).execute_with(|| { + let dispute_statements = BTreeMap::new(); + + let mut backed_and_concluding = BTreeMap::new(); + backed_and_concluding.insert(0, 1); + backed_and_concluding.insert(1, 1); + + let scenario = make_inherent_data(TestConfig { + dispute_statements, + dispute_sessions: vec![0, 0], + backed_and_concluding, + num_validators_per_core: 1, + includes_code_upgrade: None, + }); + + // We expect the scenario to have cores 0 & 1 with pending availability. The backed + // candidates are also created for cores 0 & 1, so once the pending available + // become fully available those cores are marked as free and scheduled for the backed + // candidates. + let expected_para_inherent_data = scenario.data.clone(); + + // Check the para inherent data is as expected: + // * 1 bitfield per validator (2 validators) + assert_eq!(expected_para_inherent_data.bitfields.len(), 2); + // * 1 backed candidate per core (2 cores) + assert_eq!(expected_para_inherent_data.backed_candidates.len(), 2); + // * 0 disputes. + assert_eq!(expected_para_inherent_data.disputes.len(), 0); + let mut inherent_data = InherentData::new(); + inherent_data + .put_data(PARACHAINS_INHERENT_IDENTIFIER, &expected_para_inherent_data) + .unwrap(); + + // The current schedule is empty prior to calling `create_inherent_enter`. + assert_eq!(>::scheduled(), vec![]); + + // Nothing is filtered out (including the backed candidates.) + assert_eq!( + Pallet::::create_inherent_inner(&inherent_data.clone()).unwrap(), + expected_para_inherent_data + ); + + // The schedule is still empty prior to calling `enter`. (`create_inherent_inner` should not + // alter storage, but just double checking for sanity). + assert_eq!(>::scheduled(), vec![]); + + assert_eq!(Pallet::::on_chain_votes(), None); + // Call enter with our 2 backed candidates + assert_ok!(Pallet::::enter( + frame_system::RawOrigin::None.into(), + expected_para_inherent_data + )); + assert_eq!( + // The length of this vec is equal to the number of candidates, so we know our 2 + // backed candidates did not get filtered out + Pallet::::on_chain_votes().unwrap().backing_validators_per_candidate.len(), + 2 + ); + }); + } + + #[test] + // Ensure that disputes are filtered out if the session is in the future. + fn filter_multi_dispute_data() { + new_test_ext(MockGenesisConfig::default()).execute_with(|| { + // Create the inherent data for this block + let dispute_statements = BTreeMap::new(); + + let backed_and_concluding = BTreeMap::new(); + + let scenario = make_inherent_data(TestConfig { + dispute_statements, + dispute_sessions: vec![1, 2, 3 /* Session 3 too new, will get filtered out */], + backed_and_concluding, + num_validators_per_core: 5, + includes_code_upgrade: None, + }); + + let expected_para_inherent_data = scenario.data.clone(); + + // Check the para inherent data is as expected: + // * 1 bitfield per validator (5 validators per core, 3 disputes => 3 cores, 15 validators) + assert_eq!(expected_para_inherent_data.bitfields.len(), 15); + // * 0 backed candidate per core + assert_eq!(expected_para_inherent_data.backed_candidates.len(), 0); + // * 3 disputes. + assert_eq!(expected_para_inherent_data.disputes.len(), 3); + let mut inherent_data = InherentData::new(); + inherent_data + .put_data(PARACHAINS_INHERENT_IDENTIFIER, &expected_para_inherent_data) + .unwrap(); + + // The current schedule is empty prior to calling `create_inherent_enter`. + assert_eq!(>::scheduled(), vec![]); + + let multi_dispute_inherent_data = + Pallet::::create_inherent_inner(&inherent_data.clone()).unwrap(); + // Dispute for session that lies too far in the future should be filtered out + assert!(multi_dispute_inherent_data != expected_para_inherent_data); + + assert_eq!(multi_dispute_inherent_data.disputes.len(), 2); + + // Assert that the first 2 disputes are included + assert_eq!( + &multi_dispute_inherent_data.disputes[..2], + &expected_para_inherent_data.disputes[..2], + ); + + // The schedule is still empty prior to calling `enter`. (`create_inherent_inner` should not + // alter storage, but just double checking for sanity). + assert_eq!(>::scheduled(), vec![]); + + assert_eq!(Pallet::::on_chain_votes(), None); + // Call enter with our 2 disputes + assert_ok!(Pallet::::enter( + frame_system::RawOrigin::None.into(), + multi_dispute_inherent_data, + )); + + assert_eq!( + // The length of this vec is equal to the number of candidates, so we know there + // where no backed candidates included + Pallet::::on_chain_votes().unwrap().backing_validators_per_candidate.len(), + 0 + ); + }); + } + + #[test] + // Ensure that when dispute data establishes an over weight block that we adequately + // filter out disputes according to our prioritization rule + fn limit_dispute_data() { + new_test_ext(MockGenesisConfig::default()).execute_with(|| { + // Create the inherent data for this block + let dispute_statements = BTreeMap::new(); + // No backed and concluding cores, so all cores will be fileld with disputesw + let backed_and_concluding = BTreeMap::new(); + + let scenario = make_inherent_data(TestConfig { + dispute_statements, + dispute_sessions: vec![2, 2, 1], // 3 cores, all disputes + backed_and_concluding, + num_validators_per_core: 6, + includes_code_upgrade: None, + }); + + let expected_para_inherent_data = scenario.data.clone(); + + // Check the para inherent data is as expected: + // * 1 bitfield per validator (6 validators per core, 3 disputes => 18 validators) + assert_eq!(expected_para_inherent_data.bitfields.len(), 18); + // * 0 backed candidate per core + assert_eq!(expected_para_inherent_data.backed_candidates.len(), 0); + // * 3 disputes. + assert_eq!(expected_para_inherent_data.disputes.len(), 3); + let mut inherent_data = InherentData::new(); + inherent_data + .put_data(PARACHAINS_INHERENT_IDENTIFIER, &expected_para_inherent_data) + .unwrap(); + + // The current schedule is empty prior to calling `create_inherent_enter`. + assert_eq!(>::scheduled(), vec![]); + + let limit_inherent_data = + Pallet::::create_inherent_inner(&inherent_data.clone()).unwrap(); + // Expect that inherent data is filtered to include only 2 disputes + assert!(limit_inherent_data != expected_para_inherent_data); + + // Ensure that the included disputes are sorted by session + assert_eq!(limit_inherent_data.disputes.len(), 2); + assert_eq!(limit_inherent_data.disputes[0].session, 1); + assert_eq!(limit_inherent_data.disputes[1].session, 2); + + // The schedule is still empty prior to calling `enter`. (`create_inherent_inner` should not + // alter storage, but just double checking for sanity). + assert_eq!(>::scheduled(), vec![]); + + assert_eq!(Pallet::::on_chain_votes(), None); + // Call enter with our 2 disputes + assert_ok!(Pallet::::enter( + frame_system::RawOrigin::None.into(), + limit_inherent_data, + )); + + assert_eq!( + // Ensure that our inherent data did not included backed candidates as expected + Pallet::::on_chain_votes().unwrap().backing_validators_per_candidate.len(), + 0 + ); + }); + } + + #[test] + // Ensure that when dispute data establishes an over weight block that we abort + // due to an over weight block + fn limit_dispute_data_overweight() { + new_test_ext(MockGenesisConfig::default()).execute_with(|| { + // Create the inherent data for this block + let dispute_statements = BTreeMap::new(); + // No backed and concluding cores, so all cores will be fileld with disputesw + let backed_and_concluding = BTreeMap::new(); + + let scenario = make_inherent_data(TestConfig { + dispute_statements, + dispute_sessions: vec![2, 2, 1], // 3 cores, all disputes + backed_and_concluding, + num_validators_per_core: 6, + includes_code_upgrade: None, + }); + + let expected_para_inherent_data = scenario.data.clone(); + + // Check the para inherent data is as expected: + // * 1 bitfield per validator (6 validators per core, 3 disputes => 18 validators) + assert_eq!(expected_para_inherent_data.bitfields.len(), 18); + // * 0 backed candidate per core + assert_eq!(expected_para_inherent_data.backed_candidates.len(), 0); + // * 3 disputes. + assert_eq!(expected_para_inherent_data.disputes.len(), 3); + let mut inherent_data = InherentData::new(); + inherent_data + .put_data(PARACHAINS_INHERENT_IDENTIFIER, &expected_para_inherent_data) + .unwrap(); + + // The current schedule is empty prior to calling `create_inherent_enter`. + assert_eq!(>::scheduled(), vec![]); + + assert_ok!(Pallet::::enter( + frame_system::RawOrigin::None.into(), + expected_para_inherent_data, + )); + }); + } + + #[test] + // Ensure that when a block is over weight due to disputes, but there is still sufficient + // block weight to include a number of signed bitfields, the inherent data is filtered + // as expected + fn limit_dispute_data_ignore_backed_candidates() { + new_test_ext(MockGenesisConfig::default()).execute_with(|| { + // Create the inherent data for this block + let dispute_statements = BTreeMap::new(); + + let mut backed_and_concluding = BTreeMap::new(); + // 2 backed candidates shall be scheduled + backed_and_concluding.insert(0, 2); + backed_and_concluding.insert(1, 2); + + let scenario = make_inherent_data(TestConfig { + dispute_statements, + // 2 backed candidates + 3 disputes (at sessions 2, 1 and 1) + dispute_sessions: vec![0, 0, 2, 2, 1], + backed_and_concluding, + num_validators_per_core: 4, + includes_code_upgrade: None, + }); + + let expected_para_inherent_data = scenario.data.clone(); + + // Check the para inherent data is as expected: + // * 1 bitfield per validator (4 validators per core, 2 backed candidates, 3 disputes => 4*5 = 20) + assert_eq!(expected_para_inherent_data.bitfields.len(), 20); + // * 2 backed candidates + assert_eq!(expected_para_inherent_data.backed_candidates.len(), 2); + // * 3 disputes. + assert_eq!(expected_para_inherent_data.disputes.len(), 3); + let mut inherent_data = InherentData::new(); + inherent_data + .put_data(PARACHAINS_INHERENT_IDENTIFIER, &expected_para_inherent_data) + .unwrap(); + + // The current schedule is empty prior to calling `create_inherent_enter`. + assert_eq!(>::scheduled(), vec![]); + + // Nothing is filtered out (including the backed candidates.) + let limit_inherent_data = + Pallet::::create_inherent_inner(&inherent_data.clone()).unwrap(); + assert!(limit_inherent_data != expected_para_inherent_data); + + // Three disputes is over weight (see previous test), so we expect to only see 2 disputes + assert_eq!(limit_inherent_data.disputes.len(), 2); + // Ensure disputes are filtered as expected + assert_eq!(limit_inherent_data.disputes[0].session, 1); + assert_eq!(limit_inherent_data.disputes[1].session, 2); + // Ensure all bitfields are included as these are still not over weight + assert_eq!( + limit_inherent_data.bitfields.len(), + expected_para_inherent_data.bitfields.len() + ); + // Ensure that all backed candidates are filtered out as either would make the block over weight + assert_eq!(limit_inherent_data.backed_candidates.len(), 0); + + // The schedule is still empty prior to calling `enter`. (`create_inherent_inner` should not + // alter storage, but just double checking for sanity). + assert_eq!(>::scheduled(), vec![]); + + assert_eq!(Pallet::::on_chain_votes(), None); + // Call enter with our 2 disputes + assert_ok!(Pallet::::enter( + frame_system::RawOrigin::None.into(), + limit_inherent_data, + )); + + assert_eq!( + // The length of this vec is equal to the number of candidates, so we know + // all of our candidates got filtered out + Pallet::::on_chain_votes().unwrap().backing_validators_per_candidate.len(), + 0, + ); + }); + } + + #[test] + // Ensure that we abort if we encounter an over weight block for disputes + bitfields + fn limit_dispute_data_ignore_backed_candidates_overweight() { + new_test_ext(MockGenesisConfig::default()).execute_with(|| { + // Create the inherent data for this block + let dispute_statements = BTreeMap::new(); + + let mut backed_and_concluding = BTreeMap::new(); + // 2 backed candidates shall be scheduled + backed_and_concluding.insert(0, 2); + backed_and_concluding.insert(1, 2); + + let scenario = make_inherent_data(TestConfig { + dispute_statements, + // 2 backed candidates + 3 disputes (at sessions 2, 1 and 1) + dispute_sessions: vec![0, 0, 2, 2, 1], + backed_and_concluding, + num_validators_per_core: 4, + includes_code_upgrade: None, + }); + + let expected_para_inherent_data = scenario.data.clone(); + + // Check the para inherent data is as expected: + // * 1 bitfield per validator (4 validators per core, 2 backed candidates, 3 disputes => 4*5 = 20) + assert_eq!(expected_para_inherent_data.bitfields.len(), 20); + // * 2 backed candidates + assert_eq!(expected_para_inherent_data.backed_candidates.len(), 2); + // * 3 disputes. + assert_eq!(expected_para_inherent_data.disputes.len(), 3); + let mut inherent_data = InherentData::new(); + inherent_data + .put_data(PARACHAINS_INHERENT_IDENTIFIER, &expected_para_inherent_data) + .unwrap(); + + // The current schedule is empty prior to calling `create_inherent_enter`. + assert_eq!(>::scheduled(), vec![]); + + // Ensure that calling enter with 3 disputes and 2 candidates is over weight + assert_ok!(Pallet::::enter( + frame_system::RawOrigin::None.into(), + expected_para_inherent_data, + )); + + assert_eq!( + // The length of this vec is equal to the number of candidates, so we know + // all of our candidates got filtered out + Pallet::::on_chain_votes().unwrap().backing_validators_per_candidate.len(), + 0, + ); + }); + } + + #[test] + // Ensure that when a block is over weight due to disputes and bitfields, the bitfields are + // filtered to accommodate the block size and no backed candidates are included. + fn limit_bitfields() { + new_test_ext(MockGenesisConfig::default()).execute_with(|| { + // Create the inherent data for this block + let mut dispute_statements = BTreeMap::new(); + // Cap the number of statements per dispute to 20 in order to ensure we have enough + // space in the block for some (but not all) bitfields + dispute_statements.insert(2, 20); + dispute_statements.insert(3, 20); + dispute_statements.insert(4, 20); + + let mut backed_and_concluding = BTreeMap::new(); + // Schedule 2 backed candidates + backed_and_concluding.insert(0, 2); + backed_and_concluding.insert(1, 2); + + let scenario = make_inherent_data(TestConfig { + dispute_statements, + // 2 backed candidates + 3 disputes (at sessions 2, 1 and 1) + dispute_sessions: vec![0, 0, 2, 2, 1], + backed_and_concluding, + num_validators_per_core: 5, + includes_code_upgrade: None, + }); + + let expected_para_inherent_data = scenario.data.clone(); + + // Check the para inherent data is as expected: + // * 1 bitfield per validator (5 validators per core, 2 backed candidates, 3 disputes => 4*5 = 20), + assert_eq!(expected_para_inherent_data.bitfields.len(), 25); + // * 2 backed candidates, + assert_eq!(expected_para_inherent_data.backed_candidates.len(), 2); + // * 3 disputes. + assert_eq!(expected_para_inherent_data.disputes.len(), 3); + let mut inherent_data = InherentData::new(); + inherent_data + .put_data(PARACHAINS_INHERENT_IDENTIFIER, &expected_para_inherent_data) + .unwrap(); + + // The current schedule is empty prior to calling `create_inherent_enter`. + assert_eq!(>::scheduled(), vec![]); + + // Nothing is filtered out (including the backed candidates.) + let limit_inherent_data = + Pallet::::create_inherent_inner(&inherent_data.clone()).unwrap(); + assert!(limit_inherent_data != expected_para_inherent_data); + + // Three disputes is over weight (see previous test), so we expect to only see 2 disputes + assert_eq!(limit_inherent_data.disputes.len(), 2); + // Ensure disputes are filtered as expected + assert_eq!(limit_inherent_data.disputes[0].session, 1); + assert_eq!(limit_inherent_data.disputes[1].session, 2); + // Ensure all bitfields are included as these are still not over weight + assert_eq!(limit_inherent_data.bitfields.len(), 20,); + // Ensure that all backed candidates are filtered out as either would make the block over weight + assert_eq!(limit_inherent_data.backed_candidates.len(), 0); + + // The schedule is still empty prior to calling `enter`. (`create_inherent_inner` should not + // alter storage, but just double checking for sanity). + assert_eq!(>::scheduled(), vec![]); + + assert_eq!(Pallet::::on_chain_votes(), None); + // Call enter with our 2 disputes + assert_ok!(Pallet::::enter( + frame_system::RawOrigin::None.into(), + limit_inherent_data, + )); + + assert_eq!( + // The length of this vec is equal to the number of candidates, so we know + // all of our candidates got filtered out + Pallet::::on_chain_votes().unwrap().backing_validators_per_candidate.len(), + 0, + ); + }); + } + + #[test] + // Ensure that when a block is over weight due to disputes and bitfields, we abort + fn limit_bitfields_overweight() { + new_test_ext(MockGenesisConfig::default()).execute_with(|| { + // Create the inherent data for this block + let mut dispute_statements = BTreeMap::new(); + // Control the number of statements per dispute to ensure we have enough space + // in the block for some (but not all) bitfields + dispute_statements.insert(2, 20); + dispute_statements.insert(3, 20); + dispute_statements.insert(4, 20); + + let mut backed_and_concluding = BTreeMap::new(); + // 2 backed candidates shall be scheduled + backed_and_concluding.insert(0, 2); + backed_and_concluding.insert(1, 2); + + let scenario = make_inherent_data(TestConfig { + dispute_statements, + // 2 backed candidates + 3 disputes (at sessions 2, 1 and 1) + dispute_sessions: vec![0, 0, 2, 2, 1], + backed_and_concluding, + num_validators_per_core: 5, + includes_code_upgrade: None, + }); + + let expected_para_inherent_data = scenario.data.clone(); + + // Check the para inherent data is as expected: + // * 1 bitfield per validator (5 validators per core, 2 backed candidates, 3 disputes => 5*5 = 25) + assert_eq!(expected_para_inherent_data.bitfields.len(), 25); + // * 2 backed candidates + assert_eq!(expected_para_inherent_data.backed_candidates.len(), 2); + // * 3 disputes. + assert_eq!(expected_para_inherent_data.disputes.len(), 3); + let mut inherent_data = InherentData::new(); + inherent_data + .put_data(PARACHAINS_INHERENT_IDENTIFIER, &expected_para_inherent_data) + .unwrap(); + + // The current schedule is empty prior to calling `create_inherent_enter`. + assert_eq!(>::scheduled(), vec![]); + + assert_ok!(Pallet::::enter( + frame_system::RawOrigin::None.into(), + expected_para_inherent_data, + )); + + assert_eq!( + // The length of this vec is equal to the number of candidates, so we know + // all of our candidates got filtered out + Pallet::::on_chain_votes().unwrap().backing_validators_per_candidate.len(), + 0, + ); + }); + } + + #[test] + // Ensure that when a block is over weight due to disputes and bitfields, we abort + fn limit_candidates_over_weight_1() { + new_test_ext(MockGenesisConfig::default()).execute_with(|| { + // Create the inherent data for this block + let mut dispute_statements = BTreeMap::new(); + // Control the number of statements per dispute to ensure we have enough space + // in the block for some (but not all) bitfields + dispute_statements.insert(2, 17); + dispute_statements.insert(3, 17); + dispute_statements.insert(4, 17); + + let mut backed_and_concluding = BTreeMap::new(); + // 2 backed candidates shall be scheduled + backed_and_concluding.insert(0, 16); + backed_and_concluding.insert(1, 25); + + let scenario = make_inherent_data(TestConfig { + dispute_statements, + dispute_sessions: vec![0, 0, 2, 2, 1], // 2 backed candidates, 3 disputes at sessions 2, 1 and 1 respectively + backed_and_concluding, + num_validators_per_core: 5, + includes_code_upgrade: None, + }); + + let expected_para_inherent_data = scenario.data.clone(); + + // Check the para inherent data is as expected: + // * 1 bitfield per validator (5 validators per core, 2 backed candidates, 3 disputes => 5*5 = 25) + assert_eq!(expected_para_inherent_data.bitfields.len(), 25); + // * 2 backed candidates + assert_eq!(expected_para_inherent_data.backed_candidates.len(), 2); + // * 3 disputes. + assert_eq!(expected_para_inherent_data.disputes.len(), 3); + let mut inherent_data = InherentData::new(); + inherent_data + .put_data(PARACHAINS_INHERENT_IDENTIFIER, &expected_para_inherent_data) + .unwrap(); + + let limit_inherent_data = + Pallet::::create_inherent_inner(&inherent_data.clone()).unwrap(); + // Expect that inherent data is filtered to include only 1 backed candidate and 2 disputes + assert!(limit_inherent_data != expected_para_inherent_data); + + // * 1 bitfields + assert_eq!(limit_inherent_data.bitfields.len(), 25); + // * 2 backed candidates + assert_eq!(limit_inherent_data.backed_candidates.len(), 1); + // * 3 disputes. + assert_eq!(limit_inherent_data.disputes.len(), 2); + + // The current schedule is empty prior to calling `create_inherent_enter`. + assert_eq!(>::scheduled(), vec![]); + + assert_ok!(Pallet::::enter( + frame_system::RawOrigin::None.into(), + limit_inherent_data, + )); + + assert_eq!( + // The length of this vec is equal to the number of candidates, so we know our 2 + // backed candidates did not get filtered out + Pallet::::on_chain_votes().unwrap().backing_validators_per_candidate.len(), + 1 + ); + }); + } + + #[test] + // Ensure that when a block is over weight due to disputes and bitfields, we abort + fn limit_candidates_over_weight_0() { + new_test_ext(MockGenesisConfig::default()).execute_with(|| { + // Create the inherent data for this block + let mut dispute_statements = BTreeMap::new(); + // Control the number of statements per dispute to ensure we have enough space + // in the block for some (but not all) bitfields + dispute_statements.insert(2, 17); + dispute_statements.insert(3, 17); + dispute_statements.insert(4, 17); + + let mut backed_and_concluding = BTreeMap::new(); + // 2 backed candidates shall be scheduled + backed_and_concluding.insert(0, 16); + backed_and_concluding.insert(1, 25); + + let scenario = make_inherent_data(TestConfig { + dispute_statements, + dispute_sessions: vec![0, 0, 2, 2, 1], // 2 backed candidates, 3 disputes at sessions 2, 1 and 1 respectively + backed_and_concluding, + num_validators_per_core: 5, + includes_code_upgrade: None, + }); + + let expected_para_inherent_data = scenario.data.clone(); + + // Check the para inherent data is as expected: + // * 1 bitfield per validator (5 validators per core, 2 backed candidates, 3 disputes => 5*5 = 25) + assert_eq!(expected_para_inherent_data.bitfields.len(), 25); + // * 2 backed candidates + assert_eq!(expected_para_inherent_data.backed_candidates.len(), 2); + // * 3 disputes. + assert_eq!(expected_para_inherent_data.disputes.len(), 3); + + assert_ok!(Pallet::::enter( + frame_system::RawOrigin::None.into(), + expected_para_inherent_data, + )); + + assert_eq!( + // The length of this vec is equal to the number of candidates, so we know our 2 + // backed candidates did not get filtered out + Pallet::::on_chain_votes().unwrap().backing_validators_per_candidate.len(), + 0 + ); + }); + } +} + +fn default_header() -> primitives::v1::Header { + primitives::v1::Header { + parent_hash: Default::default(), + number: 0, + state_root: Default::default(), + extrinsics_root: Default::default(), + digest: Default::default(), + } +} + +mod sanitizers { + use super::*; + + use crate::inclusion::tests::{ + back_candidate, collator_sign_candidate, BackingKind, TestCandidateBuilder, + }; + use bitvec::order::Lsb0; + use primitives::v1::{ + AvailabilityBitfield, GroupIndex, Hash, Id as ParaId, SignedAvailabilityBitfield, + ValidatorIndex, + }; + + use crate::mock::Test; + use futures::executor::block_on; + use keyring::Sr25519Keyring; + use primitives::v0::PARACHAIN_KEY_TYPE_ID; + use sc_keystore::LocalKeystore; + use sp_keystore::{SyncCryptoStore, SyncCryptoStorePtr}; + use std::sync::Arc; + + fn validator_pubkeys(val_ids: &[keyring::Sr25519Keyring]) -> Vec { + val_ids.iter().map(|v| v.public().into()).collect() + } + + #[test] + fn bitfields() { + let header = default_header(); + let parent_hash = header.hash(); + // 2 cores means two bits + let expected_bits = 2; + let session_index = SessionIndex::from(0_u32); + + let crypto_store = LocalKeystore::in_memory(); + let crypto_store = Arc::new(crypto_store) as SyncCryptoStorePtr; + let signing_context = SigningContext { parent_hash, session_index }; + + let validators = vec![ + keyring::Sr25519Keyring::Alice, + keyring::Sr25519Keyring::Bob, + keyring::Sr25519Keyring::Charlie, + keyring::Sr25519Keyring::Dave, + ]; + for validator in validators.iter() { + SyncCryptoStore::sr25519_generate_new( + &*crypto_store, + PARACHAIN_KEY_TYPE_ID, + Some(&validator.to_seed()), + ) + .unwrap(); + } + let validator_public = validator_pubkeys(&validators); + + let unchecked_bitfields = [ + BitVec::::repeat(true, expected_bits), + BitVec::::repeat(true, expected_bits), + { + let mut bv = BitVec::::repeat(false, expected_bits); + bv.set(expected_bits - 1, true); + bv + }, + ] + .iter() + .enumerate() + .map(|(vi, ab)| { + let validator_index = ValidatorIndex::from(vi as u32); + block_on(SignedAvailabilityBitfield::sign( + &crypto_store, + AvailabilityBitfield::from(ab.clone()), + &signing_context, + validator_index, + &validator_public[vi], + )) + .unwrap() + .unwrap() + .into_unchecked() + }) + .collect::>(); + + let disputed_bitfield = DisputedBitfield::zeros(expected_bits); + + { + assert_eq!( + sanitize_bitfields::( + unchecked_bitfields.clone(), + disputed_bitfield.clone(), + expected_bits, + parent_hash, + session_index, + &validator_public[..], + FullCheck::Skip, + ), + unchecked_bitfields.clone() + ); + assert_eq!( + sanitize_bitfields::( + unchecked_bitfields.clone(), + disputed_bitfield.clone(), + expected_bits, + parent_hash, + session_index, + &validator_public[..], + FullCheck::Yes + ), + unchecked_bitfields.clone() + ); + } + + // disputed bitfield is non-zero + { + let mut disputed_bitfield = DisputedBitfield::zeros(expected_bits); + // pretend the first core was freed by either a malicious validator + // or by resolved dispute + disputed_bitfield.0.set(0, true); + + assert_eq!( + sanitize_bitfields::( + unchecked_bitfields.clone(), + disputed_bitfield.clone(), + expected_bits, + parent_hash, + session_index, + &validator_public[..], + FullCheck::Yes + ) + .len(), + 1 + ); + assert_eq!( + sanitize_bitfields::( + unchecked_bitfields.clone(), + disputed_bitfield.clone(), + expected_bits, + parent_hash, + session_index, + &validator_public[..], + FullCheck::Skip + ) + .len(), + 1 + ); + } + + // bitfield size mismatch + { + assert!(sanitize_bitfields::( + unchecked_bitfields.clone(), + disputed_bitfield.clone(), + expected_bits + 1, + parent_hash, + session_index, + &validator_public[..], + FullCheck::Yes + ) + .is_empty()); + assert!(sanitize_bitfields::( + unchecked_bitfields.clone(), + disputed_bitfield.clone(), + expected_bits + 1, + parent_hash, + session_index, + &validator_public[..], + FullCheck::Skip + ) + .is_empty()); + } + + // remove the last validator + { + let shortened = validator_public.len() - 2; + assert_eq!( + &sanitize_bitfields::( + unchecked_bitfields.clone(), + disputed_bitfield.clone(), + expected_bits, + parent_hash, + session_index, + &validator_public[..shortened], + FullCheck::Yes, + )[..], + &unchecked_bitfields[..shortened] + ); + assert_eq!( + &sanitize_bitfields::( + unchecked_bitfields.clone(), + disputed_bitfield.clone(), + expected_bits, + parent_hash, + session_index, + &validator_public[..shortened], + FullCheck::Skip, + )[..], + &unchecked_bitfields[..shortened] + ); + } + + // switch ordering of bitfields + { + let mut unchecked_bitfields = unchecked_bitfields.clone(); + let x = unchecked_bitfields.swap_remove(0); + unchecked_bitfields.push(x); + assert_eq!( + &sanitize_bitfields::( + unchecked_bitfields.clone(), + disputed_bitfield.clone(), + expected_bits, + parent_hash, + session_index, + &validator_public[..], + FullCheck::Yes + )[..], + &unchecked_bitfields[..(unchecked_bitfields.len() - 2)] + ); + assert_eq!( + &sanitize_bitfields::( + unchecked_bitfields.clone(), + disputed_bitfield.clone(), + expected_bits, + parent_hash, + session_index, + &validator_public[..], + FullCheck::Skip + )[..], + &unchecked_bitfields[..(unchecked_bitfields.len() - 2)] + ); + } + + // check the validators signature + { + use primitives::v1::ValidatorSignature; + let mut unchecked_bitfields = unchecked_bitfields.clone(); + + // insert a bad signature for the last bitfield + let last_bit_idx = unchecked_bitfields.len() - 1; + unchecked_bitfields + .get_mut(last_bit_idx) + .and_then(|u| Some(u.set_signature(ValidatorSignature::default()))) + .expect("we are accessing a valid index"); + assert_eq!( + &sanitize_bitfields::( + unchecked_bitfields.clone(), + disputed_bitfield.clone(), + expected_bits, + parent_hash, + session_index, + &validator_public[..], + FullCheck::Yes + )[..], + &unchecked_bitfields[..last_bit_idx] + ); + assert_eq!( + &sanitize_bitfields::( + unchecked_bitfields.clone(), + disputed_bitfield.clone(), + expected_bits, + parent_hash, + session_index, + &validator_public[..], + FullCheck::Skip + )[..], + &unchecked_bitfields[..] + ); + } + } + + #[test] + fn candidates() { + const RELAY_PARENT_NUM: u32 = 3; + + let header = default_header(); + let relay_parent = header.hash(); + let session_index = SessionIndex::from(0_u32); + + let keystore = LocalKeystore::in_memory(); + let keystore = Arc::new(keystore) as SyncCryptoStorePtr; + let signing_context = SigningContext { parent_hash: relay_parent, session_index }; + + let validators = vec![ + keyring::Sr25519Keyring::Alice, + keyring::Sr25519Keyring::Bob, + keyring::Sr25519Keyring::Charlie, + keyring::Sr25519Keyring::Dave, + ]; + for validator in validators.iter() { + SyncCryptoStore::sr25519_generate_new( + &*keystore, + PARACHAIN_KEY_TYPE_ID, + Some(&validator.to_seed()), + ) + .unwrap(); + } + + let has_concluded_invalid = + |_idx: usize, _backed_candidate: &BackedCandidate| -> bool { false }; + + let scheduled = (0_usize..2) + .into_iter() + .map(|idx| { + let ca = CoreAssignment { + kind: scheduler::AssignmentKind::Parachain, + group_idx: GroupIndex::from(idx as u32), + para_id: ParaId::from(1_u32 + idx as u32), + core: CoreIndex::from(idx as u32), + }; + ca + }) + .collect::>(); + let scheduled = &scheduled[..]; + + let group_validators = |group_index: GroupIndex| { + match group_index { + group_index if group_index == GroupIndex::from(0) => Some(vec![0, 1]), + group_index if group_index == GroupIndex::from(1) => Some(vec![2, 3]), + _ => panic!("Group index out of bounds for 2 parachains and 1 parathread core"), + } + .map(|m| m.into_iter().map(ValidatorIndex).collect::>()) + }; + + let backed_candidates = (0_usize..2) + .into_iter() + .map(|idx0| { + let idx1 = idx0 + 1; + let mut candidate = TestCandidateBuilder { + para_id: ParaId::from(idx1), + relay_parent, + pov_hash: Hash::repeat_byte(idx1 as u8), + persisted_validation_data_hash: [42u8; 32].into(), + hrmp_watermark: RELAY_PARENT_NUM, + ..Default::default() + } + .build(); + + collator_sign_candidate(Sr25519Keyring::One, &mut candidate); + + let backed = block_on(back_candidate( + candidate, + &validators, + group_validators(GroupIndex::from(idx0 as u32)).unwrap().as_ref(), + &keystore, + &signing_context, + BackingKind::Threshold, + )); + backed + }) + .collect::>(); + + // happy path + assert_eq!( + sanitize_backed_candidates::( + relay_parent, + backed_candidates.clone(), + has_concluded_invalid, + scheduled + ), + backed_candidates + ); + + // nothing is scheduled, so no paraids match, thus all backed candidates are skipped + { + let scheduled = &[][..]; + assert!(sanitize_backed_candidates::( + relay_parent, + backed_candidates.clone(), + has_concluded_invalid, + scheduled + ) + .is_empty()); + } + + // relay parent mismatch + { + let relay_parent = Hash::repeat_byte(0xFA); + assert!(sanitize_backed_candidates::( + relay_parent, + backed_candidates.clone(), + has_concluded_invalid, + scheduled + ) + .is_empty()); + } + + // candidates that have concluded as invalid are filtered out + { + // mark every second one as concluded invalid + let set = { + let mut set = std::collections::HashSet::new(); + for (idx, backed_candidate) in backed_candidates.iter().enumerate() { + if idx & 0x01 == 0 { + set.insert(backed_candidate.hash().clone()); + } + } + set + }; + let has_concluded_invalid = + |_idx: usize, candidate: &BackedCandidate| set.contains(&candidate.hash()); + assert_eq!( + sanitize_backed_candidates::( + relay_parent, + backed_candidates.clone(), + has_concluded_invalid, + scheduled + ) + .len(), + backed_candidates.len() / 2 + ); + } + } +} diff --git a/polkadot/runtime/parachains/src/paras_inherent/weights.rs b/polkadot/runtime/parachains/src/paras_inherent/weights.rs new file mode 100644 index 0000000000..06dcbe57d1 --- /dev/null +++ b/polkadot/runtime/parachains/src/paras_inherent/weights.rs @@ -0,0 +1,118 @@ +// Copyright 2021 Parity Technologies (UK) Ltd. +// This file is part of Polkadot. + +// Polkadot is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Polkadot is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Polkadot. If not, see . +use super::{ + BackedCandidate, Config, DisputeStatementSet, UncheckedSignedAvailabilityBitfield, Weight, +}; + +pub trait WeightInfo { + /// Variant over `v`, the count of dispute statements in a dispute statement set. This gives the + /// weight of a single dispute statement set. + fn enter_variable_disputes(v: u32) -> Weight; + /// The weight of one bitfield. + fn enter_bitfields() -> Weight; + /// Variant over `v`, the count of validity votes for a backed candidate. This gives the weight + /// of a single backed candidate. + fn enter_backed_candidates_variable(v: u32) -> Weight; + /// The weight of a single backed candidate with a code upgrade. + fn enter_backed_candidate_code_upgrade() -> Weight; +} + +pub struct TestWeightInfo; +// `WeightInfo` impl for unit and integration tests. Based off of the `max_block` weight for the +// mock. +#[cfg(not(feature = "runtime-benchmarks"))] +impl WeightInfo for TestWeightInfo { + fn enter_variable_disputes(v: u32) -> Weight { + // MAX Block Weight should fit 4 disputes + 80_000 * v as Weight + 80_000 + } + fn enter_bitfields() -> Weight { + // MAX Block Weight should fit 4 backed candidates + 40_000 as Weight + } + fn enter_backed_candidates_variable(v: u32) -> Weight { + // MAX Block Weight should fit 4 backed candidates + 40_000 * v as Weight + 40_000 + } + fn enter_backed_candidate_code_upgrade() -> Weight { + 0 + } +} +// To simplify benchmarks running as tests, we set all the weights to 0. `enter` will exit early +// when if the data causes it to be over weight, but we don't want that to block a benchmark from +// running as a test. +#[cfg(feature = "runtime-benchmarks")] +impl WeightInfo for TestWeightInfo { + fn enter_variable_disputes(_v: u32) -> Weight { + 0 + } + fn enter_bitfields() -> Weight { + 0 + } + fn enter_backed_candidates_variable(_v: u32) -> Weight { + 0 + } + fn enter_backed_candidate_code_upgrade() -> Weight { + 0 + } +} + +pub fn paras_inherent_total_weight( + backed_candidates: &[BackedCandidate<::Hash>], + bitfields: &[UncheckedSignedAvailabilityBitfield], + disputes: &[DisputeStatementSet], +) -> Weight { + backed_candidates_weight::(backed_candidates) + .saturating_add(signed_bitfields_weight::(bitfields.len())) + .saturating_add(dispute_statements_weight::(disputes)) +} + +pub fn dispute_statements_weight(disputes: &[DisputeStatementSet]) -> Weight { + disputes + .iter() + .map(|d| { + <::WeightInfo as WeightInfo>::enter_variable_disputes( + d.statements.len() as u32 + ) + }) + .fold(0, |acc, x| acc.saturating_add(x)) +} + +pub fn signed_bitfields_weight(bitfields_len: usize) -> Weight { + <::WeightInfo as WeightInfo>::enter_bitfields() + .saturating_mul(bitfields_len as Weight) +} + +pub fn backed_candidate_weight( + candidate: &BackedCandidate, +) -> Weight { + if candidate.candidate.commitments.new_validation_code.is_some() { + <::WeightInfo as WeightInfo>::enter_backed_candidate_code_upgrade() + } else { + <::WeightInfo as WeightInfo>::enter_backed_candidates_variable( + candidate.validity_votes.len() as u32, + ) + } +} + +pub fn backed_candidates_weight( + candidates: &[BackedCandidate], +) -> Weight { + candidates + .iter() + .map(|c| backed_candidate_weight::(c)) + .fold(0, |acc, x| acc.saturating_add(x)) +}