pezkuwichain/pezkuwi-subxt
1
0
Fork 0
mirror of https://github.com/pezkuwichain/pezkuwi-subxt.git synced 2026-07-25 05:55:42 +00:00
Code Issues Packages Projects Releases Wiki Activity

Moare fixes for parachains (#1911)

Browse Source 
* Moare fixes for parachains

- Sending data to a job should always contain a relay parent. Done this
for the provisioner
- Fixed the `select_availability_bitfields` function. It was assuming we
have one core per validator, while we only have one core per parachain.
- Drive by async "rewrite" in proposer

* Make tests compile

* Update primitives/src/v1.rs

Co-authored-by: Peter Goodspeed-Niklaus <coriolinus@users.noreply.github.com>

Co-authored-by: Peter Goodspeed-Niklaus <coriolinus@users.noreply.github.com>
This commit is contained in:
Bastian Köcher
2020-11-03 18:16:47 +01:00
committed by GitHub
parent 04eba097b6
commit 04da99de58
11 changed files with 176 additions and 132 deletions
Download Patch File Download Diff File Expand all files Collapse all files
polkadot/node/core/provisioner/src/lib.rs
+68 -62
View File
@@ -41,7 +41,7 @@ use polkadot_primitives::v1::{
BackedCandidate, BlockNumber, CoreState, Hash, OccupiedCoreAssumption,
SignedAvailabilityBitfield,
};
use std::{collections::HashMap, convert::TryFrom, pin::Pin};
use std::{collections::HashSet, convert::TryFrom, pin::Pin};
use thiserror::Error;
struct ProvisioningJob {
@@ -211,7 +211,7 @@ impl ProvisioningJob {
ToJob::Provisioner(RequestBlockAuthorshipData(_, sender)) => {
self.provisionable_data_channels.push(sender)
}
ToJob::Provisioner(ProvisionableData(data)) => {
ToJob::Provisioner(ProvisionableData(_, data)) => {
let mut bad_indices = Vec::new();
for (idx, channel) in self.provisionable_data_channels.iter_mut().enumerate() {
match channel.send(data.clone()).await {
@@ -266,23 +266,23 @@ impl ProvisioningJob {
type CoreAvailability = BitVec<bitvec::order::Lsb0, u8>;
// The provisioner is the subsystem best suited to choosing which specific
// backed candidates and availability bitfields should be assembled into the
// block. To engage this functionality, a
// `ProvisionerMessage::RequestInherentData` is sent; the response is a set of
// non-conflicting candidates and the appropriate bitfields. Non-conflicting
// means that there are never two distinct parachain candidates included for
// the same parachain and that new parachain candidates cannot be included
// until the previous one either gets declared available or expired.
//
// The main complication here is going to be around handling
// occupied-core-assumptions. We might have candidates that are only
// includable when some bitfields are included. And we might have candidates
// that are not includable when certain bitfields are included.
//
// When we're choosing bitfields to include, the rule should be simple:
// maximize availability. So basically, include all bitfields. And then
// choose a coherent set of candidates along with that.
/// The provisioner is the subsystem best suited to choosing which specific
/// backed candidates and availability bitfields should be assembled into the
/// block. To engage this functionality, a
/// `ProvisionerMessage::RequestInherentData` is sent; the response is a set of
/// non-conflicting candidates and the appropriate bitfields. Non-conflicting
/// means that there are never two distinct parachain candidates included for
/// the same parachain and that new parachain candidates cannot be included
/// until the previous one either gets declared available or expired.
///
/// The main complication here is going to be around handling
/// occupied-core-assumptions. We might have candidates that are only
/// includable when some bitfields are included. And we might have candidates
/// that are not includable when certain bitfields are included.
///
/// When we're choosing bitfields to include, the rule should be simple:
/// maximize availability. So basically, include all bitfields. And then
/// choose a coherent set of candidates along with that.
async fn send_inherent_data(
relay_parent: Hash,
bitfields: &[SignedAvailabilityBitfield],
@@ -310,48 +310,49 @@ async fn send_inherent_data(
Ok(())
}
// in general, we want to pick all the bitfields. However, we have the following constraints:
//
// - not more than one per validator
// - each must correspond to an occupied core
//
// If we have too many, an arbitrary selection policy is fine. For purposes of maximizing availability,
// we pick the one with the greatest number of 1 bits.
//
// note: this does not enforce any sorting precondition on the output; the ordering there will be unrelated
// to the sorting of the input.
/// In general, we want to pick all the bitfields. However, we have the following constraints:
///
/// - not more than one per validator
/// - each must correspond to an occupied core
///
/// If we have too many, an arbitrary selection policy is fine. For purposes of maximizing availability,
/// we pick the one with the greatest number of 1 bits.
///
/// Note: This does not enforce any sorting precondition on the output; the ordering there will be unrelated
/// to the sorting of the input.
fn select_availability_bitfields(
cores: &[CoreState],
bitfields: &[SignedAvailabilityBitfield],
) -> Vec<SignedAvailabilityBitfield> {
let mut fields_by_core: HashMap<_, Vec<_>> = HashMap::new();
for bitfield in bitfields.iter() {
let core_idx = bitfield.validator_index() as usize;
if let CoreState::Occupied(_) = cores[core_idx] {
fields_by_core
.entry(core_idx)
// there cannot be a value list in field_by_core with len < 1
.or_default()
.push(bitfield.clone());
let mut bitfield_per_core: Vec<Option<SignedAvailabilityBitfield>> = vec![None; cores.len()];
let mut seen_validators = HashSet::new();
for mut bitfield in bitfields.iter().cloned() {
// If we have seen the validator already, ignore it.
if !seen_validators.insert(bitfield.validator_index()) {
continue;
}
for (idx, _) in cores.iter().enumerate().filter(|v| v.1.is_occupied()) {
if *bitfield.payload().0.get(idx).unwrap_or(&false) {
if let Some(ref mut occupied) = bitfield_per_core[idx] {
if occupied.payload().0.count_ones() < bitfield.payload().0.count_ones() {
// We found a better bitfield, lets swap them and search a new spot for the old
// best one
std::mem::swap(occupied, &mut bitfield);
}
} else {
bitfield_per_core[idx] = Some(bitfield);
break;
}
}
}
}
let mut out = Vec::with_capacity(fields_by_core.len());
for (_, core_bitfields) in fields_by_core.iter_mut() {
core_bitfields.sort_by_key(|bitfield| bitfield.payload().0.count_ones());
out.push(
core_bitfields
.pop()
.expect("every core bitfield has at least 1 member; qed"),
);
}
out
bitfield_per_core.into_iter().filter_map(|v| v).collect()
}
// determine which cores are free, and then to the degree possible, pick a candidate appropriate to each free core.
//
// follow the candidate selection algorithm from the guide
/// Determine which cores are free, and then to the degree possible, pick a candidate appropriate to each free core.
async fn select_candidates(
availability_cores: &[CoreState],
bitfields: &[SignedAvailabilityBitfield],
@@ -416,8 +417,8 @@ async fn select_candidates(
Ok(selected_candidates)
}
// produces a block number 1 higher than that of the relay parent
// in the event of an invalid `relay_parent`, returns `Ok(0)`
/// Produces a block number 1 higher than that of the relay parent
/// in the event of an invalid `relay_parent`, returns `Ok(0)`
async fn get_block_number_under_construction(
relay_parent: Hash,
sender: &mut mpsc::Sender<FromJob>,
@@ -437,19 +438,18 @@ async fn get_block_number_under_construction(
}
}
// the availability bitfield for a given core is the transpose
// of a set of signed availability bitfields. It goes like this:
//
// - construct a transverse slice along `core_idx`
// - bitwise-or it with the availability slice
// - count the 1 bits, compare to the total length; true on 2/3+
/// The availability bitfield for a given core is the transpose
/// of a set of signed availability bitfields. It goes like this:
///
/// - construct a transverse slice along `core_idx`
/// - bitwise-or it with the availability slice
/// - count the 1 bits, compare to the total length; true on 2/3+
fn bitfields_indicate_availability(
core_idx: usize,
bitfields: &[SignedAvailabilityBitfield],
availability: &CoreAvailability,
) -> bool {
let mut availability = availability.clone();
// we need to pre-compute this to avoid a borrow-immutable-while-borrowing-mutable error in the error message
let availability_len = availability.len();
for bitfield in bitfields {
@@ -459,12 +459,18 @@ fn bitfields_indicate_availability(
// in principle, this function might return a `Result<bool, Error>` so that we can more clearly express this error condition
// however, in practice, that would just push off an error-handling routine which would look a whole lot like this one.
// simpler to just handle the error internally here.
log::warn!(target: "provisioner", "attempted to set a transverse bit at idx {} which is greater than bitfield size {}", validator_idx, availability_len);
log::warn!(
target: "provisioner", "attempted to set a transverse bit at idx {} which is greater than bitfield size {}",
validator_idx,
availability_len,
);
return false;
}
Some(mut bit_mut) => *bit_mut |= bitfield.payload().0[core_idx],
}
}
3 * availability.count_ones() >= 2 * availability.len()
}
polkadot/node/core/provisioner/src/tests.rs
+70 -27
View File
@@ -46,8 +46,7 @@ mod select_availability_bitfields {
use std::sync::Arc;
use polkadot_primitives::v1::{SigningContext, ValidatorIndex, ValidatorId};
use sp_application_crypto::AppKey;
use sp_keystore::{CryptoStore, SyncCryptoStorePtr};
use sc_keystore::LocalKeystore;
use sp_keystore::{CryptoStore, SyncCryptoStorePtr, testing::KeyStore};
async fn signed_bitfield(
keystore: &SyncCryptoStorePtr,
@@ -68,12 +67,10 @@ mod select_availability_bitfields {
#[test]
fn not_more_than_one_per_validator() {
// Configure filesystem-based keystore as generating keys without seed
// would trigger the key to be generated on the filesystem.
let keystore_path = tempfile::tempdir().expect("Creates keystore path");
let keystore : SyncCryptoStorePtr = Arc::new(LocalKeystore::open(keystore_path.path(), None)
.expect("Creates keystore"));
let bitvec = default_bitvec();
let keystore: SyncCryptoStorePtr = Arc::new(KeyStore::new());
let mut bitvec = default_bitvec();
bitvec.set(0, true);
bitvec.set(1, true);
let cores = vec![occupied_core(0), occupied_core(1)];
@@ -96,11 +93,7 @@ mod select_availability_bitfields {
#[test]
fn each_corresponds_to_an_occupied_core() {
// Configure filesystem-based keystore as generating keys without seed
// would trigger the key to be generated on the filesystem.
let keystore_path = tempfile::tempdir().expect("Creates keystore path");
let keystore : SyncCryptoStorePtr = Arc::new(LocalKeystore::open(keystore_path.path(), None)
.expect("Creates keystore"));
let keystore: SyncCryptoStorePtr = Arc::new(KeyStore::new());
let bitvec = default_bitvec();
let cores = vec![CoreState::Free, CoreState::Scheduled(Default::default())];
@@ -120,23 +113,18 @@ mod select_availability_bitfields {
#[test]
fn more_set_bits_win_conflicts() {
// Configure filesystem-based keystore as generating keys without seed
// would trigger the key to be generated on the filesystem.
let keystore_path = tempfile::tempdir().expect("Creates keystore path");
let keystore : SyncCryptoStorePtr = Arc::new(LocalKeystore::open(keystore_path.path(), None)
.expect("Creates keystore"));
let bitvec_zero = default_bitvec();
let bitvec_one = {
let mut bitvec = bitvec_zero.clone();
bitvec.set(0, true);
bitvec
};
let keystore: SyncCryptoStorePtr = Arc::new(KeyStore::new());
let mut bitvec = default_bitvec();
bitvec.set(0, true);
let mut bitvec1 = bitvec.clone();
bitvec1.set(1, true);
let cores = vec![occupied_core(0)];
let bitfields = vec![
block_on(signed_bitfield(&keystore, bitvec_zero, 0)),
block_on(signed_bitfield(&keystore, bitvec_one.clone(), 0)),
block_on(signed_bitfield(&keystore, bitvec, 0)),
block_on(signed_bitfield(&keystore, bitvec1.clone(), 1)),
];
// this test is probablistic: chances are excellent that it does what it claims to.
@@ -145,9 +133,64 @@ mod select_availability_bitfields {
for _ in 0..64 {
let selected_bitfields = select_availability_bitfields(&cores, &bitfields);
assert_eq!(selected_bitfields.len(), 1);
assert_eq!(selected_bitfields[0].payload().0, bitvec_one);
assert_eq!(selected_bitfields[0].payload().0, bitvec1.clone());
}
}
#[test]
fn more_validators_than_parachains() {
let keystore: SyncCryptoStorePtr = Arc::new(KeyStore::new());
let mut bitvec = default_bitvec();
bitvec.set(0, true);
let cores = vec![occupied_core(0)];
let bitfields = vec![
block_on(signed_bitfield(&keystore, bitvec.clone(), 0)),
block_on(signed_bitfield(&keystore, bitvec.clone(), 1)),
block_on(signed_bitfield(&keystore, bitvec.clone(), 2)),
block_on(signed_bitfield(&keystore, bitvec.clone(), 3)),
];
let selected_bitfields = select_availability_bitfields(&cores, &bitfields);
assert_eq!(selected_bitfields.len(), 1);
assert_eq!(selected_bitfields[0].payload().0, bitvec);
}
#[test]
fn more_complex_bitfields() {
let keystore: SyncCryptoStorePtr = Arc::new(KeyStore::new());
let mut bitvec0 = default_bitvec();
bitvec0.set(0, true);
bitvec0.set(2, true);
let mut bitvec1 = default_bitvec();
bitvec1.set(1, true);
let mut bitvec2 = default_bitvec();
bitvec2.set(2, true);
let mut bitvec3 = default_bitvec();
bitvec3.set(0, true);
bitvec3.set(1, true);
bitvec3.set(2, true);
bitvec3.set(3, true);
let cores = vec![occupied_core(0), occupied_core(1), occupied_core(2), occupied_core(3)];
let bitfields = vec![
block_on(signed_bitfield(&keystore, bitvec0.clone(), 0)),
block_on(signed_bitfield(&keystore, bitvec1.clone(), 1)),
block_on(signed_bitfield(&keystore, bitvec2.clone(), 2)),
block_on(signed_bitfield(&keystore, bitvec3.clone(), 3)),
];
let selected_bitfields = select_availability_bitfields(&cores, &bitfields);
assert_eq!(selected_bitfields.len(), 3);
assert_eq!(selected_bitfields[0].payload().0, bitvec3);
assert_eq!(selected_bitfields[1].payload().0, bitvec1);
assert_eq!(selected_bitfields[2].payload().0, bitvec0);
}
}
mod select_candidates {