pezkuwichain/pezkuwi-subxt
1
0
Fork 0
mirror of https://github.com/pezkuwichain/pezkuwi-subxt.git synced 2026-06-17 04:21:01 +00:00
Code Issues Packages Projects Releases Wiki Activity

cargo +nightly fmt (#3540)

Browse Source 
* cargo +nightly fmt

* add cargo-fmt check to ci

* update ci

* fmt

* fmt

* skip macro

* ignore bridges
This commit is contained in:
Shawn Tabrizi
2021-08-02 12:47:33 +02:00
committed by GitHub
parent 30e3012270
commit ff5d56fb76
350 changed files with 20617 additions and 21266 deletions
Download Patch File Download Diff File Expand all files Collapse all files
polkadot/node/core/approval-voting/src/approval_checking.rs
+125 -154
View File
@@ -16,13 +16,14 @@
//! Utilities for checking whether a candidate has been approved under a given block.
use bitvec::{order::Lsb0 as BitOrderLsb0, slice::BitSlice};
use polkadot_node_primitives::approval::DelayTranche;
use polkadot_primitives::v1::ValidatorIndex;
use bitvec::slice::BitSlice;
use bitvec::order::Lsb0 as BitOrderLsb0;
use crate::persisted_entries::{TrancheEntry, ApprovalEntry, CandidateEntry};
use crate::time::Tick;
use crate::{
persisted_entries::{ApprovalEntry, CandidateEntry, TrancheEntry},
time::Tick,
};
/// The required tranches of assignments needed to determine whether a candidate is approved.
#[derive(Debug, PartialEq, Clone)]
@@ -57,7 +58,7 @@ pub enum RequiredTranches {
/// event that there are some assignments that don't have corresponding approval votes. If this
/// is `None`, all assignments have approvals.
next_no_show: Option<Tick>,
}
},
}
/// The result of a check.
@@ -96,12 +97,11 @@ pub fn check_approval(
approval: &ApprovalEntry,
required: RequiredTranches,
) -> Check {
// any set of size f+1 contains at least one honest node. If at least one
// honest node approves, the candidate should be approved.
let approvals = candidate.approvals();
if 3 * approvals.count_ones() > approvals.len() {
return Check::ApprovedOneThird;
return Check::ApprovedOneThird
}
match required {
@@ -134,7 +134,7 @@ pub fn check_approval(
} else {
Check::Unapproved
}
}
},
}
}
@@ -182,7 +182,7 @@ impl State {
) -> RequiredTranches {
let covering = if self.depth == 0 { 0 } else { self.covering };
if self.depth != 0 && self.assignments + covering + self.uncovered >= n_validators {
return RequiredTranches::All;
return RequiredTranches::All
}
// If we have enough assignments and all no-shows are covered, we have reached the number
@@ -192,7 +192,7 @@ impl State {
needed: tranche,
tolerated_missing: self.covered,
next_no_show: self.next_no_show,
};
}
}
// We're pending more assignments and should look at more tranches.
@@ -245,10 +245,7 @@ impl State {
self.covered + new_covered
};
let uncovered = self.uncovered + new_no_shows;
let next_no_show = super::min_prefer_some(
self.next_no_show,
next_no_show,
);
let next_no_show = super::min_prefer_some(self.next_no_show, next_no_show);
let (depth, covering, uncovered) = if covering == 0 {
if uncovered == 0 {
@@ -271,27 +268,25 @@ fn filled_tranche_iterator<'a>(
) -> impl Iterator<Item = (DelayTranche, &[(ValidatorIndex, Tick)])> {
let mut gap_end = None;
let approval_entries_filled = tranches
.iter()
.flat_map(move |tranche_entry| {
let tranche = tranche_entry.tranche();
let assignments = tranche_entry.assignments();
let approval_entries_filled = tranches.iter().flat_map(move |tranche_entry| {
let tranche = tranche_entry.tranche();
let assignments = tranche_entry.assignments();
// The new gap_start immediately follows the prior gap_end, if one exists.
// Otherwise, on the first pass, the new gap_start is set to the first
// tranche so that the range below will be empty.
let gap_start = gap_end.map(|end| end + 1).unwrap_or(tranche);
gap_end = Some(tranche);
// The new gap_start immediately follows the prior gap_end, if one exists.
// Otherwise, on the first pass, the new gap_start is set to the first
// tranche so that the range below will be empty.
let gap_start = gap_end.map(|end| end + 1).unwrap_or(tranche);
gap_end = Some(tranche);
(gap_start..tranche).map(|i| (i, &[] as &[_]))
.chain(std::iter::once((tranche, assignments)))
});
(gap_start..tranche)
.map(|i| (i, &[] as &[_]))
.chain(std::iter::once((tranche, assignments)))
});
let pre_end = tranches.first().map(|t| t.tranche());
let post_start = tranches.last().map_or(0, |t| t.tranche() + 1);
let pre = pre_end.into_iter()
.flat_map(|pre_end| (0..pre_end).map(|i| (i, &[] as &[_])));
let pre = pre_end.into_iter().flat_map(|pre_end| (0..pre_end).map(|i| (i, &[] as &[_])));
let post = (post_start..).map(|i| (i, &[] as &[_]));
pre.chain(approval_entries_filled).chain(post)
@@ -313,13 +308,14 @@ fn count_no_shows(
drifted_tick_now: Tick,
) -> (usize, Option<u64>) {
let mut next_no_show = None;
let no_shows = assignments.iter()
let no_shows = assignments
.iter()
.map(|(v_index, tick)| (v_index, tick.saturating_sub(clock_drift) + no_show_duration))
.filter(|&(v_index, no_show_at)| {
let has_approved = if let Some(approved) = approvals.get(v_index.0 as usize) {
*approved
} else {
return false;
return false
};
let is_no_show = !has_approved && no_show_at <= drifted_tick_now;
@@ -331,14 +327,12 @@ fn count_no_shows(
// *this node* should observe whether or not the validator is a no show. Recall
// that when the when drifted_tick_now is computed during that subsequent wake up,
// the clock drift will be removed again to do the comparison above.
next_no_show = super::min_prefer_some(
next_no_show,
Some(no_show_at + clock_drift),
);
next_no_show = super::min_prefer_some(next_no_show, Some(no_show_at + clock_drift));
}
is_no_show
}).count();
})
.count();
(no_shows, next_no_show)
}
@@ -430,9 +424,8 @@ pub fn tranches_to_approve(
mod tests {
use super::*;
use bitvec::{bitvec, order::Lsb0 as BitOrderLsb0};
use polkadot_primitives::v1::GroupIndex;
use bitvec::bitvec;
use bitvec::order::Lsb0 as BitOrderLsb0;
use crate::approval_db;
@@ -443,7 +436,8 @@ mod tests {
session: 0,
block_assignments: Default::default(),
approvals: Default::default(),
}.into();
}
.into();
let approval_entry = approval_db::v1::ApprovalEntry {
tranches: Vec::new(),
@@ -452,7 +446,8 @@ mod tests {
our_approval_sig: None,
backing_group: GroupIndex(0),
approved: false,
}.into();
}
.into();
assert!(!check_approval(
&candidate,
@@ -463,7 +458,8 @@ mod tests {
maximum_broadcast: 0,
clock_drift: 0,
},
).is_approved());
)
.is_approved());
}
#[test]
@@ -473,7 +469,8 @@ mod tests {
session: 0,
block_assignments: Default::default(),
approvals: bitvec![BitOrderLsb0, u8; 0; 10],
}.into();
}
.into();
for i in 0..3 {
candidate.mark_approval(ValidatorIndex(i));
@@ -499,35 +496,27 @@ mod tests {
our_approval_sig: None,
backing_group: GroupIndex(0),
approved: false,
}.into();
}
.into();
assert!(check_approval(
&candidate,
&approval_entry,
RequiredTranches::Exact {
needed: 0,
tolerated_missing: 0,
next_no_show: None,
},
).is_approved());
RequiredTranches::Exact { needed: 0, tolerated_missing: 0, next_no_show: None },
)
.is_approved());
assert!(!check_approval(
&candidate,
&approval_entry,
RequiredTranches::Exact {
needed: 1,
tolerated_missing: 0,
next_no_show: None,
},
).is_approved());
RequiredTranches::Exact { needed: 1, tolerated_missing: 0, next_no_show: None },
)
.is_approved());
assert!(check_approval(
&candidate,
&approval_entry,
RequiredTranches::Exact {
needed: 1,
tolerated_missing: 2,
next_no_show: None,
},
).is_approved());
RequiredTranches::Exact { needed: 1, tolerated_missing: 2, next_no_show: None },
)
.is_approved());
}
#[test]
@@ -537,7 +526,8 @@ mod tests {
session: 0,
block_assignments: Default::default(),
approvals: bitvec![BitOrderLsb0, u8; 0; 10],
}.into();
}
.into();
for i in 0..3 {
candidate.mark_approval(ValidatorIndex(i));
@@ -563,13 +553,11 @@ mod tests {
our_approval_sig: None,
backing_group: GroupIndex(0),
approved: false,
}.into();
}
.into();
let exact_all = RequiredTranches::Exact {
needed: 10,
tolerated_missing: 0,
next_no_show: None,
};
let exact_all =
RequiredTranches::Exact { needed: 10, tolerated_missing: 0, next_no_show: None };
let pending_all = RequiredTranches::Pending {
considered: 5,
@@ -578,44 +566,20 @@ mod tests {
clock_drift: 12,
};
assert!(!check_approval(
&candidate,
&approval_entry,
RequiredTranches::All,
).is_approved());
assert!(!check_approval(&candidate, &approval_entry, RequiredTranches::All,).is_approved());
assert!(!check_approval(
&candidate,
&approval_entry,
exact_all.clone(),
).is_approved());
assert!(!check_approval(&candidate, &approval_entry, exact_all.clone(),).is_approved());
assert!(!check_approval(
&candidate,
&approval_entry,
pending_all.clone(),
).is_approved());
assert!(!check_approval(&candidate, &approval_entry, pending_all.clone(),).is_approved());
// This creates a set of 4/10 approvals, which is always an approval.
candidate.mark_approval(ValidatorIndex(3));
assert!(check_approval(
&candidate,
&approval_entry,
RequiredTranches::All,
).is_approved());
assert!(check_approval(&candidate, &approval_entry, RequiredTranches::All,).is_approved());
assert!(check_approval(
&candidate,
&approval_entry,
exact_all,
).is_approved());
assert!(check_approval(&candidate, &approval_entry, exact_all,).is_approved());
assert!(check_approval(
&candidate,
&approval_entry,
pending_all,
).is_approved());
assert!(check_approval(&candidate, &approval_entry, pending_all,).is_approved());
}
#[test]
@@ -631,15 +595,16 @@ mod tests {
our_approval_sig: None,
backing_group: GroupIndex(0),
approved: false,
}.into();
}
.into();
approval_entry.import_assignment(0,ValidatorIndex(0), block_tick);
approval_entry.import_assignment(0,ValidatorIndex(1), block_tick);
approval_entry.import_assignment(0, ValidatorIndex(0), block_tick);
approval_entry.import_assignment(0, ValidatorIndex(1), block_tick);
approval_entry.import_assignment(1,ValidatorIndex(2), block_tick + 1);
approval_entry.import_assignment(1,ValidatorIndex(3), block_tick + 1);
approval_entry.import_assignment(1, ValidatorIndex(2), block_tick + 1);
approval_entry.import_assignment(1, ValidatorIndex(3), block_tick + 1);
approval_entry.import_assignment(2,ValidatorIndex(4), block_tick + 2);
approval_entry.import_assignment(2, ValidatorIndex(4), block_tick + 2);
let approvals = bitvec![BitOrderLsb0, u8; 1; 5];
@@ -669,7 +634,8 @@ mod tests {
our_approval_sig: None,
backing_group: GroupIndex(0),
approved: false,
}.into();
}
.into();
approval_entry.import_assignment(0, ValidatorIndex(0), block_tick);
approval_entry.import_assignment(1, ValidatorIndex(2), block_tick);
@@ -708,7 +674,8 @@ mod tests {
our_approval_sig: None,
backing_group: GroupIndex(0),
approved: false,
}.into();
}
.into();
approval_entry.import_assignment(0, ValidatorIndex(0), block_tick);
approval_entry.import_assignment(0, ValidatorIndex(1), block_tick);
@@ -752,7 +719,8 @@ mod tests {
our_approval_sig: None,
backing_group: GroupIndex(0),
approved: false,
}.into();
}
.into();
approval_entry.import_assignment(0, ValidatorIndex(0), block_tick);
approval_entry.import_assignment(0, ValidatorIndex(1), block_tick);
@@ -818,7 +786,8 @@ mod tests {
our_approval_sig: None,
backing_group: GroupIndex(0),
approved: false,
}.into();
}
.into();
approval_entry.import_assignment(0, ValidatorIndex(0), block_tick);
approval_entry.import_assignment(0, ValidatorIndex(1), block_tick);
@@ -868,7 +837,7 @@ mod tests {
RequiredTranches::Exact {
needed: 2,
tolerated_missing: 1,
next_no_show: Some(block_tick + 2*no_show_duration + 2),
next_no_show: Some(block_tick + 2 * no_show_duration + 2),
},
);
@@ -906,7 +875,8 @@ mod tests {
our_approval_sig: None,
backing_group: GroupIndex(0),
approved: false,
}.into();
}
.into();
approval_entry.import_assignment(0, ValidatorIndex(0), block_tick);
approval_entry.import_assignment(0, ValidatorIndex(1), block_tick);
@@ -935,11 +905,7 @@ mod tests {
no_show_duration,
needed_approvals,
),
RequiredTranches::Exact {
needed: 2,
tolerated_missing: 1,
next_no_show: None,
},
RequiredTranches::Exact { needed: 2, tolerated_missing: 1, next_no_show: None },
);
// Even though tranche 2 has 2 validators, it only covers 1 no-show.
@@ -977,11 +943,7 @@ mod tests {
no_show_duration,
needed_approvals,
),
RequiredTranches::Exact {
needed: 3,
tolerated_missing: 2,
next_no_show: None,
},
RequiredTranches::Exact { needed: 3, tolerated_missing: 2, next_no_show: None },
);
}
@@ -996,7 +958,8 @@ mod tests {
session: 0,
block_assignments: Default::default(),
approvals: bitvec![BitOrderLsb0, u8; 0; 3],
}.into();
}
.into();
for i in 0..3 {
candidate.mark_approval(ValidatorIndex(i));
@@ -1015,7 +978,8 @@ mod tests {
our_approval_sig: None,
backing_group: GroupIndex(0),
approved: false,
}.into();
}
.into();
let approvals = bitvec![BitOrderLsb0, u8; 0; 3];
@@ -1043,14 +1007,14 @@ mod tests {
const PREFIX: u32 = 10;
let test_tranches = vec![
vec![], // empty set
vec![0], // zero start
vec![0, 3], // zero start with gap
vec![2], // non-zero start
vec![2, 4], // non-zero start with gap
vec![0, 1, 2], // zero start with run and no gap
vec![2, 3, 4, 8], // non-zero start with run and gap
vec![0, 1, 2, 5, 6, 7], // zero start with runs and gap
vec![], // empty set
vec![0], // zero start
vec![0, 3], // zero start with gap
vec![2], // non-zero start
vec![2, 4], // non-zero start with gap
vec![0, 1, 2], // zero start with run and no gap
vec![2, 3, 4, 8], // non-zero start with run and gap
vec![0, 1, 2, 5, 6, 7], // zero start with runs and gap
];
for test_tranche in test_tranches {
@@ -1061,7 +1025,8 @@ mod tests {
our_approval_sig: None,
assignments: bitvec![BitOrderLsb0, u8; 0; 3],
approved: false,
}.into();
}
.into();
// Populate the requested tranches. The assignemnts aren't inspected in
// this test.
@@ -1072,10 +1037,8 @@ mod tests {
let filled_tranches = filled_tranche_iterator(approval_entry.tranches());
// Take the first PREFIX entries and map them to their tranche.
let tranches: Vec<DelayTranche> = filled_tranches
.take(PREFIX as usize)
.map(|e| e.0)
.collect();
let tranches: Vec<DelayTranche> =
filled_tranches.take(PREFIX as usize).map(|e| e.0).collect();
// We expect this sequence to be sequential.
let exp_tranches: Vec<DelayTranche> = (0..PREFIX).collect();
@@ -1194,7 +1157,11 @@ mod tests {
#[test]
fn count_no_shows_three_validators_one_almost_late_one_no_show_one_approving() {
test_count_no_shows(NoShowTest {
assignments: vec![(ValidatorIndex(1), 21), (ValidatorIndex(2), 20), (ValidatorIndex(3), 20)],
assignments: vec![
(ValidatorIndex(1), 21),
(ValidatorIndex(2), 20),
(ValidatorIndex(3), 20),
],
approvals: vec![3],
clock_drift: 10,
no_show_duration: 10,
@@ -1207,7 +1174,11 @@ mod tests {
#[test]
fn count_no_shows_three_no_show_validators() {
test_count_no_shows(NoShowTest {
assignments: vec![(ValidatorIndex(1), 20), (ValidatorIndex(2), 20), (ValidatorIndex(3), 20)],
assignments: vec![
(ValidatorIndex(1), 20),
(ValidatorIndex(2), 20),
(ValidatorIndex(3), 20),
],
approvals: vec![],
clock_drift: 10,
no_show_duration: 10,
@@ -1220,7 +1191,11 @@ mod tests {
#[test]
fn count_no_shows_three_approving_validators() {
test_count_no_shows(NoShowTest {
assignments: vec![(ValidatorIndex(1), 20), (ValidatorIndex(2), 20), (ValidatorIndex(3), 20)],
assignments: vec![
(ValidatorIndex(1), 20),
(ValidatorIndex(2), 20),
(ValidatorIndex(3), 20),
],
approvals: vec![1, 2, 3],
clock_drift: 10,
no_show_duration: 10,
@@ -1270,17 +1245,17 @@ mod tests {
}
#[test]
fn count_no_shows_validator_index_out_of_approvals_range_is_ignored_as_next_no_show() {
test_count_no_shows(NoShowTest {
assignments: vec![(ValidatorIndex(1000), 21)],
approvals: vec![],
clock_drift: 10,
no_show_duration: 10,
drifted_tick_now: 20,
exp_no_shows: 0,
exp_next_no_show: None,
})
}
fn count_no_shows_validator_index_out_of_approvals_range_is_ignored_as_next_no_show() {
test_count_no_shows(NoShowTest {
assignments: vec![(ValidatorIndex(1000), 21)],
approvals: vec![],
clock_drift: 10,
no_show_duration: 10,
drifted_tick_now: 20,
exp_no_shows: 0,
exp_next_no_show: None,
})
}
}
#[test]
@@ -1318,10 +1293,6 @@ fn depth_0_issued_as_exact_even_when_all() {
assert_eq!(
state.output(0, 10, 10, 20),
RequiredTranches::Exact {
needed: 0,
tolerated_missing: 0,
next_no_show: None,
},
RequiredTranches::Exact { needed: 0, tolerated_missing: 0, next_no_show: None },
);
}
polkadot/node/core/approval-voting/src/approval_db/v1/mod.rs
+34 -53
View File
@@ -17,21 +17,22 @@
//! Version 1 of the DB schema.
use kvdb::{DBTransaction, KeyValueDB};
use polkadot_node_subsystem::{SubsystemResult, SubsystemError};
use polkadot_node_primitives::approval::{DelayTranche, AssignmentCert};
use parity_scale_codec::{Decode, Encode};
use polkadot_node_primitives::approval::{AssignmentCert, DelayTranche};
use polkadot_node_subsystem::{SubsystemError, SubsystemResult};
use polkadot_primitives::v1::{
ValidatorIndex, GroupIndex, CandidateReceipt, SessionIndex, CoreIndex,
BlockNumber, Hash, CandidateHash, ValidatorSignature,
BlockNumber, CandidateHash, CandidateReceipt, CoreIndex, GroupIndex, Hash, SessionIndex,
ValidatorIndex, ValidatorSignature,
};
use sp_consensus_slots::Slot;
use parity_scale_codec::{Encode, Decode};
use std::collections::BTreeMap;
use std::sync::Arc;
use bitvec::{vec::BitVec, order::Lsb0 as BitOrderLsb0};
use bitvec::{order::Lsb0 as BitOrderLsb0, vec::BitVec};
use std::{collections::BTreeMap, sync::Arc};
use crate::backend::{Backend, BackendWriteOp};
use crate::persisted_entries;
use crate::{
backend::{Backend, BackendWriteOp},
persisted_entries,
};
const STORED_BLOCKS_KEY: &[u8] = b"Approvals_StoredBlocks";
@@ -48,10 +49,7 @@ impl DbBackend {
/// Create a new [`DbBackend`] with the supplied key-value store and
/// config.
pub fn new(db: Arc<dyn KeyValueDB>, config: Config) -> Self {
DbBackend {
inner: db,
config,
}
DbBackend { inner: db, config }
}
}
@@ -60,22 +58,17 @@ impl Backend for DbBackend {
&self,
block_hash: &Hash,
) -> SubsystemResult<Option<persisted_entries::BlockEntry>> {
load_block_entry(&*self.inner, &self.config, block_hash)
.map(|e| e.map(Into::into))
load_block_entry(&*self.inner, &self.config, block_hash).map(|e| e.map(Into::into))
}
fn load_candidate_entry(
&self,
candidate_hash: &CandidateHash,
) -> SubsystemResult<Option<persisted_entries::CandidateEntry>> {
load_candidate_entry(&*self.inner, &self.config, candidate_hash)
.map(|e| e.map(Into::into))
load_candidate_entry(&*self.inner, &self.config, candidate_hash).map(|e| e.map(Into::into))
}
fn load_blocks_at_height(
&self,
block_height: &BlockNumber
) -> SubsystemResult<Vec<Hash>> {
fn load_blocks_at_height(&self, block_height: &BlockNumber) -> SubsystemResult<Vec<Hash>> {
load_blocks_at_height(&*self.inner, &self.config, block_height)
}
@@ -89,7 +82,8 @@ impl Backend for DbBackend {
/// Atomically write the list of operations, with later operations taking precedence over prior.
fn write<I>(&mut self, ops: I) -> SubsystemResult<()>
where I: IntoIterator<Item = BackendWriteOp>
where
I: IntoIterator<Item = BackendWriteOp>,
{
let mut tx = DBTransaction::new();
for op in ops {
@@ -100,20 +94,13 @@ impl Backend for DbBackend {
&STORED_BLOCKS_KEY,
stored_block_range.encode(),
);
}
},
BackendWriteOp::WriteBlocksAtHeight(h, blocks) => {
tx.put_vec(
self.config.col_data,
&blocks_at_height_key(h),
blocks.encode(),
);
}
tx.put_vec(self.config.col_data, &blocks_at_height_key(h), blocks.encode());
},
BackendWriteOp::DeleteBlocksAtHeight(h) => {
tx.delete(
self.config.col_data,
&blocks_at_height_key(h),
);
}
tx.delete(self.config.col_data, &blocks_at_height_key(h));
},
BackendWriteOp::WriteBlockEntry(block_entry) => {
let block_entry: BlockEntry = block_entry.into();
tx.put_vec(
@@ -121,13 +108,10 @@ impl Backend for DbBackend {
&block_entry_key(&block_entry.block_hash),
block_entry.encode(),
);
}
},
BackendWriteOp::DeleteBlockEntry(hash) => {
tx.delete(
self.config.col_data,
&block_entry_key(&hash),
);
}
tx.delete(self.config.col_data, &block_entry_key(&hash));
},
BackendWriteOp::WriteCandidateEntry(candidate_entry) => {
let candidate_entry: CandidateEntry = candidate_entry.into();
tx.put_vec(
@@ -135,13 +119,10 @@ impl Backend for DbBackend {
&candidate_entry_key(&candidate_entry.candidate.hash()),
candidate_entry.encode(),
);
}
},
BackendWriteOp::DeleteCandidateEntry(candidate_hash) => {
tx.delete(
self.config.col_data,
&candidate_entry_key(&candidate_hash),
);
}
tx.delete(self.config.col_data, &candidate_entry_key(&candidate_hash));
},
}
}
@@ -257,13 +238,14 @@ impl std::error::Error for Error {}
/// Result alias for DB errors.
pub type Result<T> = std::result::Result<T, Error>;
pub(crate) fn load_decode<D: Decode>(store: &dyn KeyValueDB, col_data: u32, key: &[u8]) -> Result<Option<D>>
{
pub(crate) fn load_decode<D: Decode>(
store: &dyn KeyValueDB,
col_data: u32,
key: &[u8],
) -> Result<Option<D>> {
match store.get(col_data, key)? {
None => Ok(None),
Some(raw) => D::decode(&mut &raw[..])
.map(Some)
.map_err(Into::into),
Some(raw) => D::decode(&mut &raw[..]).map(Some).map_err(Into::into),
}
}
@@ -308,7 +290,6 @@ pub fn load_all_blocks(store: &dyn KeyValueDB, config: &Config) -> SubsystemResu
let blocks = load_blocks_at_height(store, config, &height)?;
hashes.extend(blocks);
}
}
Ok(hashes)
polkadot/node/core/approval-voting/src/approval_db/v1/tests.rs
+147 -208
View File
@@ -16,21 +16,19 @@
//! Tests for the aux-schema of approval voting.
use super::*;
use std::sync::Arc;
use std::collections::HashMap;
use polkadot_primitives::v1::Id as ParaId;
use super::{DbBackend, StoredBlockRange, *};
use crate::{
backend::{Backend, OverlayedBackend},
ops::{add_block_entry, canonicalize, force_approve, NewCandidateInfo},
};
use kvdb::KeyValueDB;
use super::{DbBackend, StoredBlockRange};
use crate::backend::{Backend, OverlayedBackend};
use crate::ops::{NewCandidateInfo, add_block_entry, force_approve, canonicalize};
use polkadot_primitives::v1::Id as ParaId;
use std::{collections::HashMap, sync::Arc};
const DATA_COL: u32 = 0;
const NUM_COLUMNS: u32 = 1;
const TEST_CONFIG: Config = Config {
col_data: DATA_COL,
};
const TEST_CONFIG: Config = Config { col_data: DATA_COL };
fn make_db() -> (DbBackend, Arc<dyn KeyValueDB>) {
let db_writer: Arc<dyn KeyValueDB> = Arc::new(kvdb_memorydb::create(NUM_COLUMNS));
@@ -80,26 +78,25 @@ fn read_write() {
let range = StoredBlockRange(10, 20);
let at_height = vec![hash_a, hash_b];
let block_entry = make_block_entry(
hash_a,
Default::default(),
1,
vec![(CoreIndex(0), candidate_hash)],
);
let block_entry =
make_block_entry(hash_a, Default::default(), 1, vec![(CoreIndex(0), candidate_hash)]);
let candidate_entry = CandidateEntry {
candidate: Default::default(),
session: 5,
block_assignments: vec![
(hash_a, ApprovalEntry {
block_assignments: vec![(
hash_a,
ApprovalEntry {
tranches: Vec::new(),
backing_group: GroupIndex(1),
our_assignment: None,
our_approval_sig: None,
assignments: Default::default(),
approved: false,
})
].into_iter().collect(),
},
)]
.into_iter()
.collect(),
approvals: Default::default(),
};
@@ -114,7 +111,10 @@ fn read_write() {
assert_eq!(load_stored_blocks(store.as_ref(), &TEST_CONFIG).unwrap(), Some(range));
assert_eq!(load_blocks_at_height(store.as_ref(), &TEST_CONFIG, &1).unwrap(), at_height);
assert_eq!(load_block_entry(store.as_ref(), &TEST_CONFIG, &hash_a).unwrap(), Some(block_entry.into()));
assert_eq!(
load_block_entry(store.as_ref(), &TEST_CONFIG, &hash_a).unwrap(),
Some(block_entry.into())
);
assert_eq!(
load_candidate_entry(store.as_ref(), &TEST_CONFIG, &candidate_hash).unwrap(),
Some(candidate_entry.into()),
@@ -129,7 +129,9 @@ fn read_write() {
assert!(load_blocks_at_height(store.as_ref(), &TEST_CONFIG, &1).unwrap().is_empty());
assert!(load_block_entry(store.as_ref(), &TEST_CONFIG, &hash_a).unwrap().is_none());
assert!(load_candidate_entry(store.as_ref(), &TEST_CONFIG, &candidate_hash).unwrap().is_none());
assert!(load_candidate_entry(store.as_ref(), &TEST_CONFIG, &candidate_hash)
.unwrap()
.is_none());
}
#[test]
@@ -165,47 +167,48 @@ fn add_block_entry_works() {
let n_validators = 10;
let mut new_candidate_info = HashMap::new();
new_candidate_info.insert(candidate_hash_a, NewCandidateInfo::new(
candidate_receipt_a,
GroupIndex(0),
None,
));
new_candidate_info
.insert(candidate_hash_a, NewCandidateInfo::new(candidate_receipt_a, GroupIndex(0), None));
let mut overlay_db = OverlayedBackend::new(&db);
add_block_entry(
&mut overlay_db,
block_entry_a.clone().into(),
n_validators,
|h| new_candidate_info.get(h).map(|x| x.clone()),
).unwrap();
add_block_entry(&mut overlay_db, block_entry_a.clone().into(), n_validators, |h| {
new_candidate_info.get(h).map(|x| x.clone())
})
.unwrap();
let write_ops = overlay_db.into_write_ops();
db.write(write_ops).unwrap();
new_candidate_info.insert(candidate_hash_b, NewCandidateInfo::new(
candidate_receipt_b,
GroupIndex(1),
None,
));
new_candidate_info
.insert(candidate_hash_b, NewCandidateInfo::new(candidate_receipt_b, GroupIndex(1), None));
let mut overlay_db = OverlayedBackend::new(&db);
add_block_entry(
&mut overlay_db,
block_entry_b.clone().into(),
n_validators,
|h| new_candidate_info.get(h).map(|x| x.clone()),
).unwrap();
add_block_entry(&mut overlay_db, block_entry_b.clone().into(), n_validators, |h| {
new_candidate_info.get(h).map(|x| x.clone())
})
.unwrap();
let write_ops = overlay_db.into_write_ops();
db.write(write_ops).unwrap();
assert_eq!(load_block_entry(store.as_ref(), &TEST_CONFIG, &block_hash_a).unwrap(), Some(block_entry_a.into()));
assert_eq!(load_block_entry(store.as_ref(), &TEST_CONFIG, &block_hash_b).unwrap(), Some(block_entry_b.into()));
assert_eq!(
load_block_entry(store.as_ref(), &TEST_CONFIG, &block_hash_a).unwrap(),
Some(block_entry_a.into())
);
assert_eq!(
load_block_entry(store.as_ref(), &TEST_CONFIG, &block_hash_b).unwrap(),
Some(block_entry_b.into())
);
let candidate_entry_a = load_candidate_entry(store.as_ref(), &TEST_CONFIG, &candidate_hash_a)
.unwrap().unwrap();
assert_eq!(candidate_entry_a.block_assignments.keys().collect::<Vec<_>>(), vec![&block_hash_a, &block_hash_b]);
.unwrap()
.unwrap();
assert_eq!(
candidate_entry_a.block_assignments.keys().collect::<Vec<_>>(),
vec![&block_hash_a, &block_hash_b]
);
let candidate_entry_b = load_candidate_entry(store.as_ref(), &TEST_CONFIG, &candidate_hash_b)
.unwrap().unwrap();
.unwrap()
.unwrap();
assert_eq!(candidate_entry_b.block_assignments.keys().collect::<Vec<_>>(), vec![&block_hash_b]);
}
@@ -217,44 +220,30 @@ fn add_block_entry_adds_child() {
let block_hash_a = Hash::repeat_byte(2);
let block_hash_b = Hash::repeat_byte(69);
let mut block_entry_a = make_block_entry(
block_hash_a,
parent_hash,
1,
Vec::new(),
);
let mut block_entry_a = make_block_entry(block_hash_a, parent_hash, 1, Vec::new());
let block_entry_b = make_block_entry(
block_hash_b,
block_hash_a,
2,
Vec::new(),
);
let block_entry_b = make_block_entry(block_hash_b, block_hash_a, 2, Vec::new());
let n_validators = 10;
let mut overlay_db = OverlayedBackend::new(&db);
add_block_entry(
&mut overlay_db,
block_entry_a.clone().into(),
n_validators,
|_| None,
).unwrap();
add_block_entry(&mut overlay_db, block_entry_a.clone().into(), n_validators, |_| None).unwrap();
add_block_entry(
&mut overlay_db,
block_entry_b.clone().into(),
n_validators,
|_| None,
).unwrap();
add_block_entry(&mut overlay_db, block_entry_b.clone().into(), n_validators, |_| None).unwrap();
let write_ops = overlay_db.into_write_ops();
db.write(write_ops).unwrap();
block_entry_a.children.push(block_hash_b);
assert_eq!(load_block_entry(store.as_ref(), &TEST_CONFIG, &block_hash_a).unwrap(), Some(block_entry_a.into()));
assert_eq!(load_block_entry(store.as_ref(), &TEST_CONFIG, &block_hash_b).unwrap(), Some(block_entry_b.into()));
assert_eq!(
load_block_entry(store.as_ref(), &TEST_CONFIG, &block_hash_a).unwrap(),
Some(block_entry_a.into())
);
assert_eq!(
load_block_entry(store.as_ref(), &TEST_CONFIG, &block_hash_b).unwrap(),
Some(block_entry_b.into())
);
}
#[test]
@@ -305,12 +294,8 @@ fn canonicalize_works() {
let block_entry_a = make_block_entry(block_hash_a, genesis, 1, Vec::new());
let block_entry_b1 = make_block_entry(block_hash_b1, block_hash_a, 2, Vec::new());
let block_entry_b2 = make_block_entry(
block_hash_b2,
block_hash_a,
2,
vec![(CoreIndex(0), cand_hash_1)],
);
let block_entry_b2 =
make_block_entry(block_hash_b2, block_hash_a, 2, vec![(CoreIndex(0), cand_hash_1)]);
let block_entry_c1 = make_block_entry(block_hash_c1, block_hash_b1, 3, Vec::new());
let block_entry_c2 = make_block_entry(
block_hash_c2,
@@ -324,44 +309,27 @@ fn canonicalize_works() {
4,
vec![(CoreIndex(0), cand_hash_3), (CoreIndex(1), cand_hash_4)],
);
let block_entry_d2 = make_block_entry(
block_hash_d2,
block_hash_c2,
4,
vec![(CoreIndex(0), cand_hash_5)],
);
let block_entry_d2 =
make_block_entry(block_hash_d2, block_hash_c2, 4, vec![(CoreIndex(0), cand_hash_5)]);
let candidate_info = {
let mut candidate_info = HashMap::new();
candidate_info.insert(cand_hash_1, NewCandidateInfo::new(
candidate_receipt_genesis,
GroupIndex(1),
None,
));
candidate_info.insert(
cand_hash_1,
NewCandidateInfo::new(candidate_receipt_genesis, GroupIndex(1), None),
);
candidate_info.insert(cand_hash_2, NewCandidateInfo::new(
candidate_receipt_a,
GroupIndex(2),
None,
));
candidate_info
.insert(cand_hash_2, NewCandidateInfo::new(candidate_receipt_a, GroupIndex(2), None));
candidate_info.insert(cand_hash_3, NewCandidateInfo::new(
candidate_receipt_b,
GroupIndex(3),
None,
));
candidate_info
.insert(cand_hash_3, NewCandidateInfo::new(candidate_receipt_b, GroupIndex(3), None));
candidate_info.insert(cand_hash_4, NewCandidateInfo::new(
candidate_receipt_b1,
GroupIndex(4),
None,
));
candidate_info
.insert(cand_hash_4, NewCandidateInfo::new(candidate_receipt_b1, GroupIndex(4), None));
candidate_info.insert(cand_hash_5, NewCandidateInfo::new(
candidate_receipt_c1,
GroupIndex(5),
None,
));
candidate_info
.insert(cand_hash_5, NewCandidateInfo::new(candidate_receipt_c1, GroupIndex(5), None));
candidate_info
};
@@ -379,12 +347,10 @@ fn canonicalize_works() {
let mut overlay_db = OverlayedBackend::new(&db);
for block_entry in blocks {
add_block_entry(
&mut overlay_db,
block_entry.into(),
n_validators,
|h| candidate_info.get(h).map(|x| x.clone()),
).unwrap();
add_block_entry(&mut overlay_db, block_entry.into(), n_validators, |h| {
candidate_info.get(h).map(|x| x.clone())
})
.unwrap();
}
let write_ops = overlay_db.into_write_ops();
db.write(write_ops).unwrap();
@@ -393,9 +359,11 @@ fn canonicalize_works() {
for (c_hash, in_blocks) in expected {
let (entry, in_blocks) = match in_blocks {
None => {
assert!(load_candidate_entry(store.as_ref(), &TEST_CONFIG, &c_hash).unwrap().is_none());
assert!(load_candidate_entry(store.as_ref(), &TEST_CONFIG, &c_hash)
.unwrap()
.is_none());
continue
}
},
Some(i) => (
load_candidate_entry(store.as_ref(), &TEST_CONFIG, &c_hash).unwrap().unwrap(),
i,
@@ -414,13 +382,13 @@ fn canonicalize_works() {
for (hash, with_candidates) in expected {
let (entry, with_candidates) = match with_candidates {
None => {
assert!(load_block_entry(store.as_ref(), &TEST_CONFIG, &hash).unwrap().is_none());
assert!(load_block_entry(store.as_ref(), &TEST_CONFIG, &hash)
.unwrap()
.is_none());
continue
}
Some(i) => (
load_block_entry(store.as_ref(), &TEST_CONFIG, &hash).unwrap().unwrap(),
i,
),
},
Some(i) =>
(load_block_entry(store.as_ref(), &TEST_CONFIG, &hash).unwrap().unwrap(), i),
};
assert_eq!(entry.candidates.len(), with_candidates.len());
@@ -454,7 +422,10 @@ fn canonicalize_works() {
let write_ops = overlay_db.into_write_ops();
db.write(write_ops).unwrap();
assert_eq!(load_stored_blocks(store.as_ref(), &TEST_CONFIG).unwrap().unwrap(), StoredBlockRange(4, 5));
assert_eq!(
load_stored_blocks(store.as_ref(), &TEST_CONFIG).unwrap().unwrap(),
StoredBlockRange(4, 5)
);
check_candidates_in_store(vec![
(cand_hash_1, None),
@@ -489,25 +460,31 @@ fn force_approve_works() {
let single_candidate_vec = vec![(CoreIndex(0), candidate_hash)];
let candidate_info = {
let mut candidate_info = HashMap::new();
candidate_info.insert(candidate_hash, NewCandidateInfo::new(
make_candidate(1.into(), Default::default()),
GroupIndex(1),
None,
));
candidate_info.insert(
candidate_hash,
NewCandidateInfo::new(
make_candidate(1.into(), Default::default()),
GroupIndex(1),
None,
),
);
candidate_info
};
let block_hash_a = Hash::repeat_byte(1); // 1
let block_hash_b = Hash::repeat_byte(2);
let block_hash_c = Hash::repeat_byte(3);
let block_hash_d = Hash::repeat_byte(4); // 4
let block_entry_a = make_block_entry(block_hash_a, Default::default(), 1, single_candidate_vec.clone());
let block_entry_b = make_block_entry(block_hash_b, block_hash_a, 2, single_candidate_vec.clone());
let block_entry_c = make_block_entry(block_hash_c, block_hash_b, 3, single_candidate_vec.clone());
let block_entry_d = make_block_entry(block_hash_d, block_hash_c, 4, single_candidate_vec.clone());
let block_entry_a =
make_block_entry(block_hash_a, Default::default(), 1, single_candidate_vec.clone());
let block_entry_b =
make_block_entry(block_hash_b, block_hash_a, 2, single_candidate_vec.clone());
let block_entry_c =
make_block_entry(block_hash_c, block_hash_b, 3, single_candidate_vec.clone());
let block_entry_d =
make_block_entry(block_hash_d, block_hash_c, 4, single_candidate_vec.clone());
let blocks = vec![
block_entry_a.clone(),
@@ -518,41 +495,36 @@ fn force_approve_works() {
let mut overlay_db = OverlayedBackend::new(&db);
for block_entry in blocks {
add_block_entry(
&mut overlay_db,
block_entry.into(),
n_validators,
|h| candidate_info.get(h).map(|x| x.clone()),
).unwrap();
add_block_entry(&mut overlay_db, block_entry.into(), n_validators, |h| {
candidate_info.get(h).map(|x| x.clone())
})
.unwrap();
}
let approved_hashes = force_approve(&mut overlay_db, block_hash_d, 2).unwrap();
let approved_hashes = force_approve(&mut overlay_db, block_hash_d, 2).unwrap();
let write_ops = overlay_db.into_write_ops();
db.write(write_ops).unwrap();
assert!(load_block_entry(
store.as_ref(),
&TEST_CONFIG,
&block_hash_a,
).unwrap().unwrap().approved_bitfield.all());
assert!(load_block_entry(
store.as_ref(),
&TEST_CONFIG,
&block_hash_b,
).unwrap().unwrap().approved_bitfield.all());
assert!(load_block_entry(
store.as_ref(),
&TEST_CONFIG,
&block_hash_c,
).unwrap().unwrap().approved_bitfield.not_any());
assert!(load_block_entry(
store.as_ref(),
&TEST_CONFIG,
&block_hash_d,
).unwrap().unwrap().approved_bitfield.not_any());
assert_eq!(
approved_hashes,
vec![block_hash_b, block_hash_a],
);
assert!(load_block_entry(store.as_ref(), &TEST_CONFIG, &block_hash_a,)
.unwrap()
.unwrap()
.approved_bitfield
.all());
assert!(load_block_entry(store.as_ref(), &TEST_CONFIG, &block_hash_b,)
.unwrap()
.unwrap()
.approved_bitfield
.all());
assert!(load_block_entry(store.as_ref(), &TEST_CONFIG, &block_hash_c,)
.unwrap()
.unwrap()
.approved_bitfield
.not_any());
assert!(load_block_entry(store.as_ref(), &TEST_CONFIG, &block_hash_d,)
.unwrap()
.unwrap()
.approved_bitfield
.not_any());
assert_eq!(approved_hashes, vec![block_hash_b, block_hash_a],);
}
#[test]
@@ -566,60 +538,27 @@ fn load_all_blocks_works() {
let block_number = 10;
let block_entry_a = make_block_entry(
block_hash_a,
parent_hash,
block_number,
vec![],
);
let block_entry_a = make_block_entry(block_hash_a, parent_hash, block_number, vec![]);
let block_entry_b = make_block_entry(
block_hash_b,
parent_hash,
block_number,
vec![],
);
let block_entry_b = make_block_entry(block_hash_b, parent_hash, block_number, vec![]);
let block_entry_c = make_block_entry(
block_hash_c,
block_hash_a,
block_number + 1,
vec![],
);
let block_entry_c = make_block_entry(block_hash_c, block_hash_a, block_number + 1, vec![]);
let n_validators = 10;
let mut overlay_db = OverlayedBackend::new(&db);
add_block_entry(
&mut overlay_db,
block_entry_a.clone().into(),
n_validators,
|_| None
).unwrap();
add_block_entry(&mut overlay_db, block_entry_a.clone().into(), n_validators, |_| None).unwrap();
// add C before B to test sorting.
add_block_entry(
&mut overlay_db,
block_entry_c.clone().into(),
n_validators,
|_| None
).unwrap();
add_block_entry(&mut overlay_db, block_entry_c.clone().into(), n_validators, |_| None).unwrap();
add_block_entry(
&mut overlay_db,
block_entry_b.clone().into(),
n_validators,
|_| None
).unwrap();
add_block_entry(&mut overlay_db, block_entry_b.clone().into(), n_validators, |_| None).unwrap();
let write_ops = overlay_db.into_write_ops();
db.write(write_ops).unwrap();
assert_eq!(
load_all_blocks(
store.as_ref(),
&TEST_CONFIG
).unwrap(),
load_all_blocks(store.as_ref(), &TEST_CONFIG).unwrap(),
vec![block_hash_a, block_hash_b, block_hash_c],
)
}
polkadot/node/core/approval-voting/src/backend.rs
+15 -6
View File
@@ -21,13 +21,15 @@
//! [`Backend`], maintaining consistency between queries and temporary writes,
//! before any commit to the underlying storage is made.
use polkadot_node_subsystem::{SubsystemResult};
use polkadot_node_subsystem::SubsystemResult;
use polkadot_primitives::v1::{BlockNumber, CandidateHash, Hash};
use std::collections::HashMap;
use super::approval_db::v1::StoredBlockRange;
use super::persisted_entries::{BlockEntry, CandidateEntry};
use super::{
approval_db::v1::StoredBlockRange,
persisted_entries::{BlockEntry, CandidateEntry},
};
#[derive(Debug)]
pub enum BackendWriteOp {
@@ -45,7 +47,10 @@ pub trait Backend {
/// Load a block entry from the DB.
fn load_block_entry(&self, hash: &Hash) -> SubsystemResult<Option<BlockEntry>>;
/// Load a candidate entry from the DB.
fn load_candidate_entry(&self, candidate_hash: &CandidateHash) -> SubsystemResult<Option<CandidateEntry>>;
fn load_candidate_entry(
&self,
candidate_hash: &CandidateHash,
) -> SubsystemResult<Option<CandidateEntry>>;
/// Load all blocks at a specific height.
fn load_blocks_at_height(&self, height: &BlockNumber) -> SubsystemResult<Vec<Hash>>;
/// Load all block from the DB.
@@ -54,7 +59,8 @@ pub trait Backend {
fn load_stored_blocks(&self) -> SubsystemResult<Option<StoredBlockRange>>;
/// Atomically write the list of operations, with later operations taking precedence over prior.
fn write<I>(&mut self, ops: I) -> SubsystemResult<()>
where I: IntoIterator<Item = BackendWriteOp>;
where
I: IntoIterator<Item = BackendWriteOp>;
}
/// An in-memory overlay over the backend.
@@ -128,7 +134,10 @@ impl<'a, B: 'a + Backend> OverlayedBackend<'a, B> {
self.inner.load_block_entry(hash)
}
pub fn load_candidate_entry(&self, candidate_hash: &CandidateHash) -> SubsystemResult<Option<CandidateEntry>> {
pub fn load_candidate_entry(
&self,
candidate_hash: &CandidateHash,
) -> SubsystemResult<Option<CandidateEntry>> {
if let Some(val) = self.candidate_entries.get(&candidate_hash) {
return Ok(val.clone())
}
polkadot/node/core/approval-voting/src/criteria.rs
+123 -117
View File
@@ -16,21 +16,20 @@
//! Assignment criteria VRF generation and checking.
use parity_scale_codec::{Decode, Encode};
use polkadot_node_primitives::approval::{
self as approval_types, AssignmentCert, AssignmentCertKind, DelayTranche, RelayVRFStory,
};
use polkadot_primitives::v1::{
CoreIndex, ValidatorIndex, SessionInfo, AssignmentPair, AssignmentId, GroupIndex, CandidateHash,
AssignmentId, AssignmentPair, CandidateHash, CoreIndex, GroupIndex, SessionInfo, ValidatorIndex,
};
use sc_keystore::LocalKeystore;
use parity_scale_codec::{Encode, Decode};
use sp_application_crypto::Public;
use merlin::Transcript;
use schnorrkel::vrf::VRFInOut;
use std::collections::HashMap;
use std::collections::hash_map::Entry;
use std::collections::{hash_map::Entry, HashMap};
use super::LOG_TARGET;
@@ -88,10 +87,7 @@ impl From<OurAssignment> for crate::approval_db::v1::OurAssignment {
}
}
fn relay_vrf_modulo_transcript(
relay_vrf_story: RelayVRFStory,
sample: u32,
) -> Transcript {
fn relay_vrf_modulo_transcript(relay_vrf_story: RelayVRFStory, sample: u32) -> Transcript {
// combine the relay VRF story with a sample number.
let mut t = Transcript::new(approval_types::RELAY_VRF_MODULO_CONTEXT);
t.append_message(b"RC-VRF", &relay_vrf_story.0);
@@ -100,10 +96,7 @@ fn relay_vrf_modulo_transcript(
t
}
fn relay_vrf_modulo_core(
vrf_in_out: &VRFInOut,
n_cores: u32,
) -> CoreIndex {
fn relay_vrf_modulo_core(vrf_in_out: &VRFInOut, n_cores: u32) -> CoreIndex {
let bytes: [u8; 4] = vrf_in_out.make_bytes(approval_types::CORE_RANDOMNESS_CONTEXT);
// interpret as little-endian u32.
@@ -111,10 +104,7 @@ fn relay_vrf_modulo_core(
CoreIndex(random_core)
}
fn relay_vrf_delay_transcript(
relay_vrf_story: RelayVRFStory,
core_index: CoreIndex,
) -> Transcript {
fn relay_vrf_delay_transcript(relay_vrf_story: RelayVRFStory, core_index: CoreIndex) -> Transcript {
let mut t = Transcript::new(approval_types::RELAY_VRF_DELAY_CONTEXT);
t.append_message(b"RC-VRF", &relay_vrf_story.0);
core_index.0.using_encoded(|s| t.append_message(b"core", s));
@@ -129,7 +119,8 @@ fn relay_vrf_delay_tranche(
let bytes: [u8; 4] = vrf_in_out.make_bytes(approval_types::TRANCHE_RANDOMNESS_CONTEXT);
// interpret as little-endian u32 and reduce by the number of tranches.
let wide_tranche = u32::from_le_bytes(bytes) % (num_delay_tranches + zeroth_delay_tranche_width);
let wide_tranche =
u32::from_le_bytes(bytes) % (num_delay_tranches + zeroth_delay_tranche_width);
// Consolidate early results to tranche zero so tranche zero is extra wide.
wide_tranche.saturating_sub(zeroth_delay_tranche_width)
@@ -202,12 +193,7 @@ impl AssignmentCriteria for RealAssignmentCriteria {
config: &Config,
leaving_cores: Vec<(CandidateHash, CoreIndex, GroupIndex)>,
) -> HashMap<CoreIndex, OurAssignment> {
compute_assignments(
keystore,
relay_vrf_story,
config,
leaving_cores,
)
compute_assignments(keystore, relay_vrf_story, config, leaving_cores)
}
fn check_assignment_cert(
@@ -246,13 +232,16 @@ pub(crate) fn compute_assignments(
config: &Config,
leaving_cores: impl IntoIterator<Item = (CandidateHash, CoreIndex, GroupIndex)> + Clone,
) -> HashMap<CoreIndex, OurAssignment> {
if config.n_cores == 0 || config.assignment_keys.is_empty() || config.validator_groups.is_empty() {
if config.n_cores == 0 ||
config.assignment_keys.is_empty() ||
config.validator_groups.is_empty()
{
return HashMap::new()
}
let (index, assignments_key): (ValidatorIndex, AssignmentPair) = {
let key = config.assignment_keys.iter().enumerate()
.find_map(|(i, p)| match keystore.key_pair(p) {
let key = config.assignment_keys.iter().enumerate().find_map(|(i, p)| {
match keystore.key_pair(p) {
Ok(Some(pair)) => Some((ValidatorIndex(i as _), pair)),
Ok(None) => None,
Err(sc_keystore::Error::Unavailable) => None,
@@ -260,8 +249,9 @@ pub(crate) fn compute_assignments(
Err(e) => {
tracing::warn!(target: LOG_TARGET, "Encountered keystore error: {:?}", e);
None
}
});
},
}
});
match key {
None => return Default::default(),
@@ -270,7 +260,8 @@ pub(crate) fn compute_assignments(
};
// Ignore any cores where the assigned group is our own.
let leaving_cores = leaving_cores.into_iter()
let leaving_cores = leaving_cores
.into_iter()
.filter(|&(_, _, ref g)| !is_in_backing_group(&config.validator_groups, index, *g))
.map(|(c_hash, core, _)| (c_hash, core))
.collect::<Vec<_>>();
@@ -322,8 +313,8 @@ fn compute_relay_vrf_modulo_assignments(
relay_vrf_modulo_transcript(relay_vrf_story.clone(), rvm_sample),
|vrf_in_out| {
*core = relay_vrf_modulo_core(&vrf_in_out, config.n_cores);
if let Some((candidate_hash, _))
= leaving_cores.clone().into_iter().find(|(_, c)| c == core)
if let Some((candidate_hash, _)) =
leaving_cores.clone().into_iter().find(|(_, c)| c == core)
{
tracing::trace!(
target: LOG_TARGET,
@@ -338,7 +329,7 @@ fn compute_relay_vrf_modulo_assignments(
} else {
None
}
}
},
)
};
@@ -347,7 +338,10 @@ fn compute_relay_vrf_modulo_assignments(
// has been executed.
let cert = AssignmentCert {
kind: AssignmentCertKind::RelayVRFModulo { sample: rvm_sample },
vrf: (approval_types::VRFOutput(vrf_in_out.to_output()), approval_types::VRFProof(vrf_proof)),
vrf: (
approval_types::VRFOutput(vrf_in_out.to_output()),
approval_types::VRFProof(vrf_proof),
),
};
// All assignments of type RelayVRFModulo have tranche 0.
@@ -370,9 +364,8 @@ fn compute_relay_vrf_delay_assignments(
assignments: &mut HashMap<CoreIndex, OurAssignment>,
) {
for (candidate_hash, core) in leaving_cores {
let (vrf_in_out, vrf_proof, _) = assignments_key.vrf_sign(
relay_vrf_delay_transcript(relay_vrf_story.clone(), core),
);
let (vrf_in_out, vrf_proof, _) =
assignments_key.vrf_sign(relay_vrf_delay_transcript(relay_vrf_story.clone(), core));
let tranche = relay_vrf_delay_tranche(
&vrf_in_out,
@@ -382,24 +375,26 @@ fn compute_relay_vrf_delay_assignments(
let cert = AssignmentCert {
kind: AssignmentCertKind::RelayVRFDelay { core_index: core },
vrf: (approval_types::VRFOutput(vrf_in_out.to_output()), approval_types::VRFProof(vrf_proof)),
vrf: (
approval_types::VRFOutput(vrf_in_out.to_output()),
approval_types::VRFProof(vrf_proof),
),
};
let our_assignment = OurAssignment {
cert,
tranche,
validator_index,
triggered: false,
};
let our_assignment = OurAssignment { cert, tranche, validator_index, triggered: false };
let used = match assignments.entry(core) {
Entry::Vacant(e) => { let _ = e.insert(our_assignment); true }
Entry::Occupied(mut e) => if e.get().tranche > our_assignment.tranche {
e.insert(our_assignment);
Entry::Vacant(e) => {
let _ = e.insert(our_assignment);
true
} else {
false
},
Entry::Occupied(mut e) =>
if e.get().tranche > our_assignment.tranche {
e.insert(our_assignment);
true
} else {
false
},
};
if used {
@@ -425,7 +420,7 @@ impl std::fmt::Display for InvalidAssignment {
}
}
impl std::error::Error for InvalidAssignment { }
impl std::error::Error for InvalidAssignment {}
/// Checks the crypto of an assignment cert. Failure conditions:
/// * Validator index out of bounds
@@ -446,7 +441,8 @@ pub(crate) fn check_assignment_cert(
assignment: &AssignmentCert,
backing_group: GroupIndex,
) -> Result<DelayTranche, InvalidAssignment> {
let validator_public = config.assignment_keys
let validator_public = config
.assignment_keys
.get(validator_index.0 as usize)
.ok_or(InvalidAssignment)?;
@@ -454,34 +450,33 @@ pub(crate) fn check_assignment_cert(
.map_err(|_| InvalidAssignment)?;
if claimed_core_index.0 >= config.n_cores {
return Err(InvalidAssignment);
return Err(InvalidAssignment)
}
// Check that the validator was not part of the backing group
// and not already assigned.
let is_in_backing = is_in_backing_group(
&config.validator_groups,
validator_index,
backing_group,
);
let is_in_backing =
is_in_backing_group(&config.validator_groups, validator_index, backing_group);
if is_in_backing {
return Err(InvalidAssignment);
return Err(InvalidAssignment)
}
let &(ref vrf_output, ref vrf_proof) = &assignment.vrf;
match assignment.kind {
AssignmentCertKind::RelayVRFModulo { sample } => {
if sample >= config.relay_vrf_modulo_samples {
return Err(InvalidAssignment);
return Err(InvalidAssignment)
}
let (vrf_in_out, _) = public.vrf_verify_extra(
relay_vrf_modulo_transcript(relay_vrf_story, sample),
&vrf_output.0,
&vrf_proof.0,
assigned_core_transcript(claimed_core_index),
).map_err(|_| InvalidAssignment)?;
let (vrf_in_out, _) = public
.vrf_verify_extra(
relay_vrf_modulo_transcript(relay_vrf_story, sample),
&vrf_output.0,
&vrf_proof.0,
assigned_core_transcript(claimed_core_index),
)
.map_err(|_| InvalidAssignment)?;
// ensure that the `vrf_in_out` actually gives us the claimed core.
if relay_vrf_modulo_core(&vrf_in_out, config.n_cores) == claimed_core_index {
@@ -489,24 +484,26 @@ pub(crate) fn check_assignment_cert(
} else {
Err(InvalidAssignment)
}
}
},
AssignmentCertKind::RelayVRFDelay { core_index } => {
if core_index != claimed_core_index {
return Err(InvalidAssignment);
return Err(InvalidAssignment)
}
let (vrf_in_out, _) = public.vrf_verify(
relay_vrf_delay_transcript(relay_vrf_story, core_index),
&vrf_output.0,
&vrf_proof.0,
).map_err(|_| InvalidAssignment)?;
let (vrf_in_out, _) = public
.vrf_verify(
relay_vrf_delay_transcript(relay_vrf_story, core_index),
&vrf_output.0,
&vrf_proof.0,
)
.map_err(|_| InvalidAssignment)?;
Ok(relay_vrf_delay_tranche(
&vrf_in_out,
config.n_delay_tranches,
config.zeroth_delay_tranche_width,
))
}
},
}
}
@@ -521,22 +518,22 @@ fn is_in_backing_group(
#[cfg(test)]
mod tests {
use super::*;
use sp_keystore::CryptoStore;
use sp_keyring::sr25519::Keyring as Sr25519Keyring;
use polkadot_node_primitives::approval::{VRFOutput, VRFProof};
use polkadot_primitives::v1::{Hash, ASSIGNMENT_KEY_TYPE_ID};
use sp_application_crypto::sr25519;
use sp_core::crypto::Pair as PairT;
use polkadot_primitives::v1::{ASSIGNMENT_KEY_TYPE_ID, Hash};
use polkadot_node_primitives::approval::{VRFOutput, VRFProof};
use sp_keyring::sr25519::Keyring as Sr25519Keyring;
use sp_keystore::CryptoStore;
// sets up a keystore with the given keyring accounts.
async fn make_keystore(accounts: &[Sr25519Keyring]) -> LocalKeystore {
let store = LocalKeystore::in_memory();
for s in accounts.iter().copied().map(|k| k.to_seed()) {
store.sr25519_generate_new(
ASSIGNMENT_KEY_TYPE_ID,
Some(s.as_str()),
).await.unwrap();
store
.sr25519_generate_new(ASSIGNMENT_KEY_TYPE_ID, Some(s.as_str()))
.await
.unwrap();
}
store
@@ -546,12 +543,15 @@ mod tests {
assignment_keys_plus_random(accounts, 0)
}
fn assignment_keys_plus_random(accounts: &[Sr25519Keyring], random: usize) -> Vec<AssignmentId> {
let gen_random = (0..random).map(|_|
AssignmentId::from(sr25519::Pair::generate().0.public())
);
fn assignment_keys_plus_random(
accounts: &[Sr25519Keyring],
random: usize,
) -> Vec<AssignmentId> {
let gen_random =
(0..random).map(|_| AssignmentId::from(sr25519::Pair::generate().0.public()));
accounts.iter()
accounts
.iter()
.map(|k| AssignmentId::from(k.public()))
.chain(gen_random)
.collect()
@@ -562,12 +562,14 @@ mod tests {
let big_groups = n_validators % n_groups;
let scraps = n_groups * size;
(0..n_groups).map(|i| {
(i * size .. (i + 1) *size)
.chain(if i < big_groups { Some(scraps + i) } else { None })
.map(|j| ValidatorIndex(j as _))
.collect::<Vec<_>>()
}).collect()
(0..n_groups)
.map(|i| {
(i * size..(i + 1) * size)
.chain(if i < big_groups { Some(scraps + i) } else { None })
.map(|j| ValidatorIndex(j as _))
.collect::<Vec<_>>()
})
.collect()
}
// used for generating assignments where the validity of the VRF doesn't matter.
@@ -581,9 +583,7 @@ mod tests {
#[test]
fn assignments_produced_for_non_backing() {
let keystore = futures::executor::block_on(
make_keystore(&[Sr25519Keyring::Alice])
);
let keystore = futures::executor::block_on(make_keystore(&[Sr25519Keyring::Alice]));
let c_a = CandidateHash(Hash::repeat_byte(0));
let c_b = CandidateHash(Hash::repeat_byte(1));
@@ -598,7 +598,10 @@ mod tests {
Sr25519Keyring::Bob,
Sr25519Keyring::Charlie,
]),
validator_groups: vec![vec![ValidatorIndex(0)], vec![ValidatorIndex(1), ValidatorIndex(2)]],
validator_groups: vec![
vec![ValidatorIndex(0)],
vec![ValidatorIndex(1), ValidatorIndex(2)],
],
n_cores: 2,
zeroth_delay_tranche_width: 10,
relay_vrf_modulo_samples: 3,
@@ -615,9 +618,7 @@ mod tests {
#[test]
fn assign_to_nonzero_core() {
let keystore = futures::executor::block_on(
make_keystore(&[Sr25519Keyring::Alice])
);
let keystore = futures::executor::block_on(make_keystore(&[Sr25519Keyring::Alice]));
let c_a = CandidateHash(Hash::repeat_byte(0));
let c_b = CandidateHash(Hash::repeat_byte(1));
@@ -632,7 +633,10 @@ mod tests {
Sr25519Keyring::Bob,
Sr25519Keyring::Charlie,
]),
validator_groups: vec![vec![ValidatorIndex(0)], vec![ValidatorIndex(1), ValidatorIndex(2)]],
validator_groups: vec![
vec![ValidatorIndex(0)],
vec![ValidatorIndex(1), ValidatorIndex(2)],
],
n_cores: 2,
zeroth_delay_tranche_width: 10,
relay_vrf_modulo_samples: 3,
@@ -647,9 +651,7 @@ mod tests {
#[test]
fn succeeds_empty_for_0_cores() {
let keystore = futures::executor::block_on(
make_keystore(&[Sr25519Keyring::Alice])
);
let keystore = futures::executor::block_on(make_keystore(&[Sr25519Keyring::Alice]));
let relay_vrf_story = RelayVRFStory([42u8; 32]);
let assignments = compute_assignments(
@@ -689,14 +691,15 @@ mod tests {
rotation_offset: usize,
f: impl Fn(&mut MutatedAssignment) -> Option<bool>, // None = skip
) {
let keystore = futures::executor::block_on(
make_keystore(&[Sr25519Keyring::Alice])
);
let keystore = futures::executor::block_on(make_keystore(&[Sr25519Keyring::Alice]));
let group_for_core = |i| GroupIndex(((i + rotation_offset) % n_cores) as _);
let config = Config {
assignment_keys: assignment_keys_plus_random(&[Sr25519Keyring::Alice], n_validators - 1),
assignment_keys: assignment_keys_plus_random(
&[Sr25519Keyring::Alice],
n_validators - 1,
),
validator_groups: basic_groups(n_validators, n_cores),
n_cores: n_cores as u32,
zeroth_delay_tranche_width: 10,
@@ -710,11 +713,13 @@ mod tests {
relay_vrf_story.clone(),
&config,
(0..n_cores)
.map(|i| (
CandidateHash(Hash::repeat_byte(i as u8)),
CoreIndex(i as u32),
group_for_core(i),
))
.map(|i| {
(
CandidateHash(Hash::repeat_byte(i as u8)),
CoreIndex(i as u32),
group_for_core(i),
)
})
.collect::<Vec<_>>(),
);
@@ -743,7 +748,8 @@ mod tests {
relay_vrf_story.clone(),
&mutated.cert,
mutated.group,
).is_ok();
)
.is_ok();
assert_eq!(expected, is_good)
}
@@ -787,7 +793,7 @@ mod tests {
AssignmentCertKind::RelayVRFDelay { .. } => {
m.cert.vrf = garbage_vrf();
Some(false)
}
},
_ => None, // skip everything else.
}
});
@@ -800,7 +806,7 @@ mod tests {
AssignmentCertKind::RelayVRFModulo { .. } => {
m.cert.vrf = garbage_vrf();
Some(false)
}
},
_ => None, // skip everything else.
}
});
@@ -813,7 +819,7 @@ mod tests {
AssignmentCertKind::RelayVRFModulo { sample } => {
m.config.relay_vrf_modulo_samples = sample;
Some(false)
}
},
_ => None, // skip everything else.
}
});
@@ -826,7 +832,7 @@ mod tests {
AssignmentCertKind::RelayVRFDelay { .. } => {
m.core = CoreIndex((m.core.0 + 1) % 100);
Some(false)
}
},
_ => None, // skip everything else.
}
});
@@ -839,7 +845,7 @@ mod tests {
AssignmentCertKind::RelayVRFModulo { .. } => {
m.core = CoreIndex((m.core.0 + 1) % 100);
Some(false)
}
},
_ => None, // skip everything else.
}
});
polkadot/node/core/approval-voting/src/import.rs
+172 -196
View File
@@ -28,43 +28,42 @@
//!
//! We maintain a rolling window of session indices. This starts as empty
use polkadot_node_subsystem::{
overseer,
messages::{
RuntimeApiMessage, RuntimeApiRequest, ChainApiMessage, ApprovalDistributionMessage,
ChainSelectionMessage,
},
SubsystemContext, SubsystemError, SubsystemResult,
};
use polkadot_node_subsystem_util::determine_new_blocks;
use polkadot_node_subsystem_util::rolling_session_window::{
RollingSessionWindow, SessionWindowUpdate,
};
use polkadot_primitives::v1::{
Hash, SessionIndex, CandidateEvent, Header, CandidateHash,
CandidateReceipt, CoreIndex, GroupIndex, BlockNumber, ConsensusLog,
};
use polkadot_node_jaeger as jaeger;
use polkadot_node_primitives::approval::{
self as approval_types, BlockApprovalMeta, RelayVRFStory,
};
use polkadot_node_jaeger as jaeger;
use polkadot_node_subsystem::{
messages::{
ApprovalDistributionMessage, ChainApiMessage, ChainSelectionMessage, RuntimeApiMessage,
RuntimeApiRequest,
},
overseer, SubsystemContext, SubsystemError, SubsystemResult,
};
use polkadot_node_subsystem_util::{
determine_new_blocks,
rolling_session_window::{RollingSessionWindow, SessionWindowUpdate},
};
use polkadot_primitives::v1::{
BlockNumber, CandidateEvent, CandidateHash, CandidateReceipt, ConsensusLog, CoreIndex,
GroupIndex, Hash, Header, SessionIndex,
};
use sc_keystore::LocalKeystore;
use sp_consensus_slots::Slot;
use futures::prelude::*;
use futures::channel::oneshot;
use bitvec::order::Lsb0 as BitOrderLsb0;
use futures::{channel::oneshot, prelude::*};
use std::collections::HashMap;
use std::convert::TryFrom;
use std::{collections::HashMap, convert::TryFrom};
use super::approval_db::v1;
use crate::backend::{Backend, OverlayedBackend};
use crate::persisted_entries::CandidateEntry;
use crate::criteria::{AssignmentCriteria, OurAssignment};
use crate::time::{slot_number_to_tick, Tick};
use crate::{
backend::{Backend, OverlayedBackend},
criteria::{AssignmentCriteria, OurAssignment},
persisted_entries::CandidateEntry,
time::{slot_number_to_tick, Tick},
};
use super::{LOG_TARGET, State};
use super::{State, LOG_TARGET};
struct ImportedBlockInfo {
included_candidates: Vec<(CandidateHash, CandidateReceipt, CoreIndex, GroupIndex)>,
@@ -99,7 +98,8 @@ async fn imported_block_info(
ctx.send_message(RuntimeApiMessage::Request(
block_hash,
RuntimeApiRequest::CandidateEvents(c_tx),
)).await;
))
.await;
let events: Vec<CandidateEvent> = match c_rx.await {
Ok(Ok(events)) => events,
@@ -107,11 +107,14 @@ async fn imported_block_info(
Err(_) => return Ok(None),
};
events.into_iter().filter_map(|e| match e {
CandidateEvent::CandidateIncluded(receipt, _, core, group)
=> Some((receipt.hash(), receipt, core, group)),
_ => None,
}).collect()
events
.into_iter()
.filter_map(|e| match e {
CandidateEvent::CandidateIncluded(receipt, _, core, group) =>
Some((receipt.hash(), receipt, core, group)),
_ => None,
})
.collect()
};
// fetch session. ignore blocks that are too old, but unless sessions are really
@@ -121,7 +124,8 @@ async fn imported_block_info(
ctx.send_message(RuntimeApiMessage::Request(
block_header.parent_hash,
RuntimeApiRequest::SessionIndexForChild(s_tx),
)).await;
))
.await;
let session_index = match s_rx.await {
Ok(Ok(s)) => s,
@@ -130,10 +134,14 @@ async fn imported_block_info(
};
if env.session_window.earliest_session().map_or(true, |e| session_index < e) {
tracing::debug!(target: LOG_TARGET, "Block {} is from ancient session {}. Skipping",
block_hash, session_index);
tracing::debug!(
target: LOG_TARGET,
"Block {} is from ancient session {}. Skipping",
block_hash,
session_index
);
return Ok(None);
return Ok(None)
}
session_index
@@ -162,7 +170,8 @@ async fn imported_block_info(
ctx.send_message(RuntimeApiMessage::Request(
block_hash,
RuntimeApiRequest::CurrentBabeEpoch(s_tx),
)).await;
))
.await;
match s_rx.await {
Ok(Ok(s)) => s,
@@ -180,8 +189,8 @@ async fn imported_block_info(
block_hash,
);
return Ok(None);
}
return Ok(None)
},
};
let (assignments, slot, relay_vrf_story) = {
@@ -201,7 +210,8 @@ async fn imported_block_info(
&env.keystore,
relay_vrf.clone(),
&crate::criteria::Config::from(session_info),
included_candidates.iter()
included_candidates
.iter()
.map(|(c_hash, _, core, group)| (*c_hash, *core, *group))
.collect(),
);
@@ -210,7 +220,7 @@ async fn imported_block_info(
},
Err(_) => return Ok(None),
}
}
},
None => {
tracing::debug!(
target: LOG_TARGET,
@@ -218,16 +228,12 @@ async fn imported_block_info(
block_hash,
);
return Ok(None);
}
return Ok(None)
},
}
};
tracing::trace!(
target: LOG_TARGET,
n_assignments = assignments.len(),
"Produced assignments"
);
tracing::trace!(target: LOG_TARGET, n_assignments = assignments.len(), "Produced assignments");
let force_approve =
block_header.digest.convert_first(|l| match ConsensusLog::from_digest_item(l) {
@@ -241,7 +247,7 @@ async fn imported_block_info(
);
Some(num)
}
},
Ok(Some(_)) => None,
Ok(None) => None,
Err(err) => {
@@ -253,7 +259,7 @@ async fn imported_block_info(
);
None
}
},
});
Ok(Some(ImportedBlockInfo {
@@ -291,8 +297,7 @@ pub(crate) async fn handle_new_head(
db: &mut OverlayedBackend<'_, impl Backend>,
head: Hash,
finalized_number: &Option<BlockNumber>,
) -> SubsystemResult<Vec<BlockImportedCandidates>>
{
) -> SubsystemResult<Vec<BlockImportedCandidates>> {
// Update session info based on most recent head.
let mut span = jaeger::Span::new(head, "approval-checking-import");
@@ -309,13 +314,13 @@ pub(crate) async fn handle_new_head(
e,
);
return Ok(Vec::new());
}
return Ok(Vec::new())
},
Ok(None) => {
tracing::warn!(target: LOG_TARGET, "Missing header for new head {}", head);
return Ok(Vec::new());
}
Ok(Some(h)) => h
return Ok(Vec::new())
},
Ok(Some(h)) => h,
}
};
@@ -329,15 +334,15 @@ pub(crate) async fn handle_new_head(
);
return Ok(Vec::new())
}
},
Ok(a @ SessionWindowUpdate::Advanced { .. }) => {
tracing::info!(
target: LOG_TARGET,
update = ?a,
"Advanced session window for approvals",
);
}
Ok(_) => {}
},
Ok(_) => {},
}
// If we've just started the node and haven't yet received any finality notifications,
@@ -351,12 +356,14 @@ pub(crate) async fn handle_new_head(
&header,
lower_bound_number,
)
.map_err(|e| SubsystemError::with_origin("approval-voting", e))
.await?;
.map_err(|e| SubsystemError::with_origin("approval-voting", e))
.await?;
span.add_uint_tag("new-blocks", new_blocks.len() as u64);
if new_blocks.is_empty() { return Ok(Vec::new()) }
if new_blocks.is_empty() {
return Ok(Vec::new())
}
let mut approval_meta: Vec<BlockApprovalMeta> = Vec::with_capacity(new_blocks.len());
let mut imported_candidates = Vec::with_capacity(new_blocks.len());
@@ -376,9 +383,11 @@ pub(crate) async fn handle_new_head(
None => {
// It's possible that we've lost a race with finality.
let (tx, rx) = oneshot::channel();
ctx.send_message(
ChainApiMessage::FinalizedBlockHash(block_header.number.clone(), tx)
).await;
ctx.send_message(ChainApiMessage::FinalizedBlockHash(
block_header.number.clone(),
tx,
))
.await;
let lost_to_finality = match rx.await {
Ok(Ok(Some(h))) if h != block_hash => true,
@@ -395,7 +404,7 @@ pub(crate) async fn handle_new_head(
);
}
return Ok(Vec::new());
return Ok(Vec::new())
},
};
}
@@ -420,7 +429,9 @@ pub(crate) async fn handle_new_head(
force_approve,
} = imported_block_info;
let session_info = state.session_window.session_info(session_index)
let session_info = state
.session_window
.session_info(session_index)
.expect("imported_block_info requires session to be available; qed");
let (block_tick, no_show_duration) = {
@@ -432,7 +443,8 @@ pub(crate) async fn handle_new_head(
(block_tick, no_show_duration)
};
let needed_approvals = session_info.needed_approvals;
let validator_group_lens: Vec<usize> = session_info.validator_groups.iter().map(|v| v.len()).collect();
let validator_group_lens: Vec<usize> =
session_info.validator_groups.iter().map(|v| v.len()).collect();
// insta-approve candidates on low-node testnets:
// cf. https://github.com/paritytech/polkadot/issues/2411
let num_candidates = included_candidates.len();
@@ -447,10 +459,10 @@ pub(crate) async fn handle_new_head(
} else {
let mut result = bitvec::bitvec![BitOrderLsb0, u8; 0; num_candidates];
for (i, &(_, _, _, backing_group)) in included_candidates.iter().enumerate() {
let backing_group_size = validator_group_lens.get(backing_group.0 as usize)
.copied()
.unwrap_or(0);
let needed_approvals = usize::try_from(needed_approvals).expect("usize is at least u32; qed");
let backing_group_size =
validator_group_lens.get(backing_group.0 as usize).copied().unwrap_or(0);
let needed_approvals =
usize::try_from(needed_approvals).expect("usize is at least u32; qed");
if n_validators.saturating_sub(backing_group_size) < needed_approvals {
result.set(i, true);
}
@@ -481,19 +493,16 @@ pub(crate) async fn handle_new_head(
session: session_index,
slot,
relay_vrf_story: relay_vrf_story.0,
candidates: included_candidates.iter()
.map(|(hash, _, core, _)| (*core, *hash)).collect(),
candidates: included_candidates
.iter()
.map(|(hash, _, core, _)| (*core, *hash))
.collect(),
approved_bitfield,
children: Vec::new(),
};
if let Some(up_to) = force_approve {
tracing::debug!(
target: LOG_TARGET,
?block_hash,
up_to,
"Enacting force-approve",
);
tracing::debug!(target: LOG_TARGET, ?block_hash, up_to, "Enacting force-approve",);
let approved_hashes = crate::ops::force_approve(db, block_hash, up_to)
.map_err(|e| SubsystemError::with_origin("approval-voting", e))?;
@@ -511,19 +520,19 @@ pub(crate) async fn handle_new_head(
"Writing BlockEntry",
);
let candidate_entries = crate::ops::add_block_entry(
db,
block_entry.into(),
n_validators,
|candidate_hash| {
included_candidates.iter().find(|(hash, _, _, _)| candidate_hash == hash)
.map(|(_, receipt, core, backing_group)| super::ops::NewCandidateInfo::new(
receipt.clone(),
*backing_group,
assignments.get(core).map(|a| a.clone().into()),
))
}
).map_err(|e| SubsystemError::with_origin("approval-voting", e))?;
let candidate_entries =
crate::ops::add_block_entry(db, block_entry.into(), n_validators, |candidate_hash| {
included_candidates.iter().find(|(hash, _, _, _)| candidate_hash == hash).map(
|(_, receipt, core, backing_group)| {
super::ops::NewCandidateInfo::new(
receipt.clone(),
*backing_group,
assignments.get(core).map(|a| a.clone().into()),
)
},
)
})
.map_err(|e| SubsystemError::with_origin("approval-voting", e))?;
approval_meta.push(BlockApprovalMeta {
hash: block_hash,
number: block_header.number,
@@ -532,18 +541,16 @@ pub(crate) async fn handle_new_head(
slot,
});
imported_candidates.push(
BlockImportedCandidates {
block_hash,
block_number: block_header.number,
block_tick,
no_show_duration,
imported_candidates: candidate_entries
.into_iter()
.map(|(h, e)| (h, e.into()))
.collect(),
}
);
imported_candidates.push(BlockImportedCandidates {
block_hash,
block_number: block_header.number,
block_tick,
no_show_duration,
imported_candidates: candidate_entries
.into_iter()
.map(|(h, e)| (h, e.into()))
.collect(),
});
}
tracing::trace!(
@@ -562,33 +569,30 @@ pub(crate) async fn handle_new_head(
pub(crate) mod tests {
use super::*;
use crate::approval_db::v1::DbBackend;
use polkadot_node_subsystem_test_helpers::make_subsystem_context;
use polkadot_node_primitives::approval::{VRFOutput, VRFProof};
use polkadot_primitives::v1::{SessionInfo, ValidatorIndex};
use polkadot_node_subsystem::messages::AllMessages;
use sp_core::testing::TaskExecutor;
pub(crate) use sp_runtime::{Digest, DigestItem};
pub(crate) use sp_consensus_babe::{
Epoch as BabeEpoch, BabeEpochConfiguration, AllowedSlots,
};
pub(crate) use sp_consensus_babe::digests::{CompatibleDigestItem, PreDigest, SecondaryVRFPreDigest};
use sp_keyring::sr25519::Keyring as Sr25519Keyring;
use assert_matches::assert_matches;
use merlin::Transcript;
use std::{pin::Pin, sync::Arc};
use kvdb::KeyValueDB;
use merlin::Transcript;
use polkadot_node_primitives::approval::{VRFOutput, VRFProof};
use polkadot_node_subsystem::messages::AllMessages;
use polkadot_node_subsystem_test_helpers::make_subsystem_context;
use polkadot_primitives::v1::{SessionInfo, ValidatorIndex};
pub(crate) use sp_consensus_babe::{
digests::{CompatibleDigestItem, PreDigest, SecondaryVRFPreDigest},
AllowedSlots, BabeEpochConfiguration, Epoch as BabeEpoch,
};
use sp_core::testing::TaskExecutor;
use sp_keyring::sr25519::Keyring as Sr25519Keyring;
pub(crate) use sp_runtime::{Digest, DigestItem};
use std::{pin::Pin, sync::Arc};
use crate::{
APPROVAL_SESSIONS, criteria, BlockEntry,
approval_db::v1::Config as DatabaseConfig,
approval_db::v1::Config as DatabaseConfig, criteria, BlockEntry, APPROVAL_SESSIONS,
};
const DATA_COL: u32 = 0;
const NUM_COLUMNS: u32 = 1;
const TEST_CONFIG: DatabaseConfig = DatabaseConfig {
col_data: DATA_COL,
};
const TEST_CONFIG: DatabaseConfig = DatabaseConfig { col_data: DATA_COL };
#[derive(Default)]
struct MockClock;
@@ -598,9 +602,7 @@ pub(crate) mod tests {
}
fn wait(&self, _tick: Tick) -> Pin<Box<dyn Future<Output = ()> + Send + 'static>> {
Box::pin(async move {
()
})
Box::pin(async move { () })
}
}
@@ -633,7 +635,11 @@ pub(crate) mod tests {
_keystore: &LocalKeystore,
_relay_vrf_story: polkadot_node_primitives::approval::RelayVRFStory,
_config: &criteria::Config,
_leaving_cores: Vec<(CandidateHash, polkadot_primitives::v1::CoreIndex, polkadot_primitives::v1::GroupIndex)>,
_leaving_cores: Vec<(
CandidateHash,
polkadot_primitives::v1::CoreIndex,
polkadot_primitives::v1::GroupIndex,
)>,
) -> HashMap<polkadot_primitives::v1::CoreIndex, criteria::OurAssignment> {
HashMap::new()
}
@@ -675,7 +681,6 @@ pub(crate) mod tests {
}
}
#[test]
fn imported_block_info_is_good() {
let pool = TaskExecutor::new();
@@ -691,12 +696,7 @@ pub(crate) mod tests {
let mut d = Digest::default();
let (vrf_output, vrf_proof) = garbage_vrf();
d.push(DigestItem::babe_pre_digest(PreDigest::SecondaryVRF(
SecondaryVRFPreDigest {
authority_index: 0,
slot,
vrf_output,
vrf_proof,
}
SecondaryVRFPreDigest { authority_index: 0, slot, vrf_output, vrf_proof },
)));
d
@@ -719,13 +719,15 @@ pub(crate) mod tests {
(make_candidate(2.into()), CoreIndex(1), GroupIndex(3)),
];
let inclusion_events = candidates.iter().cloned()
let inclusion_events = candidates
.iter()
.cloned()
.map(|(r, c, g)| CandidateEvent::CandidateIncluded(r, Vec::new().into(), c, g))
.collect::<Vec<_>>();
let test_fut = {
let included_candidates = candidates.iter()
let included_candidates = candidates
.iter()
.map(|(r, c, g)| (r.hash(), r.clone(), *c, *g))
.collect::<Vec<_>>();
@@ -743,12 +745,8 @@ pub(crate) mod tests {
keystore: &LocalKeystore::in_memory(),
};
let info = imported_block_info(
&mut ctx,
env,
hash,
&header,
).await.unwrap().unwrap();
let info =
imported_block_info(&mut ctx, env, hash, &header).await.unwrap().unwrap();
assert_eq!(info.included_candidates, included_candidates);
assert_eq!(info.session_index, session);
@@ -835,7 +833,9 @@ pub(crate) mod tests {
(make_candidate(2.into()), CoreIndex(1), GroupIndex(3)),
];
let inclusion_events = candidates.iter().cloned()
let inclusion_events = candidates
.iter()
.cloned()
.map(|(r, c, g)| CandidateEvent::CandidateIncluded(r, Vec::new().into(), c, g))
.collect::<Vec<_>>();
@@ -854,12 +854,7 @@ pub(crate) mod tests {
keystore: &LocalKeystore::in_memory(),
};
let info = imported_block_info(
&mut ctx,
env,
hash,
&header,
).await.unwrap();
let info = imported_block_info(&mut ctx, env, hash, &header).await.unwrap();
assert!(info.is_none());
})
@@ -940,7 +935,9 @@ pub(crate) mod tests {
(make_candidate(2.into()), CoreIndex(1), GroupIndex(3)),
];
let inclusion_events = candidates.iter().cloned()
let inclusion_events = candidates
.iter()
.cloned()
.map(|(r, c, g)| CandidateEvent::CandidateIncluded(r, Vec::new().into(), c, g))
.collect::<Vec<_>>();
@@ -955,12 +952,7 @@ pub(crate) mod tests {
keystore: &LocalKeystore::in_memory(),
};
let info = imported_block_info(
&mut ctx,
env,
hash,
&header,
).await.unwrap();
let info = imported_block_info(&mut ctx, env, hash, &header).await.unwrap();
assert!(info.is_none());
})
@@ -1008,12 +1000,7 @@ pub(crate) mod tests {
let mut d = Digest::default();
let (vrf_output, vrf_proof) = garbage_vrf();
d.push(DigestItem::babe_pre_digest(PreDigest::SecondaryVRF(
SecondaryVRFPreDigest {
authority_index: 0,
slot,
vrf_output,
vrf_proof,
}
SecondaryVRFPreDigest { authority_index: 0, slot, vrf_output, vrf_proof },
)));
d.push(ConsensusLog::ForceApprove(3).into());
@@ -1038,13 +1025,15 @@ pub(crate) mod tests {
(make_candidate(2.into()), CoreIndex(1), GroupIndex(3)),
];
let inclusion_events = candidates.iter().cloned()
let inclusion_events = candidates
.iter()
.cloned()
.map(|(r, c, g)| CandidateEvent::CandidateIncluded(r, Vec::new().into(), c, g))
.collect::<Vec<_>>();
let test_fut = {
let included_candidates = candidates.iter()
let included_candidates = candidates
.iter()
.map(|(r, c, g)| (r.hash(), r.clone(), *c, *g))
.collect::<Vec<_>>();
@@ -1062,12 +1051,8 @@ pub(crate) mod tests {
keystore: &LocalKeystore::in_memory(),
};
let info = imported_block_info(
&mut ctx,
env,
hash,
&header,
).await.unwrap().unwrap();
let info =
imported_block_info(&mut ctx, env, hash, &header).await.unwrap().unwrap();
assert_eq!(info.included_candidates, included_candidates);
assert_eq!(info.session_index, session);
@@ -1159,12 +1144,7 @@ pub(crate) mod tests {
let mut d = Digest::default();
let (vrf_output, vrf_proof) = garbage_vrf();
d.push(DigestItem::babe_pre_digest(PreDigest::SecondaryVRF(
SecondaryVRFPreDigest {
authority_index: 0,
slot,
vrf_output,
vrf_proof,
}
SecondaryVRFPreDigest { authority_index: 0, slot, vrf_output, vrf_proof },
)));
d
@@ -1186,7 +1166,9 @@ pub(crate) mod tests {
(make_candidate(1.into()), CoreIndex(0), GroupIndex(0)),
(make_candidate(2.into()), CoreIndex(1), GroupIndex(1)),
];
let inclusion_events = candidates.iter().cloned()
let inclusion_events = candidates
.iter()
.cloned()
.map(|(r, c, g)| CandidateEvent::CandidateIncluded(r, Vec::new().into(), c, g))
.collect::<Vec<_>>();
@@ -1202,7 +1184,8 @@ pub(crate) mod tests {
candidates: Vec::new(),
approved_bitfield: Default::default(),
children: Vec::new(),
}.into()
}
.into(),
);
let write_ops = overlay_db.into_write_ops();
@@ -1211,13 +1194,9 @@ pub(crate) mod tests {
let test_fut = {
Box::pin(async move {
let mut overlay_db = OverlayedBackend::new(&db);
let result = handle_new_head(
&mut ctx,
&mut state,
&mut overlay_db,
hash,
&Some(1),
).await.unwrap();
let result = handle_new_head(&mut ctx, &mut state, &mut overlay_db, hash, &Some(1))
.await
.unwrap();
let write_ops = overlay_db.into_write_ops();
db.write(write_ops).unwrap();
@@ -1229,14 +1208,11 @@ pub(crate) mod tests {
assert_eq!(candidates[1].1.approvals().len(), 6);
// the first candidate should be insta-approved
// the second should not
let entry: BlockEntry = v1::load_block_entry(
db_writer.as_ref(),
&TEST_CONFIG,
&hash,
)
.unwrap()
.unwrap()
.into();
let entry: BlockEntry =
v1::load_block_entry(db_writer.as_ref(), &TEST_CONFIG, &hash)
.unwrap()
.unwrap()
.into();
assert!(entry.is_candidate_approved(&candidates[0].0));
assert!(!entry.is_candidate_approved(&candidates[1].0));
})
polkadot/node/core/approval-voting/src/lib.rs
+490 -526
View File
File diff suppressed because it is too large Load Diff
polkadot/node/core/approval-voting/src/old_tests.rs
+327 -391
View File
File diff suppressed because it is too large Load Diff
polkadot/node/core/approval-voting/src/ops.rs
+33 -51
View File
@@ -19,21 +19,18 @@
use polkadot_node_subsystem::SubsystemResult;
use polkadot_primitives::v1::{
CandidateHash, CandidateReceipt, BlockNumber, GroupIndex, Hash,
use bitvec::order::Lsb0 as BitOrderLsb0;
use polkadot_primitives::v1::{BlockNumber, CandidateHash, CandidateReceipt, GroupIndex, Hash};
use std::{
collections::{hash_map::Entry, BTreeMap, HashMap},
convert::Into,
};
use bitvec::{order::Lsb0 as BitOrderLsb0};
use std::convert::Into;
use std::collections::{BTreeMap, HashMap};
use std::collections::hash_map::Entry;
use super::persisted_entries::{ApprovalEntry, CandidateEntry, BlockEntry};
use super::backend::{Backend, OverlayedBackend};
use super::approval_db::{
v1::{
OurAssignment, StoredBlockRange,
},
use super::{
approval_db::v1::{OurAssignment, StoredBlockRange},
backend::{Backend, OverlayedBackend},
persisted_entries::{ApprovalEntry, BlockEntry, CandidateEntry},
};
/// Information about a new candidate necessary to instantiate the requisite
@@ -75,7 +72,7 @@ fn visit_and_remove_block_entry(
None => continue, // Should not happen except for corrupt DB
Some(c) => c,
})
}
},
};
candidate.block_assignments.remove(&block_hash);
@@ -111,11 +108,7 @@ pub fn canonicalize(
overlay_db.delete_blocks_at_height(i);
for b in at_height {
let _ = visit_and_remove_block_entry(
b,
overlay_db,
&mut visited_candidates,
)?;
let _ = visit_and_remove_block_entry(b, overlay_db, &mut visited_candidates)?;
}
}
@@ -129,11 +122,7 @@ pub fn canonicalize(
let mut pruned_branches = Vec::new();
for b in at_height {
let children = visit_and_remove_block_entry(
b,
overlay_db,
&mut visited_candidates,
)?;
let children = visit_and_remove_block_entry(b, overlay_db, &mut visited_candidates)?;
if b != canon_hash {
pruned_branches.extend(children);
@@ -145,13 +134,11 @@ pub fn canonicalize(
// Follow all children of non-canonicalized blocks.
{
let mut frontier: Vec<(BlockNumber, Hash)> = pruned_branches.into_iter().map(|h| (canon_number + 1, h)).collect();
let mut frontier: Vec<(BlockNumber, Hash)> =
pruned_branches.into_iter().map(|h| (canon_number + 1, h)).collect();
while let Some((height, next_child)) = frontier.pop() {
let children = visit_and_remove_block_entry(
next_child,
overlay_db,
&mut visited_candidates,
)?;
let children =
visit_and_remove_block_entry(next_child, overlay_db, &mut visited_candidates)?;
// extend the frontier of branches to include the given height.
frontier.extend(children.into_iter().map(|h| (height + 1, h)));
@@ -188,10 +175,7 @@ pub fn canonicalize(
// due to the fork pruning, this range actually might go too far above where our actual highest block is,
// if a relatively short fork is canonicalized.
// TODO https://github.com/paritytech/polkadot/issues/3389
let new_range = StoredBlockRange(
canon_number + 1,
std::cmp::max(range.1, canon_number + 2),
);
let new_range = StoredBlockRange(canon_number + 1, std::cmp::max(range.1, canon_number + 2));
overlay_db.write_stored_block_range(new_range);
@@ -246,21 +230,20 @@ pub fn add_block_entry(
// read and write all updated entries.
{
for &(_, ref candidate_hash) in entry.candidates() {
let NewCandidateInfo {
candidate,
backing_group,
our_assignment,
} = match candidate_info(candidate_hash) {
None => return Ok(Vec::new()),
Some(info) => info,
};
let NewCandidateInfo { candidate, backing_group, our_assignment } =
match candidate_info(candidate_hash) {
None => return Ok(Vec::new()),
Some(info) => info,
};
let mut candidate_entry = store.load_candidate_entry(&candidate_hash)?
.unwrap_or_else(move || CandidateEntry {
candidate,
session,
block_assignments: BTreeMap::new(),
approvals: bitvec::bitvec![BitOrderLsb0, u8; 0; n_validators],
let mut candidate_entry =
store.load_candidate_entry(&candidate_hash)?.unwrap_or_else(move || {
CandidateEntry {
candidate,
session,
block_assignments: BTreeMap::new(),
approvals: bitvec::bitvec![BitOrderLsb0, u8; 0; n_validators],
}
});
candidate_entry.block_assignments.insert(
@@ -272,7 +255,7 @@ pub fn add_block_entry(
None,
bitvec::bitvec![BitOrderLsb0, u8; 0; n_validators],
false,
)
),
);
store.write_candidate_entry(candidate_entry.clone());
@@ -313,7 +296,6 @@ pub fn force_approve(
// iterate back to the `up_to` block, and then iterate backwards until all blocks
// are updated.
while let Some(mut entry) = store.load_block_entry(&cur_hash)? {
if entry.block_number() <= up_to {
state = State::Approving;
}
@@ -326,7 +308,7 @@ pub fn force_approve(
entry.approved_bitfield.iter_mut().for_each(|mut b| *b = true);
approved_hashes.push(entry.block_hash());
store.write_block_entry(entry);
}
},
}
}
polkadot/node/core/approval-voting/src/persisted_entries.rs
+42 -39
View File
@@ -20,18 +20,17 @@
//! Within that context, things are plain-old-data. Within this module,
//! data and logic are intertwined.
use polkadot_node_primitives::approval::{DelayTranche, RelayVRFStory, AssignmentCert};
use polkadot_node_primitives::approval::{AssignmentCert, DelayTranche, RelayVRFStory};
use polkadot_primitives::v1::{
ValidatorIndex, CandidateReceipt, SessionIndex, GroupIndex, CoreIndex,
Hash, CandidateHash, BlockNumber, ValidatorSignature,
BlockNumber, CandidateHash, CandidateReceipt, CoreIndex, GroupIndex, Hash, SessionIndex,
ValidatorIndex, ValidatorSignature,
};
use sp_consensus_slots::Slot;
use bitvec::{order::Lsb0 as BitOrderLsb0, slice::BitSlice, vec::BitVec};
use std::collections::BTreeMap;
use bitvec::{slice::BitSlice, vec::BitVec, order::Lsb0 as BitOrderLsb0};
use super::time::Tick;
use super::criteria::OurAssignment;
use super::{criteria::OurAssignment, time::Tick};
/// Metadata regarding a specific tranche of assignments for a specific candidate.
#[derive(Debug, Clone, PartialEq)]
@@ -105,11 +104,14 @@ impl ApprovalEntry {
}
// Note that our assignment is triggered. No-op if already triggered.
pub fn trigger_our_assignment(&mut self, tick_now: Tick)
-> Option<(AssignmentCert, ValidatorIndex, DelayTranche)>
{
pub fn trigger_our_assignment(
&mut self,
tick_now: Tick,
) -> Option<(AssignmentCert, ValidatorIndex, DelayTranche)> {
let our = self.our_assignment.as_mut().and_then(|a| {
if a.triggered() { return None }
if a.triggered() {
return None
}
a.mark_triggered();
Some(a.clone())
@@ -143,22 +145,16 @@ impl ApprovalEntry {
let idx = match self.tranches.iter().position(|t| t.tranche >= tranche) {
Some(pos) => {
if self.tranches[pos].tranche > tranche {
self.tranches.insert(pos, TrancheEntry {
tranche: tranche,
assignments: Vec::new(),
});
self.tranches.insert(pos, TrancheEntry { tranche, assignments: Vec::new() });
}
pos
}
},
None => {
self.tranches.push(TrancheEntry {
tranche: tranche,
assignments: Vec::new(),
});
self.tranches.push(TrancheEntry { tranche, assignments: Vec::new() });
self.tranches.len() - 1
}
},
};
self.tranches[idx].assignments.push((validator_index, tick_now));
@@ -168,15 +164,16 @@ impl ApprovalEntry {
// Produce a bitvec indicating the assignments of all validators up to and
// including `tranche`.
pub fn assignments_up_to(&self, tranche: DelayTranche) -> BitVec<BitOrderLsb0, u8> {
self.tranches.iter()
.take_while(|e| e.tranche <= tranche)
.fold(bitvec::bitvec![BitOrderLsb0, u8; 0; self.assignments.len()], |mut a, e| {
self.tranches.iter().take_while(|e| e.tranche <= tranche).fold(
bitvec::bitvec![BitOrderLsb0, u8; 0; self.assignments.len()],
|mut a, e| {
for &(v, _) in &e.assignments {
a.set(v.0 as _, true);
}
a
})
},
)
}
/// Whether the approval entry is approved
@@ -299,11 +296,7 @@ impl CandidateEntry {
}
#[cfg(test)]
pub fn add_approval_entry(
&mut self,
block_hash: Hash,
approval_entry: ApprovalEntry,
) {
pub fn add_approval_entry(&mut self, block_hash: Hash, approval_entry: ApprovalEntry) {
self.block_assignments.insert(block_hash, approval_entry);
}
}
@@ -313,7 +306,11 @@ impl From<crate::approval_db::v1::CandidateEntry> for CandidateEntry {
CandidateEntry {
candidate: entry.candidate,
session: entry.session,
block_assignments: entry.block_assignments.into_iter().map(|(h, ae)| (h, ae.into())).collect(),
block_assignments: entry
.block_assignments
.into_iter()
.map(|(h, ae)| (h, ae.into()))
.collect(),
approvals: entry.approvals,
}
}
@@ -324,7 +321,11 @@ impl From<CandidateEntry> for crate::approval_db::v1::CandidateEntry {
Self {
candidate: entry.candidate,
session: entry.session,
block_assignments: entry.block_assignments.into_iter().map(|(h, ae)| (h, ae.into())).collect(),
block_assignments: entry
.block_assignments
.into_iter()
.map(|(h, ae)| (h, ae.into()))
.collect(),
approvals: entry.approvals,
}
}
@@ -360,7 +361,9 @@ impl BlockEntry {
/// Whether a candidate is approved in the bitfield.
pub fn is_candidate_approved(&self, candidate_hash: &CandidateHash) -> bool {
self.candidates.iter().position(|(_, h)| h == candidate_hash)
self.candidates
.iter()
.position(|(_, h)| h == candidate_hash)
.and_then(|p| self.approved_bitfield.get(p).map(|b| *b))
.unwrap_or(false)
}
@@ -372,10 +375,12 @@ impl BlockEntry {
/// Iterate over all unapproved candidates.
pub fn unapproved_candidates(&self) -> impl Iterator<Item = CandidateHash> + '_ {
self.approved_bitfield.iter().enumerate().filter_map(move |(i, a)| if !*a {
Some(self.candidates[i].1)
} else {
None
self.approved_bitfield.iter().enumerate().filter_map(move |(i, a)| {
if !*a {
Some(self.candidates[i].1)
} else {
None
}
})
}
@@ -385,9 +390,7 @@ impl BlockEntry {
/// Panics if the core is already used.
#[cfg(test)]
pub fn add_candidate(&mut self, core: CoreIndex, candidate_hash: CandidateHash) -> usize {
let pos = self.candidates
.binary_search_by_key(&core, |(c, _)| *c)
.unwrap_err();
let pos = self.candidates.binary_search_by_key(&core, |(c, _)| *c).unwrap_err();
self.candidates.insert(pos, (core, candidate_hash));
polkadot/node/core/approval-voting/src/tests.rs
+191 -273
View File
@@ -16,34 +16,39 @@
// along with Polkadot. If not, see <http://www.gnu.org/licenses/>.
use super::*;
use std::time::Duration;
use polkadot_overseer::HeadSupportsParachains;
use polkadot_primitives::v1::{
CoreIndex, GroupIndex, ValidatorSignature, Header, CandidateEvent,
};
use polkadot_node_subsystem::{ActivatedLeaf, ActiveLeavesUpdate, LeafStatus};
use polkadot_node_primitives::approval::{
AssignmentCert, AssignmentCertKind, VRFOutput, VRFProof,
RELAY_VRF_MODULO_CONTEXT, DelayTranche,
AssignmentCert, AssignmentCertKind, DelayTranche, VRFOutput, VRFProof, RELAY_VRF_MODULO_CONTEXT,
};
use polkadot_node_subsystem::{
messages::{AllMessages, ApprovalVotingMessage, AssignmentCheckResult},
ActivatedLeaf, ActiveLeavesUpdate, LeafStatus,
};
use polkadot_node_subsystem_test_helpers as test_helpers;
use polkadot_node_subsystem::messages::{AllMessages, ApprovalVotingMessage, AssignmentCheckResult};
use polkadot_node_subsystem_util::TimeoutExt;
use polkadot_overseer::HeadSupportsParachains;
use polkadot_primitives::v1::{CandidateEvent, CoreIndex, GroupIndex, Header, ValidatorSignature};
use std::time::Duration;
use assert_matches::assert_matches;
use parking_lot::Mutex;
use std::pin::Pin;
use std::sync::Arc;
use std::sync::atomic::{AtomicBool, Ordering};
use sp_keyring::sr25519::Keyring as Sr25519Keyring;
use sp_keystore::CryptoStore;
use assert_matches::assert_matches;
use super::import::tests::{
BabeEpoch, BabeEpochConfiguration, AllowedSlots, Digest, garbage_vrf, DigestItem, PreDigest,
SecondaryVRFPreDigest, CompatibleDigestItem,
use std::{
pin::Pin,
sync::{
atomic::{AtomicBool, Ordering},
Arc,
},
};
use super::{
approval_db::v1::StoredBlockRange,
backend::BackendWriteOp,
import::tests::{
garbage_vrf, AllowedSlots, BabeEpoch, BabeEpochConfiguration, CompatibleDigestItem, Digest,
DigestItem, PreDigest, SecondaryVRFPreDigest,
},
};
use super::approval_db::v1::StoredBlockRange;
use super::backend::BackendWriteOp;
const SLOT_DURATION_MILLIS: u64 = 5000;
@@ -92,14 +97,8 @@ fn make_sync_oracle(val: bool) -> (TestSyncOracle, TestSyncOracleHandle) {
let flag = Arc::new(AtomicBool::new(val));
(
TestSyncOracle {
flag: flag.clone(),
done_syncing_sender: Arc::new(Mutex::new(Some(tx))),
},
TestSyncOracleHandle {
flag,
done_syncing_receiver: rx,
}
TestSyncOracle { flag: flag.clone(), done_syncing_sender: Arc::new(Mutex::new(Some(tx))) },
TestSyncOracleHandle { flag, done_syncing_receiver: rx },
)
}
@@ -114,9 +113,7 @@ pub mod test_constants {
const DATA_COL: u32 = 0;
pub(crate) const NUM_COLUMNS: u32 = 1;
pub(crate) const TEST_CONFIG: DatabaseConfig = DatabaseConfig {
col_data: DATA_COL,
};
pub(crate) const TEST_CONFIG: DatabaseConfig = DatabaseConfig { col_data: DATA_COL };
}
struct MockSupportsParachains;
@@ -175,10 +172,8 @@ impl MockClockInner {
fn wakeup_all(&mut self, up_to: Tick) {
// This finds the position of the first wakeup after
// the given tick, or the end of the map.
let drain_up_to = self.wakeups.binary_search_by_key(
&(up_to + 1),
|w| w.0,
).unwrap_or_else(|i| i);
let drain_up_to =
self.wakeups.binary_search_by_key(&(up_to + 1), |w| w.0).unwrap_or_else(|i| i);
for (_, wakeup) in self.wakeups.drain(..drain_up_to) {
let _ = wakeup.send(());
@@ -201,10 +196,7 @@ impl MockClockInner {
fn register_wakeup(&mut self, tick: Tick, pre_emptive: bool) -> oneshot::Receiver<()> {
let (tx, rx) = oneshot::channel();
let pos = self.wakeups.binary_search_by_key(
&tick,
|w| w.0,
).unwrap_or_else(|i| i);
let pos = self.wakeups.binary_search_by_key(&tick, |w| w.0).unwrap_or_else(|i| i);
self.wakeups.insert(pos, (tick, tx));
@@ -223,14 +215,18 @@ struct MockAssignmentCriteria<Compute, Check>(Compute, Check);
impl<Compute, Check> AssignmentCriteria for MockAssignmentCriteria<Compute, Check>
where
Compute: Fn() -> HashMap<polkadot_primitives::v1::CoreIndex, criteria::OurAssignment>,
Check: Fn() -> Result<DelayTranche, criteria::InvalidAssignment>
Check: Fn() -> Result<DelayTranche, criteria::InvalidAssignment>,
{
fn compute_assignments(
&self,
_keystore: &LocalKeystore,
_relay_vrf_story: polkadot_node_primitives::approval::RelayVRFStory,
_config: &criteria::Config,
_leaving_cores: Vec<(CandidateHash, polkadot_primitives::v1::CoreIndex, polkadot_primitives::v1::GroupIndex)>,
_leaving_cores: Vec<(
CandidateHash,
polkadot_primitives::v1::CoreIndex,
polkadot_primitives::v1::GroupIndex,
)>,
) -> HashMap<polkadot_primitives::v1::CoreIndex, criteria::OurAssignment> {
self.0()
}
@@ -248,10 +244,12 @@ where
}
}
impl<F> MockAssignmentCriteria<
fn() -> HashMap<polkadot_primitives::v1::CoreIndex, criteria::OurAssignment>,
F,
> {
impl<F>
MockAssignmentCriteria<
fn() -> HashMap<polkadot_primitives::v1::CoreIndex, criteria::OurAssignment>,
F,
>
{
fn check_only(f: F) -> Self {
MockAssignmentCriteria(Default::default, f)
}
@@ -266,10 +264,7 @@ struct TestStore {
}
impl Backend for TestStore {
fn load_block_entry(
&self,
block_hash: &Hash,
) -> SubsystemResult<Option<BlockEntry>> {
fn load_block_entry(&self, block_hash: &Hash) -> SubsystemResult<Option<BlockEntry>> {
Ok(self.block_entries.get(block_hash).cloned())
}
@@ -280,10 +275,7 @@ impl Backend for TestStore {
Ok(self.candidate_entries.get(candidate_hash).cloned())
}
fn load_blocks_at_height(
&self,
height: &BlockNumber,
) -> SubsystemResult<Vec<Hash>> {
fn load_blocks_at_height(&self, height: &BlockNumber) -> SubsystemResult<Vec<Hash>> {
Ok(self.blocks_at_height.get(height).cloned().unwrap_or_default())
}
@@ -300,31 +292,33 @@ impl Backend for TestStore {
}
fn write<I>(&mut self, ops: I) -> SubsystemResult<()>
where I: IntoIterator<Item = BackendWriteOp>
where
I: IntoIterator<Item = BackendWriteOp>,
{
for op in ops {
match op {
BackendWriteOp::WriteStoredBlockRange(stored_block_range) => {
self.stored_block_range = Some(stored_block_range);
}
},
BackendWriteOp::WriteBlocksAtHeight(h, blocks) => {
self.blocks_at_height.insert(h, blocks);
}
},
BackendWriteOp::DeleteBlocksAtHeight(h) => {
let _ = self.blocks_at_height.remove(&h);
}
},
BackendWriteOp::WriteBlockEntry(block_entry) => {
self.block_entries.insert(block_entry.block_hash(), block_entry);
}
},
BackendWriteOp::DeleteBlockEntry(hash) => {
let _ = self.block_entries.remove(&hash);
}
},
BackendWriteOp::WriteCandidateEntry(candidate_entry) => {
self.candidate_entries.insert(candidate_entry.candidate_receipt().hash(), candidate_entry);
}
self.candidate_entries
.insert(candidate_entry.candidate_receipt().hash(), candidate_entry);
},
BackendWriteOp::DeleteCandidateEntry(candidate_hash) => {
let _ = self.candidate_entries.remove(&candidate_hash);
}
},
}
}
@@ -340,10 +334,7 @@ fn garbage_assignment_cert(kind: AssignmentCertKind) -> AssignmentCert {
let (inout, proof, _) = keypair.vrf_sign(ctx.bytes(msg));
let out = inout.to_output();
AssignmentCert {
kind,
vrf: (VRFOutput(out), VRFProof(proof)),
}
AssignmentCert { kind, vrf: (VRFOutput(out), VRFProof(proof)) }
}
fn sign_approval(
@@ -383,47 +374,40 @@ fn test_harness<T: Future<Output = VirtualOverseer>>(
let store = TestStore::default();
let HarnessConfig {
tick_start,
assigned_tranche,
} = config;
let HarnessConfig { tick_start, assigned_tranche } = config;
let clock = Box::new(MockClock::new(tick_start));
let subsystem = run(
context,
ApprovalVotingSubsystem::with_config(
Config{
col_data: test_constants::TEST_CONFIG.col_data,
slot_duration_millis: 100u64,
},
Config { col_data: test_constants::TEST_CONFIG.col_data, slot_duration_millis: 100u64 },
Arc::new(kvdb_memorydb::create(test_constants::NUM_COLUMNS)),
Arc::new(keystore),
sync_oracle,
Metrics::default(),
),
clock.clone(),
Box::new(MockAssignmentCriteria::check_only(move || { Ok(assigned_tranche) })),
Box::new(MockAssignmentCriteria::check_only(move || Ok(assigned_tranche))),
store,
);
let test_fut = test(TestHarness {
virtual_overseer,
clock,
});
let test_fut = test(TestHarness { virtual_overseer, clock });
futures::pin_mut!(test_fut);
futures::pin_mut!(subsystem);
futures::executor::block_on(future::join(async move {
let mut overseer = test_fut.await;
overseer_signal(&mut overseer, OverseerSignal::Conclude).await;
}, subsystem)).1.unwrap();
futures::executor::block_on(future::join(
async move {
let mut overseer = test_fut.await;
overseer_signal(&mut overseer, OverseerSignal::Conclude).await;
},
subsystem,
))
.1
.unwrap();
}
async fn overseer_send(
overseer: &mut VirtualOverseer,
msg: FromOverseer<ApprovalVotingMessage>,
) {
async fn overseer_send(overseer: &mut VirtualOverseer, msg: FromOverseer<ApprovalVotingMessage>) {
tracing::trace!("Sending message:\n{:?}", &msg);
overseer
.send(msg)
@@ -432,9 +416,7 @@ async fn overseer_send(
.expect(&format!("{:?} is enough for sending messages.", TIMEOUT));
}
async fn overseer_recv(
overseer: &mut VirtualOverseer,
) -> AllMessages {
async fn overseer_recv(overseer: &mut VirtualOverseer) -> AllMessages {
let msg = overseer_recv_with_timeout(overseer, TIMEOUT)
.await
.expect(&format!("{:?} is enough to receive messages.", TIMEOUT));
@@ -449,17 +431,11 @@ async fn overseer_recv_with_timeout(
timeout: Duration,
) -> Option<AllMessages> {
tracing::trace!("Waiting for message...");
overseer
.recv()
.timeout(timeout)
.await
overseer.recv().timeout(timeout).await
}
const TIMEOUT: Duration = Duration::from_millis(2000);
async fn overseer_signal(
overseer: &mut VirtualOverseer,
signal: OverseerSignal,
) {
async fn overseer_signal(overseer: &mut VirtualOverseer, signal: OverseerSignal) {
overseer
.send(FromOverseer::Signal(signal))
.timeout(TIMEOUT)
@@ -471,10 +447,7 @@ async fn overseer_signal(
fn blank_subsystem_act_on_bad_block() {
let (oracle, handle) = make_sync_oracle(false);
test_harness(Default::default(), Box::new(oracle), |test_harness| async move {
let TestHarness {
mut virtual_overseer,
..
} = test_harness;
let TestHarness { mut virtual_overseer, .. } = test_harness;
let (tx, rx) = oneshot::channel();
@@ -484,18 +457,19 @@ fn blank_subsystem_act_on_bad_block() {
&mut virtual_overseer,
FromOverseer::Communication {
msg: ApprovalVotingMessage::CheckAndImportAssignment(
IndirectAssignmentCert{
IndirectAssignmentCert {
block_hash: bad_block_hash.clone(),
validator: 0u32.into(),
cert: garbage_assignment_cert(
AssignmentCertKind::RelayVRFModulo { sample: 0 }
),
cert: garbage_assignment_cert(AssignmentCertKind::RelayVRFModulo {
sample: 0,
}),
},
0u32,
tx,
)
}
).await;
),
},
)
.await;
handle.await_mode_switch().await;
@@ -516,10 +490,7 @@ fn blank_subsystem_act_on_bad_block() {
fn ss_rejects_approval_if_no_block_entry() {
let (oracle, _handle) = make_sync_oracle(false);
test_harness(Default::default(), Box::new(oracle), |test_harness| async move {
let TestHarness {
mut virtual_overseer,
..
} = test_harness;
let TestHarness { mut virtual_overseer, .. } = test_harness;
let block_hash = Hash::repeat_byte(0x01);
let candidate_index = 0;
@@ -535,7 +506,8 @@ fn ss_rejects_approval_if_no_block_entry() {
candidate_hash,
session_index,
false,
).await;
)
.await;
assert_matches!(
rx.await,
@@ -552,10 +524,7 @@ fn ss_rejects_approval_if_no_block_entry() {
fn ss_rejects_approval_before_assignment() {
let (oracle, _handle) = make_sync_oracle(false);
test_harness(Default::default(), Box::new(oracle), |test_harness| async move {
let TestHarness {
mut virtual_overseer,
..
} = test_harness;
let TestHarness { mut virtual_overseer, .. } = test_harness;
let block_hash = Hash::repeat_byte(0x01);
@@ -584,7 +553,8 @@ fn ss_rejects_approval_before_assignment() {
candidate_hash,
session_index,
false,
).await;
)
.await;
assert_matches!(
rx.await,
@@ -600,59 +570,49 @@ fn ss_rejects_approval_before_assignment() {
#[test]
fn ss_rejects_assignment_in_future() {
let (oracle, _handle) = make_sync_oracle(false);
test_harness(HarnessConfig {
tick_start: 0,
assigned_tranche: TICK_TOO_FAR_IN_FUTURE as _,
..Default::default()
}, Box::new(oracle), |test_harness| async move {
let TestHarness {
mut virtual_overseer,
clock,
} = test_harness;
test_harness(
HarnessConfig {
tick_start: 0,
assigned_tranche: TICK_TOO_FAR_IN_FUTURE as _,
..Default::default()
},
Box::new(oracle),
|test_harness| async move {
let TestHarness { mut virtual_overseer, clock } = test_harness;
let block_hash = Hash::repeat_byte(0x01);
let candidate_index = 0;
let validator = ValidatorIndex(0);
let block_hash = Hash::repeat_byte(0x01);
let candidate_index = 0;
let validator = ValidatorIndex(0);
// Add block hash 00.
ChainBuilder::new()
.add_block(block_hash, ChainBuilder::GENESIS_HASH, Slot::from(1), 1)
.build(&mut virtual_overseer)
.await;
// Add block hash 00.
ChainBuilder::new()
.add_block(block_hash, ChainBuilder::GENESIS_HASH, Slot::from(1), 1)
.build(&mut virtual_overseer)
.await;
let rx = cai_assignment(
&mut virtual_overseer,
block_hash,
candidate_index,
validator,
).await;
let rx =
cai_assignment(&mut virtual_overseer, block_hash, candidate_index, validator).await;
assert_eq!(rx.await, Ok(AssignmentCheckResult::TooFarInFuture));
assert_eq!(rx.await, Ok(AssignmentCheckResult::TooFarInFuture));
// Advance clock to make assignment reasonably near.
clock.inner.lock().set_tick(1);
// Advance clock to make assignment reasonably near.
clock.inner.lock().set_tick(1);
let rx = cai_assignment(
&mut virtual_overseer,
block_hash,
candidate_index,
validator,
).await;
let rx =
cai_assignment(&mut virtual_overseer, block_hash, candidate_index, validator).await;
assert_eq!(rx.await, Ok(AssignmentCheckResult::Accepted));
assert_eq!(rx.await, Ok(AssignmentCheckResult::Accepted));
virtual_overseer
});
virtual_overseer
},
);
}
#[test]
fn ss_accepts_duplicate_assignment() {
let (oracle, _handle) = make_sync_oracle(false);
test_harness(Default::default(), Box::new(oracle), |test_harness| async move {
let TestHarness {
mut virtual_overseer,
..
} = test_harness;
let TestHarness { mut virtual_overseer, .. } = test_harness;
let block_hash = Hash::repeat_byte(0x01);
let candidate_index = 0;
@@ -664,21 +624,13 @@ fn ss_accepts_duplicate_assignment() {
.build(&mut virtual_overseer)
.await;
let rx = cai_assignment(
&mut virtual_overseer,
block_hash,
candidate_index,
validator,
).await;
let rx =
cai_assignment(&mut virtual_overseer, block_hash, candidate_index, validator).await;
assert_eq!(rx.await, Ok(AssignmentCheckResult::Accepted));
let rx = cai_assignment(
&mut virtual_overseer,
block_hash,
candidate_index,
validator,
).await;
let rx =
cai_assignment(&mut virtual_overseer, block_hash, candidate_index, validator).await;
assert_eq!(rx.await, Ok(AssignmentCheckResult::AcceptedDuplicate));
@@ -690,10 +642,7 @@ fn ss_accepts_duplicate_assignment() {
fn ss_rejects_assignment_with_unknown_candidate() {
let (oracle, _handle) = make_sync_oracle(false);
test_harness(Default::default(), Box::new(oracle), |test_harness| async move {
let TestHarness {
mut virtual_overseer,
..
} = test_harness;
let TestHarness { mut virtual_overseer, .. } = test_harness;
let block_hash = Hash::repeat_byte(0x01);
let candidate_index = 7;
@@ -705,16 +654,14 @@ fn ss_rejects_assignment_with_unknown_candidate() {
.build(&mut virtual_overseer)
.await;
let rx = cai_assignment(
&mut virtual_overseer,
block_hash,
candidate_index,
validator,
).await;
let rx =
cai_assignment(&mut virtual_overseer, block_hash, candidate_index, validator).await;
assert_eq!(
rx.await,
Ok(AssignmentCheckResult::Bad(AssignmentCheckError::InvalidCandidateIndex(candidate_index))),
Ok(AssignmentCheckResult::Bad(AssignmentCheckError::InvalidCandidateIndex(
candidate_index
))),
);
virtual_overseer
@@ -725,10 +672,7 @@ fn ss_rejects_assignment_with_unknown_candidate() {
fn ss_accepts_and_imports_approval_after_assignment() {
let (oracle, _handle) = make_sync_oracle(false);
test_harness(Default::default(), Box::new(oracle), |test_harness| async move {
let TestHarness {
mut virtual_overseer,
..
} = test_harness;
let TestHarness { mut virtual_overseer, .. } = test_harness;
let block_hash = Hash::repeat_byte(0x01);
@@ -749,12 +693,8 @@ fn ss_accepts_and_imports_approval_after_assignment() {
.build(&mut virtual_overseer)
.await;
let rx = cai_assignment(
&mut virtual_overseer,
block_hash,
candidate_index,
validator,
).await;
let rx =
cai_assignment(&mut virtual_overseer, block_hash, candidate_index, validator).await;
assert_eq!(rx.await, Ok(AssignmentCheckResult::Accepted));
@@ -766,7 +706,8 @@ fn ss_accepts_and_imports_approval_after_assignment() {
candidate_hash,
session_index,
true,
).await;
)
.await;
assert_eq!(rx.await, Ok(ApprovalCheckResult::Accepted));
@@ -777,10 +718,7 @@ fn ss_accepts_and_imports_approval_after_assignment() {
fn ss_assignment_import_updates_candidate_entry_and_schedules_wakeup() {
let (oracle, _handle) = make_sync_oracle(false);
test_harness(Default::default(), Box::new(oracle), |test_harness| async move {
let TestHarness {
mut virtual_overseer,
..
} = test_harness;
let TestHarness { mut virtual_overseer, .. } = test_harness;
let block_hash = Hash::repeat_byte(0x01);
@@ -800,12 +738,8 @@ fn ss_assignment_import_updates_candidate_entry_and_schedules_wakeup() {
.build(&mut virtual_overseer)
.await;
let rx = cai_assignment(
&mut virtual_overseer,
block_hash,
candidate_index,
validator,
).await;
let rx =
cai_assignment(&mut virtual_overseer, block_hash, candidate_index, validator).await;
assert_eq!(rx.await, Ok(AssignmentCheckResult::Accepted));
@@ -831,16 +765,12 @@ async fn cai_approval(
overseer,
FromOverseer::Communication {
msg: ApprovalVotingMessage::CheckAndImportApproval(
IndirectSignedApprovalVote {
block_hash,
candidate_index,
validator,
signature,
},
IndirectSignedApprovalVote { block_hash, candidate_index, validator, signature },
tx,
),
}
).await;
},
)
.await;
if expect_coordinator {
assert_matches!(
@@ -870,17 +800,14 @@ async fn cai_assignment(
IndirectAssignmentCert {
block_hash,
validator,
cert: garbage_assignment_cert(
AssignmentCertKind::RelayVRFModulo {
sample: 0,
},
),
cert: garbage_assignment_cert(AssignmentCertKind::RelayVRFModulo { sample: 0 }),
},
candidate_index,
tx,
),
}
).await;
},
)
.await;
rx
}
@@ -889,16 +816,13 @@ struct ChainBuilder {
blocks_at_height: BTreeMap<u32, Vec<Hash>>,
}
impl ChainBuilder {
const GENESIS_HASH: Hash = Hash::repeat_byte(0xff);
const GENESIS_PARENT_HASH: Hash = Hash::repeat_byte(0x00);
pub fn new() -> Self {
let mut builder = Self {
blocks_by_hash: HashMap::new(),
blocks_at_height: BTreeMap::new(),
};
let mut builder =
Self { blocks_by_hash: HashMap::new(), blocks_at_height: BTreeMap::new() };
builder.add_block_inner(Self::GENESIS_HASH, Self::GENESIS_PARENT_HASH, Slot::from(0), 0);
builder
}
@@ -912,7 +836,10 @@ impl ChainBuilder {
) -> &'a mut Self {
assert!(number != 0, "cannot add duplicate genesis block");
assert!(hash != Self::GENESIS_HASH, "cannot add block with genesis hash");
assert!(parent_hash != Self::GENESIS_PARENT_HASH, "cannot add block with genesis parent hash");
assert!(
parent_hash != Self::GENESIS_PARENT_HASH,
"cannot add block with genesis parent hash"
);
assert!(self.blocks_by_hash.len() < u8::MAX.into());
self.add_block_inner(hash, parent_hash, slot, number)
}
@@ -927,7 +854,8 @@ impl ChainBuilder {
let header = ChainBuilder::make_header(parent_hash, slot, number);
assert!(
self.blocks_by_hash.insert(hash, header).is_none(),
"block with hash {:?} already exists", hash,
"block with hash {:?} already exists",
hash,
);
self.blocks_at_height.entry(number).or_insert_with(Vec::new).push(hash);
self
@@ -939,7 +867,8 @@ impl ChainBuilder {
let mut cur_hash = *hash;
let mut ancestry = Vec::new();
while cur_hash != Self::GENESIS_PARENT_HASH {
let cur_header = self.blocks_by_hash.get(&cur_hash).expect("chain is not contiguous");
let cur_header =
self.blocks_by_hash.get(&cur_hash).expect("chain is not contiguous");
ancestry.push((cur_hash, cur_header.clone()));
cur_hash = cur_header.parent_hash;
}
@@ -950,21 +879,12 @@ impl ChainBuilder {
}
}
fn make_header(
parent_hash: Hash,
slot: Slot,
number: u32,
) -> Header {
fn make_header(parent_hash: Hash, slot: Slot, number: u32) -> Header {
let digest = {
let mut digest = Digest::default();
let (vrf_output, vrf_proof) = garbage_vrf();
digest.push(DigestItem::babe_pre_digest(PreDigest::SecondaryVRF(
SecondaryVRFPreDigest {
authority_index: 0,
slot,
vrf_output,
vrf_proof,
}
SecondaryVRFPreDigest { authority_index: 0, slot, vrf_output, vrf_proof },
)));
digest
};
@@ -976,8 +896,8 @@ impl ChainBuilder {
state_root: Default::default(),
parent_hash,
}
}
}
}
}
async fn import_block(
overseer: &mut VirtualOverseer,
@@ -1003,13 +923,16 @@ async fn import_block(
let (new_head, new_header) = &hashes[hashes.len() - 1];
overseer_send(
overseer,
FromOverseer::Signal(OverseerSignal::ActiveLeaves(ActiveLeavesUpdate::start_work(ActivatedLeaf {
hash: *new_head,
number: session,
status: LeafStatus::Fresh,
span: Arc::new(jaeger::Span::Disabled),
})),
)).await;
FromOverseer::Signal(OverseerSignal::ActiveLeaves(ActiveLeavesUpdate::start_work(
ActivatedLeaf {
hash: *new_head,
number: session,
status: LeafStatus::Fresh,
span: Arc::new(jaeger::Span::Disabled),
},
))),
)
.await;
assert_matches!(
overseer_recv(overseer).await,
@@ -1070,22 +993,21 @@ async fn import_block(
let (hash, header) = hashes[i as usize].clone();
assert_eq!(hash, *new_head);
h_tx.send(Ok(Some(header))).unwrap();
}
},
AllMessages::ChainApi(ChainApiMessage::Ancestors {
hash,
k,
response_channel,
}) => {
assert_eq!(hash, *new_head);
assert_eq!(k as u32, session-1);
assert_eq!(k as u32, session - 1);
let history: Vec<Hash> = hashes.iter().map(|v| v.0).take(k).collect();
response_channel.send(Ok(history)).unwrap();
}
_ => unreachable!{},
},
_ => unreachable! {},
}
}
}
}
if session > 0 {
@@ -1184,10 +1106,7 @@ fn linear_import_act_on_leaf() {
let (oracle, _handle) = make_sync_oracle(false);
test_harness(Default::default(), Box::new(oracle), |test_harness| async move {
let TestHarness {
mut virtual_overseer,
..
} = test_harness;
let TestHarness { mut virtual_overseer, .. } = test_harness;
let mut head: Hash = ChainBuilder::GENESIS_HASH;
let mut builder = ChainBuilder::new();
@@ -1197,7 +1116,7 @@ fn linear_import_act_on_leaf() {
let hash = Hash::repeat_byte(i as u8);
builder.add_block(hash, head, slot, i);
head = hash;
}
}
builder.build(&mut virtual_overseer).await;
@@ -1207,18 +1126,19 @@ fn linear_import_act_on_leaf() {
&mut virtual_overseer,
FromOverseer::Communication {
msg: ApprovalVotingMessage::CheckAndImportAssignment(
IndirectAssignmentCert{
IndirectAssignmentCert {
block_hash: head,
validator: 0u32.into(),
cert: garbage_assignment_cert(
AssignmentCertKind::RelayVRFModulo { sample: 0 }
),
cert: garbage_assignment_cert(AssignmentCertKind::RelayVRFModulo {
sample: 0,
}),
},
0u32,
tx,
)
}
).await;
),
},
)
.await;
assert_eq!(rx.await, Ok(AssignmentCheckResult::Accepted));
@@ -1232,10 +1152,7 @@ fn forkful_import_at_same_height_act_on_leaf() {
let (oracle, _handle) = make_sync_oracle(false);
test_harness(Default::default(), Box::new(oracle), |test_harness| async move {
let TestHarness {
mut virtual_overseer,
..
} = test_harness;
let TestHarness { mut virtual_overseer, .. } = test_harness;
let mut head: Hash = ChainBuilder::GENESIS_HASH;
let mut builder = ChainBuilder::new();
@@ -1252,7 +1169,7 @@ fn forkful_import_at_same_height_act_on_leaf() {
let slot = Slot::from(session as u64);
let hash = Hash::repeat_byte(session as u8 + i);
builder.add_block(hash, head, slot, session);
}
}
builder.build(&mut virtual_overseer).await;
for head in forks.into_iter() {
@@ -1262,18 +1179,19 @@ fn forkful_import_at_same_height_act_on_leaf() {
&mut virtual_overseer,
FromOverseer::Communication {
msg: ApprovalVotingMessage::CheckAndImportAssignment(
IndirectAssignmentCert{
IndirectAssignmentCert {
block_hash: head,
validator: 0u32.into(),
cert: garbage_assignment_cert(
AssignmentCertKind::RelayVRFModulo { sample: 0 }
),
cert: garbage_assignment_cert(AssignmentCertKind::RelayVRFModulo {
sample: 0,
}),
},
0u32,
tx,
)
}
).await;
),
},
)
.await;
assert_eq!(rx.await, Ok(AssignmentCheckResult::Accepted));
}
polkadot/node/core/approval-voting/src/time.rs
+5 -3
View File
@@ -16,11 +16,13 @@
//! Time utilities for approval voting.
use futures::prelude::*;
use polkadot_node_primitives::approval::DelayTranche;
use sp_consensus_slots::Slot;
use futures::prelude::*;
use std::time::{Duration, SystemTime};
use std::pin::Pin;
use std::{
pin::Pin,
time::{Duration, SystemTime},
};
const TICK_DURATION_MILLIS: u64 = 500;
polkadot/node/core/av-store/src/lib.rs
+162 -224
View File
@@ -16,41 +16,36 @@
//! Implements a `AvailabilityStoreSubsystem`.
#![recursion_limit="256"]
#![recursion_limit = "256"]
#![warn(missing_docs)]
use std::collections::{HashMap, HashSet, BTreeSet};
use std::io;
use std::sync::Arc;
use std::time::{Duration, SystemTime, SystemTimeError, UNIX_EPOCH};
use std::{
collections::{BTreeSet, HashMap, HashSet},
io,
sync::Arc,
time::{Duration, SystemTime, SystemTimeError, UNIX_EPOCH},
};
use parity_scale_codec::{Encode, Decode, Input, Error as CodecError};
use futures::{select, channel::oneshot, future, FutureExt};
use futures::{channel::oneshot, future, select, FutureExt};
use futures_timer::Delay;
use kvdb::{KeyValueDB, DBTransaction};
use kvdb::{DBTransaction, KeyValueDB};
use parity_scale_codec::{Decode, Encode, Error as CodecError, Input};
use polkadot_primitives::v1::{
Hash, BlockNumber, CandidateEvent, ValidatorIndex, CandidateHash,
CandidateReceipt, Header,
};
use polkadot_node_primitives::{
ErasureChunk, AvailableData,
};
use polkadot_subsystem::{
FromOverseer, OverseerSignal, SubsystemError,
SubsystemContext, SpawnedSubsystem,
overseer,
ActiveLeavesUpdate,
errors::{ChainApiError, RuntimeApiError},
};
use bitvec::{order::Lsb0 as BitOrderLsb0, vec::BitVec};
use polkadot_node_primitives::{AvailableData, ErasureChunk};
use polkadot_node_subsystem_util::{
self as util,
metrics::{self, prometheus},
};
use polkadot_subsystem::messages::{
AvailabilityStoreMessage, ChainApiMessage,
use polkadot_primitives::v1::{
BlockNumber, CandidateEvent, CandidateHash, CandidateReceipt, Hash, Header, ValidatorIndex,
};
use polkadot_subsystem::{
errors::{ChainApiError, RuntimeApiError},
messages::{AvailabilityStoreMessage, ChainApiMessage},
overseer, ActiveLeavesUpdate, FromOverseer, OverseerSignal, SpawnedSubsystem, SubsystemContext,
SubsystemError,
};
use bitvec::{vec::BitVec, order::Lsb0 as BitOrderLsb0};
#[cfg(test)]
mod tests;
@@ -126,7 +121,9 @@ impl Encode for BEBlockNumber {
impl Decode for BEBlockNumber {
fn decode<I: Input>(value: &mut I) -> Result<Self, CodecError> {
<[u8; std::mem::size_of::<BlockNumber>()]>::decode(value).map(BlockNumber::from_be_bytes).map(Self)
<[u8; std::mem::size_of::<BlockNumber>()]>::decode(value)
.map(BlockNumber::from_be_bytes)
.map(Self)
}
}
@@ -143,7 +140,7 @@ enum State {
Unfinalized(BETimestamp, Vec<(BEBlockNumber, Hash)>),
/// Candidate data has appeared in a finalized block and did so at the given time.
#[codec(index = 2)]
Finalized(BETimestamp)
Finalized(BETimestamp),
}
// Meta information about a candidate.
@@ -163,12 +160,12 @@ fn query_inner<D: Decode>(
Ok(Some(raw)) => {
let res = D::decode(&mut &raw[..])?;
Ok(Some(res))
}
},
Ok(None) => Ok(None),
Err(e) => {
tracing::warn!(target: LOG_TARGET, err = ?e, "Error reading from the availability store");
Err(e.into())
}
},
}
}
@@ -193,11 +190,7 @@ fn load_available_data(
query_inner(db, config.col_data, &key)
}
fn delete_available_data(
tx: &mut DBTransaction,
config: &Config,
hash: &CandidateHash,
) {
fn delete_available_data(tx: &mut DBTransaction, config: &Config, hash: &CandidateHash) {
let key = (AVAILABLE_PREFIX, hash).encode();
tx.delete(config.col_data, &key[..])
@@ -247,12 +240,7 @@ fn load_meta(
query_inner(db, config.col_meta, &key)
}
fn write_meta(
tx: &mut DBTransaction,
config: &Config,
hash: &CandidateHash,
meta: &CandidateMeta,
) {
fn write_meta(tx: &mut DBTransaction, config: &Config, hash: &CandidateHash, meta: &CandidateMeta) {
let key = (META_PREFIX, hash).encode();
tx.put_vec(config.col_meta, &key, meta.encode());
@@ -263,11 +251,7 @@ fn delete_meta(tx: &mut DBTransaction, config: &Config, hash: &CandidateHash) {
tx.delete(config.col_meta, &key[..])
}
fn delete_unfinalized_height(
tx: &mut DBTransaction,
config: &Config,
block_number: BlockNumber,
) {
fn delete_unfinalized_height(tx: &mut DBTransaction, config: &Config, block_number: BlockNumber) {
let prefix = (UNFINALIZED_PREFIX, BEBlockNumber(block_number)).encode();
tx.delete_prefix(config.col_meta, &prefix);
}
@@ -279,12 +263,8 @@ fn delete_unfinalized_inclusion(
block_hash: &Hash,
candidate_hash: &CandidateHash,
) {
let key = (
UNFINALIZED_PREFIX,
BEBlockNumber(block_number),
block_hash,
candidate_hash,
).encode();
let key =
(UNFINALIZED_PREFIX, BEBlockNumber(block_number), block_hash, candidate_hash).encode();
tx.delete(config.col_meta, &key[..]);
}
@@ -338,7 +318,7 @@ fn pruning_range(now: impl Into<BETimestamp>) -> (Vec<u8>, Vec<u8>) {
fn decode_unfinalized_key(s: &[u8]) -> Result<(BlockNumber, Hash, CandidateHash), CodecError> {
if !s.starts_with(UNFINALIZED_PREFIX) {
return Err("missing magic string".into());
return Err("missing magic string".into())
}
<(BEBlockNumber, Hash, CandidateHash)>::decode(&mut &s[UNFINALIZED_PREFIX.len()..])
@@ -347,7 +327,7 @@ fn decode_unfinalized_key(s: &[u8]) -> Result<(BlockNumber, Hash, CandidateHash)
fn decode_pruning_key(s: &[u8]) -> Result<(Duration, CandidateHash), CodecError> {
if !s.starts_with(PRUNE_BY_TIME_PREFIX) {
return Err("missing magic string".into());
return Err("missing magic string".into())
}
<(BETimestamp, CandidateHash)>::decode(&mut &s[PRUNE_BY_TIME_PREFIX.len()..])
@@ -389,8 +369,8 @@ impl Error {
fn trace(&self) {
match self {
// don't spam the log with spurious errors
Self::RuntimeApi(_) |
Self::Oneshot(_) => tracing::debug!(target: LOG_TARGET, err = ?self),
Self::RuntimeApi(_) | Self::Oneshot(_) =>
tracing::debug!(target: LOG_TARGET, err = ?self),
// it's worth reporting otherwise
_ => tracing::warn!(target: LOG_TARGET, err = ?self),
}
@@ -457,11 +437,7 @@ pub struct AvailabilityStoreSubsystem {
impl AvailabilityStoreSubsystem {
/// Create a new `AvailabilityStoreSubsystem` with a given config on disk.
pub fn new(
db: Arc<dyn KeyValueDB>,
config: Config,
metrics: Metrics,
) -> Self {
pub fn new(db: Arc<dyn KeyValueDB>, config: Config, metrics: Metrics) -> Self {
Self::with_pruning_config_and_clock(
db,
config,
@@ -530,14 +506,9 @@ where
Context: overseer::SubsystemContext<Message = AvailabilityStoreMessage>,
{
fn start(self, ctx: Context) -> SpawnedSubsystem {
let future = run(self, ctx)
.map(|_| Ok(()))
.boxed();
let future = run(self, ctx).map(|_| Ok(())).boxed();
SpawnedSubsystem {
name: "availability-store-subsystem",
future,
}
SpawnedSubsystem { name: "availability-store-subsystem", future }
}
}
@@ -555,12 +526,12 @@ where
e.trace();
if let Error::Subsystem(SubsystemError::Context(_)) = e {
break;
break
}
}
},
Ok(true) => {
tracing::info!(target: LOG_TARGET, "received `Conclude` signal, exiting");
break;
break
},
Ok(false) => continue,
}
@@ -571,8 +542,7 @@ async fn run_iteration<Context>(
ctx: &mut Context,
subsystem: &mut AvailabilityStoreSubsystem,
mut next_pruning: &mut future::Fuse<Delay>,
)
-> Result<bool, Error>
) -> Result<bool, Error>
where
Context: SubsystemContext<Message = AvailabilityStoreMessage>,
Context: overseer::SubsystemContext<Message = AvailabilityStoreMessage>,
@@ -634,9 +604,7 @@ where
let block_header = {
let (tx, rx) = oneshot::channel();
ctx.send_message(
ChainApiMessage::BlockHeader(activated, tx)
).await;
ctx.send_message(ChainApiMessage::BlockHeader(activated, tx)).await;
match rx.await?? {
None => return Ok(()),
@@ -647,13 +615,12 @@ where
let new_blocks = util::determine_new_blocks(
ctx.sender(),
|hash| -> Result<bool, Error> {
Ok(subsystem.known_blocks.is_known(hash))
},
|hash| -> Result<bool, Error> { Ok(subsystem.known_blocks.is_known(hash)) },
activated,
&block_header,
subsystem.finalized_number.unwrap_or(block_number.saturating_sub(1)),
).await?;
)
.await?;
// determine_new_blocks is descending in block height
for (hash, header) in new_blocks.into_iter().rev() {
@@ -669,7 +636,8 @@ where
now,
hash,
header,
).await?;
)
.await?;
subsystem.known_blocks.insert(hash, block_number);
subsystem.db.write(tx)?;
}
@@ -691,18 +659,12 @@ where
Context: SubsystemContext<Message = AvailabilityStoreMessage>,
Context: overseer::SubsystemContext<Message = AvailabilityStoreMessage>,
{
let candidate_events = util::request_candidate_events(
hash,
ctx.sender(),
).await.await??;
let candidate_events = util::request_candidate_events(hash, ctx.sender()).await.await??;
// We need to request the number of validators based on the parent state,
// as that is the number of validators used to create this block.
let n_validators = util::request_validators(
header.parent_hash,
ctx.sender(),
).await.await??.len();
let n_validators =
util::request_validators(header.parent_hash, ctx.sender()).await.await??.len();
for event in candidate_events {
match event {
@@ -716,7 +678,7 @@ where
n_validators,
receipt,
)?;
}
},
CandidateEvent::CandidateIncluded(receipt, _head, _core_index, _group_index) => {
note_block_included(
db,
@@ -726,8 +688,8 @@ where
(header.number, hash),
receipt,
)?;
}
_ => {}
},
_ => {},
}
}
@@ -745,11 +707,7 @@ fn note_block_backed(
) -> Result<(), Error> {
let candidate_hash = candidate.hash();
tracing::debug!(
target: LOG_TARGET,
?candidate_hash,
"Candidate backed",
);
tracing::debug!(target: LOG_TARGET, ?candidate_hash, "Candidate backed",);
if load_meta(db, config, &candidate_hash)?.is_none() {
let meta = CandidateMeta {
@@ -771,7 +729,7 @@ fn note_block_included(
db: &Arc<dyn KeyValueDB>,
db_transaction: &mut DBTransaction,
config: &Config,
pruning_config:&PruningConfig,
pruning_config: &PruningConfig,
block: (BlockNumber, Hash),
candidate: CandidateReceipt,
) -> Result<(), Error> {
@@ -786,15 +744,11 @@ fn note_block_included(
?candidate_hash,
"Candidate included without being backed?",
);
}
},
Some(mut meta) => {
let be_block = (BEBlockNumber(block.0), block.1);
tracing::debug!(
target: LOG_TARGET,
?candidate_hash,
"Candidate included",
);
tracing::debug!(target: LOG_TARGET, ?candidate_hash, "Candidate included",);
meta.state = match meta.state {
State::Unavailable(at) => {
@@ -803,20 +757,20 @@ fn note_block_included(
delete_pruning_key(db_transaction, config, prune_at, &candidate_hash);
State::Unfinalized(at, vec![be_block])
}
},
State::Unfinalized(at, mut within) => {
if let Err(i) = within.binary_search(&be_block) {
within.insert(i, be_block);
State::Unfinalized(at, within)
} else {
return Ok(());
return Ok(())
}
}
},
State::Finalized(_at) => {
// This should never happen as a candidate would have to be included after
// finality.
return Ok(())
}
},
};
write_unfinalized_block_contains(
@@ -827,7 +781,7 @@ fn note_block_included(
&candidate_hash,
);
write_meta(db_transaction, config, &candidate_hash, &meta);
}
},
}
Ok(())
@@ -837,9 +791,9 @@ macro_rules! peek_num {
($iter:ident) => {
match $iter.peek() {
Some((k, _)) => decode_unfinalized_key(&k[..]).ok().map(|(b, _, _)| b),
None => None
None => None,
}
}
};
}
async fn process_block_finalized<Context>(
@@ -863,7 +817,9 @@ where
// as it is not `Send`. That is why we create the iterator once within this loop, drop it,
// do an asynchronous request, and then instantiate the exact same iterator again.
let batch_num = {
let mut iter = subsystem.db.iter_with_prefix(subsystem.config.col_meta, &start_prefix)
let mut iter = subsystem
.db
.iter_with_prefix(subsystem.config.col_meta, &start_prefix)
.take_while(|(k, _)| &k[..] < &end_prefix[..])
.peekable();
@@ -873,7 +829,9 @@ where
}
};
if batch_num < next_possible_batch { continue } // sanity.
if batch_num < next_possible_batch {
continue
} // sanity.
next_possible_batch = batch_num + 1;
let batch_finalized_hash = if batch_num == finalized_number {
@@ -884,19 +842,22 @@ where
match rx.await?? {
None => {
tracing::warn!(target: LOG_TARGET,
tracing::warn!(
target: LOG_TARGET,
"Availability store was informed that block #{} is finalized, \
but chain API has no finalized hash.",
batch_num,
);
break
}
},
Some(h) => h,
}
};
let iter = subsystem.db.iter_with_prefix(subsystem.config.col_meta, &start_prefix)
let iter = subsystem
.db
.iter_with_prefix(subsystem.config.col_meta, &start_prefix)
.take_while(|(k, _)| &k[..] < &end_prefix[..])
.peekable();
@@ -907,13 +868,7 @@ where
delete_unfinalized_height(&mut db_transaction, &subsystem.config, batch_num);
update_blocks_at_finalized_height(
&subsystem,
&mut db_transaction,
batch,
batch_num,
now,
)?;
update_blocks_at_finalized_height(&subsystem, &mut db_transaction, batch, batch_num, now)?;
// We need to write at the end of the loop so the prefix iterator doesn't pick up the same values again
// in the next iteration. Another unfortunate effect of having to re-initialize the iterator.
@@ -936,14 +891,14 @@ fn load_all_at_finalized_height(
// Load all candidates that were included at this height.
loop {
match peek_num!(iter) {
None => break, // end of iterator.
None => break, // end of iterator.
Some(n) if n != block_number => break, // end of batch.
_ => {}
_ => {},
}
let (k, _v) = iter.next().expect("`peek` used to check non-empty; qed");
let (_, block_hash, candidate_hash) = decode_unfinalized_key(&k[..])
.expect("`peek_num` checks validity of key; qed");
let (_, block_hash, candidate_hash) =
decode_unfinalized_key(&k[..]).expect("`peek_num` checks validity of key; qed");
if block_hash == finalized_hash {
candidates.insert(candidate_hash, true);
@@ -971,8 +926,8 @@ fn update_blocks_at_finalized_height(
candidate_hash,
);
continue;
}
continue
},
Some(c) => c,
};
@@ -985,7 +940,7 @@ fn update_blocks_at_finalized_height(
// iterating over the candidate here indicates that `State` should
// be `Unfinalized`.
delete_pruning_key(db_transaction, &subsystem.config, at, &candidate_hash);
}
},
State::Unfinalized(_, blocks) => {
for (block_num, block_hash) in blocks.iter().cloned() {
// this exact height is all getting cleared out anyway.
@@ -999,7 +954,7 @@ fn update_blocks_at_finalized_height(
);
}
}
}
},
}
meta.state = State::Finalized(now.into());
@@ -1014,7 +969,7 @@ fn update_blocks_at_finalized_height(
);
} else {
meta.state = match meta.state {
State::Finalized(_) => continue, // sanity.
State::Finalized(_) => continue, // sanity.
State::Unavailable(_) => continue, // sanity.
State::Unfinalized(at, mut blocks) => {
// Clear out everything at this height.
@@ -1035,7 +990,7 @@ fn update_blocks_at_finalized_height(
} else {
State::Unfinalized(at, blocks)
}
}
},
};
// Update the meta entry.
@@ -1053,20 +1008,22 @@ fn process_message(
match msg {
AvailabilityStoreMessage::QueryAvailableData(candidate, tx) => {
let _ = tx.send(load_available_data(&subsystem.db, &subsystem.config, &candidate)?);
}
},
AvailabilityStoreMessage::QueryDataAvailability(candidate, tx) => {
let a = load_meta(&subsystem.db, &subsystem.config, &candidate)?.map_or(false, |m| m.data_available);
let a = load_meta(&subsystem.db, &subsystem.config, &candidate)?
.map_or(false, |m| m.data_available);
let _ = tx.send(a);
}
},
AvailabilityStoreMessage::QueryChunk(candidate, validator_index, tx) => {
let _timer = subsystem.metrics.time_get_chunk();
let _ = tx.send(load_chunk(&subsystem.db, &subsystem.config, &candidate, validator_index)?);
}
let _ =
tx.send(load_chunk(&subsystem.db, &subsystem.config, &candidate, validator_index)?);
},
AvailabilityStoreMessage::QueryAllChunks(candidate, tx) => {
match load_meta(&subsystem.db, &subsystem.config, &candidate)? {
None => {
let _ = tx.send(Vec::new());
}
},
Some(meta) => {
let mut chunks = Vec::new();
@@ -1086,64 +1043,61 @@ fn process_message(
index,
"No chunk found for set bit in meta"
);
}
},
}
}
let _ = tx.send(chunks);
}
},
}
}
},
AvailabilityStoreMessage::QueryChunkAvailability(candidate, validator_index, tx) => {
let a = load_meta(&subsystem.db, &subsystem.config, &candidate)?
.map_or(false, |m|
*m.chunks_stored.get(validator_index.0 as usize).as_deref().unwrap_or(&false)
);
let a = load_meta(&subsystem.db, &subsystem.config, &candidate)?.map_or(false, |m| {
*m.chunks_stored.get(validator_index.0 as usize).as_deref().unwrap_or(&false)
});
let _ = tx.send(a);
}
AvailabilityStoreMessage::StoreChunk {
candidate_hash,
chunk,
tx,
} => {
},
AvailabilityStoreMessage::StoreChunk { candidate_hash, chunk, tx } => {
subsystem.metrics.on_chunks_received(1);
let _timer = subsystem.metrics.time_store_chunk();
match store_chunk(&subsystem.db, &subsystem.config, candidate_hash, chunk) {
Ok(true) => {
let _ = tx.send(Ok(()));
}
},
Ok(false) => {
let _ = tx.send(Err(()));
}
},
Err(e) => {
let _ = tx.send(Err(()));
return Err(e)
}
},
}
}
AvailabilityStoreMessage::StoreAvailableData(candidate, _our_index, n_validators, available_data, tx) => {
},
AvailabilityStoreMessage::StoreAvailableData(
candidate,
_our_index,
n_validators,
available_data,
tx,
) => {
subsystem.metrics.on_chunks_received(n_validators as _);
let _timer = subsystem.metrics.time_store_available_data();
let res = store_available_data(
&subsystem,
candidate,
n_validators as _,
available_data,
);
let res =
store_available_data(&subsystem, candidate, n_validators as _, available_data);
match res {
Ok(()) => {
let _ = tx.send(Ok(()));
}
},
Err(e) => {
let _ = tx.send(Err(()));
return Err(e)
}
},
}
}
},
}
Ok(())
@@ -1170,7 +1124,7 @@ fn store_chunk(
write_chunk(&mut tx, config, &candidate_hash, chunk.index, &chunk);
write_meta(&mut tx, config, &candidate_hash, &meta);
}
},
None => return Ok(false), // out of bounds.
}
@@ -1197,7 +1151,7 @@ fn store_available_data(
let mut meta = match load_meta(&subsystem.db, &subsystem.config, &candidate_hash)? {
Some(m) => {
if m.data_available {
return Ok(()); // already stored.
return Ok(()) // already stored.
}
m
@@ -1214,20 +1168,19 @@ fn store_available_data(
data_available: false,
chunks_stored: BitVec::new(),
}
}
},
};
let chunks = erasure::obtain_chunks_v1(n_validators, &available_data)?;
let branches = erasure::branches(chunks.as_ref());
let erasure_chunks = chunks.iter()
.zip(branches.map(|(proof, _)| proof))
.enumerate()
.map(|(index, (chunk, proof))| ErasureChunk {
let erasure_chunks = chunks.iter().zip(branches.map(|(proof, _)| proof)).enumerate().map(
|(index, (chunk, proof))| ErasureChunk {
chunk: chunk.clone(),
proof,
index: ValidatorIndex(index as u32),
});
},
);
for chunk in erasure_chunks {
write_chunk(&mut tx, &subsystem.config, &candidate_hash, chunk.index, &chunk);
@@ -1236,16 +1189,12 @@ fn store_available_data(
meta.data_available = true;
meta.chunks_stored = bitvec::bitvec![BitOrderLsb0, u8; 1; n_validators];
write_meta(&mut tx, &subsystem.config, &candidate_hash, &meta);
write_meta(&mut tx, &subsystem.config, &candidate_hash, &meta);
write_available_data(&mut tx, &subsystem.config, &candidate_hash, &available_data);
subsystem.db.write(tx)?;
tracing::debug!(
target: LOG_TARGET,
?candidate_hash,
"Stored data and chunks",
);
tracing::debug!(target: LOG_TARGET, ?candidate_hash, "Stored data and chunks",);
Ok(())
}
@@ -1255,7 +1204,8 @@ fn prune_all(db: &Arc<dyn KeyValueDB>, config: &Config, clock: &dyn Clock) -> Re
let (range_start, range_end) = pruning_range(now);
let mut tx = DBTransaction::new();
let iter = db.iter_with_prefix(config.col_meta, &range_start[..])
let iter = db
.iter_with_prefix(config.col_meta, &range_start[..])
.take_while(|(k, _)| &k[..] < &range_end[..]);
for (k, _v) in iter {
@@ -1334,7 +1284,9 @@ impl Metrics {
}
/// Provide a timer for `process_block_finalized` which observes on drop.
fn time_process_block_finalized(&self) -> Option<metrics::prometheus::prometheus::HistogramTimer> {
fn time_process_block_finalized(
&self,
) -> Option<metrics::prometheus::prometheus::HistogramTimer> {
self.0.as_ref().map(|metrics| metrics.process_block_finalized.start_timer())
}
@@ -1375,66 +1327,52 @@ impl metrics::Metrics for Metrics {
registry,
)?,
pruning: prometheus::register(
prometheus::Histogram::with_opts(
prometheus::HistogramOpts::new(
"parachain_av_store_pruning",
"Time spent within `av_store::prune_all`",
)
)?,
prometheus::Histogram::with_opts(prometheus::HistogramOpts::new(
"parachain_av_store_pruning",
"Time spent within `av_store::prune_all`",
))?,
registry,
)?,
process_block_finalized: prometheus::register(
prometheus::Histogram::with_opts(
prometheus::HistogramOpts::new(
"parachain_av_store_process_block_finalized",
"Time spent within `av_store::process_block_finalized`",
)
)?,
prometheus::Histogram::with_opts(prometheus::HistogramOpts::new(
"parachain_av_store_process_block_finalized",
"Time spent within `av_store::process_block_finalized`",
))?,
registry,
)?,
block_activated: prometheus::register(
prometheus::Histogram::with_opts(
prometheus::HistogramOpts::new(
"parachain_av_store_block_activated",
"Time spent within `av_store::process_block_activated`",
)
)?,
prometheus::Histogram::with_opts(prometheus::HistogramOpts::new(
"parachain_av_store_block_activated",
"Time spent within `av_store::process_block_activated`",
))?,
registry,
)?,
process_message: prometheus::register(
prometheus::Histogram::with_opts(
prometheus::HistogramOpts::new(
"parachain_av_store_process_message",
"Time spent within `av_store::process_message`",
)
)?,
prometheus::Histogram::with_opts(prometheus::HistogramOpts::new(
"parachain_av_store_process_message",
"Time spent within `av_store::process_message`",
))?,
registry,
)?,
store_available_data: prometheus::register(
prometheus::Histogram::with_opts(
prometheus::HistogramOpts::new(
"parachain_av_store_store_available_data",
"Time spent within `av_store::store_available_data`",
)
)?,
prometheus::Histogram::with_opts(prometheus::HistogramOpts::new(
"parachain_av_store_store_available_data",
"Time spent within `av_store::store_available_data`",
))?,
registry,
)?,
store_chunk: prometheus::register(
prometheus::Histogram::with_opts(
prometheus::HistogramOpts::new(
"parachain_av_store_store_chunk",
"Time spent within `av_store::store_chunk`",
)
)?,
prometheus::Histogram::with_opts(prometheus::HistogramOpts::new(
"parachain_av_store_store_chunk",
"Time spent within `av_store::store_chunk`",
))?,
registry,
)?,
get_chunk: prometheus::register(
prometheus::Histogram::with_opts(
prometheus::HistogramOpts::new(
"parachain_av_store_get_chunk",
"Time spent fetching requested chunks.`",
)
)?,
prometheus::Histogram::with_opts(prometheus::HistogramOpts::new(
"parachain_av_store_get_chunk",
"Time spent fetching requested chunks.`",
))?,
registry,
)?,
};
polkadot/node/core/av-store/src/tests.rs
+145 -220
View File
@@ -17,27 +17,23 @@
use super::*;
use assert_matches::assert_matches;
use futures::{
future,
channel::oneshot,
executor,
Future,
};
use futures::{channel::oneshot, executor, future, Future};
use polkadot_primitives::v1::{
CandidateDescriptor, CandidateReceipt, HeadData,
PersistedValidationData, Id as ParaId, CandidateHash, Header, ValidatorId,
CoreIndex, GroupIndex,
};
use polkadot_node_primitives::{AvailableData, BlockData, PoV};
use polkadot_node_subsystem_util::TimeoutExt;
use polkadot_subsystem::{
ActiveLeavesUpdate, errors::RuntimeApiError, jaeger, ActivatedLeaf,
LeafStatus, messages::{AllMessages, RuntimeApiMessage, RuntimeApiRequest},
};
use polkadot_node_subsystem_test_helpers as test_helpers;
use sp_keyring::Sr25519Keyring;
use parking_lot::Mutex;
use polkadot_node_primitives::{AvailableData, BlockData, PoV};
use polkadot_node_subsystem_test_helpers as test_helpers;
use polkadot_node_subsystem_util::TimeoutExt;
use polkadot_primitives::v1::{
CandidateDescriptor, CandidateHash, CandidateReceipt, CoreIndex, GroupIndex, HeadData, Header,
Id as ParaId, PersistedValidationData, ValidatorId,
};
use polkadot_subsystem::{
errors::RuntimeApiError,
jaeger,
messages::{AllMessages, RuntimeApiMessage, RuntimeApiRequest},
ActivatedLeaf, ActiveLeavesUpdate, LeafStatus,
};
use sp_keyring::Sr25519Keyring;
mod columns {
pub const DATA: u32 = 0;
@@ -45,10 +41,7 @@ mod columns {
pub const NUM_COLUMNS: u32 = 2;
}
const TEST_CONFIG: Config = Config {
col_data: columns::DATA,
col_meta: columns::META,
};
const TEST_CONFIG: Config = Config { col_data: columns::DATA, col_meta: columns::META };
type VirtualOverseer = test_helpers::TestSubsystemContextHandle<AvailabilityStoreMessage>;
@@ -95,7 +88,6 @@ impl Clock for TestClock {
}
}
#[derive(Clone)]
struct TestState {
persisted_validation_data: PersistedValidationData,
@@ -126,34 +118,21 @@ impl Default for TestState {
pruning_interval: Duration::from_millis(250),
};
let clock = TestClock {
inner: Arc::new(Mutex::new(Duration::from_secs(0))),
};
let clock = TestClock { inner: Arc::new(Mutex::new(Duration::from_secs(0))) };
Self {
persisted_validation_data,
pruning_config,
clock,
}
Self { persisted_validation_data, pruning_config, clock }
}
}
fn test_harness<T: Future<Output=VirtualOverseer>>(
fn test_harness<T: Future<Output = VirtualOverseer>>(
state: TestState,
store: Arc<dyn KeyValueDB>,
test: impl FnOnce(VirtualOverseer) -> T,
) {
let _ = env_logger::builder()
.is_test(true)
.filter(
Some("polkadot_node_core_av_store"),
log::LevelFilter::Trace,
)
.filter(
Some(LOG_TARGET),
log::LevelFilter::Trace,
)
.filter(Some("polkadot_node_core_av_store"), log::LevelFilter::Trace)
.filter(Some(LOG_TARGET), log::LevelFilter::Trace)
.try_init();
let pool = sp_core::testing::TaskExecutor::new();
@@ -174,21 +153,18 @@ fn test_harness<T: Future<Output=VirtualOverseer>>(
futures::pin_mut!(test_fut);
futures::pin_mut!(subsystem);
executor::block_on(future::join(async move {
let mut overseer = test_fut.await;
overseer_signal(
&mut overseer,
OverseerSignal::Conclude,
).await;
}, subsystem));
executor::block_on(future::join(
async move {
let mut overseer = test_fut.await;
overseer_signal(&mut overseer, OverseerSignal::Conclude).await;
},
subsystem,
));
}
const TIMEOUT: Duration = Duration::from_millis(100);
async fn overseer_send(
overseer: &mut VirtualOverseer,
msg: AvailabilityStoreMessage,
) {
async fn overseer_send(overseer: &mut VirtualOverseer, msg: AvailabilityStoreMessage) {
tracing::trace!(meg = ?msg, "sending message");
overseer
.send(FromOverseer::Communication { msg })
@@ -197,9 +173,7 @@ async fn overseer_send(
.expect(&format!("{:?} is more than enough for sending messages.", TIMEOUT));
}
async fn overseer_recv(
overseer: &mut VirtualOverseer,
) -> AllMessages {
async fn overseer_recv(overseer: &mut VirtualOverseer) -> AllMessages {
let msg = overseer_recv_with_timeout(overseer, TIMEOUT)
.await
.expect(&format!("{:?} is more than enough to receive messages", TIMEOUT));
@@ -214,16 +188,10 @@ async fn overseer_recv_with_timeout(
timeout: Duration,
) -> Option<AllMessages> {
tracing::trace!("waiting for message...");
overseer
.recv()
.timeout(timeout)
.await
overseer.recv().timeout(timeout).await
}
async fn overseer_signal(
overseer: &mut VirtualOverseer,
signal: OverseerSignal,
) {
async fn overseer_signal(overseer: &mut VirtualOverseer, signal: OverseerSignal) {
overseer
.send(FromOverseer::Signal(signal))
.timeout(TIMEOUT)
@@ -261,7 +229,8 @@ fn runtime_api_error_does_not_stop_the_subsystem() {
status: LeafStatus::Fresh,
span: Arc::new(jaeger::Span::Disabled),
})),
).await;
)
.await;
let header = Header {
parent_hash: Hash::zero(),
@@ -298,11 +267,7 @@ fn runtime_api_error_does_not_stop_the_subsystem() {
let (tx, rx) = oneshot::channel();
let candidate_hash = CandidateHash(Hash::repeat_byte(33));
let validator_index = ValidatorIndex(5);
let query_chunk = AvailabilityStoreMessage::QueryChunk(
candidate_hash,
validator_index,
tx,
);
let query_chunk = AvailabilityStoreMessage::QueryChunk(candidate_hash, validator_index, tx);
overseer_send(&mut virtual_overseer, query_chunk.into()).await;
@@ -328,30 +293,28 @@ fn store_chunk_works() {
// Ensure an entry already exists. In reality this would come from watching
// chain events.
with_tx(&store, |tx| {
super::write_meta(tx, &TEST_CONFIG, &candidate_hash, &CandidateMeta {
data_available: false,
chunks_stored: bitvec::bitvec![BitOrderLsb0, u8; 0; n_validators],
state: State::Unavailable(BETimestamp(0)),
});
super::write_meta(
tx,
&TEST_CONFIG,
&candidate_hash,
&CandidateMeta {
data_available: false,
chunks_stored: bitvec::bitvec![BitOrderLsb0, u8; 0; n_validators],
state: State::Unavailable(BETimestamp(0)),
},
);
});
let (tx, rx) = oneshot::channel();
let chunk_msg = AvailabilityStoreMessage::StoreChunk {
candidate_hash,
chunk: chunk.clone(),
tx,
};
let chunk_msg =
AvailabilityStoreMessage::StoreChunk { candidate_hash, chunk: chunk.clone(), tx };
overseer_send(&mut virtual_overseer, chunk_msg.into()).await;
assert_eq!(rx.await.unwrap(), Ok(()));
let (tx, rx) = oneshot::channel();
let query_chunk = AvailabilityStoreMessage::QueryChunk(
candidate_hash,
validator_index,
tx,
);
let query_chunk = AvailabilityStoreMessage::QueryChunk(candidate_hash, validator_index, tx);
overseer_send(&mut virtual_overseer, query_chunk.into()).await;
@@ -360,7 +323,6 @@ fn store_chunk_works() {
});
}
#[test]
fn store_chunk_does_nothing_if_no_entry_already() {
let store = Arc::new(kvdb_memorydb::create(columns::NUM_COLUMNS));
@@ -376,21 +338,14 @@ fn store_chunk_does_nothing_if_no_entry_already() {
let (tx, rx) = oneshot::channel();
let chunk_msg = AvailabilityStoreMessage::StoreChunk {
candidate_hash,
chunk: chunk.clone(),
tx,
};
let chunk_msg =
AvailabilityStoreMessage::StoreChunk { candidate_hash, chunk: chunk.clone(), tx };
overseer_send(&mut virtual_overseer, chunk_msg.into()).await;
assert_eq!(rx.await.unwrap(), Err(()));
let (tx, rx) = oneshot::channel();
let query_chunk = AvailabilityStoreMessage::QueryChunk(
candidate_hash,
validator_index,
tx,
);
let query_chunk = AvailabilityStoreMessage::QueryChunk(candidate_hash, validator_index, tx);
overseer_send(&mut virtual_overseer, query_chunk.into()).await;
@@ -410,23 +365,25 @@ fn query_chunk_checks_meta() {
// Ensure an entry already exists. In reality this would come from watching
// chain events.
with_tx(&store, |tx| {
super::write_meta(tx, &TEST_CONFIG, &candidate_hash, &CandidateMeta {
data_available: false,
chunks_stored: {
let mut v = bitvec::bitvec![BitOrderLsb0, u8; 0; n_validators];
v.set(validator_index.0 as usize, true);
v
super::write_meta(
tx,
&TEST_CONFIG,
&candidate_hash,
&CandidateMeta {
data_available: false,
chunks_stored: {
let mut v = bitvec::bitvec![BitOrderLsb0, u8; 0; n_validators];
v.set(validator_index.0 as usize, true);
v
},
state: State::Unavailable(BETimestamp(0)),
},
state: State::Unavailable(BETimestamp(0)),
});
);
});
let (tx, rx) = oneshot::channel();
let query_chunk = AvailabilityStoreMessage::QueryChunkAvailability(
candidate_hash,
validator_index,
tx,
);
let query_chunk =
AvailabilityStoreMessage::QueryChunkAvailability(candidate_hash, validator_index, tx);
overseer_send(&mut virtual_overseer, query_chunk.into()).await;
assert!(rx.await.unwrap());
@@ -453,16 +410,13 @@ fn store_block_works() {
let validator_index = ValidatorIndex(5);
let n_validators = 10;
let pov = PoV {
block_data: BlockData(vec![4, 5, 6]),
};
let pov = PoV { block_data: BlockData(vec![4, 5, 6]) };
let available_data = AvailableData {
pov: Arc::new(pov),
validation_data: test_state.persisted_validation_data.clone(),
};
let (tx, rx) = oneshot::channel();
let block_msg = AvailabilityStoreMessage::StoreAvailableData(
candidate_hash,
@@ -472,24 +426,23 @@ fn store_block_works() {
tx,
);
virtual_overseer.send(FromOverseer::Communication{ msg: block_msg }).await;
virtual_overseer.send(FromOverseer::Communication { msg: block_msg }).await;
assert_eq!(rx.await.unwrap(), Ok(()));
let pov = query_available_data(&mut virtual_overseer, candidate_hash).await.unwrap();
assert_eq!(pov, available_data);
let chunk = query_chunk(&mut virtual_overseer, candidate_hash, validator_index).await.unwrap();
let chunk = query_chunk(&mut virtual_overseer, candidate_hash, validator_index)
.await
.unwrap();
let chunks = erasure::obtain_chunks_v1(10, &available_data).unwrap();
let mut branches = erasure::branches(chunks.as_ref());
let branch = branches.nth(5).unwrap();
let expected_chunk = ErasureChunk {
chunk: branch.1.to_vec(),
index: ValidatorIndex(5),
proof: branch.0,
};
let expected_chunk =
ErasureChunk { chunk: branch.1.to_vec(), index: ValidatorIndex(5), proof: branch.0 };
assert_eq!(chunk, expected_chunk);
virtual_overseer
@@ -505,16 +458,15 @@ fn store_pov_and_query_chunk_works() {
let candidate_hash = CandidateHash(Hash::repeat_byte(1));
let n_validators = 10;
let pov = PoV {
block_data: BlockData(vec![4, 5, 6]),
};
let pov = PoV { block_data: BlockData(vec![4, 5, 6]) };
let available_data = AvailableData {
pov: Arc::new(pov),
validation_data: test_state.persisted_validation_data.clone(),
};
let chunks_expected = erasure::obtain_chunks_v1(n_validators as _, &available_data).unwrap();
let chunks_expected =
erasure::obtain_chunks_v1(n_validators as _, &available_data).unwrap();
let (tx, rx) = oneshot::channel();
let block_msg = AvailabilityStoreMessage::StoreAvailableData(
@@ -525,12 +477,14 @@ fn store_pov_and_query_chunk_works() {
tx,
);
virtual_overseer.send(FromOverseer::Communication{ msg: block_msg }).await;
virtual_overseer.send(FromOverseer::Communication { msg: block_msg }).await;
assert_eq!(rx.await.unwrap(), Ok(()));
for i in 0..n_validators {
let chunk = query_chunk(&mut virtual_overseer, candidate_hash, ValidatorIndex(i as _)).await.unwrap();
let chunk = query_chunk(&mut virtual_overseer, candidate_hash, ValidatorIndex(i as _))
.await
.unwrap();
assert_eq!(chunk.chunk, chunks_expected[i as usize]);
}
@@ -553,9 +507,7 @@ fn query_all_chunks_works() {
let n_validators = 10;
let pov = PoV {
block_data: BlockData(vec![4, 5, 6]),
};
let pov = PoV { block_data: BlockData(vec![4, 5, 6]) };
let available_data = AvailableData {
pov: Arc::new(pov),
@@ -578,11 +530,16 @@ fn query_all_chunks_works() {
{
with_tx(&store, |tx| {
super::write_meta(tx, &TEST_CONFIG, &candidate_hash_2, &CandidateMeta {
data_available: false,
chunks_stored: bitvec::bitvec![BitOrderLsb0, u8; 0; n_validators as _],
state: State::Unavailable(BETimestamp(0)),
});
super::write_meta(
tx,
&TEST_CONFIG,
&candidate_hash_2,
&CandidateMeta {
data_available: false,
chunks_stored: bitvec::bitvec![BitOrderLsb0, u8; 0; n_validators as _],
state: State::Unavailable(BETimestamp(0)),
},
);
});
let chunk = ErasureChunk {
@@ -598,7 +555,9 @@ fn query_all_chunks_works() {
tx,
};
virtual_overseer.send(FromOverseer::Communication { msg: store_chunk_msg }).await;
virtual_overseer
.send(FromOverseer::Communication { msg: store_chunk_msg })
.await;
assert_eq!(rx.await.unwrap(), Ok(()));
}
@@ -638,9 +597,7 @@ fn stored_but_not_included_data_is_pruned() {
let candidate_hash = CandidateHash(Hash::repeat_byte(1));
let n_validators = 10;
let pov = PoV {
block_data: BlockData(vec![4, 5, 6]),
};
let pov = PoV { block_data: BlockData(vec![4, 5, 6]) };
let available_data = AvailableData {
pov: Arc::new(pov),
@@ -656,7 +613,7 @@ fn stored_but_not_included_data_is_pruned() {
tx,
);
virtual_overseer.send(FromOverseer::Communication{ msg: block_msg }).await;
virtual_overseer.send(FromOverseer::Communication { msg: block_msg }).await;
rx.await.unwrap().unwrap();
@@ -684,16 +641,11 @@ fn stored_data_kept_until_finalized() {
test_harness(test_state.clone(), store.clone(), |mut virtual_overseer| async move {
let n_validators = 10;
let pov = PoV {
block_data: BlockData(vec![4, 5, 6]),
};
let pov = PoV { block_data: BlockData(vec![4, 5, 6]) };
let pov_hash = pov.hash();
let candidate = TestCandidateBuilder {
pov_hash,
..Default::default()
}.build();
let candidate = TestCandidateBuilder { pov_hash, ..Default::default() }.build();
let candidate_hash = candidate.hash();
@@ -714,7 +666,7 @@ fn stored_data_kept_until_finalized() {
tx,
);
virtual_overseer.send(FromOverseer::Communication{ msg: block_msg }).await;
virtual_overseer.send(FromOverseer::Communication { msg: block_msg }).await;
rx.await.unwrap().unwrap();
@@ -730,7 +682,8 @@ fn stored_data_kept_until_finalized() {
block_number,
vec![candidate_included(candidate)],
(0..n_validators).map(|_| Sr25519Keyring::Alice.public().into()).collect(),
).await;
)
.await;
// Wait until unavailable data would definitely be pruned.
test_state.clock.inc(test_state.pruning_config.keep_unavailable_for * 10);
@@ -742,14 +695,13 @@ fn stored_data_kept_until_finalized() {
available_data,
);
assert!(
has_all_chunks(&mut virtual_overseer, candidate_hash, n_validators, true).await
);
assert!(has_all_chunks(&mut virtual_overseer, candidate_hash, n_validators, true).await);
overseer_signal(
&mut virtual_overseer,
OverseerSignal::BlockFinalized(new_leaf, block_number)
).await;
OverseerSignal::BlockFinalized(new_leaf, block_number),
)
.await;
// Wait until unavailable data would definitely be pruned.
test_state.clock.inc(test_state.pruning_config.keep_finalized_for / 2);
@@ -761,22 +713,16 @@ fn stored_data_kept_until_finalized() {
available_data,
);
assert!(
has_all_chunks(&mut virtual_overseer, candidate_hash, n_validators, true).await
);
assert!(has_all_chunks(&mut virtual_overseer, candidate_hash, n_validators, true).await);
// Wait until it definitely should be gone.
test_state.clock.inc(test_state.pruning_config.keep_finalized_for);
test_state.wait_for_pruning().await;
// At this point data should be gone from the store.
assert!(
query_available_data(&mut virtual_overseer, candidate_hash).await.is_none(),
);
assert!(query_available_data(&mut virtual_overseer, candidate_hash).await.is_none(),);
assert!(
has_all_chunks(&mut virtual_overseer, candidate_hash, n_validators, false).await
);
assert!(has_all_chunks(&mut virtual_overseer, candidate_hash, n_validators, false).await);
virtual_overseer
});
}
@@ -787,10 +733,8 @@ fn we_dont_miss_anything_if_import_notifications_are_missed() {
let test_state = TestState::default();
test_harness(test_state.clone(), store.clone(), |mut virtual_overseer| async move {
overseer_signal(
&mut virtual_overseer,
OverseerSignal::BlockFinalized(Hash::zero(), 1)
).await;
overseer_signal(&mut virtual_overseer, OverseerSignal::BlockFinalized(Hash::zero(), 1))
.await;
let header = Header {
parent_hash: Hash::repeat_byte(3),
@@ -809,7 +753,8 @@ fn we_dont_miss_anything_if_import_notifications_are_missed() {
status: LeafStatus::Fresh,
span: Arc::new(jaeger::Span::Disabled),
})),
).await;
)
.await;
assert_matches!(
overseer_recv(&mut virtual_overseer).await,
@@ -915,33 +860,26 @@ fn forkfullness_works() {
let n_validators = 10;
let block_number_1 = 5;
let block_number_2 = 5;
let validators: Vec<_> = (0..n_validators).map(|_| Sr25519Keyring::Alice.public().into()).collect();
let validators: Vec<_> =
(0..n_validators).map(|_| Sr25519Keyring::Alice.public().into()).collect();
let parent_1 = Hash::repeat_byte(3);
let parent_2 = Hash::repeat_byte(4);
let pov_1 = PoV {
block_data: BlockData(vec![1, 2, 3]),
};
let pov_1 = PoV { block_data: BlockData(vec![1, 2, 3]) };
let pov_1_hash = pov_1.hash();
let pov_2 = PoV {
block_data: BlockData(vec![4, 5, 6]),
};
let pov_2 = PoV { block_data: BlockData(vec![4, 5, 6]) };
let pov_2_hash = pov_2.hash();
let candidate_1 = TestCandidateBuilder {
pov_hash: pov_1_hash,
..Default::default()
}.build();
let candidate_1 =
TestCandidateBuilder { pov_hash: pov_1_hash, ..Default::default() }.build();
let candidate_1_hash = candidate_1.hash();
let candidate_2 = TestCandidateBuilder {
pov_hash: pov_2_hash,
..Default::default()
}.build();
let candidate_2 =
TestCandidateBuilder { pov_hash: pov_2_hash, ..Default::default() }.build();
let candidate_2_hash = candidate_2.hash();
@@ -964,7 +902,7 @@ fn forkfullness_works() {
tx,
);
virtual_overseer.send(FromOverseer::Communication{ msg }).await;
virtual_overseer.send(FromOverseer::Communication { msg }).await;
rx.await.unwrap().unwrap();
@@ -977,7 +915,7 @@ fn forkfullness_works() {
tx,
);
virtual_overseer.send(FromOverseer::Communication{ msg }).await;
virtual_overseer.send(FromOverseer::Communication { msg }).await;
rx.await.unwrap().unwrap();
@@ -997,7 +935,8 @@ fn forkfullness_works() {
block_number_1,
vec![candidate_included(candidate_1)],
validators.clone(),
).await;
)
.await;
let _new_leaf_2 = import_leaf(
&mut virtual_overseer,
@@ -1005,12 +944,14 @@ fn forkfullness_works() {
block_number_2,
vec![candidate_included(candidate_2)],
validators.clone(),
).await;
)
.await;
overseer_signal(
&mut virtual_overseer,
OverseerSignal::BlockFinalized(new_leaf_1, block_number_1)
).await;
OverseerSignal::BlockFinalized(new_leaf_1, block_number_1),
)
.await;
// Data of both candidates should be still present in the DB.
assert_eq!(
@@ -1023,13 +964,9 @@ fn forkfullness_works() {
available_data_2,
);
assert!(
has_all_chunks(&mut virtual_overseer, candidate_1_hash, n_validators, true).await,
);
assert!(has_all_chunks(&mut virtual_overseer, candidate_1_hash, n_validators, true).await,);
assert!(
has_all_chunks(&mut virtual_overseer, candidate_2_hash, n_validators, true).await,
);
assert!(has_all_chunks(&mut virtual_overseer, candidate_2_hash, n_validators, true).await,);
// Candidate 2 should now be considered unavailable and will be pruned.
test_state.clock.inc(test_state.pruning_config.keep_unavailable_for);
@@ -1040,38 +977,24 @@ fn forkfullness_works() {
available_data_1,
);
assert!(
query_available_data(&mut virtual_overseer, candidate_2_hash).await.is_none(),
);
assert!(query_available_data(&mut virtual_overseer, candidate_2_hash).await.is_none(),);
assert!(
has_all_chunks(&mut virtual_overseer, candidate_1_hash, n_validators, true).await,
);
assert!(has_all_chunks(&mut virtual_overseer, candidate_1_hash, n_validators, true).await,);
assert!(
has_all_chunks(&mut virtual_overseer, candidate_2_hash, n_validators, false).await,
);
assert!(has_all_chunks(&mut virtual_overseer, candidate_2_hash, n_validators, false).await,);
// Wait for longer than finalized blocks should be kept for
test_state.clock.inc(test_state.pruning_config.keep_finalized_for);
test_state.wait_for_pruning().await;
// Everything should be pruned now.
assert!(
query_available_data(&mut virtual_overseer, candidate_1_hash).await.is_none(),
);
assert!(query_available_data(&mut virtual_overseer, candidate_1_hash).await.is_none(),);
assert!(
query_available_data(&mut virtual_overseer, candidate_2_hash).await.is_none(),
);
assert!(query_available_data(&mut virtual_overseer, candidate_2_hash).await.is_none(),);
assert!(
has_all_chunks(&mut virtual_overseer, candidate_1_hash, n_validators, false).await,
);
assert!(has_all_chunks(&mut virtual_overseer, candidate_1_hash, n_validators, false).await,);
assert!(
has_all_chunks(&mut virtual_overseer, candidate_2_hash, n_validators, false).await,
);
assert!(has_all_chunks(&mut virtual_overseer, candidate_2_hash, n_validators, false).await,);
virtual_overseer
});
}
@@ -1083,7 +1006,7 @@ async fn query_available_data(
let (tx, rx) = oneshot::channel();
let query = AvailabilityStoreMessage::QueryAvailableData(candidate_hash, tx);
virtual_overseer.send(FromOverseer::Communication{ msg: query }).await;
virtual_overseer.send(FromOverseer::Communication { msg: query }).await;
rx.await.unwrap()
}
@@ -1096,7 +1019,7 @@ async fn query_chunk(
let (tx, rx) = oneshot::channel();
let query = AvailabilityStoreMessage::QueryChunk(candidate_hash, index, tx);
virtual_overseer.send(FromOverseer::Communication{ msg: query }).await;
virtual_overseer.send(FromOverseer::Communication { msg: query }).await;
rx.await.unwrap()
}
@@ -1108,7 +1031,9 @@ async fn has_all_chunks(
expect_present: bool,
) -> bool {
for i in 0..n_validators {
if query_chunk(virtual_overseer, candidate_hash, ValidatorIndex(i)).await.is_some() != expect_present {
if query_chunk(virtual_overseer, candidate_hash, ValidatorIndex(i)).await.is_some() !=
expect_present
{
return false
}
}
@@ -1139,7 +1064,8 @@ async fn import_leaf(
status: LeafStatus::Fresh,
span: Arc::new(jaeger::Span::Disabled),
})),
).await;
)
.await;
assert_matches!(
overseer_recv(virtual_overseer).await,
@@ -1163,7 +1089,6 @@ async fn import_leaf(
}
);
assert_matches!(
overseer_recv(virtual_overseer).await,
AllMessages::RuntimeApi(RuntimeApiMessage::Request(
polkadot/node/core/backing/src/lib.rs
+240 -261
View File
@@ -18,55 +18,50 @@
#![deny(unused_crate_dependencies)]
use std::collections::{HashMap, HashSet};
use std::pin::Pin;
use std::sync::Arc;
use std::{
collections::{HashMap, HashSet},
pin::Pin,
sync::Arc,
};
use bitvec::vec::BitVec;
use futures::{channel::{mpsc, oneshot}, Future, FutureExt, SinkExt, StreamExt};
use futures::{
channel::{mpsc, oneshot},
Future, FutureExt, SinkExt, StreamExt,
};
use sp_keystore::SyncCryptoStorePtr;
use polkadot_primitives::v1::{
BackedCandidate, CandidateCommitments, CandidateDescriptor, CandidateHash,
CandidateReceipt, CollatorId, CommittedCandidateReceipt, CoreIndex, CoreState, Hash, Id as ParaId,
SigningContext, ValidatorId, ValidatorIndex, ValidatorSignature, ValidityAttestation,
SessionIndex,
};
use polkadot_node_primitives::{
Statement, SignedFullStatement, ValidationResult, PoV, AvailableData, SignedDisputeStatement,
};
use polkadot_subsystem::{
PerLeafSpan, Stage, SubsystemSender,
jaeger,
overseer,
messages::{
AllMessages, AvailabilityDistributionMessage, AvailabilityStoreMessage,
CandidateBackingMessage, CandidateValidationMessage, CollatorProtocolMessage,
ProvisionableData, ProvisionerMessage, RuntimeApiRequest,
StatementDistributionMessage, ValidationFailed, DisputeCoordinatorMessage,
ImportStatementsResult,
}
AvailableData, PoV, SignedDisputeStatement, SignedFullStatement, Statement, ValidationResult,
};
use polkadot_node_subsystem_util::{
self as util,
request_session_index_for_child,
request_validator_groups,
request_validators,
request_from_runtime,
Validator,
FromJobCommand,
JobSender,
metrics::{self, prometheus},
request_from_runtime, request_session_index_for_child, request_validator_groups,
request_validators, FromJobCommand, JobSender, Validator,
};
use polkadot_primitives::v1::{
BackedCandidate, CandidateCommitments, CandidateDescriptor, CandidateHash, CandidateReceipt,
CollatorId, CommittedCandidateReceipt, CoreIndex, CoreState, Hash, Id as ParaId, SessionIndex,
SigningContext, ValidatorId, ValidatorIndex, ValidatorSignature, ValidityAttestation,
};
use polkadot_subsystem::{
jaeger,
messages::{
AllMessages, AvailabilityDistributionMessage, AvailabilityStoreMessage,
CandidateBackingMessage, CandidateValidationMessage, CollatorProtocolMessage,
DisputeCoordinatorMessage, ImportStatementsResult, ProvisionableData, ProvisionerMessage,
RuntimeApiRequest, StatementDistributionMessage, ValidationFailed,
},
overseer, PerLeafSpan, Stage, SubsystemSender,
};
use sp_keystore::SyncCryptoStorePtr;
use statement_table::{
generic::AttestedCandidate as TableAttestedCandidate,
Context as TableContextTrait,
Table,
v1::{
SignedStatement as TableSignedStatement,
Statement as TableStatement,
SignedStatement as TableSignedStatement, Statement as TableStatement,
Summary as TableSummary,
},
Context as TableContextTrait, Table,
};
use thiserror::Error;
@@ -127,12 +122,9 @@ impl std::fmt::Debug for ValidatedCandidateCommand {
fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result {
let candidate_hash = self.candidate_hash();
match *self {
ValidatedCandidateCommand::Second(_) =>
write!(f, "Second({})", candidate_hash),
ValidatedCandidateCommand::Attest(_) =>
write!(f, "Attest({})", candidate_hash),
ValidatedCandidateCommand::AttestNoPoV(_) =>
write!(f, "Attest({})", candidate_hash),
ValidatedCandidateCommand::Second(_) => write!(f, "Second({})", candidate_hash),
ValidatedCandidateCommand::Attest(_) => write!(f, "Attest({})", candidate_hash),
ValidatedCandidateCommand::AttestNoPoV(_) => write!(f, "Attest({})", candidate_hash),
}
}
}
@@ -224,7 +216,9 @@ impl TableContextTrait for TableContext {
}
fn is_member_of(&self, authority: &ValidatorIndex, group: &ParaId) -> bool {
self.groups.get(group).map_or(false, |g| g.iter().position(|a| a == authority).is_some())
self.groups
.get(group)
.map_or(false, |g| g.iter().position(|a| a == authority).is_some())
}
fn requisite_votes(&self, group: &ParaId) -> usize {
@@ -260,10 +254,8 @@ fn table_attested_to_backed(
) -> Option<BackedCandidate> {
let TableAttestedCandidate { candidate, validity_votes, group_id: para_id } = attested;
let (ids, validity_votes): (Vec<_>, Vec<ValidityAttestation>) = validity_votes
.into_iter()
.map(|(id, vote)| (id, vote.into()))
.unzip();
let (ids, validity_votes): (Vec<_>, Vec<ValidityAttestation>) =
validity_votes.into_iter().map(|(id, vote)| (id, vote.into())).unzip();
let group = table_context.groups.get(&para_id)?;
@@ -285,14 +277,15 @@ fn table_attested_to_backed(
"Logic error: Validity vote from table does not correspond to group",
);
return None;
return None
}
}
vote_positions.sort_by_key(|(_orig, pos_in_group)| *pos_in_group);
Some(BackedCandidate {
candidate,
validity_votes: vote_positions.into_iter()
validity_votes: vote_positions
.into_iter()
.map(|(pos_in_votes, _pos_in_group)| validity_votes[pos_in_votes].clone())
.collect(),
validator_indices,
@@ -307,13 +300,15 @@ async fn store_available_data(
available_data: AvailableData,
) -> Result<(), Error> {
let (tx, rx) = oneshot::channel();
sender.send_message(AvailabilityStoreMessage::StoreAvailableData(
candidate_hash,
id,
n_validators,
available_data,
tx,
)).await;
sender
.send_message(AvailabilityStoreMessage::StoreAvailableData(
candidate_hash,
id,
n_validators,
available_data,
tx,
))
.await;
let _ = rx.await.map_err(Error::StoreAvailableData)?;
@@ -334,33 +329,23 @@ async fn make_pov_available(
expected_erasure_root: Hash,
span: Option<&jaeger::Span>,
) -> Result<Result<(), InvalidErasureRoot>, Error> {
let available_data = AvailableData {
pov,
validation_data,
};
let available_data = AvailableData { pov, validation_data };
{
let _span = span.as_ref().map(|s| {
s.child("erasure-coding").with_candidate(candidate_hash)
});
let _span = span.as_ref().map(|s| s.child("erasure-coding").with_candidate(candidate_hash));
let chunks = erasure_coding::obtain_chunks_v1(
n_validators,
&available_data,
)?;
let chunks = erasure_coding::obtain_chunks_v1(n_validators, &available_data)?;
let branches = erasure_coding::branches(chunks.as_ref());
let erasure_root = branches.root();
if erasure_root != expected_erasure_root {
return Ok(Err(InvalidErasureRoot));
return Ok(Err(InvalidErasureRoot))
}
}
{
let _span = span.as_ref().map(|s|
s.child("store-data").with_candidate(candidate_hash)
);
let _span = span.as_ref().map(|s| s.child("store-data").with_candidate(candidate_hash));
store_available_data(
sender,
@@ -368,7 +353,8 @@ async fn make_pov_available(
n_validators as u32,
candidate_hash,
available_data,
).await?;
)
.await?;
}
Ok(Ok(()))
@@ -381,15 +367,16 @@ async fn request_pov(
candidate_hash: CandidateHash,
pov_hash: Hash,
) -> Result<Arc<PoV>, Error> {
let (tx, rx) = oneshot::channel();
sender.send_message(AvailabilityDistributionMessage::FetchPoV {
relay_parent,
from_validator,
candidate_hash,
pov_hash,
tx,
}).await;
sender
.send_message(AvailabilityDistributionMessage::FetchPoV {
relay_parent,
from_validator,
candidate_hash,
pov_hash,
tx,
})
.await;
let pov = rx.await.map_err(|_| Error::FetchPoV)?;
Ok(Arc::new(pov))
@@ -402,13 +389,9 @@ async fn request_candidate_validation(
) -> Result<ValidationResult, Error> {
let (tx, rx) = oneshot::channel();
sender.send_message(
CandidateValidationMessage::ValidateFromChainState(
candidate,
pov,
tx,
)
).await;
sender
.send_message(CandidateValidationMessage::ValidateFromChainState(candidate, pov, tx))
.await;
match rx.await {
Ok(Ok(validation_result)) => Ok(validation_result),
@@ -417,7 +400,8 @@ async fn request_candidate_validation(
}
}
type BackgroundValidationResult = Result<(CandidateReceipt, CandidateCommitments, Arc<PoV>), CandidateReceipt>;
type BackgroundValidationResult =
Result<(CandidateReceipt, CandidateCommitments, Arc<PoV>), CandidateReceipt>;
struct BackgroundValidationParams<S: overseer::SubsystemSender<AllMessages>, F> {
sender: JobSender<S>,
@@ -435,7 +419,7 @@ async fn validate_and_make_available(
params: BackgroundValidationParams<
impl SubsystemSender,
impl Fn(BackgroundValidationResult) -> ValidatedCandidateCommand + Sync,
>
>,
) -> Result<(), Error> {
let BackgroundValidationParams {
mut sender,
@@ -451,27 +435,22 @@ async fn validate_and_make_available(
let pov = match pov {
PoVData::Ready(pov) => pov,
PoVData::FetchFromValidator {
from_validator,
candidate_hash,
pov_hash,
} => {
PoVData::FetchFromValidator { from_validator, candidate_hash, pov_hash } => {
let _span = span.as_ref().map(|s| s.child("request-pov"));
match request_pov(
&mut sender,
relay_parent,
from_validator,
candidate_hash,
pov_hash,
).await {
match request_pov(&mut sender, relay_parent, from_validator, candidate_hash, pov_hash)
.await
{
Err(Error::FetchPoV) => {
tx_command.send(ValidatedCandidateCommand::AttestNoPoV(candidate.hash())).await.map_err(Error::Mpsc)?;
tx_command
.send(ValidatedCandidateCommand::AttestNoPoV(candidate.hash()))
.await
.map_err(Error::Mpsc)?;
return Ok(())
}
},
Err(err) => return Err(err),
Ok(pov) => pov,
}
}
},
};
let v = {
@@ -512,7 +491,8 @@ async fn validate_and_make_available(
validation_data,
candidate.descriptor.erasure_root,
span.as_ref(),
).await?;
)
.await?;
match erasure_valid {
Ok(()) => Ok((candidate, commitments, pov.clone())),
@@ -527,7 +507,7 @@ async fn validate_and_make_available(
},
}
}
}
},
ValidationResult::Invalid(reason) => {
tracing::debug!(
target: LOG_TARGET,
@@ -536,7 +516,7 @@ async fn validate_and_make_available(
"Validation yielded an invalid candidate",
);
Err(candidate)
}
},
};
tx_command.send(make_command(res)).await.map_err(Into::into)
@@ -591,7 +571,9 @@ impl CandidateBackingJob {
match res {
Ok((candidate, commitments, _)) => {
// sanity check.
if self.seconded.is_none() && !self.issued_statements.contains(&candidate_hash) {
if self.seconded.is_none() &&
!self.issued_statements.contains(&candidate_hash)
{
self.seconded = Some(candidate_hash);
self.issued_statements.insert(candidate_hash);
self.metrics.on_candidate_seconded();
@@ -600,24 +582,26 @@ impl CandidateBackingJob {
descriptor: candidate.descriptor.clone(),
commitments,
});
if let Some(stmt) = self.sign_import_and_distribute_statement(
sender,
statement,
root_span,
).await? {
sender.send_message(
CollatorProtocolMessage::Seconded(self.parent, stmt)
).await;
if let Some(stmt) = self
.sign_import_and_distribute_statement(sender, statement, root_span)
.await?
{
sender
.send_message(CollatorProtocolMessage::Seconded(
self.parent,
stmt,
))
.await;
}
}
}
},
Err(candidate) => {
sender.send_message(
CollatorProtocolMessage::Invalid(self.parent, candidate)
).await;
}
sender
.send_message(CollatorProtocolMessage::Invalid(self.parent, candidate))
.await;
},
}
}
},
ValidatedCandidateCommand::Attest(res) => {
// We are done - avoid new validation spawns:
self.fallbacks.remove(&candidate_hash);
@@ -625,11 +609,12 @@ impl CandidateBackingJob {
if !self.issued_statements.contains(&candidate_hash) {
if res.is_ok() {
let statement = Statement::Valid(candidate_hash);
self.sign_import_and_distribute_statement(sender, statement, &root_span).await?;
self.sign_import_and_distribute_statement(sender, statement, &root_span)
.await?;
}
self.issued_statements.insert(candidate_hash);
}
}
},
ValidatedCandidateCommand::AttestNoPoV(candidate_hash) => {
if let Some((attesting, span)) = self.fallbacks.get_mut(&candidate_hash) {
if let Some(index) = attesting.backing.pop() {
@@ -639,7 +624,6 @@ impl CandidateBackingJob {
let attesting = attesting.clone();
self.kick_off_validation_work(sender, attesting, c_span).await?
}
} else {
tracing::warn!(
target: LOG_TARGET,
@@ -647,7 +631,7 @@ impl CandidateBackingJob {
);
debug_assert!(false);
}
}
},
}
Ok(())
@@ -658,7 +642,7 @@ impl CandidateBackingJob {
sender: &mut JobSender<impl SubsystemSender>,
params: BackgroundValidationParams<
impl SubsystemSender,
impl Fn(BackgroundValidationResult) -> ValidatedCandidateCommand + Send + 'static + Sync
impl Fn(BackgroundValidationResult) -> ValidatedCandidateCommand + Send + 'static + Sync,
>,
) -> Result<(), Error> {
let candidate_hash = params.candidate.hash();
@@ -666,10 +650,16 @@ impl CandidateBackingJob {
// spawn background task.
let bg = async move {
if let Err(e) = validate_and_make_available(params).await {
tracing::error!(target: LOG_TARGET, "Failed to validate and make available: {:?}", e);
tracing::error!(
target: LOG_TARGET,
"Failed to validate and make available: {:?}",
e
);
}
};
sender.send_command(FromJobCommand::Spawn("Backing Validation", bg.boxed())).await?;
sender
.send_command(FromJobCommand::Spawn("Backing Validation", bg.boxed()))
.await?;
}
Ok(())
@@ -685,13 +675,15 @@ impl CandidateBackingJob {
pov: Arc<PoV>,
) -> Result<(), Error> {
// Check that candidate is collated by the right collator.
if self.required_collator.as_ref()
if self
.required_collator
.as_ref()
.map_or(false, |c| c != &candidate.descriptor().collator)
{
sender.send_message(
CollatorProtocolMessage::Invalid(self.parent, candidate.clone())
).await;
return Ok(());
sender
.send_message(CollatorProtocolMessage::Invalid(self.parent, candidate.clone()))
.await;
return Ok(())
}
let candidate_hash = candidate.hash();
@@ -723,8 +715,9 @@ impl CandidateBackingJob {
n_validators: self.table_context.validators.len(),
span,
make_command: ValidatedCandidateCommand::Second,
}
).await?;
},
)
.await?;
Ok(())
}
@@ -751,12 +744,12 @@ impl CandidateBackingJob {
// collect the misbehaviors to avoid double mutable self borrow issues
let misbehaviors: Vec<_> = self.table.drain_misbehaviors().collect();
for (validator_id, report) in misbehaviors {
sender.send_message(
ProvisionerMessage::ProvisionableData(
sender
.send_message(ProvisionerMessage::ProvisionableData(
self.parent,
ProvisionableData::MisbehaviorReport(self.parent, validator_id, report)
)
).await;
ProvisionableData::MisbehaviorReport(self.parent, validator_id, report),
))
.await;
}
}
@@ -777,14 +770,17 @@ impl CandidateBackingJob {
let candidate_hash = statement.payload().candidate_hash();
let import_statement_span = {
// create a span only for candidates we're already aware of.
self.get_unbacked_statement_child(root_span, candidate_hash, statement.validator_index())
self.get_unbacked_statement_child(
root_span,
candidate_hash,
statement.validator_index(),
)
};
if let Err(ValidatorIndexOutOfBounds) = self.dispatch_new_statement_to_dispute_coordinator(
sender,
candidate_hash,
&statement,
).await {
if let Err(ValidatorIndexOutOfBounds) = self
.dispatch_new_statement_to_dispute_coordinator(sender, candidate_hash, &statement)
.await
{
tracing::warn!(
target: LOG_TARGET,
session_index = ?self.session_index,
@@ -793,14 +789,15 @@ impl CandidateBackingJob {
"Supposedly 'Signed' statement has validator index out of bounds."
);
return Ok(None);
return Ok(None)
}
let stmt = primitive_statement_to_table(statement);
let summary = self.table.import_statement(&self.table_context, stmt);
let unbacked_span = if let Some(attested) = summary.as_ref()
let unbacked_span = if let Some(attested) = summary
.as_ref()
.and_then(|s| self.table.attested_candidate(&s.candidate, &self.table_context))
{
let candidate_hash = attested.candidate.hash();
@@ -808,9 +805,7 @@ impl CandidateBackingJob {
if self.backed.insert(candidate_hash) {
let span = self.remove_unbacked_span(&candidate_hash);
if let Some(backed) =
table_attested_to_backed(attested, &self.table_context)
{
if let Some(backed) = table_attested_to_backed(attested, &self.table_context) {
tracing::debug!(
target: LOG_TARGET,
candidate_hash = ?candidate_hash,
@@ -866,18 +861,11 @@ impl CandidateBackingJob {
) -> Result<(), ValidatorIndexOutOfBounds> {
// Dispatch the statement to the dispute coordinator.
let validator_index = statement.validator_index();
let signing_context = SigningContext {
parent_hash: self.parent,
session_index: self.session_index,
};
let signing_context =
SigningContext { parent_hash: self.parent, session_index: self.session_index };
let validator_public = match self.table_context
.validators
.get(validator_index.0 as usize)
{
None => {
return Err(ValidatorIndexOutOfBounds);
}
let validator_public = match self.table_context.validators.get(validator_index.0 as usize) {
None => return Err(ValidatorIndexOutOfBounds),
Some(v) => v,
};
@@ -887,39 +875,36 @@ impl CandidateBackingJob {
// Valid statements are only supposed to be imported
// once we've seen at least one `Seconded` statement.
self.table.get_candidate(&candidate_hash).map(|c| c.to_plain())
}
},
};
let maybe_signed_dispute_statement = SignedDisputeStatement::from_backing_statement(
statement.as_unchecked(),
signing_context,
validator_public.clone(),
).ok();
)
.ok();
if let (Some(candidate_receipt), Some(dispute_statement))
= (maybe_candidate_receipt, maybe_signed_dispute_statement)
if let (Some(candidate_receipt), Some(dispute_statement)) =
(maybe_candidate_receipt, maybe_signed_dispute_statement)
{
let (pending_confirmation, confirmation_rx) = oneshot::channel();
sender.send_message(
DisputeCoordinatorMessage::ImportStatements {
sender
.send_message(DisputeCoordinatorMessage::ImportStatements {
candidate_hash,
candidate_receipt,
session: self.session_index,
statements: vec![(dispute_statement, validator_index)],
pending_confirmation,
}
).await;
})
.await;
match confirmation_rx.await {
Err(oneshot::Canceled) => tracing::warn!(
target: LOG_TARGET,
"Dispute coordinator confirmation lost",
),
Ok(ImportStatementsResult::ValidImport) => {}
Ok(ImportStatementsResult::InvalidImport) => tracing::warn!(
target: LOG_TARGET,
"Failed to import statements of validity",
),
Err(oneshot::Canceled) =>
tracing::warn!(target: LOG_TARGET, "Dispute coordinator confirmation lost",),
Ok(ImportStatementsResult::ValidImport) => {},
Ok(ImportStatementsResult::InvalidImport) =>
tracing::warn!(target: LOG_TARGET, "Failed to import statements of validity",),
}
}
@@ -936,7 +921,8 @@ impl CandidateBackingJob {
CandidateBackingMessage::Second(relay_parent, candidate, pov) => {
let _timer = self.metrics.time_process_second();
let span = root_span.child("second")
let span = root_span
.child("second")
.with_stage(jaeger::Stage::CandidateBacking)
.with_pov(&pov)
.with_candidate(candidate.hash())
@@ -944,7 +930,7 @@ impl CandidateBackingJob {
// Sanity check that candidate is from our assignment.
if Some(candidate.descriptor().para_id) != self.assignment {
return Ok(());
return Ok(())
}
// If the message is a `CandidateBackingMessage::Second`, sign and dispatch a
@@ -956,13 +942,15 @@ impl CandidateBackingJob {
if !self.issued_statements.contains(&candidate_hash) {
let pov = Arc::new(pov);
self.validate_and_second(&span, &root_span, sender, &candidate, pov).await?;
self.validate_and_second(&span, &root_span, sender, &candidate, pov)
.await?;
}
}
}
},
CandidateBackingMessage::Statement(_relay_parent, statement) => {
let _timer = self.metrics.time_process_statement();
let _span = root_span.child("statement")
let _span = root_span
.child("statement")
.with_stage(jaeger::Stage::CandidateBacking)
.with_candidate(statement.payload().candidate_hash())
.with_relay_parent(_relay_parent);
@@ -972,20 +960,21 @@ impl CandidateBackingJob {
Err(e) => return Err(e),
Ok(()) => (),
}
}
},
CandidateBackingMessage::GetBackedCandidates(_, requested_candidates, tx) => {
let _timer = self.metrics.time_get_backed_candidates();
let backed = requested_candidates
.into_iter()
.filter_map(|hash| {
self.table.attested_candidate(&hash, &self.table_context)
.and_then(|attested| table_attested_to_backed(attested, &self.table_context))
self.table.attested_candidate(&hash, &self.table_context).and_then(
|attested| table_attested_to_backed(attested, &self.table_context),
)
})
.collect();
tx.send(backed).map_err(|data| Error::Send(data))?;
}
},
}
Ok(())
@@ -1013,15 +1002,13 @@ impl CandidateBackingJob {
);
// Check that candidate is collated by the right collator.
if self.required_collator.as_ref()
.map_or(false, |c| c != &descriptor.collator)
{
if self.required_collator.as_ref().map_or(false, |c| c != &descriptor.collator) {
// If not, we've got the statement in the table but we will
// not issue validation work for it.
//
// Act as though we've issued a statement.
self.issued_statements.insert(candidate_hash);
return Ok(());
return Ok(())
}
let bg_sender = sender.clone();
@@ -1043,7 +1030,8 @@ impl CandidateBackingJob {
span,
make_command: ValidatedCandidateCommand::Attest,
},
).await
)
.await
}
/// Import the statement and kick off validation work if it is a part of our assignment.
@@ -1068,7 +1056,11 @@ impl CandidateBackingJob {
);
let attesting = AttestingData {
candidate: self.table.get_candidate(&candidate_hash).ok_or(Error::CandidateNotFound)?.to_plain(),
candidate: self
.table
.get_candidate(&candidate_hash)
.ok_or(Error::CandidateNotFound)?
.to_plain(),
pov_hash: receipt.descriptor.pov_hash,
from_validator: statement.validator_index(),
backing: Vec::new(),
@@ -1076,10 +1068,9 @@ impl CandidateBackingJob {
let child = span.as_ref().map(|s| s.child("try"));
self.fallbacks.insert(summary.candidate, (attesting.clone(), span));
(attesting, child)
}
},
Statement::Valid(candidate_hash) => {
if let Some((attesting, span)) = self.fallbacks.get_mut(candidate_hash) {
let our_index = self.table_context.validator.as_ref().map(|v| v.index());
if our_index == Some(statement.validator_index()) {
return Ok(())
@@ -1097,20 +1088,17 @@ impl CandidateBackingJob {
} else {
return Ok(())
}
}
},
};
self.kick_off_validation_work(
sender,
attesting,
span,
).await?;
self.kick_off_validation_work(sender, attesting, span).await?;
}
Ok(())
}
async fn sign_statement(&self, statement: Statement) -> Option<SignedFullStatement> {
let signed = self.table_context
let signed = self
.table_context
.validator
.as_ref()?
.sign(self.keystore.clone(), statement)
@@ -1126,7 +1114,7 @@ impl CandidateBackingJob {
&mut self,
parent_span: &jaeger::Span,
hash: CandidateHash,
para_id: Option<ParaId>
para_id: Option<ParaId>,
) -> Option<&jaeger::Span> {
if !self.backed.contains(&hash) {
// only add if we don't consider this backed.
@@ -1150,12 +1138,11 @@ impl CandidateBackingJob {
hash: CandidateHash,
para_id: ParaId,
) -> Option<jaeger::Span> {
self.insert_or_get_unbacked_span(parent_span, hash, Some(para_id))
.map(|span| {
span.child("validation")
.with_candidate(hash)
.with_stage(Stage::CandidateBacking)
})
self.insert_or_get_unbacked_span(parent_span, hash, Some(para_id)).map(|span| {
span.child("validation")
.with_candidate(hash)
.with_stage(Stage::CandidateBacking)
})
}
fn get_unbacked_statement_child(
@@ -1220,12 +1207,12 @@ impl util::JobTrait for CandidateBackingJob {
request_validators(parent, &mut sender).await,
request_validator_groups(parent, &mut sender).await,
request_session_index_for_child(parent, &mut sender).await,
request_from_runtime(
parent,
&mut sender,
|tx| RuntimeApiRequest::AvailabilityCores(tx),
).await,
).map_err(Error::JoinMultiple)?;
request_from_runtime(parent, &mut sender, |tx| {
RuntimeApiRequest::AvailabilityCores(tx)
},)
.await,
)
.map_err(Error::JoinMultiple)?;
let validators = try_runtime_api!(validators);
let (validator_groups, group_rotation_info) = try_runtime_api!(groups);
@@ -1236,23 +1223,22 @@ impl util::JobTrait for CandidateBackingJob {
let _span = span.child("validator-construction");
let signing_context = SigningContext { parent_hash: parent, session_index };
let validator = match Validator::construct(
&validators,
signing_context.clone(),
keystore.clone(),
).await {
Ok(v) => Some(v),
Err(util::Error::NotAValidator) => None,
Err(e) => {
tracing::warn!(
target: LOG_TARGET,
err = ?e,
"Cannot participate in candidate backing",
);
let validator =
match Validator::construct(&validators, signing_context.clone(), keystore.clone())
.await
{
Ok(v) => Some(v),
Err(util::Error::NotAValidator) => None,
Err(e) => {
tracing::warn!(
target: LOG_TARGET,
err = ?e,
"Cannot participate in candidate backing",
);
return Ok(())
}
};
return Ok(())
},
};
drop(_span);
let mut assignments_span = span.child("compute-assignments");
@@ -1277,22 +1263,18 @@ impl util::JobTrait for CandidateBackingJob {
}
}
let table_context = TableContext {
groups,
validators,
validator,
};
let table_context = TableContext { groups, validators, validator };
let (assignment, required_collator) = match assignment {
None => {
assignments_span.add_string_tag("assigned", "false");
(None, None)
}
},
Some((assignment, required_collator)) => {
assignments_span.add_string_tag("assigned", "true");
assignments_span.add_para_id(assignment);
(Some(assignment), required_collator)
}
},
};
drop(assignments_span);
@@ -1320,7 +1302,8 @@ impl util::JobTrait for CandidateBackingJob {
drop(_span);
job.run_loop(sender, rx_to, span).await
}.boxed()
}
.boxed()
}
}
@@ -1361,7 +1344,9 @@ impl Metrics {
}
/// Provide a timer for handling `CandidateBackingMessage::GetBackedCandidates` which observes on drop.
fn time_get_backed_candidates(&self) -> Option<metrics::prometheus::prometheus::HistogramTimer> {
fn time_get_backed_candidates(
&self,
) -> Option<metrics::prometheus::prometheus::HistogramTimer> {
self.0.as_ref().map(|metrics| metrics.get_backed_candidates.start_timer())
}
}
@@ -1384,30 +1369,24 @@ impl metrics::Metrics for Metrics {
registry,
)?,
process_second: prometheus::register(
prometheus::Histogram::with_opts(
prometheus::HistogramOpts::new(
"parachain_candidate_backing_process_second",
"Time spent within `candidate_backing::process_second`",
)
)?,
prometheus::Histogram::with_opts(prometheus::HistogramOpts::new(
"parachain_candidate_backing_process_second",
"Time spent within `candidate_backing::process_second`",
))?,
registry,
)?,
process_statement: prometheus::register(
prometheus::Histogram::with_opts(
prometheus::HistogramOpts::new(
"parachain_candidate_backing_process_statement",
"Time spent within `candidate_backing::process_statement`",
)
)?,
prometheus::Histogram::with_opts(prometheus::HistogramOpts::new(
"parachain_candidate_backing_process_statement",
"Time spent within `candidate_backing::process_statement`",
))?,
registry,
)?,
get_backed_candidates: prometheus::register(
prometheus::Histogram::with_opts(
prometheus::HistogramOpts::new(
"parachain_candidate_backing_get_backed_candidates",
"Time spent within `candidate_backing::get_backed_candidates`",
)
)?,
prometheus::Histogram::with_opts(prometheus::HistogramOpts::new(
"parachain_candidate_backing_get_backed_candidates",
"Time spent within `candidate_backing::get_backed_candidates`",
))?,
registry,
)?,
};
@@ -1416,5 +1395,5 @@ impl metrics::Metrics for Metrics {
}
/// The candidate backing subsystem.
pub type CandidateBackingSubsystem<Spawner>
= polkadot_node_subsystem_util::JobSubsystem<CandidateBackingJob, Spawner>;
pub type CandidateBackingSubsystem<Spawner> =
polkadot_node_subsystem_util::JobSubsystem<CandidateBackingJob, Spawner>;
polkadot/node/core/backing/src/tests.rs
+369 -297
View File
File diff suppressed because it is too large Load Diff
polkadot/node/core/bitfield-signing/src/lib.rs
+49 -39
View File
@@ -18,24 +18,32 @@
#![deny(unused_crate_dependencies)]
#![warn(missing_docs)]
#![recursion_limit="256"]
#![recursion_limit = "256"]
use futures::{channel::{mpsc, oneshot}, lock::Mutex, prelude::*, future, Future};
use sp_keystore::{Error as KeystoreError, SyncCryptoStorePtr};
use futures::{
channel::{mpsc, oneshot},
future,
lock::Mutex,
prelude::*,
Future,
};
use polkadot_node_subsystem::{
jaeger, PerLeafSpan, SubsystemSender,
messages::{
AvailabilityStoreMessage, BitfieldDistributionMessage,
BitfieldSigningMessage, RuntimeApiMessage, RuntimeApiRequest,
},
errors::RuntimeApiError,
jaeger,
messages::{
AvailabilityStoreMessage, BitfieldDistributionMessage, BitfieldSigningMessage,
RuntimeApiMessage, RuntimeApiRequest,
},
PerLeafSpan, SubsystemSender,
};
use polkadot_node_subsystem_util::{
self as util, JobSubsystem, JobTrait, Validator, metrics::{self, prometheus},
JobSender,
self as util,
metrics::{self, prometheus},
JobSender, JobSubsystem, JobTrait, Validator,
};
use polkadot_primitives::v1::{AvailabilityBitfield, CoreState, Hash, ValidatorIndex};
use std::{pin::Pin, time::Duration, iter::FromIterator, sync::Arc};
use sp_keystore::{Error as KeystoreError, SyncCryptoStorePtr};
use std::{iter::FromIterator, pin::Pin, sync::Arc, time::Duration};
use wasm_timer::{Delay, Instant};
#[cfg(test)]
@@ -45,7 +53,6 @@ mod tests;
const JOB_DELAY: Duration = Duration::from_millis(1500);
const LOG_TARGET: &str = "parachain::bitfield-signing";
/// Each `BitfieldSigningJob` prepares a signed bitfield for a single relay parent.
pub struct BitfieldSigningJob;
@@ -92,7 +99,8 @@ async fn get_core_availability(
core.candidate_hash,
validator_idx,
tx,
).into(),
)
.into(),
)
.await;
@@ -119,15 +127,15 @@ async fn get_availability_cores(
) -> Result<Vec<CoreState>, Error> {
let (tx, rx) = oneshot::channel();
sender
.send_message(RuntimeApiMessage::Request(
relay_parent,
RuntimeApiRequest::AvailabilityCores(tx),
).into())
.send_message(
RuntimeApiMessage::Request(relay_parent, RuntimeApiRequest::AvailabilityCores(tx))
.into(),
)
.await;
match rx.await {
Ok(Ok(out)) => Ok(out),
Ok(Err(runtime_err)) => Err(runtime_err.into()),
Err(err) => Err(err.into())
Err(err) => Err(err.into()),
}
}
@@ -157,9 +165,11 @@ async fn construct_availability_bitfield(
// Handle all cores concurrently
// `try_join_all` returns all results in the same order as the input futures.
let results = future::try_join_all(
availability_cores.iter()
availability_cores
.iter()
.map(|core| get_core_availability(core, validator_idx, &sender, span)),
).await?;
)
.await?;
tracing::debug!(
target: LOG_TARGET,
@@ -206,12 +216,10 @@ impl metrics::Metrics for Metrics {
registry,
)?,
run: prometheus::register(
prometheus::Histogram::with_opts(
prometheus::HistogramOpts::new(
"parachain_bitfield_signing_run",
"Time spent within `bitfield_signing::run`",
)
)?,
prometheus::Histogram::with_opts(prometheus::HistogramOpts::new(
"parachain_bitfield_signing_run",
"Time spent within `bitfield_signing::run`",
))?,
registry,
)?,
};
@@ -244,7 +252,8 @@ impl JobTrait for BitfieldSigningJob {
// now do all the work we can before we need to wait for the availability store
// if we're not a validator, we can just succeed effortlessly
let validator = match Validator::new(relay_parent, keystore.clone(), &mut sender).await {
let validator = match Validator::new(relay_parent, keystore.clone(), &mut sender).await
{
Ok(validator) => validator,
Err(util::Error::NotAValidator) => return Ok(()),
Err(err) => return Err(Error::Util(err)),
@@ -260,19 +269,19 @@ impl JobTrait for BitfieldSigningJob {
drop(_span);
let span_availability = span.child("availability");
let bitfield =
match construct_availability_bitfield(
relay_parent,
&span_availability,
validator.index(),
sender.subsystem_sender(),
).await
let bitfield = match construct_availability_bitfield(
relay_parent,
&span_availability,
validator.index(),
sender.subsystem_sender(),
)
.await
{
Err(Error::Runtime(runtime_err)) => {
// Don't take down the node on runtime API errors.
tracing::warn!(target: LOG_TARGET, err = ?runtime_err, "Encountered a runtime API error");
return Ok(());
}
return Ok(())
},
Err(err) => return Err(err),
Ok(bitfield) => bitfield,
};
@@ -280,7 +289,8 @@ impl JobTrait for BitfieldSigningJob {
drop(span_availability);
let _span = span.child("signing");
let signed_bitfield = match validator.sign(keystore.clone(), bitfield)
let signed_bitfield = match validator
.sign(keystore.clone(), bitfield)
.await
.map_err(|e| Error::Keystore(e))?
{
@@ -290,8 +300,8 @@ impl JobTrait for BitfieldSigningJob {
target: LOG_TARGET,
"Key was found at construction, but while signing it could not be found.",
);
return Ok(());
}
return Ok(())
},
};
metrics.on_bitfield_signed();
polkadot/node/core/bitfield-signing/src/tests.rs
+4 -3
View File
@@ -15,9 +15,9 @@
// along with Polkadot. If not, see <http://www.gnu.org/licenses/>.
use super::*;
use futures::{pin_mut, executor::block_on};
use polkadot_primitives::v1::{CandidateHash, OccupiedCore};
use futures::{executor::block_on, pin_mut};
use polkadot_node_subsystem::messages::AllMessages;
use polkadot_primitives::v1::{CandidateHash, OccupiedCore};
fn occupied_core(para_id: u32, candidate_hash: CandidateHash) -> CoreState {
CoreState::Occupied(OccupiedCore {
@@ -44,7 +44,8 @@ fn construct_availability_bitfield_works() {
&jaeger::Span::Disabled,
validator_index,
&mut sender,
).fuse();
)
.fuse();
pin_mut!(future);
let hash_a = CandidateHash(Hash::repeat_byte(1));
polkadot/node/core/candidate-validation/src/lib.rs
+128 -144
View File
@@ -23,34 +23,32 @@
#![deny(unused_crate_dependencies, unused_results)]
#![warn(missing_docs)]
use polkadot_node_core_pvf::{
InvalidCandidate as WasmInvalidCandidate, Pvf, ValidationError, ValidationHost,
};
use polkadot_node_primitives::{
BlockData, InvalidCandidate, PoV, ValidationResult, POV_BOMB_LIMIT, VALIDATION_CODE_BOMB_LIMIT,
};
use polkadot_node_subsystem::{
overseer,
SubsystemContext, SpawnedSubsystem, SubsystemResult, SubsystemError,
FromOverseer, OverseerSignal,
messages::{
CandidateValidationMessage, RuntimeApiMessage,
ValidationFailed, RuntimeApiRequest,
},
errors::RuntimeApiError,
messages::{
CandidateValidationMessage, RuntimeApiMessage, RuntimeApiRequest, ValidationFailed,
},
overseer, FromOverseer, OverseerSignal, SpawnedSubsystem, SubsystemContext, SubsystemError,
SubsystemResult,
};
use polkadot_node_subsystem_util::metrics::{self, prometheus};
use polkadot_node_primitives::{
VALIDATION_CODE_BOMB_LIMIT, POV_BOMB_LIMIT, ValidationResult, InvalidCandidate, PoV, BlockData,
};
use polkadot_primitives::v1::{
ValidationCode, CandidateDescriptor, PersistedValidationData,
OccupiedCoreAssumption, Hash, CandidateCommitments,
};
use polkadot_parachain::primitives::{ValidationParams, ValidationResult as WasmValidationResult};
use polkadot_node_core_pvf::{Pvf, ValidationHost, ValidationError, InvalidCandidate as WasmInvalidCandidate};
use polkadot_primitives::v1::{
CandidateCommitments, CandidateDescriptor, Hash, OccupiedCoreAssumption,
PersistedValidationData, ValidationCode,
};
use parity_scale_codec::Encode;
use futures::channel::oneshot;
use futures::prelude::*;
use futures::{channel::oneshot, prelude::*};
use std::sync::Arc;
use std::path::PathBuf;
use std::{path::PathBuf, sync::Arc};
use async_trait::async_trait;
@@ -81,7 +79,7 @@ impl CandidateValidationSubsystem {
///
/// Check out [`IsolationStrategy`] to get more details.
pub fn with_config(config: Config, metrics: Metrics) -> Self {
CandidateValidationSubsystem { config, metrics, }
CandidateValidationSubsystem { config, metrics }
}
}
@@ -91,13 +89,11 @@ where
Context: overseer::SubsystemContext<Message = CandidateValidationMessage>,
{
fn start(self, ctx: Context) -> SpawnedSubsystem {
let future = run(ctx, self.metrics, self.config.artifacts_cache_path, self.config.program_path)
.map_err(|e| SubsystemError::with_origin("candidate-validation", e))
.boxed();
SpawnedSubsystem {
name: "candidate-validation-subsystem",
future,
}
let future =
run(ctx, self.metrics, self.config.artifacts_cache_path, self.config.program_path)
.map_err(|e| SubsystemError::with_origin("candidate-validation", e))
.boxed();
SpawnedSubsystem { name: "candidate-validation-subsystem", future }
}
}
@@ -118,8 +114,8 @@ where
loop {
match ctx.recv().await? {
FromOverseer::Signal(OverseerSignal::ActiveLeaves(_)) => {}
FromOverseer::Signal(OverseerSignal::BlockFinalized(..)) => {}
FromOverseer::Signal(OverseerSignal::ActiveLeaves(_)) => {},
FromOverseer::Signal(OverseerSignal::BlockFinalized(..)) => {},
FromOverseer::Signal(OverseerSignal::Conclude) => return Ok(()),
FromOverseer::Communication { msg } => match msg {
CandidateValidationMessage::ValidateFromChainState(
@@ -135,16 +131,17 @@ where
descriptor,
pov,
&metrics,
).await;
)
.await;
match res {
Ok(x) => {
metrics.on_validation_event(&x);
let _ = response_sender.send(x);
}
},
Err(e) => return Err(e),
}
}
},
CandidateValidationMessage::ValidateFromExhaustive(
persisted_validation_data,
validation_code,
@@ -161,7 +158,8 @@ where
descriptor,
pov,
&metrics,
).await;
)
.await;
match res {
Ok(x) => {
@@ -175,8 +173,8 @@ where
},
Err(e) => return Err(e),
}
}
}
},
},
}
}
}
@@ -191,12 +189,7 @@ where
Context: SubsystemContext<Message = CandidateValidationMessage>,
Context: overseer::SubsystemContext<Message = CandidateValidationMessage>,
{
ctx.send_message(
RuntimeApiMessage::Request(
relay_parent,
request,
)
).await;
ctx.send_message(RuntimeApiMessage::Request(relay_parent, request)).await;
receiver.await.map_err(Into::into)
}
@@ -222,44 +215,38 @@ where
let d = runtime_api_request(
ctx,
descriptor.relay_parent,
RuntimeApiRequest::PersistedValidationData(
descriptor.para_id,
assumption,
tx,
),
RuntimeApiRequest::PersistedValidationData(descriptor.para_id, assumption, tx),
rx,
).await?;
)
.await?;
match d {
Ok(None) | Err(_) => {
return Ok(AssumptionCheckOutcome::BadRequest);
}
Ok(None) | Err(_) => return Ok(AssumptionCheckOutcome::BadRequest),
Ok(Some(d)) => d,
}
};
let persisted_validation_data_hash = validation_data.hash();
SubsystemResult::Ok(if descriptor.persisted_validation_data_hash == persisted_validation_data_hash {
let (code_tx, code_rx) = oneshot::channel();
let validation_code = runtime_api_request(
ctx,
descriptor.relay_parent,
RuntimeApiRequest::ValidationCode(
descriptor.para_id,
assumption,
code_tx,
),
code_rx,
).await?;
SubsystemResult::Ok(
if descriptor.persisted_validation_data_hash == persisted_validation_data_hash {
let (code_tx, code_rx) = oneshot::channel();
let validation_code = runtime_api_request(
ctx,
descriptor.relay_parent,
RuntimeApiRequest::ValidationCode(descriptor.para_id, assumption, code_tx),
code_rx,
)
.await?;
match validation_code {
Ok(None) | Err(_) => AssumptionCheckOutcome::BadRequest,
Ok(Some(v)) => AssumptionCheckOutcome::Matches(validation_data, v),
}
} else {
AssumptionCheckOutcome::DoesNotMatch
})
match validation_code {
Ok(None) | Err(_) => AssumptionCheckOutcome::BadRequest,
Ok(Some(v)) => AssumptionCheckOutcome::Matches(validation_data, v),
}
} else {
AssumptionCheckOutcome::DoesNotMatch
},
)
}
async fn find_assumed_validation_data<Context>(
@@ -310,18 +297,16 @@ where
{
let (validation_data, validation_code) =
match find_assumed_validation_data(ctx, &descriptor).await? {
AssumptionCheckOutcome::Matches(validation_data, validation_code) => {
(validation_data, validation_code)
}
AssumptionCheckOutcome::Matches(validation_data, validation_code) =>
(validation_data, validation_code),
AssumptionCheckOutcome::DoesNotMatch => {
// If neither the assumption of the occupied core having the para included or the assumption
// of the occupied core timing out are valid, then the persisted_validation_data_hash in the descriptor
// is not based on the relay parent and is thus invalid.
return Ok(Ok(ValidationResult::Invalid(InvalidCandidate::BadParent)));
}
AssumptionCheckOutcome::BadRequest => {
return Ok(Err(ValidationFailed("Assumption Check: Bad request".into())));
}
return Ok(Ok(ValidationResult::Invalid(InvalidCandidate::BadParent)))
},
AssumptionCheckOutcome::BadRequest =>
return Ok(Err(ValidationFailed("Assumption Check: Bad request".into()))),
};
let validation_result = validate_candidate_exhaustive(
@@ -344,15 +329,10 @@ where
)
.await?
{
Ok(true) => {}
Ok(false) => {
return Ok(Ok(ValidationResult::Invalid(
InvalidCandidate::InvalidOutputs,
)));
}
Err(_) => {
return Ok(Err(ValidationFailed("Check Validation Outputs: Bad request".into())));
}
Ok(true) => {},
Ok(false) => return Ok(Ok(ValidationResult::Invalid(InvalidCandidate::InvalidOutputs))),
Err(_) =>
return Ok(Err(ValidationFailed("Check Validation Outputs: Bad request".into()))),
}
}
@@ -375,7 +355,7 @@ async fn validate_candidate_exhaustive(
&*pov,
&validation_code,
) {
return Ok(Ok(ValidationResult::Invalid(e)));
return Ok(Ok(ValidationResult::Invalid(e)))
}
let raw_validation_code = match sp_maybe_compressed_blob::decompress(
@@ -387,22 +367,20 @@ async fn validate_candidate_exhaustive(
tracing::debug!(target: LOG_TARGET, err=?e, "Invalid validation code");
// If the validation code is invalid, the candidate certainly is.
return Ok(Ok(ValidationResult::Invalid(InvalidCandidate::CodeDecompressionFailure)));
}
return Ok(Ok(ValidationResult::Invalid(InvalidCandidate::CodeDecompressionFailure)))
},
};
let raw_block_data = match sp_maybe_compressed_blob::decompress(
&pov.block_data.0,
POV_BOMB_LIMIT,
) {
Ok(block_data) => BlockData(block_data.to_vec()),
Err(e) => {
tracing::debug!(target: LOG_TARGET, err=?e, "Invalid PoV code");
let raw_block_data =
match sp_maybe_compressed_blob::decompress(&pov.block_data.0, POV_BOMB_LIMIT) {
Ok(block_data) => BlockData(block_data.to_vec()),
Err(e) => {
tracing::debug!(target: LOG_TARGET, err=?e, "Invalid PoV code");
// If the PoV is invalid, the candidate certainly is.
return Ok(Ok(ValidationResult::Invalid(InvalidCandidate::PoVDecompressionFailure)));
}
};
// If the PoV is invalid, the candidate certainly is.
return Ok(Ok(ValidationResult::Invalid(InvalidCandidate::PoVDecompressionFailure)))
},
};
let params = ValidationParams {
parent_head: persisted_validation_data.parent_head.clone(),
@@ -411,11 +389,8 @@ async fn validate_candidate_exhaustive(
relay_parent_storage_root: persisted_validation_data.relay_parent_storage_root,
};
let result =
validation_backend.validate_candidate(
raw_validation_code.to_vec(),
params
)
let result = validation_backend
.validate_candidate(raw_validation_code.to_vec(), params)
.await;
if let Err(ref e) = result {
@@ -434,9 +409,11 @@ async fn validate_candidate_exhaustive(
Err(ValidationError::InvalidCandidate(WasmInvalidCandidate::WorkerReportedError(e))) =>
Ok(ValidationResult::Invalid(InvalidCandidate::ExecutionError(e))),
Err(ValidationError::InvalidCandidate(WasmInvalidCandidate::AmbigiousWorkerDeath)) =>
Ok(ValidationResult::Invalid(InvalidCandidate::ExecutionError("ambigious worker death".to_string()))),
Ok(ValidationResult::Invalid(InvalidCandidate::ExecutionError(
"ambigious worker death".to_string(),
))),
Ok(res) => {
Ok(res) =>
if res.head_data.hash() != descriptor.para_head {
Ok(ValidationResult::Invalid(InvalidCandidate::ParaHeadHashMismatch))
} else {
@@ -449,8 +426,7 @@ async fn validate_candidate_exhaustive(
hrmp_watermark: res.hrmp_watermark,
};
Ok(ValidationResult::Valid(outputs, persisted_validation_data))
}
}
},
};
Ok(result)
@@ -461,7 +437,7 @@ trait ValidationBackend {
async fn validate_candidate(
&mut self,
raw_validation_code: Vec<u8>,
params: ValidationParams
params: ValidationParams,
) -> Result<WasmValidationResult, ValidationError>;
}
@@ -470,16 +446,22 @@ impl ValidationBackend for &'_ mut ValidationHost {
async fn validate_candidate(
&mut self,
raw_validation_code: Vec<u8>,
params: ValidationParams
params: ValidationParams,
) -> Result<WasmValidationResult, ValidationError> {
let (tx, rx) = oneshot::channel();
if let Err(err) = self.execute_pvf(
Pvf::from_code(raw_validation_code),
params.encode(),
polkadot_node_core_pvf::Priority::Normal,
tx,
).await {
return Err(ValidationError::InternalError(format!("cannot send pvf to the validation host: {:?}", err)));
if let Err(err) = self
.execute_pvf(
Pvf::from_code(raw_validation_code),
params.encode(),
polkadot_node_core_pvf::Priority::Normal,
tx,
)
.await
{
return Err(ValidationError::InternalError(format!(
"cannot send pvf to the validation host: {:?}",
err
)))
}
let validation_result = rx
@@ -503,19 +485,19 @@ fn perform_basic_checks(
let encoded_pov_size = pov.encoded_size();
if encoded_pov_size > max_pov_size as usize {
return Err(InvalidCandidate::ParamsTooLarge(encoded_pov_size as u64));
return Err(InvalidCandidate::ParamsTooLarge(encoded_pov_size as u64))
}
if pov_hash != candidate.pov_hash {
return Err(InvalidCandidate::PoVHashMismatch);
return Err(InvalidCandidate::PoVHashMismatch)
}
if validation_code_hash != candidate.validation_code_hash {
return Err(InvalidCandidate::CodeHashMismatch);
return Err(InvalidCandidate::CodeHashMismatch)
}
if let Err(()) = candidate.check_collator_signature() {
return Err(InvalidCandidate::BadSignature);
return Err(InvalidCandidate::BadSignature)
}
Ok(())
@@ -551,18 +533,26 @@ impl Metrics {
}
/// Provide a timer for `validate_from_chain_state` which observes on drop.
fn time_validate_from_chain_state(&self) -> Option<metrics::prometheus::prometheus::HistogramTimer> {
fn time_validate_from_chain_state(
&self,
) -> Option<metrics::prometheus::prometheus::HistogramTimer> {
self.0.as_ref().map(|metrics| metrics.validate_from_chain_state.start_timer())
}
/// Provide a timer for `validate_from_exhaustive` which observes on drop.
fn time_validate_from_exhaustive(&self) -> Option<metrics::prometheus::prometheus::HistogramTimer> {
fn time_validate_from_exhaustive(
&self,
) -> Option<metrics::prometheus::prometheus::HistogramTimer> {
self.0.as_ref().map(|metrics| metrics.validate_from_exhaustive.start_timer())
}
/// Provide a timer for `validate_candidate_exhaustive` which observes on drop.
fn time_validate_candidate_exhaustive(&self) -> Option<metrics::prometheus::prometheus::HistogramTimer> {
self.0.as_ref().map(|metrics| metrics.validate_candidate_exhaustive.start_timer())
fn time_validate_candidate_exhaustive(
&self,
) -> Option<metrics::prometheus::prometheus::HistogramTimer> {
self.0
.as_ref()
.map(|metrics| metrics.validate_candidate_exhaustive.start_timer())
}
}
@@ -580,30 +570,24 @@ impl metrics::Metrics for Metrics {
registry,
)?,
validate_from_chain_state: prometheus::register(
prometheus::Histogram::with_opts(
prometheus::HistogramOpts::new(
"parachain_candidate_validation_validate_from_chain_state",
"Time spent within `candidate_validation::validate_from_chain_state`",
)
)?,
prometheus::Histogram::with_opts(prometheus::HistogramOpts::new(
"parachain_candidate_validation_validate_from_chain_state",
"Time spent within `candidate_validation::validate_from_chain_state`",
))?,
registry,
)?,
validate_from_exhaustive: prometheus::register(
prometheus::Histogram::with_opts(
prometheus::HistogramOpts::new(
"parachain_candidate_validation_validate_from_exhaustive",
"Time spent within `candidate_validation::validate_from_exhaustive`",
)
)?,
prometheus::Histogram::with_opts(prometheus::HistogramOpts::new(
"parachain_candidate_validation_validate_from_exhaustive",
"Time spent within `candidate_validation::validate_from_exhaustive`",
))?,
registry,
)?,
validate_candidate_exhaustive: prometheus::register(
prometheus::Histogram::with_opts(
prometheus::HistogramOpts::new(
"parachain_candidate_validation_validate_candidate_exhaustive",
"Time spent within `candidate_validation::validate_candidate_exhaustive`",
)
)?,
prometheus::Histogram::with_opts(prometheus::HistogramOpts::new(
"parachain_candidate_validation_validate_candidate_exhaustive",
"Time spent within `candidate_validation::validate_candidate_exhaustive`",
))?,
registry,
)?,
};
polkadot/node/core/candidate-validation/src/tests.rs
+46 -90
View File
@@ -15,12 +15,12 @@
// along with Polkadot. If not, see <http://www.gnu.org/licenses/>.
use super::*;
use assert_matches::assert_matches;
use futures::executor;
use polkadot_node_subsystem::messages::AllMessages;
use polkadot_node_subsystem_test_helpers as test_helpers;
use polkadot_primitives::v1::{HeadData, UpwardMessage};
use sp_core::testing::TaskExecutor;
use futures::executor;
use assert_matches::assert_matches;
use sp_keyring::Sr25519Keyring;
fn collator_sign(descriptor: &mut CandidateDescriptor, collator: Sr25519Keyring) {
@@ -54,11 +54,9 @@ fn correctly_checks_included_assumption() {
let pool = TaskExecutor::new();
let (mut ctx, mut ctx_handle) = test_helpers::make_subsystem_context(pool.clone());
let (check_fut, check_result) = check_assumption_validation_data(
&mut ctx,
&candidate,
OccupiedCoreAssumption::Included,
).remote_handle();
let (check_fut, check_result) =
check_assumption_validation_data(&mut ctx, &candidate, OccupiedCoreAssumption::Included)
.remote_handle();
let test_fut = async move {
assert_matches!(
@@ -118,11 +116,9 @@ fn correctly_checks_timed_out_assumption() {
let pool = TaskExecutor::new();
let (mut ctx, mut ctx_handle) = test_helpers::make_subsystem_context(pool.clone());
let (check_fut, check_result) = check_assumption_validation_data(
&mut ctx,
&candidate,
OccupiedCoreAssumption::TimedOut,
).remote_handle();
let (check_fut, check_result) =
check_assumption_validation_data(&mut ctx, &candidate, OccupiedCoreAssumption::TimedOut)
.remote_handle();
let test_fut = async move {
assert_matches!(
@@ -180,11 +176,9 @@ fn check_is_bad_request_if_no_validation_data() {
let pool = TaskExecutor::new();
let (mut ctx, mut ctx_handle) = test_helpers::make_subsystem_context(pool.clone());
let (check_fut, check_result) = check_assumption_validation_data(
&mut ctx,
&candidate,
OccupiedCoreAssumption::Included,
).remote_handle();
let (check_fut, check_result) =
check_assumption_validation_data(&mut ctx, &candidate, OccupiedCoreAssumption::Included)
.remote_handle();
let test_fut = async move {
assert_matches!(
@@ -226,11 +220,9 @@ fn check_is_bad_request_if_no_validation_code() {
let pool = TaskExecutor::new();
let (mut ctx, mut ctx_handle) = test_helpers::make_subsystem_context(pool.clone());
let (check_fut, check_result) = check_assumption_validation_data(
&mut ctx,
&candidate,
OccupiedCoreAssumption::TimedOut,
).remote_handle();
let (check_fut, check_result) =
check_assumption_validation_data(&mut ctx, &candidate, OccupiedCoreAssumption::TimedOut)
.remote_handle();
let test_fut = async move {
assert_matches!(
@@ -284,11 +276,9 @@ fn check_does_not_match() {
let pool = TaskExecutor::new();
let (mut ctx, mut ctx_handle) = test_helpers::make_subsystem_context(pool.clone());
let (check_fut, check_result) = check_assumption_validation_data(
&mut ctx,
&candidate,
OccupiedCoreAssumption::Included,
).remote_handle();
let (check_fut, check_result) =
check_assumption_validation_data(&mut ctx, &candidate, OccupiedCoreAssumption::Included)
.remote_handle();
let test_fut = async move {
assert_matches!(
@@ -321,9 +311,7 @@ struct MockValidatorBackend {
impl MockValidatorBackend {
fn with_hardcoded_result(result: Result<WasmValidationResult, ValidationError>) -> Self {
Self {
result,
}
Self { result }
}
}
@@ -332,7 +320,7 @@ impl ValidationBackend for MockValidatorBackend {
async fn validate_candidate(
&mut self,
_raw_validation_code: Vec<u8>,
_params: ValidationParams
_params: ValidationParams,
) -> Result<WasmValidationResult, ValidationError> {
self.result.clone()
}
@@ -352,12 +340,8 @@ fn candidate_validation_ok_is_ok() {
descriptor.validation_code_hash = validation_code.hash();
collator_sign(&mut descriptor, Sr25519Keyring::Alice);
let check = perform_basic_checks(
&descriptor,
validation_data.max_pov_size,
&pov,
&validation_code,
);
let check =
perform_basic_checks(&descriptor, validation_data.max_pov_size, &pov, &validation_code);
assert!(check.is_ok());
let validation_result = WasmValidationResult {
@@ -402,18 +386,14 @@ fn candidate_validation_bad_return_is_invalid() {
descriptor.validation_code_hash = validation_code.hash();
collator_sign(&mut descriptor, Sr25519Keyring::Alice);
let check = perform_basic_checks(
&descriptor,
validation_data.max_pov_size,
&pov,
&validation_code,
);
let check =
perform_basic_checks(&descriptor, validation_data.max_pov_size, &pov, &validation_code);
assert!(check.is_ok());
let v = executor::block_on(validate_candidate_exhaustive(
MockValidatorBackend::with_hardcoded_result(
Err(ValidationError::InvalidCandidate(WasmInvalidCandidate::AmbigiousWorkerDeath))
),
MockValidatorBackend::with_hardcoded_result(Err(ValidationError::InvalidCandidate(
WasmInvalidCandidate::AmbigiousWorkerDeath,
))),
validation_data,
validation_code,
descriptor,
@@ -438,18 +418,14 @@ fn candidate_validation_timeout_is_internal_error() {
descriptor.validation_code_hash = validation_code.hash();
collator_sign(&mut descriptor, Sr25519Keyring::Alice);
let check = perform_basic_checks(
&descriptor,
validation_data.max_pov_size,
&pov,
&validation_code,
);
let check =
perform_basic_checks(&descriptor, validation_data.max_pov_size, &pov, &validation_code);
assert!(check.is_ok());
let v = executor::block_on(validate_candidate_exhaustive(
MockValidatorBackend::with_hardcoded_result(
Err(ValidationError::InvalidCandidate(WasmInvalidCandidate::HardTimeout)),
),
MockValidatorBackend::with_hardcoded_result(Err(ValidationError::InvalidCandidate(
WasmInvalidCandidate::HardTimeout,
))),
validation_data,
validation_code,
descriptor,
@@ -473,18 +449,14 @@ fn candidate_validation_code_mismatch_is_invalid() {
descriptor.validation_code_hash = ValidationCode(vec![1; 16]).hash();
collator_sign(&mut descriptor, Sr25519Keyring::Alice);
let check = perform_basic_checks(
&descriptor,
validation_data.max_pov_size,
&pov,
&validation_code,
);
let check =
perform_basic_checks(&descriptor, validation_data.max_pov_size, &pov, &validation_code);
assert_matches!(check, Err(InvalidCandidate::CodeHashMismatch));
let v = executor::block_on(validate_candidate_exhaustive(
MockValidatorBackend::with_hardcoded_result(
Err(ValidationError::InvalidCandidate(WasmInvalidCandidate::HardTimeout)),
),
MockValidatorBackend::with_hardcoded_result(Err(ValidationError::InvalidCandidate(
WasmInvalidCandidate::HardTimeout,
))),
validation_data,
validation_code,
descriptor,
@@ -504,10 +476,7 @@ fn compressed_code_works() {
let head_data = HeadData(vec![1, 1, 1]);
let raw_code = vec![2u8; 16];
let validation_code = sp_maybe_compressed_blob::compress(
&raw_code,
VALIDATION_CODE_BOMB_LIMIT,
)
let validation_code = sp_maybe_compressed_blob::compress(&raw_code, VALIDATION_CODE_BOMB_LIMIT)
.map(ValidationCode)
.unwrap();
@@ -546,12 +515,10 @@ fn code_decompression_failure_is_invalid() {
let head_data = HeadData(vec![1, 1, 1]);
let raw_code = vec![2u8; VALIDATION_CODE_BOMB_LIMIT + 1];
let validation_code = sp_maybe_compressed_blob::compress(
&raw_code,
VALIDATION_CODE_BOMB_LIMIT + 1,
)
.map(ValidationCode)
.unwrap();
let validation_code =
sp_maybe_compressed_blob::compress(&raw_code, VALIDATION_CODE_BOMB_LIMIT + 1)
.map(ValidationCode)
.unwrap();
let mut descriptor = CandidateDescriptor::default();
descriptor.pov_hash = pov.hash();
@@ -578,25 +545,17 @@ fn code_decompression_failure_is_invalid() {
))
.unwrap();
assert_matches!(
v,
Ok(ValidationResult::Invalid(InvalidCandidate::CodeDecompressionFailure))
);
assert_matches!(v, Ok(ValidationResult::Invalid(InvalidCandidate::CodeDecompressionFailure)));
}
#[test]
fn pov_decompression_failure_is_invalid() {
let validation_data = PersistedValidationData {
max_pov_size: POV_BOMB_LIMIT as u32,
..Default::default()
};
let validation_data =
PersistedValidationData { max_pov_size: POV_BOMB_LIMIT as u32, ..Default::default() };
let head_data = HeadData(vec![1, 1, 1]);
let raw_block_data = vec![2u8; POV_BOMB_LIMIT + 1];
let pov = sp_maybe_compressed_blob::compress(
&raw_block_data,
POV_BOMB_LIMIT + 1,
)
let pov = sp_maybe_compressed_blob::compress(&raw_block_data, POV_BOMB_LIMIT + 1)
.map(|raw| PoV { block_data: BlockData(raw) })
.unwrap();
@@ -627,8 +586,5 @@ fn pov_decompression_failure_is_invalid() {
))
.unwrap();
assert_matches!(
v,
Ok(ValidationResult::Invalid(InvalidCandidate::PoVDecompressionFailure))
);
assert_matches!(v, Ok(ValidationResult::Invalid(InvalidCandidate::PoVDecompressionFailure)));
}
polkadot/node/core/chain-api/src/lib.rs
+35 -52
View File
@@ -40,9 +40,7 @@ use sp_blockchain::HeaderBackend;
use polkadot_node_subsystem_util::metrics::{self, prometheus};
use polkadot_primitives::v1::{Block, BlockId};
use polkadot_subsystem::{
overseer,
messages::ChainApiMessage,
FromOverseer, OverseerSignal, SpawnedSubsystem,
messages::ChainApiMessage, overseer, FromOverseer, OverseerSignal, SpawnedSubsystem,
SubsystemContext, SubsystemError, SubsystemResult,
};
@@ -60,10 +58,7 @@ pub struct ChainApiSubsystem<Client> {
impl<Client> ChainApiSubsystem<Client> {
/// Create a new Chain API subsystem with the given client.
pub fn new(client: Arc<Client>, metrics: Metrics) -> Self {
ChainApiSubsystem {
client,
metrics,
}
ChainApiSubsystem { client, metrics }
}
}
@@ -77,10 +72,7 @@ where
let future = run::<Client, Context>(ctx, self)
.map_err(|e| SubsystemError::with_origin("chain-api", e))
.boxed();
SpawnedSubsystem {
future,
name: "chain-api-subsystem",
}
SpawnedSubsystem { future, name: "chain-api-subsystem" }
}
}
@@ -107,7 +99,8 @@ where
},
ChainApiMessage::BlockHeader(hash, response_channel) => {
let _timer = subsystem.metrics.time_block_header();
let result = subsystem.client
let result = subsystem
.client
.header(BlockId::Hash(hash))
.map_err(|e| e.to_string().into());
subsystem.metrics.on_request(result.is_ok());
@@ -119,7 +112,7 @@ where
.map_err(|e| e.to_string().into());
subsystem.metrics.on_request(result.is_ok());
let _ = response_channel.send(result);
}
},
ChainApiMessage::FinalizedBlockHash(number, response_channel) => {
let _timer = subsystem.metrics.time_finalized_block_hash();
// Note: we don't verify it's finalized
@@ -158,7 +151,7 @@ where
hash = header.parent_hash;
Some(Ok(hash))
}
}
},
}
});
@@ -166,7 +159,7 @@ where
subsystem.metrics.on_request(result.is_ok());
let _ = response_channel.send(result);
},
}
},
}
}
}
@@ -218,7 +211,9 @@ impl Metrics {
}
/// Provide a timer for `finalized_block_number` which observes on drop.
fn time_finalized_block_number(&self) -> Option<metrics::prometheus::prometheus::HistogramTimer> {
fn time_finalized_block_number(
&self,
) -> Option<metrics::prometheus::prometheus::HistogramTimer> {
self.0.as_ref().map(|metrics| metrics.finalized_block_number.start_timer())
}
@@ -242,57 +237,45 @@ impl metrics::Metrics for Metrics {
registry,
)?,
block_number: prometheus::register(
prometheus::Histogram::with_opts(
prometheus::HistogramOpts::new(
"parachain_chain_api_block_number",
"Time spent within `chain_api::block_number`",
)
)?,
prometheus::Histogram::with_opts(prometheus::HistogramOpts::new(
"parachain_chain_api_block_number",
"Time spent within `chain_api::block_number`",
))?,
registry,
)?,
block_header: prometheus::register(
prometheus::Histogram::with_opts(
prometheus::HistogramOpts::new(
"parachain_chain_api_block_headers",
"Time spent within `chain_api::block_headers`",
)
)?,
prometheus::Histogram::with_opts(prometheus::HistogramOpts::new(
"parachain_chain_api_block_headers",
"Time spent within `chain_api::block_headers`",
))?,
registry,
)?,
block_weight: prometheus::register(
prometheus::Histogram::with_opts(
prometheus::HistogramOpts::new(
"parachain_chain_api_block_weight",
"Time spent within `chain_api::block_weight`",
)
)?,
prometheus::Histogram::with_opts(prometheus::HistogramOpts::new(
"parachain_chain_api_block_weight",
"Time spent within `chain_api::block_weight`",
))?,
registry,
)?,
finalized_block_hash: prometheus::register(
prometheus::Histogram::with_opts(
prometheus::HistogramOpts::new(
"parachain_chain_api_finalized_block_hash",
"Time spent within `chain_api::finalized_block_hash`",
)
)?,
prometheus::Histogram::with_opts(prometheus::HistogramOpts::new(
"parachain_chain_api_finalized_block_hash",
"Time spent within `chain_api::finalized_block_hash`",
))?,
registry,
)?,
finalized_block_number: prometheus::register(
prometheus::Histogram::with_opts(
prometheus::HistogramOpts::new(
"parachain_chain_api_finalized_block_number",
"Time spent within `chain_api::finalized_block_number`",
)
)?,
prometheus::Histogram::with_opts(prometheus::HistogramOpts::new(
"parachain_chain_api_finalized_block_number",
"Time spent within `chain_api::finalized_block_number`",
))?,
registry,
)?,
ancestors: prometheus::register(
prometheus::Histogram::with_opts(
prometheus::HistogramOpts::new(
"parachain_chain_api_ancestors",
"Time spent within `chain_api::ancestors`",
)
)?,
prometheus::Histogram::with_opts(prometheus::HistogramOpts::new(
"parachain_chain_api_ancestors",
"Time spent within `chain_api::ancestors`",
))?,
registry,
)?,
};
polkadot/node/core/chain-api/src/tests.rs
+67 -45
View File
@@ -1,12 +1,12 @@
use super::*;
use std::collections::BTreeMap;
use futures::{future::BoxFuture, channel::oneshot};
use futures::{channel::oneshot, future::BoxFuture};
use parity_scale_codec::Encode;
use std::collections::BTreeMap;
use polkadot_primitives::v1::{Hash, BlockNumber, BlockId, Header};
use polkadot_node_primitives::BlockWeight;
use polkadot_node_subsystem_test_helpers::{make_subsystem_context, TestSubsystemContextHandle};
use polkadot_primitives::v1::{BlockId, BlockNumber, Hash, Header};
use sp_blockchain::Info as BlockInfo;
use sp_core::testing::TaskExecutor;
@@ -79,10 +79,7 @@ impl Default for TestClient {
fn last_key_value<K: Clone, V: Clone>(map: &BTreeMap<K, V>) -> (K, V) {
assert!(!map.is_empty());
map.iter()
.last()
.map(|(k, v)| (k.clone(), v.clone()))
.unwrap()
map.iter().last().map(|(k, v)| (k.clone(), v.clone())).unwrap()
}
impl HeaderBackend<Block> for TestClient {
@@ -110,12 +107,9 @@ impl HeaderBackend<Block> for TestClient {
fn header(&self, id: BlockId) -> sp_blockchain::Result<Option<Header>> {
match id {
// for error path testing
BlockId::Hash(hash) if hash.is_zero() => {
Err(sp_blockchain::Error::Backend("Zero hashes are illegal!".into()))
}
BlockId::Hash(hash) => {
Ok(self.headers.get(&hash).cloned())
}
BlockId::Hash(hash) if hash.is_zero() =>
Err(sp_blockchain::Error::Backend("Zero hashes are illegal!".into())),
BlockId::Hash(hash) => Ok(self.headers.get(&hash).cloned()),
_ => unreachable!(),
}
}
@@ -125,8 +119,10 @@ impl HeaderBackend<Block> for TestClient {
}
fn test_harness(
test: impl FnOnce(Arc<TestClient>, TestSubsystemContextHandle<ChainApiMessage>)
-> BoxFuture<'static, ()>,
test: impl FnOnce(
Arc<TestClient>,
TestSubsystemContextHandle<ChainApiMessage>,
) -> BoxFuture<'static, ()>,
) {
let (ctx, ctx_handle) = make_subsystem_context(TaskExecutor::new());
let client = Arc::new(TestClient::default());
@@ -174,15 +170,18 @@ fn request_block_number() {
for (hash, expected) in &test_cases {
let (tx, rx) = oneshot::channel();
sender.send(FromOverseer::Communication {
msg: ChainApiMessage::BlockNumber(*hash, tx),
}).await;
sender
.send(FromOverseer::Communication {
msg: ChainApiMessage::BlockNumber(*hash, tx),
})
.await;
assert_eq!(rx.await.unwrap().unwrap(), *expected);
}
sender.send(FromOverseer::Signal(OverseerSignal::Conclude)).await;
}.boxed()
}
.boxed()
})
}
@@ -198,15 +197,18 @@ fn request_block_header() {
for (hash, expected) in &test_cases {
let (tx, rx) = oneshot::channel();
sender.send(FromOverseer::Communication {
msg: ChainApiMessage::BlockHeader(*hash, tx),
}).await;
sender
.send(FromOverseer::Communication {
msg: ChainApiMessage::BlockHeader(*hash, tx),
})
.await;
assert_eq!(rx.await.unwrap().unwrap(), *expected);
}
sender.send(FromOverseer::Signal(OverseerSignal::Conclude)).await;
}.boxed()
}
.boxed()
})
}
@@ -223,15 +225,18 @@ fn request_block_weight() {
for (hash, expected) in &test_cases {
let (tx, rx) = oneshot::channel();
sender.send(FromOverseer::Communication {
msg: ChainApiMessage::BlockWeight(*hash, tx),
}).await;
sender
.send(FromOverseer::Communication {
msg: ChainApiMessage::BlockWeight(*hash, tx),
})
.await;
assert_eq!(rx.await.unwrap().unwrap(), *expected);
}
sender.send(FromOverseer::Signal(OverseerSignal::Conclude)).await;
}.boxed()
}
.boxed()
})
}
@@ -246,15 +251,18 @@ fn request_finalized_hash() {
for (number, expected) in &test_cases {
let (tx, rx) = oneshot::channel();
sender.send(FromOverseer::Communication {
msg: ChainApiMessage::FinalizedBlockHash(*number, tx),
}).await;
sender
.send(FromOverseer::Communication {
msg: ChainApiMessage::FinalizedBlockHash(*number, tx),
})
.await;
assert_eq!(rx.await.unwrap().unwrap(), *expected);
}
sender.send(FromOverseer::Signal(OverseerSignal::Conclude)).await;
}.boxed()
}
.boxed()
})
}
@@ -265,14 +273,17 @@ fn request_last_finalized_number() {
let (tx, rx) = oneshot::channel();
let expected = client.info().finalized_number;
sender.send(FromOverseer::Communication {
msg: ChainApiMessage::FinalizedBlockNumber(tx),
}).await;
sender
.send(FromOverseer::Communication {
msg: ChainApiMessage::FinalizedBlockNumber(tx),
})
.await;
assert_eq!(rx.await.unwrap().unwrap(), expected);
sender.send(FromOverseer::Signal(OverseerSignal::Conclude)).await;
}.boxed()
}
.boxed()
})
}
@@ -281,24 +292,35 @@ fn request_ancestors() {
test_harness(|_client, mut sender| {
async move {
let (tx, rx) = oneshot::channel();
sender.send(FromOverseer::Communication {
msg: ChainApiMessage::Ancestors { hash: THREE, k: 4, response_channel: tx },
}).await;
sender
.send(FromOverseer::Communication {
msg: ChainApiMessage::Ancestors { hash: THREE, k: 4, response_channel: tx },
})
.await;
assert_eq!(rx.await.unwrap().unwrap(), vec![TWO, ONE]);
let (tx, rx) = oneshot::channel();
sender.send(FromOverseer::Communication {
msg: ChainApiMessage::Ancestors { hash: TWO, k: 1, response_channel: tx },
}).await;
sender
.send(FromOverseer::Communication {
msg: ChainApiMessage::Ancestors { hash: TWO, k: 1, response_channel: tx },
})
.await;
assert_eq!(rx.await.unwrap().unwrap(), vec![ONE]);
let (tx, rx) = oneshot::channel();
sender.send(FromOverseer::Communication {
msg: ChainApiMessage::Ancestors { hash: ERROR_PATH, k: 2, response_channel: tx },
}).await;
sender
.send(FromOverseer::Communication {
msg: ChainApiMessage::Ancestors {
hash: ERROR_PATH,
k: 2,
response_channel: tx,
},
})
.await;
assert!(rx.await.unwrap().is_err());
sender.send(FromOverseer::Signal(OverseerSignal::Conclude)).await;
}.boxed()
}
.boxed()
})
}
polkadot/node/core/chain-selection/src/backend.rs
+15 -14
View File
@@ -25,7 +25,7 @@ use polkadot_primitives::v1::{BlockNumber, Hash};
use std::collections::HashMap;
use crate::{Error, LeafEntrySet, BlockEntry, Timestamp};
use crate::{BlockEntry, Error, LeafEntrySet, Timestamp};
pub(super) enum BackendWriteOp {
WriteBlockEntry(BlockEntry),
@@ -47,8 +47,10 @@ pub(super) trait Backend {
fn load_stagnant_at(&self, timestamp: Timestamp) -> Result<Vec<Hash>, Error>;
/// Load all stagnant lists up to and including the given Unix timestamp
/// in ascending order.
fn load_stagnant_at_up_to(&self, up_to: Timestamp)
-> Result<Vec<(Timestamp, Vec<Hash>)>, Error>;
fn load_stagnant_at_up_to(
&self,
up_to: Timestamp,
) -> Result<Vec<(Timestamp, Vec<Hash>)>, Error>;
/// Load the earliest kept block number.
fn load_first_block_number(&self) -> Result<Option<BlockNumber>, Error>;
/// Load blocks by number.
@@ -56,7 +58,8 @@ pub(super) trait Backend {
/// Atomically write the list of operations, with later operations taking precedence over prior.
fn write<I>(&mut self, ops: I) -> Result<(), Error>
where I: IntoIterator<Item = BackendWriteOp>;
where
I: IntoIterator<Item = BackendWriteOp>;
}
/// An in-memory overlay over the backend.
@@ -98,7 +101,7 @@ impl<'a, B: 'a + Backend> OverlayedBackend<'a, B> {
pub(super) fn load_blocks_by_number(&self, number: BlockNumber) -> Result<Vec<Hash>, Error> {
if let Some(val) = self.blocks_by_number.get(&number) {
return Ok(val.as_ref().map_or(Vec::new(), Clone::clone));
return Ok(val.as_ref().map_or(Vec::new(), Clone::clone))
}
self.inner.load_blocks_by_number(number)
@@ -114,7 +117,7 @@ impl<'a, B: 'a + Backend> OverlayedBackend<'a, B> {
pub(super) fn load_stagnant_at(&self, timestamp: Timestamp) -> Result<Vec<Hash>, Error> {
if let Some(val) = self.stagnant_at.get(&timestamp) {
return Ok(val.as_ref().map_or(Vec::new(), Clone::clone));
return Ok(val.as_ref().map_or(Vec::new(), Clone::clone))
}
self.inner.load_stagnant_at(timestamp)
@@ -188,17 +191,15 @@ impl<'a, B: 'a + Backend> OverlayedBackend<'a, B> {
/// return true if `ancestor` is in `head`'s chain.
///
/// If the ancestor is an older finalized block, this will return `false`.
fn contains_ancestor(
backend: &impl Backend,
head: Hash,
ancestor: Hash,
) -> Result<bool, Error> {
fn contains_ancestor(backend: &impl Backend, head: Hash, ancestor: Hash) -> Result<bool, Error> {
let mut current_hash = head;
loop {
if current_hash == ancestor { return Ok(true) }
if current_hash == ancestor {
return Ok(true)
}
match backend.load_block_entry(&current_hash)? {
Some(e) => { current_hash = e.parent_hash }
None => break
Some(e) => current_hash = e.parent_hash,
None => break,
}
}
polkadot/node/core/chain-selection/src/db_backend/v1.rs
+113 -177
View File
@@ -32,14 +32,16 @@
//! The `Vec`s stored are always non-empty. Empty `Vec`s are not stored on disk so there is no
//! semantic difference between `None` and an empty `Vec`.
use crate::backend::{Backend, BackendWriteOp};
use crate::Error;
use crate::{
backend::{Backend, BackendWriteOp},
Error,
};
use polkadot_primitives::v1::{BlockNumber, Hash};
use polkadot_node_primitives::BlockWeight;
use polkadot_primitives::v1::{BlockNumber, Hash};
use kvdb::{DBTransaction, KeyValueDB};
use parity_scale_codec::{Encode, Decode};
use parity_scale_codec::{Decode, Encode};
use std::sync::Arc;
@@ -116,11 +118,7 @@ struct LeafEntry {
impl From<crate::LeafEntry> for LeafEntry {
fn from(x: crate::LeafEntry) -> Self {
LeafEntry {
weight: x.weight,
block_number: x.block_number,
block_hash: x.block_hash,
}
LeafEntry { weight: x.weight, block_number: x.block_number, block_hash: x.block_hash }
}
}
@@ -141,17 +139,13 @@ struct LeafEntrySet {
impl From<crate::LeafEntrySet> for LeafEntrySet {
fn from(x: crate::LeafEntrySet) -> Self {
LeafEntrySet {
inner: x.inner.into_iter().map(Into::into).collect(),
}
LeafEntrySet { inner: x.inner.into_iter().map(Into::into).collect() }
}
}
impl From<LeafEntrySet> for crate::LeafEntrySet {
fn from(x: LeafEntrySet) -> crate::LeafEntrySet {
crate::LeafEntrySet {
inner: x.inner.into_iter().map(Into::into).collect(),
}
crate::LeafEntrySet { inner: x.inner.into_iter().map(Into::into).collect() }
}
}
@@ -208,28 +202,19 @@ impl DbBackend {
/// Create a new [`DbBackend`] with the supplied key-value store and
/// config.
pub fn new(db: Arc<dyn KeyValueDB>, config: Config) -> Self {
DbBackend {
inner: db,
config,
}
DbBackend { inner: db, config }
}
}
impl Backend for DbBackend {
fn load_block_entry(&self, hash: &Hash) -> Result<Option<crate::BlockEntry>, Error> {
load_decode::<BlockEntry>(
&*self.inner,
self.config.col_data,
&block_entry_key(hash),
).map(|o| o.map(Into::into))
load_decode::<BlockEntry>(&*self.inner, self.config.col_data, &block_entry_key(hash))
.map(|o| o.map(Into::into))
}
fn load_leaves(&self) -> Result<crate::LeafEntrySet, Error> {
load_decode::<LeafEntrySet>(
&*self.inner,
self.config.col_data,
LEAVES_KEY,
).map(|o| o.map(Into::into).unwrap_or_default())
load_decode::<LeafEntrySet>(&*self.inner, self.config.col_data, LEAVES_KEY)
.map(|o| o.map(Into::into).unwrap_or_default())
}
fn load_stagnant_at(&self, timestamp: crate::Timestamp) -> Result<Vec<Hash>, Error> {
@@ -237,16 +222,16 @@ impl Backend for DbBackend {
&*self.inner,
self.config.col_data,
&stagnant_at_key(timestamp.into()),
).map(|o| o.unwrap_or_default())
)
.map(|o| o.unwrap_or_default())
}
fn load_stagnant_at_up_to(&self, up_to: crate::Timestamp)
-> Result<Vec<(crate::Timestamp, Vec<Hash>)>, Error>
{
let stagnant_at_iter = self.inner.iter_with_prefix(
self.config.col_data,
&STAGNANT_AT_PREFIX[..],
);
fn load_stagnant_at_up_to(
&self,
up_to: crate::Timestamp,
) -> Result<Vec<(crate::Timestamp, Vec<Hash>)>, Error> {
let stagnant_at_iter =
self.inner.iter_with_prefix(self.config.col_data, &STAGNANT_AT_PREFIX[..]);
let val = stagnant_at_iter
.filter_map(|(k, v)| {
@@ -262,10 +247,8 @@ impl Backend for DbBackend {
}
fn load_first_block_number(&self) -> Result<Option<BlockNumber>, Error> {
let blocks_at_height_iter = self.inner.iter_with_prefix(
self.config.col_data,
&BLOCK_HEIGHT_PREFIX[..],
);
let blocks_at_height_iter =
self.inner.iter_with_prefix(self.config.col_data, &BLOCK_HEIGHT_PREFIX[..]);
let val = blocks_at_height_iter
.filter_map(|(k, _)| decode_block_height_key(&k[..]))
@@ -275,16 +258,14 @@ impl Backend for DbBackend {
}
fn load_blocks_by_number(&self, number: BlockNumber) -> Result<Vec<Hash>, Error> {
load_decode::<Vec<Hash>>(
&*self.inner,
self.config.col_data,
&block_height_key(number),
).map(|o| o.unwrap_or_default())
load_decode::<Vec<Hash>>(&*self.inner, self.config.col_data, &block_height_key(number))
.map(|o| o.unwrap_or_default())
}
/// Atomically write the list of operations, with later operations taking precedence over prior.
fn write<I>(&mut self, ops: I) -> Result<(), Error>
where I: IntoIterator<Item = BackendWriteOp>
where
I: IntoIterator<Item = BackendWriteOp>,
{
let mut tx = DBTransaction::new();
for op in ops {
@@ -296,43 +277,29 @@ impl Backend for DbBackend {
&block_entry_key(&block_entry.block_hash),
block_entry.encode(),
);
}
BackendWriteOp::WriteBlocksByNumber(block_number, v) => {
},
BackendWriteOp::WriteBlocksByNumber(block_number, v) =>
if v.is_empty() {
tx.delete(
self.config.col_data,
&block_height_key(block_number),
);
tx.delete(self.config.col_data, &block_height_key(block_number));
} else {
tx.put_vec(
self.config.col_data,
&block_height_key(block_number),
v.encode(),
);
}
}
},
BackendWriteOp::WriteViableLeaves(leaves) => {
let leaves: LeafEntrySet = leaves.into();
if leaves.inner.is_empty() {
tx.delete(
self.config.col_data,
&LEAVES_KEY[..],
);
tx.delete(self.config.col_data, &LEAVES_KEY[..]);
} else {
tx.put_vec(
self.config.col_data,
&LEAVES_KEY[..],
leaves.encode(),
);
tx.put_vec(self.config.col_data, &LEAVES_KEY[..], leaves.encode());
}
}
},
BackendWriteOp::WriteStagnantAt(timestamp, stagnant_at) => {
let timestamp: Timestamp = timestamp.into();
if stagnant_at.is_empty() {
tx.delete(
self.config.col_data,
&stagnant_at_key(timestamp),
);
tx.delete(self.config.col_data, &stagnant_at_key(timestamp));
} else {
tx.put_vec(
self.config.col_data,
@@ -340,26 +307,17 @@ impl Backend for DbBackend {
stagnant_at.encode(),
);
}
}
},
BackendWriteOp::DeleteBlocksByNumber(block_number) => {
tx.delete(
self.config.col_data,
&block_height_key(block_number),
);
}
tx.delete(self.config.col_data, &block_height_key(block_number));
},
BackendWriteOp::DeleteBlockEntry(hash) => {
tx.delete(
self.config.col_data,
&block_entry_key(&hash),
);
}
tx.delete(self.config.col_data, &block_entry_key(&hash));
},
BackendWriteOp::DeleteStagnantAt(timestamp) => {
let timestamp: Timestamp = timestamp.into();
tx.delete(
self.config.col_data,
&stagnant_at_key(timestamp),
);
}
tx.delete(self.config.col_data, &stagnant_at_key(timestamp));
},
}
}
@@ -374,9 +332,7 @@ fn load_decode<D: Decode>(
) -> Result<Option<D>, Error> {
match db.get(col_data, key)? {
None => Ok(None),
Some(raw) => D::decode(&mut &raw[..])
.map(Some)
.map_err(Into::into),
Some(raw) => D::decode(&mut &raw[..]).map(Some).map_err(Into::into),
}
}
@@ -402,8 +358,12 @@ fn stagnant_at_key(timestamp: Timestamp) -> [u8; 14 + 8] {
}
fn decode_block_height_key(key: &[u8]) -> Option<BlockNumber> {
if key.len() != 15 + 4 { return None }
if !key.starts_with(BLOCK_HEIGHT_PREFIX) { return None }
if key.len() != 15 + 4 {
return None
}
if !key.starts_with(BLOCK_HEIGHT_PREFIX) {
return None
}
let mut bytes = [0; 4];
bytes.copy_from_slice(&key[15..]);
@@ -411,8 +371,12 @@ fn decode_block_height_key(key: &[u8]) -> Option<BlockNumber> {
}
fn decode_stagnant_at_key(key: &[u8]) -> Option<Timestamp> {
if key.len() != 14 + 8 { return None }
if !key.starts_with(STAGNANT_AT_PREFIX) { return None }
if key.len() != 14 + 8 {
return None
}
if !key.starts_with(STAGNANT_AT_PREFIX) {
return None
}
let mut bytes = [0; 8];
bytes.copy_from_slice(&key[14..]);
@@ -479,9 +443,9 @@ mod tests {
weight: 100,
};
backend.write(vec![
BackendWriteOp::WriteBlockEntry(block_entry.clone().into())
]).unwrap();
backend
.write(vec![BackendWriteOp::WriteBlockEntry(block_entry.clone().into())])
.unwrap();
assert_eq!(
backend.load_block_entry(&block_entry.block_hash).unwrap().map(BlockEntry::from),
@@ -509,17 +473,15 @@ mod tests {
weight: 100,
};
backend.write(vec![
BackendWriteOp::WriteBlockEntry(block_entry.clone().into())
]).unwrap();
backend
.write(vec![BackendWriteOp::WriteBlockEntry(block_entry.clone().into())])
.unwrap();
backend.write(vec![
BackendWriteOp::DeleteBlockEntry(block_entry.block_hash),
]).unwrap();
backend
.write(vec![BackendWriteOp::DeleteBlockEntry(block_entry.block_hash)])
.unwrap();
assert!(
backend.load_block_entry(&block_entry.block_hash).unwrap().is_none(),
);
assert!(backend.load_block_entry(&block_entry.block_hash).unwrap().is_none(),);
}
#[test]
@@ -529,30 +491,26 @@ mod tests {
let mut backend = DbBackend::new(db, config);
assert!(
backend.load_first_block_number().unwrap().is_none(),
);
assert!(backend.load_first_block_number().unwrap().is_none(),);
backend.write(vec![
BackendWriteOp::WriteBlocksByNumber(2, vec![Hash::repeat_byte(0)]),
BackendWriteOp::WriteBlocksByNumber(5, vec![Hash::repeat_byte(0)]),
BackendWriteOp::WriteBlocksByNumber(10, vec![Hash::repeat_byte(0)]),
]).unwrap();
backend
.write(vec![
BackendWriteOp::WriteBlocksByNumber(2, vec![Hash::repeat_byte(0)]),
BackendWriteOp::WriteBlocksByNumber(5, vec![Hash::repeat_byte(0)]),
BackendWriteOp::WriteBlocksByNumber(10, vec![Hash::repeat_byte(0)]),
])
.unwrap();
assert_eq!(
backend.load_first_block_number().unwrap(),
Some(2),
);
assert_eq!(backend.load_first_block_number().unwrap(), Some(2),);
backend.write(vec![
BackendWriteOp::WriteBlocksByNumber(2, vec![]),
BackendWriteOp::DeleteBlocksByNumber(5),
]).unwrap();
backend
.write(vec![
BackendWriteOp::WriteBlocksByNumber(2, vec![]),
BackendWriteOp::DeleteBlocksByNumber(5),
])
.unwrap();
assert_eq!(
backend.load_first_block_number().unwrap(),
Some(10),
);
assert_eq!(backend.load_first_block_number().unwrap(), Some(10),);
}
#[test]
@@ -563,15 +521,15 @@ mod tests {
let mut backend = DbBackend::new(db, config);
// Prove that it's cheap
assert!(
backend.load_stagnant_at_up_to(Timestamp::max_value()).unwrap().is_empty(),
);
assert!(backend.load_stagnant_at_up_to(Timestamp::max_value()).unwrap().is_empty(),);
backend.write(vec![
BackendWriteOp::WriteStagnantAt(2, vec![Hash::repeat_byte(1)]),
BackendWriteOp::WriteStagnantAt(5, vec![Hash::repeat_byte(2)]),
BackendWriteOp::WriteStagnantAt(10, vec![Hash::repeat_byte(3)]),
]).unwrap();
backend
.write(vec![
BackendWriteOp::WriteStagnantAt(2, vec![Hash::repeat_byte(1)]),
BackendWriteOp::WriteStagnantAt(5, vec![Hash::repeat_byte(2)]),
BackendWriteOp::WriteStagnantAt(10, vec![Hash::repeat_byte(3)]),
])
.unwrap();
assert_eq!(
backend.load_stagnant_at_up_to(Timestamp::max_value()).unwrap(),
@@ -593,32 +551,21 @@ mod tests {
assert_eq!(
backend.load_stagnant_at_up_to(9).unwrap(),
vec![
(2, vec![Hash::repeat_byte(1)]),
(5, vec![Hash::repeat_byte(2)]),
]
vec![(2, vec![Hash::repeat_byte(1)]), (5, vec![Hash::repeat_byte(2)]),]
);
backend.write(vec![
BackendWriteOp::DeleteStagnantAt(2),
]).unwrap();
backend.write(vec![BackendWriteOp::DeleteStagnantAt(2)]).unwrap();
assert_eq!(
backend.load_stagnant_at_up_to(5).unwrap(),
vec![
(5, vec![Hash::repeat_byte(2)]),
]
vec![(5, vec![Hash::repeat_byte(2)]),]
);
backend.write(vec![
BackendWriteOp::WriteStagnantAt(5, vec![]),
]).unwrap();
backend.write(vec![BackendWriteOp::WriteStagnantAt(5, vec![])]).unwrap();
assert_eq!(
backend.load_stagnant_at_up_to(10).unwrap(),
vec![
(10, vec![Hash::repeat_byte(3)]),
]
vec![(10, vec![Hash::repeat_byte(3)]),]
);
}
@@ -629,40 +576,29 @@ mod tests {
let mut backend = DbBackend::new(db, config);
backend.write(vec![
BackendWriteOp::WriteBlocksByNumber(2, vec![Hash::repeat_byte(1)]),
BackendWriteOp::WriteBlocksByNumber(5, vec![Hash::repeat_byte(2)]),
BackendWriteOp::WriteBlocksByNumber(10, vec![Hash::repeat_byte(3)]),
]).unwrap();
backend
.write(vec![
BackendWriteOp::WriteBlocksByNumber(2, vec![Hash::repeat_byte(1)]),
BackendWriteOp::WriteBlocksByNumber(5, vec![Hash::repeat_byte(2)]),
BackendWriteOp::WriteBlocksByNumber(10, vec![Hash::repeat_byte(3)]),
])
.unwrap();
assert_eq!(
backend.load_blocks_by_number(2).unwrap(),
vec![Hash::repeat_byte(1)],
);
assert_eq!(backend.load_blocks_by_number(2).unwrap(), vec![Hash::repeat_byte(1)],);
assert_eq!(
backend.load_blocks_by_number(3).unwrap(),
vec![],
);
assert_eq!(backend.load_blocks_by_number(3).unwrap(), vec![],);
backend.write(vec![
BackendWriteOp::WriteBlocksByNumber(2, vec![]),
BackendWriteOp::DeleteBlocksByNumber(5),
]).unwrap();
backend
.write(vec![
BackendWriteOp::WriteBlocksByNumber(2, vec![]),
BackendWriteOp::DeleteBlocksByNumber(5),
])
.unwrap();
assert_eq!(
backend.load_blocks_by_number(2).unwrap(),
vec![],
);
assert_eq!(backend.load_blocks_by_number(2).unwrap(), vec![],);
assert_eq!(
backend.load_blocks_by_number(5).unwrap(),
vec![],
);
assert_eq!(backend.load_blocks_by_number(5).unwrap(), vec![],);
assert_eq!(
backend.load_blocks_by_number(10).unwrap(),
vec![Hash::repeat_byte(3)],
);
assert_eq!(backend.load_blocks_by_number(10).unwrap(), vec![Hash::repeat_byte(3)],);
}
}
polkadot/node/core/chain-selection/src/lib.rs
+58 -91
View File
@@ -16,25 +16,24 @@
//! Implements the Chain Selection Subsystem.
use polkadot_primitives::v1::{BlockNumber, Hash, Header, ConsensusLog};
use polkadot_node_primitives::BlockWeight;
use polkadot_node_subsystem::{
overseer, SubsystemContext, SubsystemError, SpawnedSubsystem,
OverseerSignal, FromOverseer,
messages::{ChainSelectionMessage, ChainApiMessage},
errors::ChainApiError,
messages::{ChainApiMessage, ChainSelectionMessage},
overseer, FromOverseer, OverseerSignal, SpawnedSubsystem, SubsystemContext, SubsystemError,
};
use polkadot_primitives::v1::{BlockNumber, ConsensusLog, Hash, Header};
use futures::{channel::oneshot, future::Either, prelude::*};
use kvdb::KeyValueDB;
use parity_scale_codec::Error as CodecError;
use futures::channel::oneshot;
use futures::future::Either;
use futures::prelude::*;
use std::time::{UNIX_EPOCH, Duration,SystemTime};
use std::sync::Arc;
use std::{
sync::Arc,
time::{Duration, SystemTime, UNIX_EPOCH},
};
use crate::backend::{Backend, OverlayedBackend, BackendWriteOp};
use crate::backend::{Backend, BackendWriteOp, OverlayedBackend};
mod backend;
mod db_backend;
@@ -108,16 +107,19 @@ struct LeafEntry {
impl PartialOrd for LeafEntry {
fn partial_cmp(&self, other: &Self) -> Option<std::cmp::Ordering> {
let ord = self.weight.cmp(&other.weight)
.then(self.block_number.cmp(&other.block_number));
let ord = self.weight.cmp(&other.weight).then(self.block_number.cmp(&other.block_number));
if !matches!(ord, std::cmp::Ordering::Equal) { Some(ord) } else { None }
if !matches!(ord, std::cmp::Ordering::Equal) {
Some(ord)
} else {
None
}
}
}
#[derive(Debug, Default, Clone)]
struct LeafEntrySet {
inner: Vec<LeafEntry>
inner: Vec<LeafEntry>,
}
impl LeafEntrySet {
@@ -127,14 +129,16 @@ impl LeafEntrySet {
Some(i) => {
self.inner.remove(i);
true
}
},
}
}
fn insert(&mut self, new: LeafEntry) {
let mut pos = None;
for (i, e) in self.inner.iter().enumerate() {
if e == &new { return }
if e == &new {
return
}
if e < &new {
pos = Some(i);
break
@@ -238,7 +242,7 @@ impl Clock for SystemClock {
);
0
}
},
}
}
}
@@ -309,10 +313,7 @@ impl ChainSelectionSubsystem {
/// Create a new instance of the subsystem with the given config
/// and key-value store.
pub fn new(config: Config, db: Arc<dyn KeyValueDB>) -> Self {
ChainSelectionSubsystem {
config,
db,
}
ChainSelectionSubsystem { config, db }
}
}
@@ -328,12 +329,7 @@ where
);
SpawnedSubsystem {
future: run(
ctx,
backend,
self.config.stagnant_check_interval,
Box::new(SystemClock),
)
future: run(ctx, backend, self.config.stagnant_check_interval, Box::new(SystemClock))
.map(Ok)
.boxed(),
name: "chain-selection-subsystem",
@@ -346,31 +342,25 @@ async fn run<Context, B>(
mut backend: B,
stagnant_check_interval: StagnantCheckInterval,
clock: Box<dyn Clock + Send + Sync>,
)
where
Context: SubsystemContext<Message = ChainSelectionMessage>,
Context: overseer::SubsystemContext<Message = ChainSelectionMessage>,
B: Backend,
) where
Context: SubsystemContext<Message = ChainSelectionMessage>,
Context: overseer::SubsystemContext<Message = ChainSelectionMessage>,
B: Backend,
{
loop {
let res = run_iteration(
&mut ctx,
&mut backend,
&stagnant_check_interval,
&*clock,
).await;
let res = run_iteration(&mut ctx, &mut backend, &stagnant_check_interval, &*clock).await;
match res {
Err(e) => {
e.trace();
if let Error::Subsystem(SubsystemError::Context(_)) = e {
break;
break
}
}
},
Ok(()) => {
tracing::info!(target: LOG_TARGET, "received `Conclude` signal, exiting");
break;
}
break
},
}
}
}
@@ -385,12 +375,11 @@ async fn run_iteration<Context, B>(
backend: &mut B,
stagnant_check_interval: &StagnantCheckInterval,
clock: &(dyn Clock + Sync),
)
-> Result<(), Error>
where
Context: SubsystemContext<Message = ChainSelectionMessage>,
Context: overseer::SubsystemContext<Message = ChainSelectionMessage>,
B: Backend,
) -> Result<(), Error>
where
Context: SubsystemContext<Message = ChainSelectionMessage>,
Context: overseer::SubsystemContext<Message = ChainSelectionMessage>,
B: Backend,
{
let mut stagnant_check_stream = stagnant_check_interval.timeout_stream();
loop {
@@ -461,15 +450,11 @@ async fn fetch_finalized(
match hash_rx.await?? {
None => {
tracing::warn!(
target: LOG_TARGET,
number,
"Missing hash for finalized block number"
);
tracing::warn!(target: LOG_TARGET, number, "Missing hash for finalized block number");
return Ok(None)
}
Some(h) => Ok(Some((h, number)))
},
Some(h) => Ok(Some((h, number))),
}
}
@@ -512,13 +497,9 @@ async fn handle_active_leaf(
let header = match fetch_header(ctx, hash).await? {
None => {
tracing::warn!(
target: LOG_TARGET,
?hash,
"Missing header for new head",
);
tracing::warn!(target: LOG_TARGET, ?hash, "Missing header for new head",);
return Ok(Vec::new())
}
},
Some(h) => h,
};
@@ -528,7 +509,8 @@ async fn handle_active_leaf(
hash,
&header,
lower_bound,
).await?;
)
.await?;
let mut overlay = OverlayedBackend::new(backend);
@@ -545,8 +527,8 @@ async fn handle_active_leaf(
// If we don't know the weight, we can't import the block.
// And none of its descendants either.
break;
}
break
},
Some(w) => w,
};
@@ -570,7 +552,9 @@ async fn handle_active_leaf(
// Ignores logs with number >= the block header number.
fn extract_reversion_logs(header: &Header) -> Vec<BlockNumber> {
let number = header.number;
let mut logs = header.digest.logs()
let mut logs = header
.digest
.logs()
.iter()
.enumerate()
.filter_map(|(i, d)| match ConsensusLog::from_digest_item(d) {
@@ -584,7 +568,7 @@ fn extract_reversion_logs(header: &Header) -> Vec<BlockNumber> {
);
None
}
},
Ok(Some(ConsensusLog::Revert(b))) if b < number => Some(b),
Ok(Some(ConsensusLog::Revert(b))) => {
tracing::warn!(
@@ -596,7 +580,7 @@ fn extract_reversion_logs(header: &Header) -> Vec<BlockNumber> {
);
None
}
},
Ok(_) => None,
})
.collect::<Vec<_>>();
@@ -612,27 +596,18 @@ fn handle_finalized_block(
finalized_hash: Hash,
finalized_number: BlockNumber,
) -> Result<(), Error> {
let ops = crate::tree::finalize_block(
&*backend,
finalized_hash,
finalized_number,
)?.into_write_ops();
let ops =
crate::tree::finalize_block(&*backend, finalized_hash, finalized_number)?.into_write_ops();
backend.write(ops)
}
// Handle an approved block event.
fn handle_approved_block(
backend: &mut impl Backend,
approved_block: Hash,
) -> Result<(), Error> {
fn handle_approved_block(backend: &mut impl Backend, approved_block: Hash) -> Result<(), Error> {
let ops = {
let mut overlay = OverlayedBackend::new(&*backend);
crate::tree::approve_block(
&mut overlay,
approved_block,
)?;
crate::tree::approve_block(&mut overlay, approved_block)?;
overlay.into_write_ops()
};
@@ -640,15 +615,9 @@ fn handle_approved_block(
backend.write(ops)
}
fn detect_stagnant(
backend: &mut impl Backend,
now: Timestamp,
) -> Result<(), Error> {
fn detect_stagnant(backend: &mut impl Backend, now: Timestamp) -> Result<(), Error> {
let ops = {
let overlay = crate::tree::detect_stagnant(
&*backend,
now,
)?;
let overlay = crate::tree::detect_stagnant(&*backend, now)?;
overlay.into_write_ops()
};
@@ -662,9 +631,7 @@ async fn load_leaves(
ctx: &mut impl SubsystemContext,
backend: &impl Backend,
) -> Result<Vec<Hash>, Error> {
let leaves: Vec<_> = backend.load_leaves()?
.into_hashes_descending()
.collect();
let leaves: Vec<_> = backend.load_leaves()?.into_hashes_descending().collect();
if leaves.is_empty() {
Ok(fetch_finalized(ctx).await?.map_or(Vec::new(), |(h, _)| vec![h]))
polkadot/node/core/chain-selection/src/tests.rs
+378 -670
View File
File diff suppressed because it is too large Load Diff
polkadot/node/core/chain-selection/src/tree.rs
+60 -80
View File
@@ -23,17 +23,12 @@
//! Each direct descendant of the finalized block acts as its own sub-tree,
//! and as the finalized block advances, orphaned sub-trees are entirely pruned.
use polkadot_primitives::v1::{BlockNumber, Hash};
use polkadot_node_primitives::BlockWeight;
use polkadot_primitives::v1::{BlockNumber, Hash};
use std::collections::HashMap;
use super::{
LOG_TARGET,
Approval, BlockEntry, Error, LeafEntry, ViabilityCriteria,
Timestamp,
};
use super::{Approval, BlockEntry, Error, LeafEntry, Timestamp, ViabilityCriteria, LOG_TARGET};
use crate::backend::{Backend, OverlayedBackend};
// A viability update to be applied to a block.
@@ -43,10 +38,7 @@ impl ViabilityUpdate {
// Apply the viability update to a single block, yielding the updated
// block entry along with a vector of children and the updates to apply
// to them.
fn apply(self, mut entry: BlockEntry) -> (
BlockEntry,
Vec<(Hash, ViabilityUpdate)>
) {
fn apply(self, mut entry: BlockEntry) -> (BlockEntry, Vec<(Hash, ViabilityUpdate)>) {
// 1. When an ancestor has changed from unviable to viable,
// we erase the `earliest_unviable_ancestor` of all descendants
// until encountering a explicitly unviable descendant D.
@@ -81,7 +73,9 @@ impl ViabilityUpdate {
};
entry.viability.earliest_unviable_ancestor = maybe_earliest_unviable;
let recurse = entry.children.iter()
let recurse = entry
.children
.iter()
.cloned()
.map(move |c| (c, ViabilityUpdate(next_earliest_unviable)))
.collect();
@@ -152,10 +146,10 @@ fn propagate_viability_update(
"Missing expected block entry"
);
continue;
}
continue
},
Some(entry) => entry,
}
},
};
let (new_entry, children) = update.apply(entry);
@@ -184,9 +178,8 @@ fn propagate_viability_update(
backend.write_block_entry(new_entry);
tree_frontier.extend(
children.into_iter().map(|(h, update)| (BlockEntryRef::Hash(h), update))
);
tree_frontier
.extend(children.into_iter().map(|(h, update)| (BlockEntryRef::Hash(h), update)));
}
// Revisit the viability pivots now that we've traversed the entire subtree.
@@ -229,12 +222,11 @@ fn propagate_viability_update(
// Furthermore, if the set of viable leaves is empty, the
// finalized block is implicitly the viable leaf.
continue
}
Some(entry) => {
},
Some(entry) =>
if entry.children.len() == pivot_count {
viable_leaves.insert(entry.leaf_entry());
}
}
},
}
}
@@ -254,12 +246,7 @@ pub(crate) fn import_block(
stagnant_at: Timestamp,
) -> Result<(), Error> {
add_block(backend, block_hash, block_number, parent_hash, weight, stagnant_at)?;
apply_reversions(
backend,
block_hash,
block_number,
reversion_logs,
)?;
apply_reversions(backend, block_hash, block_number, reversion_logs)?;
Ok(())
}
@@ -276,7 +263,9 @@ fn load_ancestor(
block_number: BlockNumber,
ancestor_number: BlockNumber,
) -> Result<Option<BlockEntry>, Error> {
if block_number <= ancestor_number { return Ok(None) }
if block_number <= ancestor_number {
return Ok(None)
}
let mut current_hash = block_hash;
let mut current_entry = None;
@@ -289,7 +278,7 @@ fn load_ancestor(
let parent_hash = entry.parent_hash;
current_entry = Some(entry);
current_hash = parent_hash;
}
},
}
}
@@ -314,24 +303,22 @@ fn add_block(
let mut leaves = backend.load_leaves()?;
let parent_entry = backend.load_block_entry(&parent_hash)?;
let inherited_viability = parent_entry.as_ref()
.and_then(|parent| parent.non_viable_ancestor_for_child());
let inherited_viability =
parent_entry.as_ref().and_then(|parent| parent.non_viable_ancestor_for_child());
// 1. Add the block to the DB assuming it's not reverted.
backend.write_block_entry(
BlockEntry {
block_hash,
block_number,
parent_hash,
children: Vec::new(),
viability: ViabilityCriteria {
earliest_unviable_ancestor: inherited_viability,
explicitly_reverted: false,
approval: Approval::Unapproved,
},
weight,
}
);
backend.write_block_entry(BlockEntry {
block_hash,
block_number,
parent_hash,
children: Vec::new(),
viability: ViabilityCriteria {
earliest_unviable_ancestor: inherited_viability,
explicitly_reverted: false,
approval: Approval::Unapproved,
},
weight,
});
// 2. Update leaves if inherited viability is fine.
if inherited_viability.is_none() {
@@ -370,38 +357,34 @@ fn apply_reversions(
// Note: since revert numbers are in ascending order, the expensive propagation
// of unviability is only heavy on the first log.
for revert_number in reversions {
let mut ancestor_entry = match load_ancestor(
backend,
block_hash,
block_number,
revert_number,
)? {
None => {
tracing::warn!(
target: LOG_TARGET,
?block_hash,
block_number,
revert_target = revert_number,
"The hammer has dropped. \
let mut ancestor_entry =
match load_ancestor(backend, block_hash, block_number, revert_number)? {
None => {
tracing::warn!(
target: LOG_TARGET,
?block_hash,
block_number,
revert_target = revert_number,
"The hammer has dropped. \
A block has indicated that its finalized ancestor be reverted. \
Please inform an adult.",
);
);
continue
}
Some(ancestor_entry) => {
tracing::info!(
target: LOG_TARGET,
?block_hash,
block_number,
revert_target = revert_number,
revert_hash = ?ancestor_entry.block_hash,
"A block has signaled that its ancestor be reverted due to a bad parachain block.",
);
continue
},
Some(ancestor_entry) => {
tracing::info!(
target: LOG_TARGET,
?block_hash,
block_number,
revert_target = revert_number,
revert_hash = ?ancestor_entry.block_hash,
"A block has signaled that its ancestor be reverted due to a bad parachain block.",
);
ancestor_entry
}
};
ancestor_entry
},
};
ancestor_entry.viability.explicitly_reverted = true;
propagate_viability_update(backend, ancestor_entry)?;
@@ -431,8 +414,8 @@ pub(super) fn finalize_block<'a, B: Backend + 'a>(
None => {
// This implies that there are no unfinalized blocks and hence nothing
// to update.
return Ok(backend);
}
return Ok(backend)
},
Some(e) => e,
};
@@ -474,9 +457,7 @@ pub(super) fn finalize_block<'a, B: Backend + 'a>(
// Add all children to the frontier.
let next_height = dead_number + 1;
frontier.extend(
entry.into_iter().flat_map(|e| e.children).map(|h| (h, next_height))
);
frontier.extend(entry.into_iter().flat_map(|e| e.children).map(|h| (h, next_height)));
}
}
@@ -533,7 +514,6 @@ pub(super) fn approve_block(
} else {
backend.write_block_entry(entry);
}
} else {
tracing::debug!(
target: LOG_TARGET,
polkadot/node/core/dispute-coordinator/src/backend.rs
+19 -24
View File
@@ -21,12 +21,12 @@
//! [`Backend`], maintaining consistency between queries and temporary writes,
//! before any commit to the underlying storage is made.
use polkadot_primitives::v1::{CandidateHash, SessionIndex};
use polkadot_node_subsystem::SubsystemResult;
use polkadot_primitives::v1::{CandidateHash, SessionIndex};
use std::collections::HashMap;
use super::db::v1::{RecentDisputes, CandidateVotes};
use super::db::v1::{CandidateVotes, RecentDisputes};
#[derive(Debug)]
pub enum BackendWriteOp {
@@ -54,7 +54,8 @@ pub trait Backend {
/// Atomically writes the list of operations, with later operations taking precedence over
/// prior.
fn write<I>(&mut self, ops: I) -> SubsystemResult<()>
where I: IntoIterator<Item = BackendWriteOp>;
where
I: IntoIterator<Item = BackendWriteOp>;
}
/// An in-memory overlay for the backend.
@@ -112,7 +113,7 @@ impl<'a, B: 'a + Backend> OverlayedBackend<'a, B> {
pub fn load_candidate_votes(
&self,
session: SessionIndex,
candidate_hash: &CandidateHash
candidate_hash: &CandidateHash,
) -> SubsystemResult<Option<CandidateVotes>> {
if let Some(val) = self.candidate_votes.get(&(session, *candidate_hash)) {
return Ok(val.clone())
@@ -142,7 +143,7 @@ impl<'a, B: 'a + Backend> OverlayedBackend<'a, B> {
&mut self,
session: SessionIndex,
candidate_hash: CandidateHash,
votes: CandidateVotes
votes: CandidateVotes,
) {
self.candidate_votes.insert((session, candidate_hash), Some(votes));
}
@@ -150,34 +151,28 @@ impl<'a, B: 'a + Backend> OverlayedBackend<'a, B> {
/// Prepare a deletion of the candidate votes under the indicated candidate.
///
/// Later calls to this function for the same candidate will override earlier ones.
pub fn delete_candidate_votes(
&mut self,
session: SessionIndex,
candidate_hash: CandidateHash,
) {
pub fn delete_candidate_votes(&mut self, session: SessionIndex, candidate_hash: CandidateHash) {
self.candidate_votes.insert((session, candidate_hash), None);
}
/// Transform this backend into a set of write-ops to be written to the inner backend.
pub fn into_write_ops(self) -> impl Iterator<Item = BackendWriteOp> {
let earliest_session_ops = self.earliest_session
let earliest_session_ops = self
.earliest_session
.map(|s| BackendWriteOp::WriteEarliestSession(s))
.into_iter();
let recent_dispute_ops = self.recent_disputes
.map(|d| BackendWriteOp::WriteRecentDisputes(d))
.into_iter();
let recent_dispute_ops =
self.recent_disputes.map(|d| BackendWriteOp::WriteRecentDisputes(d)).into_iter();
let candidate_vote_ops = self.candidate_votes
.into_iter()
.map(|((session, candidate), votes)| match votes {
Some(votes) => BackendWriteOp::WriteCandidateVotes(session, candidate, votes),
None => BackendWriteOp::DeleteCandidateVotes(session, candidate),
});
earliest_session_ops
.chain(recent_dispute_ops)
.chain(candidate_vote_ops)
let candidate_vote_ops =
self.candidate_votes
.into_iter()
.map(|((session, candidate), votes)| match votes {
Some(votes) => BackendWriteOp::WriteCandidateVotes(session, candidate, votes),
None => BackendWriteOp::DeleteCandidateVotes(session, candidate),
});
earliest_session_ops.chain(recent_dispute_ops).chain(candidate_vote_ops)
}
}
polkadot/node/core/dispute-coordinator/src/db/v1.rs
+92 -115
View File
@@ -16,19 +16,21 @@
//! `V1` database for the dispute coordinator.
use polkadot_node_subsystem::{SubsystemError, SubsystemResult};
use polkadot_primitives::v1::{
CandidateReceipt, ValidDisputeStatementKind, InvalidDisputeStatementKind, ValidatorIndex,
ValidatorSignature, SessionIndex, CandidateHash, Hash,
CandidateHash, CandidateReceipt, Hash, InvalidDisputeStatementKind, SessionIndex,
ValidDisputeStatementKind, ValidatorIndex, ValidatorSignature,
};
use polkadot_node_subsystem::{SubsystemResult, SubsystemError};
use std::sync::Arc;
use kvdb::{KeyValueDB, DBTransaction};
use parity_scale_codec::{Encode, Decode};
use kvdb::{DBTransaction, KeyValueDB};
use parity_scale_codec::{Decode, Encode};
use crate::{DISPUTE_WINDOW, DisputeStatus};
use crate::backend::{Backend, BackendWriteOp, OverlayedBackend};
use crate::{
backend::{Backend, BackendWriteOp, OverlayedBackend},
DisputeStatus, DISPUTE_WINDOW,
};
const RECENT_DISPUTES_KEY: &[u8; 15] = b"recent-disputes";
const EARLIEST_SESSION_KEY: &[u8; 16] = b"earliest-session";
@@ -41,10 +43,7 @@ pub struct DbBackend {
impl DbBackend {
pub fn new(db: Arc<dyn KeyValueDB>, config: ColumnConfiguration) -> Self {
Self {
inner: db,
config,
}
Self { inner: db, config }
}
}
@@ -71,38 +70,28 @@ impl Backend for DbBackend {
/// Atomically writes the list of operations, with later operations taking precedence over
/// prior.
fn write<I>(&mut self, ops: I) -> SubsystemResult<()>
where I: IntoIterator<Item = BackendWriteOp>
where
I: IntoIterator<Item = BackendWriteOp>,
{
let mut tx = DBTransaction::new();
for op in ops {
match op {
BackendWriteOp::WriteEarliestSession(session) => {
tx.put_vec(
self.config.col_data,
EARLIEST_SESSION_KEY,
session.encode(),
);
}
tx.put_vec(self.config.col_data, EARLIEST_SESSION_KEY, session.encode());
},
BackendWriteOp::WriteRecentDisputes(recent_disputes) => {
tx.put_vec(
self.config.col_data,
RECENT_DISPUTES_KEY,
recent_disputes.encode(),
);
}
tx.put_vec(self.config.col_data, RECENT_DISPUTES_KEY, recent_disputes.encode());
},
BackendWriteOp::WriteCandidateVotes(session, candidate_hash, votes) => {
tx.put_vec(
self.config.col_data,
&candidate_votes_key(session, &candidate_hash),
votes.encode(),
);
}
},
BackendWriteOp::DeleteCandidateVotes(session, candidate_hash) => {
tx.delete(
self.config.col_data,
&candidate_votes_key(session, &candidate_hash),
);
}
tx.delete(self.config.col_data, &candidate_votes_key(session, &candidate_hash));
},
}
}
@@ -174,14 +163,10 @@ pub enum Error {
/// Result alias for DB errors.
pub type Result<T> = std::result::Result<T, Error>;
fn load_decode<D: Decode>(db: &dyn KeyValueDB, col_data: u32, key: &[u8])
-> Result<Option<D>>
{
fn load_decode<D: Decode>(db: &dyn KeyValueDB, col_data: u32, key: &[u8]) -> Result<Option<D>> {
match db.get(col_data, key)? {
None => Ok(None),
Some(raw) => D::decode(&mut &raw[..])
.map(Some)
.map_err(Into::into),
Some(raw) => D::decode(&mut &raw[..]).map(Some).map_err(Into::into),
}
}
@@ -231,7 +216,7 @@ pub(crate) fn note_current_session(
None => {
// First launch - write new-earliest.
overlay_db.write_earliest_session(new_earliest);
}
},
Some(prev_earliest) if new_earliest > prev_earliest => {
// Prune all data in the outdated sessions.
overlay_db.write_earliest_session(new_earliest);
@@ -240,10 +225,7 @@ pub(crate) fn note_current_session(
{
let mut recent_disputes = overlay_db.load_recent_disputes()?.unwrap_or_default();
let lower_bound = (
new_earliest,
CandidateHash(Hash::repeat_byte(0x00)),
);
let lower_bound = (new_earliest, CandidateHash(Hash::repeat_byte(0x00)));
let new_recent_disputes = recent_disputes.split_off(&lower_bound);
// Any remanining disputes are considered ancient and must be pruned.
@@ -256,10 +238,10 @@ pub(crate) fn note_current_session(
}
}
}
}
},
Some(_) => {
// nothing to do.
}
},
}
Ok(())
@@ -285,13 +267,17 @@ mod tests {
overlay_db.write_earliest_session(0);
overlay_db.write_earliest_session(1);
overlay_db.write_recent_disputes(vec![
((0, CandidateHash(Hash::repeat_byte(0))), DisputeStatus::Active),
].into_iter().collect());
overlay_db.write_recent_disputes(
vec![((0, CandidateHash(Hash::repeat_byte(0))), DisputeStatus::Active)]
.into_iter()
.collect(),
);
overlay_db.write_recent_disputes(vec![
((1, CandidateHash(Hash::repeat_byte(1))), DisputeStatus::Active),
].into_iter().collect());
overlay_db.write_recent_disputes(
vec![((1, CandidateHash(Hash::repeat_byte(1))), DisputeStatus::Active)]
.into_iter()
.collect(),
);
overlay_db.write_candidate_votes(
1,
@@ -318,23 +304,23 @@ mod tests {
);
// Test that overlay returns the correct values before committing.
assert_eq!(
overlay_db.load_earliest_session().unwrap().unwrap(),
1,
);
assert_eq!(overlay_db.load_earliest_session().unwrap().unwrap(), 1,);
assert_eq!(
overlay_db.load_recent_disputes().unwrap().unwrap(),
vec![
((1, CandidateHash(Hash::repeat_byte(1))), DisputeStatus::Active),
].into_iter().collect()
vec![((1, CandidateHash(Hash::repeat_byte(1))), DisputeStatus::Active),]
.into_iter()
.collect()
);
assert_eq!(
overlay_db.load_candidate_votes(
1,
&CandidateHash(Hash::repeat_byte(1))
).unwrap().unwrap().candidate_receipt.descriptor.para_id,
overlay_db
.load_candidate_votes(1, &CandidateHash(Hash::repeat_byte(1)))
.unwrap()
.unwrap()
.candidate_receipt
.descriptor
.para_id,
ParaId::from(5),
);
@@ -342,23 +328,23 @@ mod tests {
backend.write(write_ops).unwrap();
// Test that subsequent writes were written.
assert_eq!(
backend.load_earliest_session().unwrap().unwrap(),
1,
);
assert_eq!(backend.load_earliest_session().unwrap().unwrap(), 1,);
assert_eq!(
backend.load_recent_disputes().unwrap().unwrap(),
vec![
((1, CandidateHash(Hash::repeat_byte(1))), DisputeStatus::Active),
].into_iter().collect()
vec![((1, CandidateHash(Hash::repeat_byte(1))), DisputeStatus::Active),]
.into_iter()
.collect()
);
assert_eq!(
backend.load_candidate_votes(
1,
&CandidateHash(Hash::repeat_byte(1))
).unwrap().unwrap().candidate_receipt.descriptor.para_id,
backend
.load_candidate_votes(1, &CandidateHash(Hash::repeat_byte(1)))
.unwrap()
.unwrap()
.candidate_receipt
.descriptor
.para_id,
ParaId::from(5),
);
}
@@ -377,17 +363,20 @@ mod tests {
candidate_receipt: Default::default(),
valid: Vec::new(),
invalid: Vec::new(),
}
},
);
let write_ops = overlay_db.into_write_ops();
backend.write(write_ops).unwrap();
assert_eq!(
backend.load_candidate_votes(
1,
&CandidateHash(Hash::repeat_byte(1))
).unwrap().unwrap().candidate_receipt.descriptor.para_id,
backend
.load_candidate_votes(1, &CandidateHash(Hash::repeat_byte(1)))
.unwrap()
.unwrap()
.candidate_receipt
.descriptor
.para_id,
ParaId::from(0),
);
@@ -404,7 +393,7 @@ mod tests {
},
valid: Vec::new(),
invalid: Vec::new(),
}
},
);
overlay_db.delete_candidate_votes(1, CandidateHash(Hash::repeat_byte(1)));
@@ -412,12 +401,10 @@ mod tests {
let write_ops = overlay_db.into_write_ops();
backend.write(write_ops).unwrap();
assert!(
backend.load_candidate_votes(
1,
&CandidateHash(Hash::repeat_byte(1))
).unwrap().is_none()
);
assert!(backend
.load_candidate_votes(1, &CandidateHash(Hash::repeat_byte(1)))
.unwrap()
.is_none());
}
#[test]
@@ -445,36 +432,24 @@ mod tests {
let mut overlay_db = OverlayedBackend::new(&backend);
overlay_db.write_earliest_session(prev_earliest_session);
overlay_db.write_recent_disputes(vec![
((very_old, hash_a), DisputeStatus::Active),
((slightly_old, hash_b), DisputeStatus::Active),
((new_earliest_session, hash_c), DisputeStatus::Active),
((very_recent, hash_d), DisputeStatus::Active),
].into_iter().collect());
overlay_db.write_candidate_votes(
very_old,
hash_a,
blank_candidate_votes(),
overlay_db.write_recent_disputes(
vec![
((very_old, hash_a), DisputeStatus::Active),
((slightly_old, hash_b), DisputeStatus::Active),
((new_earliest_session, hash_c), DisputeStatus::Active),
((very_recent, hash_d), DisputeStatus::Active),
]
.into_iter()
.collect(),
);
overlay_db.write_candidate_votes(
slightly_old,
hash_b,
blank_candidate_votes(),
);
overlay_db.write_candidate_votes(very_old, hash_a, blank_candidate_votes());
overlay_db.write_candidate_votes(
new_earliest_session,
hash_c,
blank_candidate_votes(),
);
overlay_db.write_candidate_votes(slightly_old, hash_b, blank_candidate_votes());
overlay_db.write_candidate_votes(
very_recent,
hash_d,
blank_candidate_votes(),
);
overlay_db.write_candidate_votes(new_earliest_session, hash_c, blank_candidate_votes());
overlay_db.write_candidate_votes(very_recent, hash_d, blank_candidate_votes());
let write_ops = overlay_db.into_write_ops();
backend.write(write_ops).unwrap();
@@ -482,22 +457,24 @@ mod tests {
let mut overlay_db = OverlayedBackend::new(&backend);
note_current_session(&mut overlay_db, current_session).unwrap();
assert_eq!(
overlay_db.load_earliest_session().unwrap(),
Some(new_earliest_session),
);
assert_eq!(overlay_db.load_earliest_session().unwrap(), Some(new_earliest_session),);
assert_eq!(
overlay_db.load_recent_disputes().unwrap().unwrap(),
vec![
((new_earliest_session, hash_c), DisputeStatus::Active),
((very_recent, hash_d), DisputeStatus::Active),
].into_iter().collect(),
]
.into_iter()
.collect(),
);
assert!(overlay_db.load_candidate_votes(very_old, &hash_a).unwrap().is_none());
assert!(overlay_db.load_candidate_votes(slightly_old, &hash_b).unwrap().is_none());
assert!(overlay_db.load_candidate_votes(new_earliest_session, &hash_c).unwrap().is_some());
assert!(overlay_db
.load_candidate_votes(new_earliest_session, &hash_c)
.unwrap()
.is_some());
assert!(overlay_db.load_candidate_votes(very_recent, &hash_d).unwrap().is_some());
}
}
polkadot/node/core/dispute-coordinator/src/lib.rs
+174 -171
View File
@@ -25,38 +25,42 @@
//! another node, this will trigger the dispute participation subsystem to recover and validate the block and call
//! back to this subsystem.
use std::collections::HashSet;
use std::sync::Arc;
use std::time::{SystemTime, UNIX_EPOCH};
use std::{
collections::HashSet,
sync::Arc,
time::{SystemTime, UNIX_EPOCH},
};
use polkadot_node_primitives::{CandidateVotes, DISPUTE_WINDOW, DisputeMessage, SignedDisputeStatement, DisputeMessageCheckError};
use polkadot_node_primitives::{
CandidateVotes, DisputeMessage, DisputeMessageCheckError, SignedDisputeStatement,
DISPUTE_WINDOW,
};
use polkadot_node_subsystem::{
overseer, SubsystemContext, FromOverseer, OverseerSignal, SpawnedSubsystem, SubsystemError,
errors::{ChainApiError, RuntimeApiError},
messages::{
ChainApiMessage, DisputeCoordinatorMessage, DisputeDistributionMessage,
DisputeParticipationMessage, ImportStatementsResult, BlockDescription,
}
BlockDescription, ChainApiMessage, DisputeCoordinatorMessage, DisputeDistributionMessage,
DisputeParticipationMessage, ImportStatementsResult,
},
overseer, FromOverseer, OverseerSignal, SpawnedSubsystem, SubsystemContext, SubsystemError,
};
use polkadot_node_subsystem_util::rolling_session_window::{
RollingSessionWindow, SessionWindowUpdate,
};
use polkadot_primitives::v1::{
BlockNumber, CandidateHash, CandidateReceipt, DisputeStatement, Hash,
SessionIndex, SessionInfo, ValidatorIndex, ValidatorPair, ValidatorSignature
BlockNumber, CandidateHash, CandidateReceipt, DisputeStatement, Hash, SessionIndex,
SessionInfo, ValidatorIndex, ValidatorPair, ValidatorSignature,
};
use futures::prelude::*;
use futures::channel::oneshot;
use futures::{channel::oneshot, prelude::*};
use kvdb::KeyValueDB;
use parity_scale_codec::{Encode, Decode, Error as CodecError};
use parity_scale_codec::{Decode, Encode, Error as CodecError};
use sc_keystore::LocalKeystore;
use db::v1::{RecentDisputes, DbBackend};
use backend::{Backend, OverlayedBackend};
use db::v1::{DbBackend, RecentDisputes};
mod db;
mod backend;
mod db;
#[cfg(test)]
mod tests;
@@ -116,11 +120,7 @@ pub struct DisputeCoordinatorSubsystem {
impl DisputeCoordinatorSubsystem {
/// Create a new instance of the subsystem.
pub fn new(
store: Arc<dyn KeyValueDB>,
config: Config,
keystore: Arc<LocalKeystore>,
) -> Self {
pub fn new(store: Arc<dyn KeyValueDB>, config: Config, keystore: Arc<LocalKeystore>) -> Self {
DisputeCoordinatorSubsystem { store, config, keystore }
}
}
@@ -132,14 +132,9 @@ where
{
fn start(self, ctx: Context) -> SpawnedSubsystem {
let backend = DbBackend::new(self.store.clone(), self.config.column_config());
let future = run(self, ctx, backend, Box::new(SystemClock))
.map(|_| Ok(()))
.boxed();
let future = run(self, ctx, backend, Box::new(SystemClock)).map(|_| Ok(())).boxed();
SpawnedSubsystem {
name: "dispute-coordinator-subsystem",
future,
}
SpawnedSubsystem { name: "dispute-coordinator-subsystem", future }
}
}
@@ -167,7 +162,7 @@ impl Clock for SystemClock {
);
0
}
},
}
}
}
@@ -210,8 +205,8 @@ impl Error {
fn trace(&self) {
match self {
// don't spam the log with spurious errors
Self::RuntimeApi(_) |
Self::Oneshot(_) => tracing::debug!(target: LOG_TARGET, err = ?self),
Self::RuntimeApi(_) | Self::Oneshot(_) =>
tracing::debug!(target: LOG_TARGET, err = ?self),
// it's worth reporting otherwise
_ => tracing::warn!(target: LOG_TARGET, err = ?self),
}
@@ -260,8 +255,10 @@ impl DisputeStatus {
pub fn concluded_against(self, now: Timestamp) -> DisputeStatus {
match self {
DisputeStatus::Active => DisputeStatus::ConcludedAgainst(now),
DisputeStatus::ConcludedFor(at) => DisputeStatus::ConcludedAgainst(std::cmp::min(at, now)),
DisputeStatus::ConcludedAgainst(at) => DisputeStatus::ConcludedAgainst(std::cmp::min(at, now)),
DisputeStatus::ConcludedFor(at) =>
DisputeStatus::ConcludedAgainst(std::cmp::min(at, now)),
DisputeStatus::ConcludedAgainst(at) =>
DisputeStatus::ConcludedAgainst(std::cmp::min(at, now)),
}
}
@@ -287,8 +284,7 @@ async fn run<B, Context>(
mut ctx: Context,
mut backend: B,
clock: Box<dyn Clock>,
)
where
) where
Context: overseer::SubsystemContext<Message = DisputeCoordinatorMessage>,
Context: SubsystemContext<Message = DisputeCoordinatorMessage>,
B: Backend,
@@ -300,13 +296,13 @@ where
e.trace();
if let Error::Subsystem(SubsystemError::Context(_)) = e {
break;
break
}
}
},
Ok(()) => {
tracing::info!(target: LOG_TARGET, "received `Conclude` signal, exiting");
break;
}
break
},
}
}
}
@@ -337,34 +333,22 @@ where
loop {
let mut overlay_db = OverlayedBackend::new(backend);
match ctx.recv().await? {
FromOverseer::Signal(OverseerSignal::Conclude) => {
return Ok(())
}
FromOverseer::Signal(OverseerSignal::Conclude) => return Ok(()),
FromOverseer::Signal(OverseerSignal::ActiveLeaves(update)) => {
handle_new_activations(
ctx,
&mut overlay_db,
&mut state,
update.activated.into_iter().map(|a| a.hash),
).await?;
)
.await?;
if !state.recovery_state.complete() {
handle_startup(
ctx,
&mut overlay_db,
&mut state,
).await?;
handle_startup(ctx, &mut overlay_db, &mut state).await?;
}
}
},
FromOverseer::Signal(OverseerSignal::BlockFinalized(_, _)) => {},
FromOverseer::Communication { msg } => {
handle_incoming(
ctx,
&mut overlay_db,
&mut state,
msg,
clock.now(),
).await?
}
FromOverseer::Communication { msg } =>
handle_incoming(ctx, &mut overlay_db, &mut state, msg, clock.now()).await?,
}
if !overlay_db.is_empty() {
@@ -395,12 +379,13 @@ where
Ok(None) => return Ok(()),
Err(e) => {
tracing::error!(target: LOG_TARGET, "Failed initial load of recent disputes: {:?}", e);
return Err(e.into());
return Err(e.into())
},
};
// Filter out disputes that have already concluded.
let active_disputes = recent_disputes.into_iter()
let active_disputes = recent_disputes
.into_iter()
.filter(|(_, status)| *status == DisputeStatus::Active)
.collect::<RecentDisputes>();
@@ -409,7 +394,11 @@ where
Ok(Some(votes)) => votes.into(),
Ok(None) => continue,
Err(e) => {
tracing::error!(target: LOG_TARGET, "Failed initial load of candidate votes: {:?}", e);
tracing::error!(
target: LOG_TARGET,
"Failed initial load of candidate votes: {:?}",
e
);
continue
},
};
@@ -422,7 +411,7 @@ where
"Missing info for session which has an active dispute",
);
continue
}
},
Some(info) => info.validators.clone(),
};
@@ -434,16 +423,18 @@ where
// 1) their statement already exists, or
// 2) the validator key is not in the local keystore (i.e. the validator is remote).
// The remaining set only contains local validators that are also missing statements.
let missing_local_statement = validators.iter()
let missing_local_statement = validators
.iter()
.enumerate()
.map(|(index, validator)| (ValidatorIndex(index as _), validator))
.any(|(index, validator)|
.any(|(index, validator)| {
!voted_indices.contains(&index) &&
state.keystore
.key_pair::<ValidatorPair>(validator)
.ok()
.map_or(false, |v| v.is_some())
);
state
.keystore
.key_pair::<ValidatorPair>(validator)
.ok()
.map_or(false, |v| v.is_some())
});
// Send a `DisputeParticipationMessage` for all non-concluded disputes which do not have a
// recorded local statement.
@@ -455,10 +446,14 @@ where
session,
n_validators: n_validators as u32,
report_availability,
}).await;
})
.await;
if !receive_availability.await? {
tracing::debug!(target: LOG_TARGET, "Participation failed. Candidate not available");
tracing::debug!(
target: LOG_TARGET,
"Participation failed. Candidate not available"
);
}
}
}
@@ -467,7 +462,8 @@ where
}
async fn handle_new_activations(
ctx: &mut (impl SubsystemContext<Message = DisputeCoordinatorMessage> + overseer::SubsystemContext<Message = DisputeCoordinatorMessage>),
ctx: &mut (impl SubsystemContext<Message = DisputeCoordinatorMessage>
+ overseer::SubsystemContext<Message = DisputeCoordinatorMessage>),
overlay_db: &mut OverlayedBackend<'_, impl Backend>,
state: &mut State,
new_activations: impl IntoIterator<Item = Hash>,
@@ -476,9 +472,7 @@ async fn handle_new_activations(
let block_header = {
let (tx, rx) = oneshot::channel();
ctx.send_message(
ChainApiMessage::BlockHeader(new_leaf, tx)
).await;
ctx.send_message(ChainApiMessage::BlockHeader(new_leaf, tx)).await;
match rx.await?? {
None => continue,
@@ -486,11 +480,11 @@ async fn handle_new_activations(
}
};
match state.rolling_session_window.cache_session_info_for_head(
ctx,
new_leaf,
&block_header,
).await {
match state
.rolling_session_window
.cache_session_info_for_head(ctx, new_leaf, &block_header)
.await
{
Err(e) => {
tracing::warn!(
target: LOG_TARGET,
@@ -499,24 +493,19 @@ async fn handle_new_activations(
);
continue
}
Ok(SessionWindowUpdate::Initialized { window_end, .. })
| Ok(SessionWindowUpdate::Advanced { new_window_end: window_end, .. })
=> {
},
Ok(SessionWindowUpdate::Initialized { window_end, .. }) |
Ok(SessionWindowUpdate::Advanced { new_window_end: window_end, .. }) => {
let session = window_end;
if state.highest_session.map_or(true, |s| s < session) {
tracing::trace!(
target: LOG_TARGET,
session,
"Observed new session. Pruning",
);
tracing::trace!(target: LOG_TARGET, session, "Observed new session. Pruning",);
state.highest_session = Some(session);
db::v1::note_current_session(overlay_db, session)?;
}
}
_ => {}
},
_ => {},
}
}
@@ -548,26 +537,21 @@ async fn handle_incoming(
statements,
now,
pending_confirmation,
).await?;
}
)
.await?;
},
DisputeCoordinatorMessage::RecentDisputes(rx) => {
let recent_disputes = overlay_db.load_recent_disputes()?.unwrap_or_default();
let _ = rx.send(recent_disputes.keys().cloned().collect());
}
},
DisputeCoordinatorMessage::ActiveDisputes(rx) => {
let recent_disputes = overlay_db.load_recent_disputes()?.unwrap_or_default();
let _ = rx.send(collect_active(recent_disputes, now));
}
DisputeCoordinatorMessage::QueryCandidateVotes(
query,
rx
) => {
},
DisputeCoordinatorMessage::QueryCandidateVotes(query, rx) => {
let mut query_output = Vec::new();
for (session_index, candidate_hash) in query.into_iter() {
if let Some(v) = overlay_db.load_candidate_votes(
session_index,
&candidate_hash,
)? {
if let Some(v) = overlay_db.load_candidate_votes(session_index, &candidate_hash)? {
query_output.push((session_index, candidate_hash, v.into()));
} else {
tracing::debug!(
@@ -578,7 +562,7 @@ async fn handle_incoming(
}
}
let _ = rx.send(query_output);
}
},
DisputeCoordinatorMessage::IssueLocalStatement(
session,
candidate_hash,
@@ -594,33 +578,37 @@ async fn handle_incoming(
session,
valid,
now,
).await?;
}
)
.await?;
},
DisputeCoordinatorMessage::DetermineUndisputedChain {
base_number,
block_descriptions,
tx,
} => {
let undisputed_chain = determine_undisputed_chain(
overlay_db,
base_number,
block_descriptions
)?;
let undisputed_chain =
determine_undisputed_chain(overlay_db, base_number, block_descriptions)?;
let _ = tx.send(undisputed_chain);
}
},
}
Ok(())
}
fn collect_active(recent_disputes: RecentDisputes, now: Timestamp) -> Vec<(SessionIndex, CandidateHash)> {
recent_disputes.iter().filter_map(|(disputed, status)|
status.concluded_at().filter(|at| at + ACTIVE_DURATION_SECS < now).map_or(
Some(*disputed),
|_| None,
)
).collect()
fn collect_active(
recent_disputes: RecentDisputes,
now: Timestamp,
) -> Vec<(SessionIndex, CandidateHash)> {
recent_disputes
.iter()
.filter_map(|(disputed, status)| {
status
.concluded_at()
.filter(|at| at + ACTIVE_DURATION_SECS < now)
.map_or(Some(*disputed), |_| None)
})
.collect()
}
fn insert_into_statement_vec<T>(
@@ -649,11 +637,12 @@ async fn handle_import_statements(
pending_confirmation: oneshot::Sender<ImportStatementsResult>,
) -> Result<(), Error> {
if state.highest_session.map_or(true, |h| session + DISPUTE_WINDOW < h) {
// It is not valid to participate in an ancient dispute (spam?).
pending_confirmation.send(ImportStatementsResult::InvalidImport).map_err(|_| Error::OneshotSend)?;
pending_confirmation
.send(ImportStatementsResult::InvalidImport)
.map_err(|_| Error::OneshotSend)?;
return Ok(());
return Ok(())
}
let validators = match state.rolling_session_window.session_info(session) {
@@ -669,7 +658,7 @@ async fn handle_import_statements(
.map_err(|_| Error::OneshotSend)?;
return Ok(())
}
},
Some(info) => info.validators.clone(),
};
@@ -677,7 +666,8 @@ async fn handle_import_statements(
let supermajority_threshold = polkadot_primitives::v1::supermajority_threshold(n_validators);
let mut votes = overlay_db.load_candidate_votes(session, &candidate_hash)?
let mut votes = overlay_db
.load_candidate_votes(session, &candidate_hash)?
.map(CandidateVotes::from)
.unwrap_or_else(|| CandidateVotes {
candidate_receipt: candidate_receipt.clone(),
@@ -687,7 +677,8 @@ async fn handle_import_statements(
// Update candidate votes.
for (statement, val_index) in statements {
if validators.get(val_index.0 as usize)
if validators
.get(val_index.0 as usize)
.map_or(true, |v| v != statement.validator_public())
{
tracing::debug!(
@@ -709,7 +700,7 @@ async fn handle_import_statements(
val_index,
statement.validator_signature().clone(),
);
}
},
DisputeStatement::Invalid(invalid_kind) => {
insert_into_statement_vec(
&mut votes.invalid,
@@ -717,7 +708,7 @@ async fn handle_import_statements(
val_index,
statement.validator_signature().clone(),
);
}
},
}
}
@@ -766,7 +757,8 @@ async fn handle_import_statements(
session,
n_validators: n_validators as u32,
report_availability,
}).await;
})
.await;
if !receive_availability.await.map_err(Error::Oneshot)? {
// If the data is not available, we disregard the dispute votes.
@@ -775,7 +767,8 @@ async fn handle_import_statements(
//
// We expect that if the candidate is truly disputed that the higher-level network
// code will retry.
pending_confirmation.send(ImportStatementsResult::InvalidImport)
pending_confirmation
.send(ImportStatementsResult::InvalidImport)
.map_err(|_| Error::OneshotSend)?;
tracing::debug!(
@@ -819,13 +812,14 @@ async fn issue_local_statement(
);
return Ok(())
}
},
Some(info) => info,
};
let validators = info.validators.clone();
let votes = overlay_db.load_candidate_votes(session, &candidate_hash)?
let votes = overlay_db
.load_candidate_votes(session, &candidate_hash)?
.map(CandidateVotes::from)
.unwrap_or_else(|| CandidateVotes {
candidate_receipt: candidate_receipt.clone(),
@@ -841,7 +835,9 @@ async fn issue_local_statement(
let voted_indices: HashSet<_> = voted_indices.into_iter().collect();
for (index, validator) in validators.iter().enumerate() {
let index = ValidatorIndex(index as _);
if voted_indices.contains(&index) { continue }
if voted_indices.contains(&index) {
continue
}
if state.keystore.key_pair::<ValidatorPair>(validator).ok().flatten().is_none() {
continue
}
@@ -853,20 +849,21 @@ async fn issue_local_statement(
candidate_hash,
session,
validator.clone(),
).await;
)
.await;
match res {
Ok(Some(signed_dispute_statement)) => {
statements.push((signed_dispute_statement, index));
}
Ok(None) => {}
},
Ok(None) => {},
Err(e) => {
tracing::error!(
target: LOG_TARGET,
err = ?e,
"Encountered keystore error while signing dispute statement",
);
}
},
}
}
@@ -874,20 +871,15 @@ async fn issue_local_statement(
for (statement, index) in &statements {
let dispute_message = match make_dispute_message(info, &votes, statement.clone(), *index) {
Err(err) => {
tracing::debug!(
target: LOG_TARGET,
?err,
"Creating dispute message failed."
);
tracing::debug!(target: LOG_TARGET, ?err, "Creating dispute message failed.");
continue
}
},
Ok(dispute_message) => dispute_message,
};
ctx.send_message(DisputeDistributionMessage::SendDispute(dispute_message)).await;
}
// Do import
if !statements.is_empty() {
let (pending_confirmation, _rx) = oneshot::channel();
@@ -901,7 +893,8 @@ async fn issue_local_statement(
statements,
now,
pending_confirmation,
).await?;
)
.await?;
}
Ok(())
@@ -923,42 +916,52 @@ fn make_dispute_message(
info: &SessionInfo,
votes: &CandidateVotes,
our_vote: SignedDisputeStatement,
our_index: ValidatorIndex
our_index: ValidatorIndex,
) -> Result<DisputeMessage, MakeDisputeMessageError> {
let validators = &info.validators;
let (valid_statement, valid_index, invalid_statement, invalid_index) =
if let DisputeStatement::Valid(_) = our_vote.statement() {
let (statement_kind, validator_index, validator_signature)
= votes.invalid.get(0).ok_or(MakeDisputeMessageError::NoOppositeVote)?.clone();
let (statement_kind, validator_index, validator_signature) =
votes.invalid.get(0).ok_or(MakeDisputeMessageError::NoOppositeVote)?.clone();
let other_vote = SignedDisputeStatement::new_checked(
DisputeStatement::Invalid(statement_kind),
our_vote.candidate_hash().clone(),
our_vote.session_index(),
validators.get(validator_index.0 as usize).ok_or(MakeDisputeMessageError::InvalidValidatorIndex)?.clone(),
validators
.get(validator_index.0 as usize)
.ok_or(MakeDisputeMessageError::InvalidValidatorIndex)?
.clone(),
validator_signature,
).map_err(|()| MakeDisputeMessageError::InvalidStoredStatement)?;
)
.map_err(|()| MakeDisputeMessageError::InvalidStoredStatement)?;
(our_vote, our_index, other_vote, validator_index)
} else {
let (statement_kind, validator_index, validator_signature)
= votes.valid.get(0).ok_or(MakeDisputeMessageError::NoOppositeVote)?.clone();
let other_vote = SignedDisputeStatement::new_checked(
DisputeStatement::Valid(statement_kind),
our_vote.candidate_hash().clone(),
our_vote.session_index(),
validators.get(validator_index.0 as usize).ok_or(MakeDisputeMessageError::InvalidValidatorIndex)?.clone(),
validator_signature,
).map_err(|()| MakeDisputeMessageError::InvalidStoredStatement)?;
(other_vote, validator_index, our_vote, our_index)
};
} else {
let (statement_kind, validator_index, validator_signature) =
votes.valid.get(0).ok_or(MakeDisputeMessageError::NoOppositeVote)?.clone();
let other_vote = SignedDisputeStatement::new_checked(
DisputeStatement::Valid(statement_kind),
our_vote.candidate_hash().clone(),
our_vote.session_index(),
validators
.get(validator_index.0 as usize)
.ok_or(MakeDisputeMessageError::InvalidValidatorIndex)?
.clone(),
validator_signature,
)
.map_err(|()| MakeDisputeMessageError::InvalidStoredStatement)?;
(other_vote, validator_index, our_vote, our_index)
};
DisputeMessage::from_signed_statements(
valid_statement, valid_index,
invalid_statement, invalid_index,
valid_statement,
valid_index,
invalid_statement,
invalid_index,
votes.candidate_receipt.clone(),
info,
).map_err(MakeDisputeMessageError::InvalidStatementCombination)
)
.map_err(MakeDisputeMessageError::InvalidStatementCombination)
}
/// Determine the the best block and its block number.
@@ -969,7 +972,8 @@ fn determine_undisputed_chain(
base_number: BlockNumber,
block_descriptions: Vec<BlockDescription>,
) -> Result<Option<(BlockNumber, Hash)>, Error> {
let last = block_descriptions.last()
let last = block_descriptions
.last()
.map(|e| (base_number + block_descriptions.len() as BlockNumber, e.block_hash));
// Fast path for no disputes.
@@ -980,21 +984,20 @@ fn determine_undisputed_chain(
};
let is_possibly_invalid = |session, candidate_hash| {
recent_disputes.get(&(session, candidate_hash)).map_or(
false,
|status| status.is_possibly_invalid(),
)
recent_disputes
.get(&(session, candidate_hash))
.map_or(false, |status| status.is_possibly_invalid())
};
for (i, BlockDescription { session, candidates, .. }) in block_descriptions.iter().enumerate() {
if candidates.iter().any(|c| is_possibly_invalid(*session, *c)) {
if i == 0 {
return Ok(None);
return Ok(None)
} else {
return Ok(Some((
base_number + i as BlockNumber,
block_descriptions[i - 1].block_hash,
)));
)))
}
}
}
polkadot/node/core/dispute-coordinator/src/tests.rs
+1036 -1094
View File
File diff suppressed because it is too large Load Diff
polkadot/node/core/dispute-participation/src/lib.rs
+62 -78
View File
@@ -20,20 +20,18 @@
//! notified of a dispute, we recover the candidate data, validate the
//! candidate, and cast our vote in the dispute.
use futures::channel::oneshot;
use futures::prelude::*;
use futures::{channel::oneshot, prelude::*};
use polkadot_node_primitives::ValidationResult;
use polkadot_node_subsystem::{
errors::{RecoveryError, RuntimeApiError},
overseer,
messages::{
AvailabilityRecoveryMessage, AvailabilityStoreMessage,
CandidateValidationMessage, DisputeCoordinatorMessage, DisputeParticipationMessage,
RuntimeApiMessage, RuntimeApiRequest,
AvailabilityRecoveryMessage, AvailabilityStoreMessage, CandidateValidationMessage,
DisputeCoordinatorMessage, DisputeParticipationMessage, RuntimeApiMessage,
RuntimeApiRequest,
},
ActiveLeavesUpdate, FromOverseer, OverseerSignal, SpawnedSubsystem,
SubsystemContext, SubsystemError,
overseer, ActiveLeavesUpdate, FromOverseer, OverseerSignal, SpawnedSubsystem, SubsystemContext,
SubsystemError,
};
use polkadot_primitives::v1::{BlockNumber, CandidateHash, CandidateReceipt, Hash, SessionIndex};
@@ -64,10 +62,7 @@ where
fn start(self, ctx: Context) -> SpawnedSubsystem {
let future = run(ctx).map(|_| Ok(())).boxed();
SpawnedSubsystem {
name: "dispute-participation-subsystem",
future,
}
SpawnedSubsystem { name: "dispute-participation-subsystem", future }
}
}
@@ -106,7 +101,7 @@ impl Error {
// don't spam the log with spurious errors
Self::RuntimeApi(_) | Self::Oneshot(_) => {
tracing::debug!(target: LOG_TARGET, err = ?self)
}
},
// it's worth reporting otherwise
_ => tracing::warn!(target: LOG_TARGET, err = ?self),
}
@@ -125,20 +120,20 @@ where
Err(_) => return,
Ok(FromOverseer::Signal(OverseerSignal::Conclude)) => {
tracing::info!(target: LOG_TARGET, "Received `Conclude` signal, exiting");
return;
}
Ok(FromOverseer::Signal(OverseerSignal::BlockFinalized(_, _))) => {}
return
},
Ok(FromOverseer::Signal(OverseerSignal::BlockFinalized(_, _))) => {},
Ok(FromOverseer::Signal(OverseerSignal::ActiveLeaves(update))) => {
update_state(&mut state, update);
}
},
Ok(FromOverseer::Communication { msg }) => {
if let Err(err) = handle_incoming(&mut ctx, &mut state, msg).await {
err.trace();
if let Error::Subsystem(SubsystemError::Context(_)) = err {
return;
return
}
}
}
},
}
}
}
@@ -163,7 +158,7 @@ async fn handle_incoming(
session,
n_validators,
report_availability,
} => {
} =>
if let Some((_, block_hash)) = state.recent_block {
participate(
ctx,
@@ -176,9 +171,8 @@ async fn handle_incoming(
)
.await
} else {
return Err(ParticipationError::MissingRecentBlockState.into());
}
}
return Err(ParticipationError::MissingRecentBlockState.into())
},
}
}
@@ -198,47 +192,43 @@ async fn participate(
// in order to validate a candidate we need to start by recovering the
// available data
ctx.send_message(
AvailabilityRecoveryMessage::RecoverAvailableData(
candidate_receipt.clone(),
session,
None,
recover_available_data_tx,
)
)
ctx.send_message(AvailabilityRecoveryMessage::RecoverAvailableData(
candidate_receipt.clone(),
session,
None,
recover_available_data_tx,
))
.await;
let available_data = match recover_available_data_rx.await? {
Ok(data) => {
report_availability.send(true).map_err(|_| Error::OneshotSendFailed)?;
data
}
},
Err(RecoveryError::Invalid) => {
report_availability.send(true).map_err(|_| Error::OneshotSendFailed)?;
// the available data was recovered but it is invalid, therefore we'll
// vote negatively for the candidate dispute
cast_invalid_vote(ctx, candidate_hash, candidate_receipt, session).await;
return Ok(());
}
return Ok(())
},
Err(RecoveryError::Unavailable) => {
report_availability.send(false).map_err(|_| Error::OneshotSendFailed)?;
return Err(ParticipationError::MissingAvailableData(candidate_hash).into());
}
return Err(ParticipationError::MissingAvailableData(candidate_hash).into())
},
};
// we also need to fetch the validation code which we can reference by its
// hash as taken from the candidate descriptor
ctx.send_message(
RuntimeApiMessage::Request(
block_hash,
RuntimeApiRequest::ValidationCodeByHash(
candidate_receipt.descriptor.validation_code_hash,
code_tx,
),
)
)
ctx.send_message(RuntimeApiMessage::Request(
block_hash,
RuntimeApiRequest::ValidationCodeByHash(
candidate_receipt.descriptor.validation_code_hash,
code_tx,
),
))
.await;
let validation_code = match code_rx.await?? {
@@ -251,22 +241,20 @@ async fn participate(
block_hash,
);
return Err(ParticipationError::MissingValidationCode(candidate_hash).into());
}
return Err(ParticipationError::MissingValidationCode(candidate_hash).into())
},
};
// we dispatch a request to store the available data for the candidate. we
// want to maximize data availability for other potential checkers involved
// in the dispute
ctx.send_message(
AvailabilityStoreMessage::StoreAvailableData(
candidate_hash,
None,
n_validators,
available_data.clone(),
store_available_data_tx,
)
)
ctx.send_message(AvailabilityStoreMessage::StoreAvailableData(
candidate_hash,
None,
n_validators,
available_data.clone(),
store_available_data_tx,
))
.await;
match store_available_data_rx.await? {
@@ -276,21 +264,19 @@ async fn participate(
"Failed to store available data for candidate {:?}",
candidate_hash,
);
}
Ok(()) => {}
},
Ok(()) => {},
}
// we issue a request to validate the candidate with the provided exhaustive
// parameters
ctx.send_message(
CandidateValidationMessage::ValidateFromExhaustive(
available_data.validation_data,
validation_code,
candidate_receipt.descriptor.clone(),
available_data.pov,
validation_tx,
)
)
ctx.send_message(CandidateValidationMessage::ValidateFromExhaustive(
available_data.validation_data,
validation_code,
candidate_receipt.descriptor.clone(),
available_data.pov,
validation_tx,
))
.await;
// we cast votes (either positive or negative) depending on the outcome of
@@ -305,7 +291,7 @@ async fn participate(
);
cast_invalid_vote(ctx, candidate_hash, candidate_receipt, session).await;
}
},
Ok(ValidationResult::Invalid(invalid)) => {
tracing::warn!(
target: LOG_TARGET,
@@ -315,7 +301,7 @@ async fn participate(
);
cast_invalid_vote(ctx, candidate_hash, candidate_receipt, session).await;
}
},
Ok(ValidationResult::Valid(commitments, _)) => {
if commitments.hash() != candidate_receipt.commitments_hash {
tracing::warn!(
@@ -329,7 +315,7 @@ async fn participate(
} else {
cast_valid_vote(ctx, candidate_hash, candidate_receipt, session).await;
}
}
},
}
Ok(())
@@ -372,13 +358,11 @@ async fn issue_local_statement(
session: SessionIndex,
valid: bool,
) {
ctx.send_message(
DisputeCoordinatorMessage::IssueLocalStatement(
session,
candidate_hash,
candidate_receipt,
valid,
),
)
ctx.send_message(DisputeCoordinatorMessage::IssueLocalStatement(
session,
candidate_hash,
candidate_receipt,
valid,
))
.await
}
polkadot/node/core/dispute-participation/src/tests.rs
+21 -19
View File
@@ -24,8 +24,10 @@ use super::*;
use parity_scale_codec::Encode;
use polkadot_node_primitives::{AvailableData, BlockData, InvalidCandidate, PoV};
use polkadot_node_subsystem::{
jaeger,
messages::{AllMessages, ValidationFailed},
overseer::Subsystem,
jaeger, messages::{AllMessages, ValidationFailed}, ActivatedLeaf, ActiveLeavesUpdate, LeafStatus,
ActivatedLeaf, ActiveLeavesUpdate, LeafStatus,
};
use polkadot_node_subsystem_test_helpers::{make_subsystem_context, TestSubsystemContextHandle};
use polkadot_primitives::v1::{BlakeTwo256, CandidateCommitments, HashT, Header, ValidationCode};
@@ -45,9 +47,7 @@ where
let (subsystem_result, _) =
futures::executor::block_on(future::join(spawned_subsystem.future, async move {
let mut ctx_handle = test_future.await;
ctx_handle
.send(FromOverseer::Signal(OverseerSignal::Conclude))
.await;
ctx_handle.send(FromOverseer::Signal(OverseerSignal::Conclude)).await;
// no further request is received by the overseer which means that
// no further attempt to participate was made
@@ -69,14 +69,14 @@ async fn activate_leaf(virtual_overseer: &mut VirtualOverseer, block_number: Blo
let block_hash = block_header.hash();
virtual_overseer
.send(FromOverseer::Signal(OverseerSignal::ActiveLeaves(
ActiveLeavesUpdate::start_work(ActivatedLeaf {
.send(FromOverseer::Signal(OverseerSignal::ActiveLeaves(ActiveLeavesUpdate::start_work(
ActivatedLeaf {
hash: block_hash,
span: Arc::new(jaeger::Span::Disabled),
number: block_number,
status: LeafStatus::Fresh,
}),
)))
},
))))
.await;
}
@@ -102,20 +102,19 @@ async fn participate(virtual_overseer: &mut VirtualOverseer) -> oneshot::Receive
n_validators,
report_availability,
},
})
.await;
})
.await;
receive_availability
}
async fn recover_available_data(virtual_overseer: &mut VirtualOverseer, receive_availability: oneshot::Receiver<bool>) {
let pov_block = PoV {
block_data: BlockData(Vec::new()),
};
async fn recover_available_data(
virtual_overseer: &mut VirtualOverseer,
receive_availability: oneshot::Receiver<bool>,
) {
let pov_block = PoV { block_data: BlockData(Vec::new()) };
let available_data = AvailableData {
pov: Arc::new(pov_block),
validation_data: Default::default(),
};
let available_data =
AvailableData { pov: Arc::new(pov_block), validation_data: Default::default() };
assert_matches!(
virtual_overseer.recv().await,
@@ -217,7 +216,10 @@ fn cannot_participate_if_cannot_recover_available_data() {
"overseer did not receive recover available data message",
);
assert_eq!(receive_availability.await.expect("Availability should get reported"), false);
assert_eq!(
receive_availability.await.expect("Availability should get reported"),
false
);
virtual_overseer
})
polkadot/node/core/parachains-inherent/src/lib.rs
+19 -16
View File
@@ -26,12 +26,9 @@
use futures::{select, FutureExt};
use polkadot_node_subsystem::{
overseer::Handle,
messages::ProvisionerMessage, errors::SubsystemError,
};
use polkadot_primitives::v1::{
Block, Hash, InherentData as ParachainsInherentData,
errors::SubsystemError, messages::ProvisionerMessage, overseer::Handle,
};
use polkadot_primitives::v1::{Block, Hash, InherentData as ParachainsInherentData};
use sp_blockchain::HeaderBackend;
use sp_runtime::generic::BlockId;
use std::time;
@@ -54,13 +51,18 @@ impl ParachainsInherentDataProvider {
let pid = async {
let (sender, receiver) = futures::channel::oneshot::channel();
overseer.wait_for_activation(parent, sender).await;
receiver.await.map_err(|_| Error::ClosedChannelAwaitingActivation)?.map_err(|e| Error::Subsystem(e))?;
receiver
.await
.map_err(|_| Error::ClosedChannelAwaitingActivation)?
.map_err(|e| Error::Subsystem(e))?;
let (sender, receiver) = futures::channel::oneshot::channel();
overseer.send_msg(
ProvisionerMessage::RequestInherentData(parent, sender),
std::any::type_name::<Self>(),
).await;
overseer
.send_msg(
ProvisionerMessage::RequestInherentData(parent, sender),
std::any::type_name::<Self>(),
)
.await;
receiver.await.map_err(|_| Error::ClosedChannelAwaitingInherentData)
};
@@ -96,7 +98,7 @@ impl ParachainsInherentDataProvider {
disputes: Vec::new(),
parent_header,
}
}
},
};
Ok(Self { inherent_data })
@@ -105,11 +107,12 @@ impl ParachainsInherentDataProvider {
#[async_trait::async_trait]
impl sp_inherents::InherentDataProvider for ParachainsInherentDataProvider {
fn provide_inherent_data(&self, inherent_data: &mut sp_inherents::InherentData) -> Result<(), sp_inherents::Error> {
inherent_data.put_data(
polkadot_primitives::v1::PARACHAINS_INHERENT_IDENTIFIER,
&self.inherent_data,
)
fn provide_inherent_data(
&self,
inherent_data: &mut sp_inherents::InherentData,
) -> Result<(), sp_inherents::Error> {
inherent_data
.put_data(polkadot_primitives::v1::PARACHAINS_INHERENT_IDENTIFIER, &self.inherent_data)
}
async fn try_handle_error(
polkadot/node/core/provisioner/src/lib.rs
+109 -99
View File
@@ -24,25 +24,29 @@ use futures::{
channel::{mpsc, oneshot},
prelude::*,
};
use futures_timer::Delay;
use polkadot_node_subsystem::{
errors::{ChainApiError, RuntimeApiError}, PerLeafSpan, SubsystemSender, jaeger,
errors::{ChainApiError, RuntimeApiError},
jaeger,
messages::{
CandidateBackingMessage, ChainApiMessage, ProvisionableData, ProvisionerInherentData,
ProvisionerMessage, DisputeCoordinatorMessage,
CandidateBackingMessage, ChainApiMessage, DisputeCoordinatorMessage, ProvisionableData,
ProvisionerInherentData, ProvisionerMessage,
},
PerLeafSpan, SubsystemSender,
};
use polkadot_node_subsystem_util::{
self as util, JobSubsystem, JobSender,
request_availability_cores, request_persisted_validation_data, JobTrait, metrics::{self, prometheus},
self as util,
metrics::{self, prometheus},
request_availability_cores, request_persisted_validation_data, JobSender, JobSubsystem,
JobTrait,
};
use polkadot_primitives::v1::{
BackedCandidate, BlockNumber, CandidateReceipt, CoreState, Hash, OccupiedCoreAssumption,
SignedAvailabilityBitfield, ValidatorIndex, MultiDisputeStatementSet, DisputeStatementSet,
DisputeStatement,
BackedCandidate, BlockNumber, CandidateReceipt, CoreState, DisputeStatement,
DisputeStatementSet, Hash, MultiDisputeStatementSet, OccupiedCoreAssumption,
SignedAvailabilityBitfield, ValidatorIndex,
};
use std::{pin::Pin, collections::BTreeMap, sync::Arc};
use std::{collections::BTreeMap, pin::Pin, sync::Arc};
use thiserror::Error;
use futures_timer::Delay;
#[cfg(test)]
mod tests;
@@ -92,7 +96,7 @@ pub struct ProvisioningJob {
signed_bitfields: Vec<SignedAvailabilityBitfield>,
metrics: Metrics,
inherent_after: InherentAfter,
awaiting_inherent: Vec<oneshot::Sender<ProvisionerInherentData>>
awaiting_inherent: Vec<oneshot::Sender<ProvisionerInherentData>>,
}
/// Errors in the provisioner.
@@ -132,7 +136,9 @@ pub enum Error {
#[error("failed to send return message with Inherents")]
InherentDataReturnChannel,
#[error("backed candidate does not correspond to selected candidate; check logic in provisioner")]
#[error(
"backed candidate does not correspond to selected candidate; check logic in provisioner"
)]
BackedCandidateOrderingProblem,
}
@@ -156,13 +162,10 @@ impl JobTrait for ProvisioningJob {
mut sender: JobSender<S>,
) -> Pin<Box<dyn Future<Output = Result<(), Self::Error>> + Send>> {
async move {
let job = ProvisioningJob::new(
relay_parent,
metrics,
receiver,
);
let job = ProvisioningJob::new(relay_parent, metrics, receiver);
job.run_loop(sender.subsystem_sender(), PerLeafSpan::new(span, "provisioner")).await
job.run_loop(sender.subsystem_sender(), PerLeafSpan::new(span, "provisioner"))
.await
}
.boxed()
}
@@ -190,9 +193,7 @@ impl ProvisioningJob {
sender: &mut impl SubsystemSender,
span: PerLeafSpan,
) -> Result<(), Error> {
use ProvisionerMessage::{
ProvisionableData, RequestInherentData,
};
use ProvisionerMessage::{ProvisionableData, RequestInherentData};
loop {
futures::select! {
msg = self.receiver.next() => match msg {
@@ -248,17 +249,21 @@ impl ProvisioningJob {
}
}
fn note_provisionable_data(&mut self, span: &jaeger::Span, provisionable_data: ProvisionableData) {
fn note_provisionable_data(
&mut self,
span: &jaeger::Span,
provisionable_data: ProvisionableData,
) {
match provisionable_data {
ProvisionableData::Bitfield(_, signed_bitfield) => {
self.signed_bitfields.push(signed_bitfield)
}
ProvisionableData::Bitfield(_, signed_bitfield) =>
self.signed_bitfields.push(signed_bitfield),
ProvisionableData::BackedCandidate(backed_candidate) => {
let _span = span.child("provisionable-backed")
let _span = span
.child("provisionable-backed")
.with_para_id(backed_candidate.descriptor().para_id);
self.backed_candidates.push(backed_candidate)
}
_ => {}
},
_ => {},
}
}
}
@@ -291,27 +296,23 @@ async fn send_inherent_data(
) -> Result<(), Error> {
let availability_cores = request_availability_cores(relay_parent, from_job)
.await
.await.map_err(|err| Error::CanceledAvailabilityCores(err))??;
.await
.map_err(|err| Error::CanceledAvailabilityCores(err))??;
let bitfields = select_availability_bitfields(&availability_cores, bitfields);
let candidates = select_candidates(
&availability_cores,
&bitfields,
candidates,
relay_parent,
from_job,
).await?;
let candidates =
select_candidates(&availability_cores, &bitfields, candidates, relay_parent, from_job)
.await?;
let disputes = select_disputes(from_job).await?;
let inherent_data = ProvisionerInherentData {
bitfields,
backed_candidates: candidates,
disputes,
};
let inherent_data =
ProvisionerInherentData { bitfields, backed_candidates: candidates, disputes };
for return_sender in return_senders {
return_sender.send(inherent_data.clone()).map_err(|_data| Error::InherentDataReturnChannel)?;
return_sender
.send(inherent_data.clone())
.map_err(|_data| Error::InherentDataReturnChannel)?;
}
Ok(())
@@ -333,16 +334,18 @@ fn select_availability_bitfields(
) -> Vec<SignedAvailabilityBitfield> {
let mut selected: BTreeMap<ValidatorIndex, SignedAvailabilityBitfield> = BTreeMap::new();
'a:
for bitfield in bitfields.iter().cloned() {
'a: for bitfield in bitfields.iter().cloned() {
if bitfield.payload().0.len() != cores.len() {
continue
}
let is_better = selected.get(&bitfield.validator_index())
let is_better = selected
.get(&bitfield.validator_index())
.map_or(true, |b| b.payload().0.count_ones() < bitfield.payload().0.count_ones());
if !is_better { continue }
if !is_better {
continue
}
for (idx, _) in cores.iter().enumerate().filter(|v| !v.1.is_occupied()) {
// Bit is set for an unoccupied core - invalid
@@ -374,23 +377,24 @@ async fn select_candidates(
let (scheduled_core, assumption) = match core {
CoreState::Scheduled(scheduled_core) => (scheduled_core, OccupiedCoreAssumption::Free),
CoreState::Occupied(occupied_core) => {
if bitfields_indicate_availability(core_idx, bitfields, &occupied_core.availability) {
if bitfields_indicate_availability(core_idx, bitfields, &occupied_core.availability)
{
if let Some(ref scheduled_core) = occupied_core.next_up_on_available {
(scheduled_core, OccupiedCoreAssumption::Included)
} else {
continue;
continue
}
} else {
if occupied_core.time_out_at != block_number {
continue;
continue
}
if let Some(ref scheduled_core) = occupied_core.next_up_on_time_out {
(scheduled_core, OccupiedCoreAssumption::TimedOut)
} else {
continue;
continue
}
}
}
},
CoreState::Free => continue,
};
@@ -401,7 +405,8 @@ async fn select_candidates(
sender,
)
.await
.await.map_err(|err| Error::CanceledPersistedValidationData(err))??
.await
.map_err(|err| Error::CanceledPersistedValidationData(err))??
{
Some(v) => v,
None => continue,
@@ -412,8 +417,8 @@ async fn select_candidates(
// we arbitrarily pick the first of the backed candidates which match the appropriate selection criteria
if let Some(candidate) = candidates.iter().find(|backed_candidate| {
let descriptor = &backed_candidate.descriptor;
descriptor.para_id == scheduled_core.para_id
&& descriptor.persisted_validation_data_hash == computed_validation_data_hash
descriptor.para_id == scheduled_core.para_id &&
descriptor.persisted_validation_data_hash == computed_validation_data_hash
}) {
let candidate_hash = candidate.hash();
tracing::trace!(
@@ -430,11 +435,16 @@ async fn select_candidates(
// now get the backed candidates corresponding to these candidate receipts
let (tx, rx) = oneshot::channel();
sender.send_message(CandidateBackingMessage::GetBackedCandidates(
relay_parent,
selected_candidates.clone(),
tx,
).into()).await;
sender
.send_message(
CandidateBackingMessage::GetBackedCandidates(
relay_parent,
selected_candidates.clone(),
tx,
)
.into(),
)
.await;
let mut candidates = rx.await.map_err(|err| Error::CanceledBackedCandidates(err))?;
// `selected_candidates` is generated in ascending order by core index, and `GetBackedCandidates`
@@ -445,7 +455,9 @@ async fn select_candidates(
// in order, we can ensure that the backed candidates are also in order.
let mut backed_idx = 0;
for selected in selected_candidates {
if selected == candidates.get(backed_idx).ok_or(Error::BackedCandidateOrderingProblem)?.hash() {
if selected ==
candidates.get(backed_idx).ok_or(Error::BackedCandidateOrderingProblem)?.hash()
{
backed_idx += 1;
}
}
@@ -484,12 +496,7 @@ async fn get_block_number_under_construction(
sender: &mut impl SubsystemSender,
) -> Result<BlockNumber, Error> {
let (tx, rx) = oneshot::channel();
sender
.send_message(ChainApiMessage::BlockNumber(
relay_parent,
tx,
).into())
.await;
sender.send_message(ChainApiMessage::BlockNumber(relay_parent, tx).into()).await;
match rx.await.map_err(|err| Error::CanceledBlockNumber(err))? {
Ok(Some(n)) => Ok(n + 1),
@@ -528,8 +535,8 @@ fn bitfields_indicate_availability(
availability_len,
);
return false;
}
return false
},
Some(mut bit_mut) => *bit_mut |= bitfield.payload().0[core_idx],
}
}
@@ -562,16 +569,17 @@ async fn select_disputes(
);
Vec::new()
}
},
};
// Load all votes for all disputes from the coordinator.
let dispute_candidate_votes = {
let (tx, rx) = oneshot::channel();
sender.send_message(DisputeCoordinatorMessage::QueryCandidateVotes(
recent_disputes,
tx,
).into()).await;
sender
.send_message(
DisputeCoordinatorMessage::QueryCandidateVotes(recent_disputes, tx).into(),
)
.await;
match rx.await {
Ok(v) => v,
@@ -581,24 +589,29 @@ async fn select_disputes(
"Unable to query candidate votes - subsystem disconnected?",
);
Vec::new()
}
},
}
};
// Transform all `CandidateVotes` into `MultiDisputeStatementSet`.
Ok(dispute_candidate_votes.into_iter().map(|(session_index, candidate_hash, votes)| {
let valid_statements = votes.valid.into_iter()
.map(|(s, i, sig)| (DisputeStatement::Valid(s), i, sig));
Ok(dispute_candidate_votes
.into_iter()
.map(|(session_index, candidate_hash, votes)| {
let valid_statements =
votes.valid.into_iter().map(|(s, i, sig)| (DisputeStatement::Valid(s), i, sig));
let invalid_statements = votes.invalid.into_iter()
.map(|(s, i, sig)| (DisputeStatement::Invalid(s), i, sig));
let invalid_statements = votes
.invalid
.into_iter()
.map(|(s, i, sig)| (DisputeStatement::Invalid(s), i, sig));
DisputeStatementSet {
candidate_hash,
session: session_index,
statements: valid_statements.chain(invalid_statements).collect(),
}
}).collect())
DisputeStatementSet {
candidate_hash,
session: session_index,
statements: valid_statements.chain(invalid_statements).collect(),
}
})
.collect())
}
#[derive(Clone)]
@@ -623,7 +636,9 @@ impl Metrics {
}
/// Provide a timer for `request_inherent_data` which observes on drop.
fn time_request_inherent_data(&self) -> Option<metrics::prometheus::prometheus::HistogramTimer> {
fn time_request_inherent_data(
&self,
) -> Option<metrics::prometheus::prometheus::HistogramTimer> {
self.0.as_ref().map(|metrics| metrics.request_inherent_data.start_timer())
}
@@ -647,21 +662,17 @@ impl metrics::Metrics for Metrics {
registry,
)?,
request_inherent_data: prometheus::register(
prometheus::Histogram::with_opts(
prometheus::HistogramOpts::new(
"parachain_provisioner_request_inherent_data",
"Time spent within `provisioner::request_inherent_data`",
)
)?,
prometheus::Histogram::with_opts(prometheus::HistogramOpts::new(
"parachain_provisioner_request_inherent_data",
"Time spent within `provisioner::request_inherent_data`",
))?,
registry,
)?,
provisionable_data: prometheus::register(
prometheus::Histogram::with_opts(
prometheus::HistogramOpts::new(
"parachain_provisioner_provisionable_data",
"Time spent within `provisioner::provisionable_data`",
)
)?,
prometheus::Histogram::with_opts(prometheus::HistogramOpts::new(
"parachain_provisioner_provisionable_data",
"Time spent within `provisioner::provisionable_data`",
))?,
registry,
)?,
};
@@ -669,6 +680,5 @@ impl metrics::Metrics for Metrics {
}
}
/// The provisioning subsystem.
pub type ProvisioningSubsystem<Spawner> = JobSubsystem<ProvisioningJob, Spawner>;
polkadot/node/core/provisioner/src/tests.rs
+72 -62
View File
@@ -34,20 +34,16 @@ pub fn default_bitvec(n_cores: usize) -> CoreAvailability {
}
pub fn scheduled_core(id: u32) -> ScheduledCore {
ScheduledCore {
para_id: id.into(),
..Default::default()
}
ScheduledCore { para_id: id.into(), ..Default::default() }
}
mod select_availability_bitfields {
use super::super::*;
use super::{default_bitvec, occupied_core};
use super::{super::*, default_bitvec, occupied_core};
use futures::executor::block_on;
use std::sync::Arc;
use polkadot_primitives::v1::{SigningContext, ValidatorIndex, ValidatorId};
use polkadot_primitives::v1::{SigningContext, ValidatorId, ValidatorIndex};
use sp_application_crypto::AppKey;
use sp_keystore::{CryptoStore, SyncCryptoStorePtr, testing::KeyStore};
use sp_keystore::{testing::KeyStore, CryptoStore, SyncCryptoStorePtr};
use std::sync::Arc;
async fn signed_bitfield(
keystore: &SyncCryptoStorePtr,
@@ -63,7 +59,11 @@ mod select_availability_bitfields {
&<SigningContext<Hash>>::default(),
validator_idx,
&public.into(),
).await.ok().flatten().expect("Should be signed")
)
.await
.ok()
.flatten()
.expect("Should be signed")
}
#[test]
@@ -109,11 +109,8 @@ mod select_availability_bitfields {
let mut bitvec2 = bitvec.clone();
bitvec2.set(2, true);
let cores = vec![
CoreState::Free,
CoreState::Scheduled(Default::default()),
occupied_core(2),
];
let cores =
vec![CoreState::Free, CoreState::Scheduled(Default::default()), occupied_core(2)];
let bitfields = vec![
block_on(signed_bitfield(&keystore, bitvec0, ValidatorIndex(0))),
@@ -191,16 +188,18 @@ mod select_availability_bitfields {
}
mod select_candidates {
use super::super::*;
use super::{build_occupied_core, occupied_core, scheduled_core, default_bitvec};
use super::{super::*, build_occupied_core, default_bitvec, occupied_core, scheduled_core};
use polkadot_node_subsystem::messages::{
AllMessages, RuntimeApiMessage,
RuntimeApiRequest::{AvailabilityCores, PersistedValidationData as PersistedValidationDataReq},
};
use polkadot_primitives::v1::{
BlockNumber, CandidateDescriptor, PersistedValidationData, CommittedCandidateReceipt, CandidateCommitments,
RuntimeApiRequest::{
AvailabilityCores, PersistedValidationData as PersistedValidationDataReq,
},
};
use polkadot_node_subsystem_test_helpers::TestSubsystemSender;
use polkadot_primitives::v1::{
BlockNumber, CandidateCommitments, CandidateDescriptor, CommittedCandidateReceipt,
PersistedValidationData,
};
const BLOCK_UNDER_PRODUCTION: BlockNumber = 128;
@@ -307,21 +306,21 @@ mod select_candidates {
while let Some(from_job) = receiver.next().await {
match from_job {
AllMessages::ChainApi(BlockNumber(_relay_parent, tx)) => {
tx.send(Ok(Some(BLOCK_UNDER_PRODUCTION - 1))).unwrap()
}
AllMessages::ChainApi(BlockNumber(_relay_parent, tx)) =>
tx.send(Ok(Some(BLOCK_UNDER_PRODUCTION - 1))).unwrap(),
AllMessages::RuntimeApi(Request(
_parent_hash,
PersistedValidationDataReq(_para_id, _assumption, tx),
)) => tx.send(Ok(Some(Default::default()))).unwrap(),
AllMessages::RuntimeApi(Request(_parent_hash, AvailabilityCores(tx))) => {
tx.send(Ok(mock_availability_cores())).unwrap()
}
AllMessages::CandidateBacking(
CandidateBackingMessage::GetBackedCandidates(_, _, sender)
) => {
AllMessages::RuntimeApi(Request(_parent_hash, AvailabilityCores(tx))) =>
tx.send(Ok(mock_availability_cores())).unwrap(),
AllMessages::CandidateBacking(CandidateBackingMessage::GetBackedCandidates(
_,
_,
sender,
)) => {
let _ = sender.send(expected.clone());
}
},
_ => panic!("Unexpected message: {:?}", from_job),
}
}
@@ -329,9 +328,12 @@ mod select_candidates {
#[test]
fn can_succeed() {
test_harness(|r| mock_overseer(r, Vec::new()), |mut tx: TestSubsystemSender| async move {
select_candidates(&[], &[], &[], Default::default(), &mut tx).await.unwrap();
})
test_harness(
|r| mock_overseer(r, Vec::new()),
|mut tx: TestSubsystemSender| async move {
select_candidates(&[], &[], &[], Default::default(), &mut tx).await.unwrap();
},
)
}
// this tests that only the appropriate candidates get selected.
@@ -368,8 +370,7 @@ mod select_candidates {
candidate
} else if idx < mock_cores.len() * 2 {
// for the second repetition of the candidates, give them the wrong hash
candidate.descriptor.persisted_validation_data_hash
= Default::default();
candidate.descriptor.persisted_validation_data_hash = Default::default();
candidate
} else {
// third go-around: right hash, wrong para_id
@@ -380,34 +381,38 @@ mod select_candidates {
.collect();
// why those particular indices? see the comments on mock_availability_cores()
let expected_candidates: Vec<_> = [1, 4, 7, 8, 10]
.iter()
.map(|&idx| candidates[idx].clone())
.collect();
let expected_candidates: Vec<_> =
[1, 4, 7, 8, 10].iter().map(|&idx| candidates[idx].clone()).collect();
let expected_backed = expected_candidates
.iter()
.map(|c| BackedCandidate {
candidate: CommittedCandidateReceipt { descriptor: c.descriptor.clone(), ..Default::default() },
candidate: CommittedCandidateReceipt {
descriptor: c.descriptor.clone(),
..Default::default()
},
validity_votes: Vec::new(),
validator_indices: default_bitvec(n_cores),
})
.collect();
test_harness(|r| mock_overseer(r, expected_backed), |mut tx: TestSubsystemSender| async move {
let result =
select_candidates(&mock_cores, &[], &candidates, Default::default(), &mut tx)
.await.unwrap();
test_harness(
|r| mock_overseer(r, expected_backed),
|mut tx: TestSubsystemSender| async move {
let result =
select_candidates(&mock_cores, &[], &candidates, Default::default(), &mut tx)
.await
.unwrap();
result.into_iter()
.for_each(|c|
result.into_iter().for_each(|c| {
assert!(
expected_candidates.iter().any(|c2| c.candidate.corresponds_to(c2)),
"Failed to find candidate: {:?}",
c,
)
);
})
});
},
)
}
#[test]
@@ -430,7 +435,11 @@ mod select_candidates {
..Default::default()
},
commitments: CandidateCommitments {
new_validation_code: if cores_with_code.contains(&i) { Some(vec![].into()) } else { None },
new_validation_code: if cores_with_code.contains(&i) {
Some(vec![].into())
} else {
None
},
..Default::default()
},
..Default::default()
@@ -439,10 +448,8 @@ mod select_candidates {
let candidates: Vec<_> = committed_receipts.iter().map(|r| r.to_plain()).collect();
let expected_candidates: Vec<_> = cores
.iter()
.map(|&idx| candidates[idx].clone())
.collect();
let expected_candidates: Vec<_> =
cores.iter().map(|&idx| candidates[idx].clone()).collect();
let expected_backed: Vec<_> = cores
.iter()
@@ -453,19 +460,22 @@ mod select_candidates {
})
.collect();
test_harness(|r| mock_overseer(r, expected_backed), |mut tx: TestSubsystemSender| async move {
let result =
select_candidates(&mock_cores, &[], &candidates, Default::default(), &mut tx)
.await.unwrap();
test_harness(
|r| mock_overseer(r, expected_backed),
|mut tx: TestSubsystemSender| async move {
let result =
select_candidates(&mock_cores, &[], &candidates, Default::default(), &mut tx)
.await
.unwrap();
result.into_iter()
.for_each(|c|
result.into_iter().for_each(|c| {
assert!(
expected_candidates.iter().any(|c2| c.candidate.corresponds_to(c2)),
"Failed to find candidate: {:?}",
c,
)
);
})
});
},
)
}
}
polkadot/node/core/pvf/src/artifacts.rs
+17 -19
View File
@@ -15,15 +15,13 @@
// along with Polkadot. If not, see <http://www.gnu.org/licenses/>.
use always_assert::always;
use async_std::{
path::{Path, PathBuf},
};
use async_std::path::{Path, PathBuf};
use parity_scale_codec::{Decode, Encode};
use polkadot_parachain::primitives::ValidationCodeHash;
use std::{
collections::HashMap,
time::{Duration, SystemTime},
};
use parity_scale_codec::{Encode, Decode};
/// A final product of preparation process. Contains either a ready to run compiled artifact or
/// a description what went wrong.
@@ -70,8 +68,8 @@ impl ArtifactId {
/// Tries to recover the artifact id from the given file name.
#[cfg(test)]
pub fn from_file_name(file_name: &str) -> Option<Self> {
use std::str::FromStr as _;
use polkadot_core_primitives::Hash;
use std::str::FromStr as _;
let file_name = file_name.strip_prefix(Self::PREFIX)?;
let code_hash = Hash::from_str(file_name).ok()?.into();
@@ -123,9 +121,7 @@ impl Artifacts {
#[cfg(test)]
pub(crate) fn empty() -> Self {
Self {
artifacts: HashMap::new(),
}
Self { artifacts: HashMap::new() }
}
/// Returns the state of the given artifact by its ID.
@@ -139,10 +135,7 @@ impl Artifacts {
/// replacing existing ones.
pub fn insert_preparing(&mut self, artifact_id: ArtifactId) {
// See the precondition.
always!(self
.artifacts
.insert(artifact_id, ArtifactState::Preparing)
.is_none());
always!(self.artifacts.insert(artifact_id, ArtifactState::Preparing).is_none());
}
/// Insert an artifact with the given ID as "prepared".
@@ -164,9 +157,7 @@ impl Artifacts {
let mut to_remove = vec![];
for (k, v) in self.artifacts.iter() {
if let ArtifactState::Prepared {
last_time_needed, ..
} = *v {
if let ArtifactState::Prepared { last_time_needed, .. } = *v {
if now
.duration_since(last_time_needed)
.map(|age| age > artifact_ttl)
@@ -187,8 +178,8 @@ impl Artifacts {
#[cfg(test)]
mod tests {
use super::{ArtifactId, Artifacts};
use async_std::path::Path;
use super::{Artifacts, ArtifactId};
use sp_core::H256;
use std::str::FromStr;
@@ -213,17 +204,24 @@ mod tests {
#[test]
fn path() {
let path = Path::new("/test");
let hash = H256::from_str("1234567890123456789012345678901234567890123456789012345678901234").unwrap().into();
let hash =
H256::from_str("1234567890123456789012345678901234567890123456789012345678901234")
.unwrap()
.into();
assert_eq!(
ArtifactId::new(hash).path(path).to_str(),
Some("/test/wasmtime_0x1234567890123456789012345678901234567890123456789012345678901234"),
Some(
"/test/wasmtime_0x1234567890123456789012345678901234567890123456789012345678901234"
),
);
}
#[test]
fn artifacts_removes_cache_on_startup() {
let fake_cache_path = async_std::task::block_on(async move { crate::worker_common::tmpfile("test-cache").await.unwrap() });
let fake_cache_path = async_std::task::block_on(async move {
crate::worker_common::tmpfile("test-cache").await.unwrap()
});
let fake_artifact_path = {
let mut p = fake_cache_path.clone();
p.push("wasmtime_0x1234567890123456789012345678901234567890123456789012345678901234");
polkadot/node/core/pvf/src/execute/mod.rs
+1 -1
View File
@@ -23,5 +23,5 @@
mod queue;
mod worker;
pub use queue::{ToQueue, start};
pub use queue::{start, ToQueue};
pub use worker::worker_entrypoint;
polkadot/node/core/pvf/src/execute/queue.rs
+31 -79
View File
@@ -16,31 +16,27 @@
//! A queue that handles requests for PVF execution.
use crate::{
worker_common::{IdleWorker, WorkerHandle},
host::ResultSender,
LOG_TARGET, InvalidCandidate, ValidationError,
};
use super::worker::Outcome;
use std::{collections::VecDeque, fmt, time::Duration};
use crate::{
host::ResultSender,
worker_common::{IdleWorker, WorkerHandle},
InvalidCandidate, ValidationError, LOG_TARGET,
};
use async_std::path::PathBuf;
use futures::{
Future, FutureExt,
channel::mpsc,
future::BoxFuture,
stream::{FuturesUnordered, StreamExt as _},
Future, FutureExt,
};
use async_std::path::PathBuf;
use slotmap::HopSlotMap;
use std::{collections::VecDeque, fmt, time::Duration};
slotmap::new_key_type! { struct Worker; }
#[derive(Debug)]
pub enum ToQueue {
Enqueue {
artifact_path: PathBuf,
params: Vec<u8>,
result_tx: ResultSender,
},
Enqueue { artifact_path: PathBuf, params: Vec<u8>, result_tx: ResultSender },
}
struct ExecuteJob {
@@ -86,11 +82,7 @@ impl Workers {
///
/// Returns `None` if either worker is not recognized or idle token is absent.
fn claim_idle(&mut self, worker: Worker) -> Option<IdleWorker> {
self
.running
.get_mut(worker)?
.idle
.take()
self.running.get_mut(worker)?.idle.take()
}
}
@@ -167,17 +159,9 @@ async fn purge_dead(workers: &mut Workers) {
}
fn handle_to_queue(queue: &mut Queue, to_queue: ToQueue) {
let ToQueue::Enqueue {
artifact_path,
params,
result_tx,
} = to_queue;
let ToQueue::Enqueue { artifact_path, params, result_tx } = to_queue;
let job = ExecuteJob {
artifact_path,
params,
result_tx,
};
let job = ExecuteJob { artifact_path, params, result_tx };
if let Some(available) = queue.workers.find_available() {
assign(queue, available, job);
@@ -194,18 +178,15 @@ async fn handle_mux(queue: &mut Queue, event: QueueEvent) {
QueueEvent::Spawn((idle, handle)) => {
queue.workers.spawn_inflight -= 1;
let worker = queue.workers.running.insert(WorkerData {
idle: Some(idle),
handle,
});
let worker = queue.workers.running.insert(WorkerData { idle: Some(idle), handle });
if let Some(job) = queue.queue.pop_front() {
assign(queue, worker, job);
}
}
},
QueueEvent::StartWork(worker, outcome, result_tx) => {
handle_job_finish(queue, worker, outcome, result_tx);
}
},
}
}
@@ -213,38 +194,22 @@ async fn handle_mux(queue: &mut Queue, event: QueueEvent) {
/// worker. Otherwise, puts back into the available workers list.
fn handle_job_finish(queue: &mut Queue, worker: Worker, outcome: Outcome, result_tx: ResultSender) {
let (idle_worker, result) = match outcome {
Outcome::Ok {
result_descriptor,
duration_ms,
idle_worker,
} => {
Outcome::Ok { result_descriptor, duration_ms, idle_worker } => {
// TODO: propagate the soft timeout
drop(duration_ms);
(Some(idle_worker), Ok(result_descriptor))
}
},
Outcome::InvalidCandidate { err, idle_worker } => (
Some(idle_worker),
Err(ValidationError::InvalidCandidate(
InvalidCandidate::WorkerReportedError(err),
)),
),
Outcome::InternalError { err, idle_worker } => (
Some(idle_worker),
Err(ValidationError::InternalError(err)),
),
Outcome::HardTimeout => (
None,
Err(ValidationError::InvalidCandidate(
InvalidCandidate::HardTimeout,
)),
),
Outcome::IoErr => (
None,
Err(ValidationError::InvalidCandidate(
InvalidCandidate::AmbigiousWorkerDeath,
)),
Err(ValidationError::InvalidCandidate(InvalidCandidate::WorkerReportedError(err))),
),
Outcome::InternalError { err, idle_worker } =>
(Some(idle_worker), Err(ValidationError::InternalError(err))),
Outcome::HardTimeout =>
(None, Err(ValidationError::InvalidCandidate(InvalidCandidate::HardTimeout))),
Outcome::IoErr =>
(None, Err(ValidationError::InvalidCandidate(InvalidCandidate::AmbigiousWorkerDeath))),
};
// First we send the result. It may fail due the other end of the channel being dropped, that's
@@ -293,15 +258,11 @@ async fn spawn_worker_task(program_path: PathBuf, spawn_timeout: Duration) -> Qu
match super::worker::spawn(&program_path, spawn_timeout).await {
Ok((idle, handle)) => break QueueEvent::Spawn((idle, handle)),
Err(err) => {
tracing::warn!(
target: LOG_TARGET,
"failed to spawn an execute worker: {:?}",
err,
);
tracing::warn!(target: LOG_TARGET, "failed to spawn an execute worker: {:?}", err,);
// Assume that the failure intermittent and retry after a delay.
Delay::new(Duration::from_secs(3)).await;
}
},
}
}
}
@@ -310,14 +271,11 @@ async fn spawn_worker_task(program_path: PathBuf, spawn_timeout: Duration) -> Qu
///
/// The worker must be running and idle.
fn assign(queue: &mut Queue, worker: Worker, job: ExecuteJob) {
let idle = queue
.workers
.claim_idle(worker)
.expect(
"this caller must supply a worker which is idle and running;
let idle = queue.workers.claim_idle(worker).expect(
"this caller must supply a worker which is idle and running;
thus claim_idle cannot return None;
qed."
);
qed.",
);
queue.mux.push(
async move {
let outcome = super::worker::start_work(idle, job.artifact_path, job.params).await;
@@ -333,12 +291,6 @@ pub fn start(
spawn_timeout: Duration,
) -> (mpsc::Sender<ToQueue>, impl Future<Output = ()>) {
let (to_queue_tx, to_queue_rx) = mpsc::channel(20);
let run = Queue::new(
program_path,
worker_capacity,
spawn_timeout,
to_queue_rx,
)
.run();
let run = Queue::new(program_path, worker_capacity, spawn_timeout, to_queue_rx).run();
(to_queue_tx, run)
}
polkadot/node/core/pvf/src/execute/worker.rs
+35 -84
View File
@@ -16,14 +16,13 @@
use crate::{
artifacts::Artifact,
LOG_TARGET,
executor_intf::TaskExecutor,
worker_common::{
IdleWorker, SpawnErr, WorkerHandle, bytes_to_path, framed_recv, framed_send, path_to_bytes,
spawn_with_program_path, worker_event_loop,
bytes_to_path, framed_recv, framed_send, path_to_bytes, spawn_with_program_path,
worker_event_loop, IdleWorker, SpawnErr, WorkerHandle,
},
LOG_TARGET,
};
use std::time::{Duration, Instant};
use async_std::{
io,
os::unix::net::UnixStream,
@@ -31,8 +30,9 @@ use async_std::{
};
use futures::FutureExt;
use futures_timer::Delay;
use parity_scale_codec::{Decode, Encode};
use polkadot_parachain::primitives::ValidationResult;
use parity_scale_codec::{Encode, Decode};
use std::time::{Duration, Instant};
const EXECUTION_TIMEOUT: Duration = Duration::from_secs(3);
@@ -43,36 +43,20 @@ pub async fn spawn(
program_path: &Path,
spawn_timeout: Duration,
) -> Result<(IdleWorker, WorkerHandle), SpawnErr> {
spawn_with_program_path(
"execute",
program_path,
&["execute-worker"],
spawn_timeout,
)
.await
spawn_with_program_path("execute", program_path, &["execute-worker"], spawn_timeout).await
}
/// Outcome of PVF execution.
pub enum Outcome {
/// PVF execution completed successfully and the result is returned. The worker is ready for
/// another job.
Ok {
result_descriptor: ValidationResult,
duration_ms: u64,
idle_worker: IdleWorker,
},
Ok { result_descriptor: ValidationResult, duration_ms: u64, idle_worker: IdleWorker },
/// The candidate validation failed. It may be for example because the preparation process
/// produced an error or the wasm execution triggered a trap.
InvalidCandidate {
err: String,
idle_worker: IdleWorker,
},
InvalidCandidate { err: String, idle_worker: IdleWorker },
/// An internal error happened during the validation. Such an error is most likely related to
/// some transient glitch.
InternalError {
err: String,
idle_worker: IdleWorker,
},
InternalError { err: String, idle_worker: IdleWorker },
/// The execution time exceeded the hard limit. The worker is terminated.
HardTimeout,
/// An I/O error happened during communication with the worker. This may mean that the worker
@@ -97,7 +81,7 @@ pub async fn start_work(
);
if send_request(&mut stream, &artifact_path, &validation_params).await.is_err() {
return Outcome::IoErr;
return Outcome::IoErr
}
let response = futures::select! {
@@ -111,22 +95,12 @@ pub async fn start_work(
};
match response {
Response::Ok {
result_descriptor,
duration_ms,
} => Outcome::Ok {
result_descriptor,
duration_ms,
idle_worker: IdleWorker { stream, pid },
},
Response::InvalidCandidate(err) => Outcome::InvalidCandidate {
err,
idle_worker: IdleWorker { stream, pid },
},
Response::InternalError(err) => Outcome::InternalError {
err,
idle_worker: IdleWorker { stream, pid },
},
Response::Ok { result_descriptor, duration_ms } =>
Outcome::Ok { result_descriptor, duration_ms, idle_worker: IdleWorker { stream, pid } },
Response::InvalidCandidate(err) =>
Outcome::InvalidCandidate { err, idle_worker: IdleWorker { stream, pid } },
Response::InternalError(err) =>
Outcome::InternalError { err, idle_worker: IdleWorker { stream, pid } },
}
}
@@ -167,10 +141,7 @@ async fn recv_response(stream: &mut UnixStream) -> io::Result<Response> {
#[derive(Encode, Decode)]
enum Response {
Ok {
result_descriptor: ValidationResult,
duration_ms: u64,
},
Ok { result_descriptor: ValidationResult, duration_ms: u64 },
InvalidCandidate(String),
InternalError(String),
}
@@ -190,10 +161,7 @@ impl Response {
pub fn worker_entrypoint(socket_path: &str) {
worker_event_loop("execute", socket_path, |mut stream| async move {
let executor = TaskExecutor::new().map_err(|e| {
io::Error::new(
io::ErrorKind::Other,
format!("cannot create task executor: {}", e),
)
io::Error::new(io::ErrorKind::Other, format!("cannot create task executor: {}", e))
})?;
loop {
let (artifact_path, params) = recv_request(&mut stream).await?;
@@ -215,13 +183,12 @@ async fn validate_using_artifact(
spawner: &TaskExecutor,
) -> Response {
let artifact_bytes = match async_std::fs::read(artifact_path).await {
Err(e) => {
Err(e) =>
return Response::InternalError(format!(
"failed to read the artifact at {}: {:?}",
artifact_path.display(),
e,
))
}
)),
Ok(b) => b,
};
@@ -231,47 +198,31 @@ async fn validate_using_artifact(
};
let compiled_artifact = match &artifact {
Artifact::PrevalidationErr(msg) => {
return Response::format_invalid("prevalidation", msg);
}
Artifact::PreparationErr(msg) => {
return Response::format_invalid("preparation", msg);
}
Artifact::DidntMakeIt => {
return Response::format_invalid("preparation timeout", "");
}
Artifact::PrevalidationErr(msg) => return Response::format_invalid("prevalidation", msg),
Artifact::PreparationErr(msg) => return Response::format_invalid("preparation", msg),
Artifact::DidntMakeIt => return Response::format_invalid("preparation timeout", ""),
Artifact::Compiled { compiled_artifact } => compiled_artifact,
};
let validation_started_at = Instant::now();
let descriptor_bytes =
match unsafe {
// SAFETY: this should be safe since the compiled artifact passed here comes from the
// file created by the prepare workers. These files are obtained by calling
// [`executor_intf::prepare`].
crate::executor_intf::execute(compiled_artifact, params, spawner.clone())
} {
Err(err) => {
return Response::format_invalid("execute", &err.to_string());
}
Ok(d) => d,
};
let descriptor_bytes = match unsafe {
// SAFETY: this should be safe since the compiled artifact passed here comes from the
// file created by the prepare workers. These files are obtained by calling
// [`executor_intf::prepare`].
crate::executor_intf::execute(compiled_artifact, params, spawner.clone())
} {
Err(err) => return Response::format_invalid("execute", &err.to_string()),
Ok(d) => d,
};
let duration_ms = validation_started_at.elapsed().as_millis() as u64;
let result_descriptor = match ValidationResult::decode(&mut &descriptor_bytes[..]) {
Err(err) => {
return Response::InvalidCandidate(format!(
"validation result decoding failed: {}",
err
))
}
Err(err) =>
return Response::InvalidCandidate(format!("validation result decoding failed: {}", err)),
Ok(r) => r,
};
Response::Ok {
result_descriptor,
duration_ms,
}
Response::Ok { result_descriptor, duration_ms }
}
polkadot/node/core/pvf/src/executor_intf.rs
+4 -8
View File
@@ -16,16 +16,14 @@
//! Interface to the Substrate Executor
use std::any::{TypeId, Any};
use sc_executor_common::{
runtime_blob::RuntimeBlob,
wasm_runtime::{InvokeMethod, WasmModule as _},
};
use sc_executor_wasmtime::{Config, Semantics, DeterministicStackLimit};
use sp_core::{
storage::{ChildInfo, TrackedStorageKey},
};
use sc_executor_wasmtime::{Config, DeterministicStackLimit, Semantics};
use sp_core::storage::{ChildInfo, TrackedStorageKey};
use sp_wasm_interface::HostFunctions as _;
use std::any::{Any, TypeId};
const CONFIG: Config = Config {
// TODO: Make sure we don't use more than 1GB: https://github.com/paritytech/polkadot/issues/699
@@ -95,9 +93,7 @@ pub unsafe fn execute(
CONFIG,
HostFunctions::host_functions(),
)?;
runtime
.new_instance()?
.call(InvokeMethod::Export("validate_block"), params)
runtime.new_instance()?.call(InvokeMethod::Export("validate_block"), params)
})?
}
polkadot/node/core/pvf/src/host.rs
+63 -159
View File
@@ -21,23 +21,20 @@
//! [`ValidationHost`], that allows communication with that event-loop.
use crate::{
Priority, Pvf, ValidationError,
artifacts::{Artifacts, ArtifactState, ArtifactId},
execute, prepare,
artifacts::{ArtifactId, ArtifactState, Artifacts},
execute, prepare, Priority, Pvf, ValidationError,
};
use always_assert::never;
use async_std::path::{Path, PathBuf};
use futures::{
channel::{mpsc, oneshot},
Future, FutureExt, SinkExt, StreamExt,
};
use polkadot_parachain::primitives::ValidationResult;
use std::{
collections::HashMap,
time::{Duration, SystemTime},
};
use always_assert::never;
use async_std::{
path::{Path, PathBuf},
};
use polkadot_parachain::primitives::ValidationResult;
use futures::{
Future, FutureExt, SinkExt, StreamExt,
channel::{mpsc, oneshot},
};
/// An alias to not spell the type for the oneshot sender for the PVF execution result.
pub(crate) type ResultSender = oneshot::Sender<Result<ValidationResult, ValidationError>>;
@@ -64,12 +61,7 @@ impl ValidationHost {
result_tx: ResultSender,
) -> Result<(), String> {
self.to_host_tx
.send(ToHost::ExecutePvf {
pvf,
params,
priority,
result_tx,
})
.send(ToHost::ExecutePvf { pvf, params, priority, result_tx })
.await
.map_err(|_| "the inner loop hung up".to_string())
}
@@ -89,15 +81,8 @@ impl ValidationHost {
}
enum ToHost {
ExecutePvf {
pvf: Pvf,
params: Vec<u8>,
priority: Priority,
result_tx: ResultSender,
},
HeadsUp {
active_pvfs: Vec<Pvf>,
},
ExecutePvf { pvf: Pvf, params: Vec<u8>, priority: Priority, result_tx: ResultSender },
HeadsUp { active_pvfs: Vec<Pvf> },
}
/// Configuration for the validation host.
@@ -180,12 +165,7 @@ pub fn start(config: Config) -> (ValidationHost, impl Future<Output = ()>) {
let run = async move {
let artifacts = Artifacts::new(&config.cache_path).await;
futures::pin_mut!(
run_prepare_queue,
run_prepare_pool,
run_execute_queue,
run_sweeper
);
futures::pin_mut!(run_prepare_queue, run_prepare_pool, run_execute_queue, run_sweeper);
run(
Inner {
@@ -375,12 +355,7 @@ async fn handle_to_host(
to_host: ToHost,
) -> Result<(), Fatal> {
match to_host {
ToHost::ExecutePvf {
pvf,
params,
priority,
result_tx,
} => {
ToHost::ExecutePvf { pvf, params, priority, result_tx } => {
handle_execute_pvf(
cache_path,
artifacts,
@@ -393,10 +368,10 @@ async fn handle_to_host(
result_tx,
)
.await?;
}
},
ToHost::HeadsUp { active_pvfs } => {
handle_heads_up(artifacts, prepare_queue, active_pvfs).await?;
}
},
}
Ok(())
@@ -417,9 +392,7 @@ async fn handle_execute_pvf(
if let Some(state) = artifacts.artifact_state_mut(&artifact_id) {
match state {
ArtifactState::Prepared {
ref mut last_time_needed,
} => {
ArtifactState::Prepared { ref mut last_time_needed } => {
*last_time_needed = SystemTime::now();
send_execute(
@@ -431,19 +404,16 @@ async fn handle_execute_pvf(
},
)
.await?;
}
},
ArtifactState::Preparing => {
send_prepare(
prepare_queue,
prepare::ToQueue::Amend {
priority,
artifact_id: artifact_id.clone(),
},
prepare::ToQueue::Amend { priority, artifact_id: artifact_id.clone() },
)
.await?;
awaiting_prepare.add(artifact_id, params, result_tx);
}
},
}
} else {
// Artifact is unknown: register it and enqueue a job with the corresponding priority and
@@ -454,7 +424,7 @@ async fn handle_execute_pvf(
awaiting_prepare.add(artifact_id, params, result_tx);
}
return Ok(());
return Ok(())
}
async fn handle_heads_up(
@@ -468,15 +438,13 @@ async fn handle_heads_up(
let artifact_id = active_pvf.as_artifact_id();
if let Some(state) = artifacts.artifact_state_mut(&artifact_id) {
match state {
ArtifactState::Prepared {
last_time_needed, ..
} => {
ArtifactState::Prepared { last_time_needed, .. } => {
*last_time_needed = now;
}
},
ArtifactState::Preparing => {
// Already preparing. We don't need to send a priority amend either because
// it can't get any lower than the background.
}
},
}
} else {
// The artifact is unknown: register it and put a background job into the prepare queue.
@@ -484,10 +452,7 @@ async fn handle_heads_up(
send_prepare(
prepare_queue,
prepare::ToQueue::Enqueue {
priority: Priority::Background,
pvf: active_pvf,
},
prepare::ToQueue::Enqueue { priority: Priority::Background, pvf: active_pvf },
)
.await?;
}
@@ -512,8 +477,8 @@ async fn handle_prepare_done(
// thus the artifact cannot be unknown, only preparing;
// qed.
never!("an unknown artifact was prepared: {:?}", artifact_id);
return Ok(());
}
return Ok(())
},
Some(ArtifactState::Prepared { .. }) => {
// before sending request to prepare, the artifact is inserted with `preparing` state;
// the requests are deduplicated for the same artifact id;
@@ -521,8 +486,8 @@ async fn handle_prepare_done(
// thus the artifact cannot be prepared, only preparing;
// qed.
never!("the artifact is already prepared: {:?}", artifact_id);
return Ok(());
}
return Ok(())
},
Some(state @ ArtifactState::Preparing) => state,
};
@@ -534,24 +499,18 @@ async fn handle_prepare_done(
if result_tx.is_canceled() {
// Preparation could've taken quite a bit of time and the requester may be not interested
// in execution anymore, in which case we just skip the request.
continue;
continue
}
send_execute(
execute_queue,
execute::ToQueue::Enqueue {
artifact_path: artifact_path.clone(),
params,
result_tx,
},
execute::ToQueue::Enqueue { artifact_path: artifact_path.clone(), params, result_tx },
)
.await?;
}
// Now consider the artifact prepared.
*state = ArtifactState::Prepared {
last_time_needed: SystemTime::now(),
};
*state = ArtifactState::Prepared { last_time_needed: SystemTime::now() };
Ok(())
}
@@ -592,7 +551,7 @@ async fn sweeper_task(mut sweeper_rx: mpsc::Receiver<PathBuf>) {
None => break,
Some(condemned) => {
let _ = async_std::fs::remove_file(condemned).await;
}
},
}
}
}
@@ -611,8 +570,8 @@ fn pulse_every(interval: std::time::Duration) -> impl futures::Stream<Item = ()>
#[cfg(test)]
mod tests {
use super::*;
use futures::future::BoxFuture;
use assert_matches::assert_matches;
use futures::future::BoxFuture;
#[async_std::test]
async fn pulse_test() {
@@ -634,9 +593,7 @@ mod tests {
}
fn artifact_path(descriminator: u32) -> PathBuf {
artifact_id(descriminator)
.path(&PathBuf::from(std::env::temp_dir()))
.to_owned()
artifact_id(descriminator).path(&PathBuf::from(std::env::temp_dir())).to_owned()
}
struct Builder {
@@ -673,13 +630,7 @@ mod tests {
}
impl Test {
fn new(
Builder {
cleanup_pulse_interval,
artifact_ttl,
artifacts,
}: Builder,
) -> Self {
fn new(Builder { cleanup_pulse_interval, artifact_ttl, artifacts }: Builder) -> Self {
let cache_path = PathBuf::from(std::env::temp_dir());
let (to_host_tx, to_host_rx) = mpsc::channel(10);
@@ -727,20 +678,14 @@ mod tests {
async fn poll_and_recv_to_prepare_queue(&mut self) -> prepare::ToQueue {
let to_prepare_queue_rx = &mut self.to_prepare_queue_rx;
run_until(
&mut self.run,
async { to_prepare_queue_rx.next().await.unwrap() }.boxed(),
)
.await
run_until(&mut self.run, async { to_prepare_queue_rx.next().await.unwrap() }.boxed())
.await
}
async fn poll_and_recv_to_execute_queue(&mut self) -> execute::ToQueue {
let to_execute_queue_rx = &mut self.to_execute_queue_rx;
run_until(
&mut self.run,
async { to_execute_queue_rx.next().await.unwrap() }.boxed(),
)
.await
run_until(&mut self.run, async { to_execute_queue_rx.next().await.unwrap() }.boxed())
.await
}
async fn poll_ensure_to_execute_queue_is_empty(&mut self) {
@@ -798,7 +743,7 @@ mod tests {
}
if let Poll::Ready(r) = futures::poll!(&mut *fut) {
break r;
break r
}
if futures::poll!(&mut *task).is_ready() {
@@ -831,9 +776,7 @@ mod tests {
let mut test = builder.build();
let mut host = test.host_handle();
host.heads_up(vec![Pvf::from_discriminator(1)])
.await
.unwrap();
host.heads_up(vec![Pvf::from_discriminator(1)]).await.unwrap();
let to_sweeper_rx = &mut test.to_sweeper_rx;
run_until(
@@ -847,9 +790,7 @@ mod tests {
// Extend TTL for the first artifact and make sure we don't receive another file removal
// request.
host.heads_up(vec![Pvf::from_discriminator(1)])
.await
.unwrap();
host.heads_up(vec![Pvf::from_discriminator(1)]).await.unwrap();
test.poll_ensure_to_sweeper_is_empty().await;
}
@@ -858,9 +799,7 @@ mod tests {
let mut test = Builder::default().build();
let mut host = test.host_handle();
host.heads_up(vec![Pvf::from_discriminator(1)])
.await
.unwrap();
host.heads_up(vec![Pvf::from_discriminator(1)]).await.unwrap();
// Run until we receive a prepare request.
let prepare_q_rx = &mut test.to_prepare_queue_rx;
@@ -877,22 +816,14 @@ mod tests {
.await;
let (result_tx, _result_rx) = oneshot::channel();
host.execute_pvf(
Pvf::from_discriminator(1),
vec![],
Priority::Critical,
result_tx,
)
.await
.unwrap();
host.execute_pvf(Pvf::from_discriminator(1), vec![], Priority::Critical, result_tx)
.await
.unwrap();
run_until(
&mut test.run,
async {
assert_matches!(
prepare_q_rx.next().await.unwrap(),
prepare::ToQueue::Amend { .. }
);
assert_matches!(prepare_q_rx.next().await.unwrap(), prepare::ToQueue::Amend { .. });
}
.boxed(),
)
@@ -907,14 +838,9 @@ mod tests {
let mut host = test.host_handle();
let (result_tx, result_rx_pvf_1_1) = oneshot::channel();
host.execute_pvf(
Pvf::from_discriminator(1),
b"pvf1".to_vec(),
Priority::Normal,
result_tx,
)
.await
.unwrap();
host.execute_pvf(Pvf::from_discriminator(1), b"pvf1".to_vec(), Priority::Normal, result_tx)
.await
.unwrap();
let (result_tx, result_rx_pvf_1_2) = oneshot::channel();
host.execute_pvf(
@@ -927,14 +853,9 @@ mod tests {
.unwrap();
let (result_tx, result_rx_pvf_2) = oneshot::channel();
host.execute_pvf(
Pvf::from_discriminator(2),
b"pvf2".to_vec(),
Priority::Normal,
result_tx,
)
.await
.unwrap();
host.execute_pvf(Pvf::from_discriminator(2), b"pvf2".to_vec(), Priority::Normal, result_tx)
.await
.unwrap();
assert_matches!(
test.poll_and_recv_to_prepare_queue().await,
@@ -972,39 +893,27 @@ mod tests {
);
result_tx_pvf_1_1
.send(Err(ValidationError::InvalidCandidate(
InvalidCandidate::AmbigiousWorkerDeath,
)))
.send(Err(ValidationError::InvalidCandidate(InvalidCandidate::AmbigiousWorkerDeath)))
.unwrap();
assert_matches!(
result_rx_pvf_1_1.now_or_never().unwrap().unwrap(),
Err(ValidationError::InvalidCandidate(
InvalidCandidate::AmbigiousWorkerDeath,
))
Err(ValidationError::InvalidCandidate(InvalidCandidate::AmbigiousWorkerDeath,))
);
result_tx_pvf_1_2
.send(Err(ValidationError::InvalidCandidate(
InvalidCandidate::AmbigiousWorkerDeath,
)))
.send(Err(ValidationError::InvalidCandidate(InvalidCandidate::AmbigiousWorkerDeath)))
.unwrap();
assert_matches!(
result_rx_pvf_1_2.now_or_never().unwrap().unwrap(),
Err(ValidationError::InvalidCandidate(
InvalidCandidate::AmbigiousWorkerDeath,
))
Err(ValidationError::InvalidCandidate(InvalidCandidate::AmbigiousWorkerDeath,))
);
result_tx_pvf_2
.send(Err(ValidationError::InvalidCandidate(
InvalidCandidate::AmbigiousWorkerDeath,
)))
.send(Err(ValidationError::InvalidCandidate(InvalidCandidate::AmbigiousWorkerDeath)))
.unwrap();
assert_matches!(
result_rx_pvf_2.now_or_never().unwrap().unwrap(),
Err(ValidationError::InvalidCandidate(
InvalidCandidate::AmbigiousWorkerDeath,
))
Err(ValidationError::InvalidCandidate(InvalidCandidate::AmbigiousWorkerDeath,))
);
}
@@ -1014,14 +923,9 @@ mod tests {
let mut host = test.host_handle();
let (result_tx, result_rx) = oneshot::channel();
host.execute_pvf(
Pvf::from_discriminator(1),
b"pvf1".to_vec(),
Priority::Normal,
result_tx,
)
.await
.unwrap();
host.execute_pvf(Pvf::from_discriminator(1), b"pvf1".to_vec(), Priority::Normal, result_tx)
.await
.unwrap();
assert_matches!(
test.poll_and_recv_to_prepare_queue().await,
polkadot/node/core/pvf/src/lib.rs
+1 -1
View File
@@ -91,7 +91,7 @@ pub mod testing;
#[doc(hidden)]
pub use sp_tracing;
pub use error::{ValidationError, InvalidCandidate};
pub use error::{InvalidCandidate, ValidationError};
pub use priority::Priority;
pub use pvf::Pvf;
polkadot/node/core/pvf/src/prepare/mod.rs
+1 -1
View File
@@ -26,6 +26,6 @@ mod pool;
mod queue;
mod worker;
pub use queue::{ToQueue, FromQueue, start as start_queue};
pub use pool::start as start_pool;
pub use queue::{start as start_queue, FromQueue, ToQueue};
pub use worker::worker_entrypoint;
polkadot/node/core/pvf/src/prepare/pool.rs
+24 -43
View File
@@ -14,21 +14,19 @@
// You should have received a copy of the GNU General Public License
// along with Polkadot. If not, see <http://www.gnu.org/licenses/>.
use super::worker::{self, Outcome};
use crate::{
worker_common::{IdleWorker, WorkerHandle},
LOG_TARGET,
};
use super::{
worker::{self, Outcome},
};
use std::{fmt, sync::Arc, task::Poll, time::Duration};
use always_assert::never;
use assert_matches::assert_matches;
use async_std::path::{Path, PathBuf};
use futures::{
Future, FutureExt, StreamExt, channel::mpsc, future::BoxFuture, stream::FuturesUnordered,
channel::mpsc, future::BoxFuture, stream::FuturesUnordered, Future, FutureExt, StreamExt,
};
use slotmap::HopSlotMap;
use assert_matches::assert_matches;
use always_assert::never;
use std::{fmt, sync::Arc, task::Poll, time::Duration};
slotmap::new_key_type! { pub struct Worker; }
@@ -170,7 +168,7 @@ async fn purge_dead(
// The idle token is missing, meaning this worker is now occupied: skip it. This is
// because the worker process is observed by the work task and should it reach the
// deadline or be terminated it will be handled by the corresponding mux event.
continue;
continue
}
if let Poll::Ready(()) = futures::poll!(&mut data.handle) {
@@ -197,13 +195,8 @@ fn handle_to_pool(
match to_pool {
ToPool::Spawn => {
mux.push(spawn_worker_task(program_path.to_owned(), spawn_timeout).boxed());
}
ToPool::StartWork {
worker,
code,
artifact_path,
background_priority,
} => {
},
ToPool::StartWork { worker, code, artifact_path, background_priority } => {
if let Some(data) = spawned.get_mut(worker) {
if let Some(idle) = data.idle.take() {
mux.push(
@@ -213,7 +206,7 @@ fn handle_to_pool(
code,
cache_path.to_owned(),
artifact_path,
background_priority
background_priority,
)
.boxed(),
);
@@ -229,16 +222,15 @@ fn handle_to_pool(
// That's a relatively normal situation since the queue may send `start_work` and
// before receiving it the pool would report that the worker died.
}
}
},
ToPool::Kill(worker) => {
// It may be absent if it were previously already removed by `purge_dead`.
let _ = spawned.remove(worker);
}
ToPool::BumpPriority(worker) => {
},
ToPool::BumpPriority(worker) =>
if let Some(data) = spawned.get(worker) {
worker::bump_priority(&data.handle);
}
}
},
}
}
@@ -249,15 +241,11 @@ async fn spawn_worker_task(program_path: PathBuf, spawn_timeout: Duration) -> Po
match worker::spawn(&program_path, spawn_timeout).await {
Ok((idle, handle)) => break PoolEvent::Spawn(idle, handle),
Err(err) => {
tracing::warn!(
target: LOG_TARGET,
"failed to spawn a prepare worker: {:?}",
err,
);
tracing::warn!(target: LOG_TARGET, "failed to spawn a prepare worker: {:?}", err,);
// Assume that the failure intermittent and retry after a delay.
Delay::new(Duration::from_secs(3)).await;
}
},
}
}
}
@@ -282,15 +270,12 @@ fn handle_mux(
) -> Result<(), Fatal> {
match event {
PoolEvent::Spawn(idle, handle) => {
let worker = spawned.insert(WorkerData {
idle: Some(idle),
handle,
});
let worker = spawned.insert(WorkerData { idle: Some(idle), handle });
reply(from_pool, FromPool::Spawned(worker))?;
Ok(())
}
},
PoolEvent::StartWork(worker, outcome) => {
match outcome {
Outcome::Concluded(idle) => {
@@ -298,8 +283,8 @@ fn handle_mux(
None => {
// Perhaps the worker was killed meanwhile and the result is no longer
// relevant.
return Ok(());
}
return Ok(())
},
Some(data) => data,
};
@@ -311,23 +296,23 @@ fn handle_mux(
reply(from_pool, FromPool::Concluded(worker, false))?;
Ok(())
}
},
Outcome::Unreachable => {
if spawned.remove(worker).is_some() {
reply(from_pool, FromPool::Rip(worker))?;
}
Ok(())
}
},
Outcome::DidntMakeIt => {
if spawned.remove(worker).is_some() {
reply(from_pool, FromPool::Concluded(worker, true))?;
}
Ok(())
}
},
}
}
},
}
}
@@ -340,11 +325,7 @@ pub fn start(
program_path: PathBuf,
cache_path: PathBuf,
spawn_timeout: Duration,
) -> (
mpsc::Sender<ToPool>,
mpsc::UnboundedReceiver<FromPool>,
impl Future<Output = ()>,
) {
) -> (mpsc::Sender<ToPool>, mpsc::UnboundedReceiver<FromPool>, impl Future<Output = ()>) {
let (to_pool_tx, to_pool_rx) = mpsc::channel(10);
let (from_pool_tx, from_pool_rx) = mpsc::unbounded();
polkadot/node/core/pvf/src/prepare/queue.rs
+62 -185
View File
@@ -16,14 +16,12 @@
//! A queue that handles requests for PVF preparation.
use super::{
pool::{self, Worker},
};
use crate::{LOG_TARGET, Priority, Pvf, artifacts::ArtifactId};
use futures::{Future, SinkExt, channel::mpsc, stream::StreamExt as _};
use std::collections::{HashMap, VecDeque};
use async_std::path::PathBuf;
use super::pool::{self, Worker};
use crate::{artifacts::ArtifactId, Priority, Pvf, LOG_TARGET};
use always_assert::{always, never};
use async_std::path::PathBuf;
use futures::{channel::mpsc, stream::StreamExt as _, Future, SinkExt};
use std::collections::{HashMap, VecDeque};
/// A request to pool.
#[derive(Debug)]
@@ -35,10 +33,7 @@ pub enum ToQueue {
/// [`ToQueue::Amend`].
Enqueue { priority: Priority, pvf: Pvf },
/// Amends the priority for the given [`ArtifactId`] if it is running. If it's not, then it's noop.
Amend {
priority: Priority,
artifact_id: ArtifactId,
},
Amend { priority: Priority, artifact_id: ArtifactId },
}
/// A response from queue.
@@ -62,11 +57,7 @@ struct Limits {
impl Limits {
/// Returns `true` if the queue is allowed to request one more worker.
fn can_afford_one_more(&self, spawned_num: usize, critical: bool) -> bool {
let cap = if critical {
self.hard_capacity
} else {
self.soft_capacity
};
let cap = if critical { self.hard_capacity } else { self.soft_capacity };
spawned_num < cap
}
@@ -179,10 +170,7 @@ impl Queue {
from_pool_rx,
cache_path,
spawn_inflight: 0,
limits: Limits {
hard_capacity,
soft_capacity,
},
limits: Limits { hard_capacity, soft_capacity },
jobs: slotmap::SlotMap::with_key(),
unscheduled: Unscheduled::default(),
artifact_id_to_job: HashMap::new(),
@@ -194,7 +182,7 @@ impl Queue {
macro_rules! break_if_fatal {
($expr:expr) => {
if let Err(Fatal) = $expr {
break;
break
}
};
}
@@ -215,13 +203,10 @@ async fn handle_to_queue(queue: &mut Queue, to_queue: ToQueue) -> Result<(), Fat
match to_queue {
ToQueue::Enqueue { priority, pvf } => {
handle_enqueue(queue, priority, pvf).await?;
}
ToQueue::Amend {
priority,
artifact_id,
} => {
},
ToQueue::Amend { priority, artifact_id } => {
handle_amend(queue, priority, artifact_id).await?;
}
},
}
Ok(())
}
@@ -241,14 +226,10 @@ async fn handle_enqueue(queue: &mut Queue, priority: Priority, pvf: Pvf) -> Resu
"duplicate `enqueue` command received for {:?}",
artifact_id,
);
return Ok(());
return Ok(())
}
let job = queue.jobs.insert(JobData {
priority,
pvf,
worker: None,
});
let job = queue.jobs.insert(JobData { priority, pvf, worker: None });
queue.artifact_id_to_job.insert(artifact_id, job);
if let Some(available) = find_idle_worker(queue) {
@@ -264,12 +245,7 @@ async fn handle_enqueue(queue: &mut Queue, priority: Priority, pvf: Pvf) -> Resu
}
fn find_idle_worker(queue: &mut Queue) -> Option<Worker> {
queue
.workers
.iter()
.filter(|(_, data)| data.is_idle())
.map(|(k, _)| k)
.next()
queue.workers.iter().filter(|(_, data)| data.is_idle()).map(|(k, _)| k).next()
}
async fn handle_amend(
@@ -336,8 +312,8 @@ async fn handle_worker_concluded(
// Assume the conditions holds, then this never is not hit;
// qed.
never!("never_none, {}", stringify!($expr));
return Ok(());
}
return Ok(())
},
}
};
}
@@ -388,10 +364,7 @@ async fn handle_worker_concluded(
spawn_extra_worker(queue, false).await?;
}
} else {
if queue
.limits
.should_cull(queue.workers.len() + queue.spawn_inflight)
{
if queue.limits.should_cull(queue.workers.len() + queue.spawn_inflight) {
// We no longer need services of this worker. Kill it.
queue.workers.remove(worker);
send_pool(&mut queue.to_pool_tx, pool::ToPool::Kill(worker)).await?;
@@ -412,20 +385,16 @@ async fn handle_worker_rip(queue: &mut Queue, worker: Worker) -> Result<(), Fata
if let Some(WorkerData { job: Some(job), .. }) = worker_data {
// This is an edge case where the worker ripped after we sent assignment but before it
// was received by the pool.
let priority = queue
.jobs
.get(job)
.map(|data| data.priority)
.unwrap_or_else(|| {
// job is inserted upon enqueue and removed on concluded signal;
// this is enclosed in the if statement that narrows the situation to before
// conclusion;
// that means that the job still exists and is known;
// this path cannot be hit;
// qed.
never!("the job of the ripped worker must be known but it is not");
Priority::Normal
});
let priority = queue.jobs.get(job).map(|data| data.priority).unwrap_or_else(|| {
// job is inserted upon enqueue and removed on concluded signal;
// this is enclosed in the if statement that narrows the situation to before
// conclusion;
// that means that the job still exists and is known;
// this path cannot be hit;
// qed.
never!("the job of the ripped worker must be known but it is not");
Priority::Normal
});
queue.unscheduled.readd(priority, job);
}
@@ -500,11 +469,7 @@ pub fn start(
cache_path: PathBuf,
to_pool_tx: mpsc::Sender<pool::ToPool>,
from_pool_rx: mpsc::UnboundedReceiver<pool::FromPool>,
) -> (
mpsc::Sender<ToQueue>,
mpsc::UnboundedReceiver<FromQueue>,
impl Future<Output = ()>,
) {
) -> (mpsc::Sender<ToQueue>, mpsc::UnboundedReceiver<FromQueue>, impl Future<Output = ()>) {
let (to_queue_tx, to_queue_rx) = mpsc::channel(150);
let (from_queue_tx, from_queue_rx) = mpsc::unbounded();
@@ -524,11 +489,11 @@ pub fn start(
#[cfg(test)]
mod tests {
use slotmap::SlotMap;
use assert_matches::assert_matches;
use futures::{FutureExt, future::BoxFuture};
use std::task::Poll;
use super::*;
use assert_matches::assert_matches;
use futures::{future::BoxFuture, FutureExt};
use slotmap::SlotMap;
use std::task::Poll;
/// Creates a new PVF which artifact id can be uniquely identified by the given number.
fn pvf(descriminator: u32) -> Pvf {
@@ -549,7 +514,7 @@ mod tests {
}
if let Poll::Ready(r) = futures::poll!(&mut *fut) {
break r;
break r
}
if futures::poll!(&mut *task).is_ready() {
@@ -597,37 +562,21 @@ mod tests {
}
fn send_queue(&mut self, to_queue: ToQueue) {
self.to_queue_tx
.send(to_queue)
.now_or_never()
.unwrap()
.unwrap();
self.to_queue_tx.send(to_queue).now_or_never().unwrap().unwrap();
}
async fn poll_and_recv_from_queue(&mut self) -> FromQueue {
let from_queue_rx = &mut self.from_queue_rx;
run_until(
&mut self.run,
async { from_queue_rx.next().await.unwrap() }.boxed(),
)
.await
run_until(&mut self.run, async { from_queue_rx.next().await.unwrap() }.boxed()).await
}
fn send_from_pool(&mut self, from_pool: pool::FromPool) {
self.from_pool_tx
.send(from_pool)
.now_or_never()
.unwrap()
.unwrap();
self.from_pool_tx.send(from_pool).now_or_never().unwrap().unwrap();
}
async fn poll_and_recv_to_pool(&mut self) -> pool::ToPool {
let to_pool_rx = &mut self.to_pool_rx;
run_until(
&mut self.run,
async { to_pool_rx.next().await.unwrap() }.boxed(),
)
.await
run_until(&mut self.run, async { to_pool_rx.next().await.unwrap() }.boxed()).await
}
async fn poll_ensure_to_pool_is_empty(&mut self) {
@@ -655,10 +604,7 @@ mod tests {
async fn properly_concludes() {
let mut test = Test::new(2, 2);
test.send_queue(ToQueue::Enqueue {
priority: Priority::Background,
pvf: pvf(1),
});
test.send_queue(ToQueue::Enqueue { priority: Priority::Background, pvf: pvf(1) });
assert_eq!(test.poll_and_recv_to_pool().await, pool::ToPool::Spawn);
let w = test.workers.insert(());
@@ -675,18 +621,9 @@ mod tests {
async fn dont_spawn_over_soft_limit_unless_critical() {
let mut test = Test::new(2, 3);
test.send_queue(ToQueue::Enqueue {
priority: Priority::Normal,
pvf: pvf(1),
});
test.send_queue(ToQueue::Enqueue {
priority: Priority::Normal,
pvf: pvf(2),
});
test.send_queue(ToQueue::Enqueue {
priority: Priority::Normal,
pvf: pvf(3),
});
test.send_queue(ToQueue::Enqueue { priority: Priority::Normal, pvf: pvf(1) });
test.send_queue(ToQueue::Enqueue { priority: Priority::Normal, pvf: pvf(2) });
test.send_queue(ToQueue::Enqueue { priority: Priority::Normal, pvf: pvf(3) });
// Receive only two spawns.
assert_eq!(test.poll_and_recv_to_pool().await, pool::ToPool::Spawn);
@@ -699,27 +636,15 @@ mod tests {
test.send_from_pool(pool::FromPool::Spawned(w2));
// Get two start works.
assert_matches!(
test.poll_and_recv_to_pool().await,
pool::ToPool::StartWork { .. }
);
assert_matches!(
test.poll_and_recv_to_pool().await,
pool::ToPool::StartWork { .. }
);
assert_matches!(test.poll_and_recv_to_pool().await, pool::ToPool::StartWork { .. });
assert_matches!(test.poll_and_recv_to_pool().await, pool::ToPool::StartWork { .. });
test.send_from_pool(pool::FromPool::Concluded(w1, false));
assert_matches!(
test.poll_and_recv_to_pool().await,
pool::ToPool::StartWork { .. }
);
assert_matches!(test.poll_and_recv_to_pool().await, pool::ToPool::StartWork { .. });
// Enqueue a critical job.
test.send_queue(ToQueue::Enqueue {
priority: Priority::Critical,
pvf: pvf(4),
});
test.send_queue(ToQueue::Enqueue { priority: Priority::Critical, pvf: pvf(4) });
// 2 out of 2 are working, but there is a critical job incoming. That means that spawning
// another worker is warranted.
@@ -730,23 +655,14 @@ mod tests {
async fn cull_unwanted() {
let mut test = Test::new(1, 2);
test.send_queue(ToQueue::Enqueue {
priority: Priority::Normal,
pvf: pvf(1),
});
test.send_queue(ToQueue::Enqueue { priority: Priority::Normal, pvf: pvf(1) });
assert_eq!(test.poll_and_recv_to_pool().await, pool::ToPool::Spawn);
let w1 = test.workers.insert(());
test.send_from_pool(pool::FromPool::Spawned(w1));
assert_matches!(
test.poll_and_recv_to_pool().await,
pool::ToPool::StartWork { .. }
);
assert_matches!(test.poll_and_recv_to_pool().await, pool::ToPool::StartWork { .. });
// Enqueue a critical job, which warrants spawning over the soft limit.
test.send_queue(ToQueue::Enqueue {
priority: Priority::Critical,
pvf: pvf(2),
});
test.send_queue(ToQueue::Enqueue { priority: Priority::Critical, pvf: pvf(2) });
assert_eq!(test.poll_and_recv_to_pool().await, pool::ToPool::Spawn);
// However, before the new worker had a chance to spawn, the first worker finishes with its
@@ -764,47 +680,29 @@ mod tests {
async fn bump_prio_on_urgency_change() {
let mut test = Test::new(2, 2);
test.send_queue(ToQueue::Enqueue {
priority: Priority::Background,
pvf: pvf(1),
});
test.send_queue(ToQueue::Enqueue { priority: Priority::Background, pvf: pvf(1) });
assert_eq!(test.poll_and_recv_to_pool().await, pool::ToPool::Spawn);
let w = test.workers.insert(());
test.send_from_pool(pool::FromPool::Spawned(w));
assert_matches!(
test.poll_and_recv_to_pool().await,
pool::ToPool::StartWork { .. }
);
assert_matches!(test.poll_and_recv_to_pool().await, pool::ToPool::StartWork { .. });
test.send_queue(ToQueue::Amend {
priority: Priority::Normal,
artifact_id: pvf(1).as_artifact_id(),
});
assert_eq!(
test.poll_and_recv_to_pool().await,
pool::ToPool::BumpPriority(w)
);
assert_eq!(test.poll_and_recv_to_pool().await, pool::ToPool::BumpPriority(w));
}
#[async_std::test]
async fn worker_mass_die_out_doesnt_stall_queue() {
let mut test = Test::new(2, 2);
test.send_queue(ToQueue::Enqueue {
priority: Priority::Normal,
pvf: pvf(1),
});
test.send_queue(ToQueue::Enqueue {
priority: Priority::Normal,
pvf: pvf(2),
});
test.send_queue(ToQueue::Enqueue {
priority: Priority::Normal,
pvf: pvf(3),
});
test.send_queue(ToQueue::Enqueue { priority: Priority::Normal, pvf: pvf(1) });
test.send_queue(ToQueue::Enqueue { priority: Priority::Normal, pvf: pvf(2) });
test.send_queue(ToQueue::Enqueue { priority: Priority::Normal, pvf: pvf(3) });
assert_eq!(test.poll_and_recv_to_pool().await, pool::ToPool::Spawn);
assert_eq!(test.poll_and_recv_to_pool().await, pool::ToPool::Spawn);
@@ -815,14 +713,8 @@ mod tests {
test.send_from_pool(pool::FromPool::Spawned(w1));
test.send_from_pool(pool::FromPool::Spawned(w2));
assert_matches!(
test.poll_and_recv_to_pool().await,
pool::ToPool::StartWork { .. }
);
assert_matches!(
test.poll_and_recv_to_pool().await,
pool::ToPool::StartWork { .. }
);
assert_matches!(test.poll_and_recv_to_pool().await, pool::ToPool::StartWork { .. });
assert_matches!(test.poll_and_recv_to_pool().await, pool::ToPool::StartWork { .. });
// Conclude worker 1 and rip it.
test.send_from_pool(pool::FromPool::Concluded(w1, true));
@@ -840,20 +732,14 @@ mod tests {
async fn doesnt_resurrect_ripped_worker_if_no_work() {
let mut test = Test::new(2, 2);
test.send_queue(ToQueue::Enqueue {
priority: Priority::Normal,
pvf: pvf(1),
});
test.send_queue(ToQueue::Enqueue { priority: Priority::Normal, pvf: pvf(1) });
assert_eq!(test.poll_and_recv_to_pool().await, pool::ToPool::Spawn);
let w1 = test.workers.insert(());
test.send_from_pool(pool::FromPool::Spawned(w1));
assert_matches!(
test.poll_and_recv_to_pool().await,
pool::ToPool::StartWork { .. }
);
assert_matches!(test.poll_and_recv_to_pool().await, pool::ToPool::StartWork { .. });
test.send_from_pool(pool::FromPool::Concluded(w1, true));
test.poll_ensure_to_pool_is_empty().await;
@@ -863,10 +749,7 @@ mod tests {
async fn rip_for_start_work() {
let mut test = Test::new(2, 2);
test.send_queue(ToQueue::Enqueue {
priority: Priority::Normal,
pvf: pvf(1),
});
test.send_queue(ToQueue::Enqueue { priority: Priority::Normal, pvf: pvf(1) });
assert_eq!(test.poll_and_recv_to_pool().await, pool::ToPool::Spawn);
@@ -875,10 +758,7 @@ mod tests {
// Now, to the interesting part. After the queue normally issues the start_work command to
// the pool, before receiving the command the queue may report that the worker ripped.
assert_matches!(
test.poll_and_recv_to_pool().await,
pool::ToPool::StartWork { .. }
);
assert_matches!(test.poll_and_recv_to_pool().await, pool::ToPool::StartWork { .. });
test.send_from_pool(pool::FromPool::Rip(w1));
// In this case, the pool should spawn a new worker and request it to work on the item.
@@ -886,9 +766,6 @@ mod tests {
let w2 = test.workers.insert(());
test.send_from_pool(pool::FromPool::Spawned(w2));
assert_matches!(
test.poll_and_recv_to_pool().await,
pool::ToPool::StartWork { .. }
);
assert_matches!(test.poll_and_recv_to_pool().await, pool::ToPool::StartWork { .. });
}
}
polkadot/node/core/pvf/src/prepare/worker.rs
+12 -20
View File
@@ -15,17 +15,17 @@
// along with Polkadot. If not, see <http://www.gnu.org/licenses/>.
use crate::{
LOG_TARGET,
artifacts::Artifact,
worker_common::{
IdleWorker, SpawnErr, WorkerHandle, bytes_to_path, framed_recv, framed_send, path_to_bytes,
spawn_with_program_path, tmpfile_in, worker_event_loop,
bytes_to_path, framed_recv, framed_send, path_to_bytes, spawn_with_program_path,
tmpfile_in, worker_event_loop, IdleWorker, SpawnErr, WorkerHandle,
},
LOG_TARGET,
};
use async_std::{
io,
os::unix::net::UnixStream,
path::{PathBuf, Path},
path::{Path, PathBuf},
};
use futures::FutureExt as _;
use futures_timer::Delay;
@@ -43,13 +43,7 @@ pub async fn spawn(
program_path: &Path,
spawn_timeout: Duration,
) -> Result<(IdleWorker, WorkerHandle), SpawnErr> {
spawn_with_program_path(
"prepare",
program_path,
&["prepare-worker"],
spawn_timeout,
)
.await
spawn_with_program_path("prepare", program_path, &["prepare-worker"], spawn_timeout).await
}
pub enum Outcome {
@@ -99,7 +93,7 @@ pub async fn start_work(
"failed to send a prepare request: {:?}",
err,
);
return Outcome::Unreachable;
return Outcome::Unreachable
}
// Wait for the result from the worker, keeping in mind that there may be a timeout, the
@@ -172,13 +166,13 @@ pub async fn start_work(
Selected::Done => {
renice(pid, NICENESS_FOREGROUND);
Outcome::Concluded(IdleWorker { stream, pid })
}
},
Selected::IoErr | Selected::Deadline => {
let bytes = Artifact::DidntMakeIt.serialize();
// best effort: there is nothing we can do here if the write fails.
let _ = async_std::fs::write(&artifact_path, &bytes).await;
Outcome::DidntMakeIt
}
},
}
})
.await
@@ -202,8 +196,8 @@ where
"failed to create a temp file for the artifact: {:?}",
err,
);
return Outcome::DidntMakeIt;
}
return Outcome::DidntMakeIt
},
};
let outcome = f(tmp_file.clone()).await;
@@ -223,7 +217,7 @@ where
"failed to remove the tmp file: {:?}",
err,
);
}
},
}
outcome
@@ -304,9 +298,7 @@ pub fn worker_entrypoint(socket_path: &str) {
fn prepare_artifact(code: &[u8]) -> Artifact {
let blob = match crate::executor_intf::prevalidate(code) {
Err(err) => {
return Artifact::PrevalidationErr(format!("{:?}", err));
}
Err(err) => return Artifact::PrevalidationErr(format!("{:?}", err)),
Ok(b) => b,
};
polkadot/node/core/pvf/src/testing.rs
+6 -5
View File
@@ -29,9 +29,10 @@ pub fn validate_candidate(
code: &[u8],
params: &[u8],
) -> Result<Vec<u8>, Box<dyn std::error::Error>> {
use crate::executor_intf::{prevalidate, prepare, execute, TaskExecutor};
use crate::executor_intf::{execute, prepare, prevalidate, TaskExecutor};
let code = sp_maybe_compressed_blob::decompress(code, 10 * 1024 * 1024).expect("Decompressing code failed");
let code = sp_maybe_compressed_blob::decompress(code, 10 * 1024 * 1024)
.expect("Decompressing code failed");
let blob = prevalidate(&*code)?;
let artifact = prepare(blob)?;
@@ -61,15 +62,15 @@ macro_rules! decl_puppet_worker_main {
match subcommand.as_ref() {
"sleep" => {
std::thread::sleep(std::time::Duration::from_secs(5));
}
},
"prepare-worker" => {
let socket_path = &args[2];
$crate::prepare_worker_entrypoint(socket_path);
}
},
"execute-worker" => {
let socket_path = &args[2];
$crate::execute_worker_entrypoint(socket_path);
}
},
other => panic!("unknown subcommand: {}", other),
}
}
polkadot/node/core/pvf/src/worker_common.rs
+9 -17
View File
@@ -20,12 +20,13 @@ use crate::LOG_TARGET;
use async_std::{
io,
os::unix::net::{UnixListener, UnixStream},
path::{PathBuf, Path},
path::{Path, PathBuf},
};
use futures::{
AsyncRead, AsyncWrite, AsyncReadExt as _, AsyncWriteExt as _, FutureExt as _, never::Never,
never::Never, AsyncRead, AsyncReadExt as _, AsyncWrite, AsyncWriteExt as _, FutureExt as _,
};
use futures_timer::Delay;
use pin_project::pin_project;
use rand::Rng;
use std::{
fmt, mem,
@@ -33,7 +34,6 @@ use std::{
task::{Context, Poll},
time::Duration,
};
use pin_project::pin_project;
/// This is publicly exposed only for integration tests.
#[doc(hidden)]
@@ -47,9 +47,7 @@ pub async fn spawn_with_program_path(
with_transient_socket_path(debug_id, |socket_path| {
let socket_path = socket_path.to_owned();
async move {
let listener = UnixListener::bind(&socket_path)
.await
.map_err(|_| SpawnErr::Bind)?;
let listener = UnixListener::bind(&socket_path).await.map_err(|_| SpawnErr::Bind)?;
let handle = WorkerHandle::spawn(program_path, extra_args, socket_path)
.map_err(|_| SpawnErr::ProcessSpawn)?;
@@ -97,11 +95,7 @@ pub async fn tmpfile_in(prefix: &str, dir: &Path) -> io::Result<PathBuf> {
let mut buf = Vec::with_capacity(prefix.len() + DESCRIMINATOR_LEN);
buf.extend(prefix.as_bytes());
buf.extend(
rand::thread_rng()
.sample_iter(&Alphanumeric)
.take(DESCRIMINATOR_LEN),
);
buf.extend(rand::thread_rng().sample_iter(&Alphanumeric).take(DESCRIMINATOR_LEN));
let s = std::str::from_utf8(&buf)
.expect("the string is collected from a valid utf-8 sequence; qed");
@@ -120,9 +114,7 @@ pub async fn tmpfile_in(prefix: &str, dir: &Path) -> io::Result<PathBuf> {
}
}
Err(
io::Error::new(io::ErrorKind::Other, "failed to create a temporary file")
)
Err(io::Error::new(io::ErrorKind::Other, "failed to create a temporary file"))
}
/// The same as [`tmpfile_in`], but uses [`std::env::temp_dir`] as the directory.
@@ -245,17 +237,17 @@ impl futures::Future for WorkerHandle {
Ok(0) => {
// 0 means EOF means the child was terminated. Resolve.
Poll::Ready(())
}
},
Ok(_bytes_read) => {
// weird, we've read something. Pretend that never happened and reschedule ourselves.
cx.waker().wake_by_ref();
Poll::Pending
}
},
Err(_) => {
// The implementation is guaranteed to not to return WouldBlock and Interrupted. This
// leaves us with a legit errors which we suppose were due to termination.
Poll::Ready(())
}
},
}
}
}
polkadot/node/core/pvf/tests/it/adder.rs
+9 -26
View File
@@ -15,22 +15,16 @@
// along with Polkadot. If not, see <http://www.gnu.org/licenses/>.
use super::TestHost;
use polkadot_parachain::{
primitives::{
RelayChainBlockNumber, BlockData as GenericBlockData, HeadData as GenericHeadData,
ValidationParams,
},
};
use adder::{hash_state, BlockData, HeadData};
use parity_scale_codec::{Decode, Encode};
use adder::{HeadData, BlockData, hash_state};
use polkadot_parachain::primitives::{
BlockData as GenericBlockData, HeadData as GenericHeadData, RelayChainBlockNumber,
ValidationParams,
};
#[async_std::test]
async fn execute_good_on_parent() {
let parent_head = HeadData {
number: 0,
parent_hash: [0; 32],
post_state: hash_state(0),
};
let parent_head = HeadData { number: 0, parent_hash: [0; 32], post_state: hash_state(0) };
let block_data = BlockData { state: 0, add: 512 };
@@ -65,16 +59,9 @@ async fn execute_good_chain_on_parent() {
let host = TestHost::new();
for add in 0..10 {
let parent_head = HeadData {
number,
parent_hash,
post_state: hash_state(last_state),
};
let parent_head = HeadData { number, parent_hash, post_state: hash_state(last_state) };
let block_data = BlockData {
state: last_state,
add,
};
let block_data = BlockData { state: last_state, add };
let ret = host
.validate_candidate(
@@ -103,11 +90,7 @@ async fn execute_good_chain_on_parent() {
#[async_std::test]
async fn execute_bad_on_parent() {
let parent_head = HeadData {
number: 0,
parent_hash: [0; 32],
post_state: hash_state(0),
};
let parent_head = HeadData { number: 0, parent_hash: [0; 32], post_state: hash_state(0) };
let block_data = BlockData {
state: 256, // start state is wrong.
polkadot/node/core/pvf/tests/it/main.rs
+11 -11
View File
@@ -14,10 +14,12 @@
// You should have received a copy of the GNU General Public License
// along with Polkadot. If not, see <http://www.gnu.org/licenses/>.
use polkadot_node_core_pvf::{Pvf, ValidationHost, start, Config, InvalidCandidate, ValidationError};
use polkadot_parachain::primitives::{BlockData, ValidationParams, ValidationResult};
use parity_scale_codec::Encode as _;
use async_std::sync::Mutex;
use parity_scale_codec::Encode as _;
use polkadot_node_core_pvf::{
start, Config, InvalidCandidate, Pvf, ValidationError, ValidationHost,
};
use polkadot_parachain::primitives::{BlockData, ValidationParams, ValidationResult};
mod adder;
mod worker_common;
@@ -44,10 +46,7 @@ impl TestHost {
f(&mut config);
let (host, task) = start(config);
let _ = async_std::task::spawn(task);
Self {
_cache_dir: cache_dir,
host: Mutex::new(host),
}
Self { _cache_dir: cache_dir, host: Mutex::new(host) }
}
async fn validate_candidate(
@@ -57,7 +56,8 @@ impl TestHost {
) -> Result<ValidationResult, ValidationError> {
let (result_tx, result_rx) = futures::channel::oneshot::channel();
let code = sp_maybe_compressed_blob::decompress(code, 16 * 1024 * 1024).expect("Compression works");
let code = sp_maybe_compressed_blob::decompress(code, 16 * 1024 * 1024)
.expect("Compression works");
self.host
.lock()
@@ -91,7 +91,7 @@ async fn terminates_on_timeout() {
.await;
match result {
Err(ValidationError::InvalidCandidate(InvalidCandidate::HardTimeout)) => {}
Err(ValidationError::InvalidCandidate(InvalidCandidate::HardTimeout)) => {},
r => panic!("{:?}", r),
}
}
@@ -124,8 +124,8 @@ async fn parallel_execution() {
// total time should be < 2 x EXECUTION_TIMEOUT_SEC
const EXECUTION_TIMEOUT_SEC: u64 = 3;
assert!(
std::time::Instant::now().duration_since(start)
< std::time::Duration::from_secs(EXECUTION_TIMEOUT_SEC * 2)
std::time::Instant::now().duration_since(start) <
std::time::Duration::from_secs(EXECUTION_TIMEOUT_SEC * 2)
);
}
polkadot/node/core/pvf/tests/it/worker_common.rs
+3 -7
View File
@@ -20,13 +20,9 @@ use std::time::Duration;
#[async_std::test]
async fn spawn_timeout() {
let result = spawn_with_program_path(
"integration-test",
PUPPET_EXE,
&["sleep"],
Duration::from_secs(2),
)
.await;
let result =
spawn_with_program_path("integration-test", PUPPET_EXE, &["sleep"], Duration::from_secs(2))
.await;
assert!(matches!(result, Err(SpawnErr::AcceptTimeout)));
}
polkadot/node/core/runtime-api/src/cache.rs
+125 -31
View File
@@ -71,18 +71,32 @@ impl<T> ResidentSize for VecOfDoesNotAllocate<T> {
pub(crate) struct RequestResultCache {
authorities: MemoryLruCache<Hash, VecOfDoesNotAllocate<AuthorityDiscoveryId>>,
validators: MemoryLruCache<Hash, ResidentSizeOf<Vec<ValidatorId>>>,
validator_groups: MemoryLruCache<Hash, ResidentSizeOf<(Vec<Vec<ValidatorIndex>>, GroupRotationInfo)>>,
validator_groups:
MemoryLruCache<Hash, ResidentSizeOf<(Vec<Vec<ValidatorIndex>>, GroupRotationInfo)>>,
availability_cores: MemoryLruCache<Hash, ResidentSizeOf<Vec<CoreState>>>,
persisted_validation_data: MemoryLruCache<(Hash, ParaId, OccupiedCoreAssumption), ResidentSizeOf<Option<PersistedValidationData>>>,
check_validation_outputs: MemoryLruCache<(Hash, ParaId, CandidateCommitments), ResidentSizeOf<bool>>,
persisted_validation_data: MemoryLruCache<
(Hash, ParaId, OccupiedCoreAssumption),
ResidentSizeOf<Option<PersistedValidationData>>,
>,
check_validation_outputs:
MemoryLruCache<(Hash, ParaId, CandidateCommitments), ResidentSizeOf<bool>>,
session_index_for_child: MemoryLruCache<Hash, ResidentSizeOf<SessionIndex>>,
validation_code: MemoryLruCache<(Hash, ParaId, OccupiedCoreAssumption), ResidentSizeOf<Option<ValidationCode>>>,
validation_code_by_hash: MemoryLruCache<ValidationCodeHash, ResidentSizeOf<Option<ValidationCode>>>,
candidate_pending_availability: MemoryLruCache<(Hash, ParaId), ResidentSizeOf<Option<CommittedCandidateReceipt>>>,
validation_code: MemoryLruCache<
(Hash, ParaId, OccupiedCoreAssumption),
ResidentSizeOf<Option<ValidationCode>>,
>,
validation_code_by_hash:
MemoryLruCache<ValidationCodeHash, ResidentSizeOf<Option<ValidationCode>>>,
candidate_pending_availability:
MemoryLruCache<(Hash, ParaId), ResidentSizeOf<Option<CommittedCandidateReceipt>>>,
candidate_events: MemoryLruCache<Hash, ResidentSizeOf<Vec<CandidateEvent>>>,
session_info: MemoryLruCache<SessionIndex, ResidentSizeOf<Option<SessionInfo>>>,
dmq_contents: MemoryLruCache<(Hash, ParaId), ResidentSizeOf<Vec<InboundDownwardMessage<BlockNumber>>>>,
inbound_hrmp_channels_contents: MemoryLruCache<(Hash, ParaId), ResidentSizeOf<BTreeMap<ParaId, Vec<InboundHrmpMessage<BlockNumber>>>>>,
dmq_contents:
MemoryLruCache<(Hash, ParaId), ResidentSizeOf<Vec<InboundDownwardMessage<BlockNumber>>>>,
inbound_hrmp_channels_contents: MemoryLruCache<
(Hash, ParaId),
ResidentSizeOf<BTreeMap<ParaId, Vec<InboundHrmpMessage<BlockNumber>>>>,
>,
current_babe_epoch: MemoryLruCache<Hash, DoesNotAllocate<Epoch>>,
}
@@ -98,7 +112,9 @@ impl Default for RequestResultCache {
session_index_for_child: MemoryLruCache::new(SESSION_INDEX_FOR_CHILD_CACHE_SIZE),
validation_code: MemoryLruCache::new(VALIDATION_CODE_CACHE_SIZE),
validation_code_by_hash: MemoryLruCache::new(VALIDATION_CODE_CACHE_SIZE),
candidate_pending_availability: MemoryLruCache::new(CANDIDATE_PENDING_AVAILABILITY_CACHE_SIZE),
candidate_pending_availability: MemoryLruCache::new(
CANDIDATE_PENDING_AVAILABILITY_CACHE_SIZE,
),
candidate_events: MemoryLruCache::new(CANDIDATE_EVENTS_CACHE_SIZE),
session_info: MemoryLruCache::new(SESSION_INFO_CACHE_SIZE),
dmq_contents: MemoryLruCache::new(DMQ_CONTENTS_CACHE_SIZE),
@@ -109,11 +125,18 @@ impl Default for RequestResultCache {
}
impl RequestResultCache {
pub(crate) fn authorities(&mut self, relay_parent: &Hash) -> Option<&Vec<AuthorityDiscoveryId>> {
pub(crate) fn authorities(
&mut self,
relay_parent: &Hash,
) -> Option<&Vec<AuthorityDiscoveryId>> {
self.authorities.get(relay_parent).map(|v| &v.0)
}
pub(crate) fn cache_authorities(&mut self, relay_parent: Hash, authorities: Vec<AuthorityDiscoveryId>) {
pub(crate) fn cache_authorities(
&mut self,
relay_parent: Hash,
authorities: Vec<AuthorityDiscoveryId>,
) {
self.authorities.insert(relay_parent, VecOfDoesNotAllocate(authorities));
}
@@ -125,11 +148,18 @@ impl RequestResultCache {
self.validators.insert(relay_parent, ResidentSizeOf(validators));
}
pub(crate) fn validator_groups(&mut self, relay_parent: &Hash) -> Option<&(Vec<Vec<ValidatorIndex>>, GroupRotationInfo)> {
pub(crate) fn validator_groups(
&mut self,
relay_parent: &Hash,
) -> Option<&(Vec<Vec<ValidatorIndex>>, GroupRotationInfo)> {
self.validator_groups.get(relay_parent).map(|v| &v.0)
}
pub(crate) fn cache_validator_groups(&mut self, relay_parent: Hash, groups: (Vec<Vec<ValidatorIndex>>, GroupRotationInfo)) {
pub(crate) fn cache_validator_groups(
&mut self,
relay_parent: Hash,
groups: (Vec<Vec<ValidatorIndex>>, GroupRotationInfo),
) {
self.validator_groups.insert(relay_parent, ResidentSizeOf(groups));
}
@@ -141,19 +171,33 @@ impl RequestResultCache {
self.availability_cores.insert(relay_parent, ResidentSizeOf(cores));
}
pub(crate) fn persisted_validation_data(&mut self, key: (Hash, ParaId, OccupiedCoreAssumption)) -> Option<&Option<PersistedValidationData>> {
pub(crate) fn persisted_validation_data(
&mut self,
key: (Hash, ParaId, OccupiedCoreAssumption),
) -> Option<&Option<PersistedValidationData>> {
self.persisted_validation_data.get(&key).map(|v| &v.0)
}
pub(crate) fn cache_persisted_validation_data(&mut self, key: (Hash, ParaId, OccupiedCoreAssumption), data: Option<PersistedValidationData>) {
pub(crate) fn cache_persisted_validation_data(
&mut self,
key: (Hash, ParaId, OccupiedCoreAssumption),
data: Option<PersistedValidationData>,
) {
self.persisted_validation_data.insert(key, ResidentSizeOf(data));
}
pub(crate) fn check_validation_outputs(&mut self, key: (Hash, ParaId, CandidateCommitments)) -> Option<&bool> {
pub(crate) fn check_validation_outputs(
&mut self,
key: (Hash, ParaId, CandidateCommitments),
) -> Option<&bool> {
self.check_validation_outputs.get(&key).map(|v| &v.0)
}
pub(crate) fn cache_check_validation_outputs(&mut self, key: (Hash, ParaId, CandidateCommitments), value: bool) {
pub(crate) fn cache_check_validation_outputs(
&mut self,
key: (Hash, ParaId, CandidateCommitments),
value: bool,
) {
self.check_validation_outputs.insert(key, ResidentSizeOf(value));
}
@@ -161,33 +205,58 @@ impl RequestResultCache {
self.session_index_for_child.get(relay_parent).map(|v| &v.0)
}
pub(crate) fn cache_session_index_for_child(&mut self, relay_parent: Hash, index: SessionIndex) {
pub(crate) fn cache_session_index_for_child(
&mut self,
relay_parent: Hash,
index: SessionIndex,
) {
self.session_index_for_child.insert(relay_parent, ResidentSizeOf(index));
}
pub(crate) fn validation_code(&mut self, key: (Hash, ParaId, OccupiedCoreAssumption)) -> Option<&Option<ValidationCode>> {
pub(crate) fn validation_code(
&mut self,
key: (Hash, ParaId, OccupiedCoreAssumption),
) -> Option<&Option<ValidationCode>> {
self.validation_code.get(&key).map(|v| &v.0)
}
pub(crate) fn cache_validation_code(&mut self, key: (Hash, ParaId, OccupiedCoreAssumption), value: Option<ValidationCode>) {
pub(crate) fn cache_validation_code(
&mut self,
key: (Hash, ParaId, OccupiedCoreAssumption),
value: Option<ValidationCode>,
) {
self.validation_code.insert(key, ResidentSizeOf(value));
}
// the actual key is `ValidationCodeHash` (`Hash` is ignored),
// but we keep the interface that way to keep the macro simple
pub(crate) fn validation_code_by_hash(&mut self, key: (Hash, ValidationCodeHash)) -> Option<&Option<ValidationCode>> {
pub(crate) fn validation_code_by_hash(
&mut self,
key: (Hash, ValidationCodeHash),
) -> Option<&Option<ValidationCode>> {
self.validation_code_by_hash.get(&key.1).map(|v| &v.0)
}
pub(crate) fn cache_validation_code_by_hash(&mut self, key: ValidationCodeHash, value: Option<ValidationCode>) {
pub(crate) fn cache_validation_code_by_hash(
&mut self,
key: ValidationCodeHash,
value: Option<ValidationCode>,
) {
self.validation_code_by_hash.insert(key, ResidentSizeOf(value));
}
pub(crate) fn candidate_pending_availability(&mut self, key: (Hash, ParaId)) -> Option<&Option<CommittedCandidateReceipt>> {
pub(crate) fn candidate_pending_availability(
&mut self,
key: (Hash, ParaId),
) -> Option<&Option<CommittedCandidateReceipt>> {
self.candidate_pending_availability.get(&key).map(|v| &v.0)
}
pub(crate) fn cache_candidate_pending_availability(&mut self, key: (Hash, ParaId), value: Option<CommittedCandidateReceipt>) {
pub(crate) fn cache_candidate_pending_availability(
&mut self,
key: (Hash, ParaId),
value: Option<CommittedCandidateReceipt>,
) {
self.candidate_pending_availability.insert(key, ResidentSizeOf(value));
}
@@ -195,11 +264,18 @@ impl RequestResultCache {
self.candidate_events.get(relay_parent).map(|v| &v.0)
}
pub(crate) fn cache_candidate_events(&mut self, relay_parent: Hash, events: Vec<CandidateEvent>) {
pub(crate) fn cache_candidate_events(
&mut self,
relay_parent: Hash,
events: Vec<CandidateEvent>,
) {
self.candidate_events.insert(relay_parent, ResidentSizeOf(events));
}
pub(crate) fn session_info(&mut self, key: (Hash, SessionIndex)) -> Option<&Option<SessionInfo>> {
pub(crate) fn session_info(
&mut self,
key: (Hash, SessionIndex),
) -> Option<&Option<SessionInfo>> {
self.session_info.get(&key.1).map(|v| &v.0)
}
@@ -207,19 +283,33 @@ impl RequestResultCache {
self.session_info.insert(key, ResidentSizeOf(value));
}
pub(crate) fn dmq_contents(&mut self, key: (Hash, ParaId)) -> Option<&Vec<InboundDownwardMessage<BlockNumber>>> {
pub(crate) fn dmq_contents(
&mut self,
key: (Hash, ParaId),
) -> Option<&Vec<InboundDownwardMessage<BlockNumber>>> {
self.dmq_contents.get(&key).map(|v| &v.0)
}
pub(crate) fn cache_dmq_contents(&mut self, key: (Hash, ParaId), value: Vec<InboundDownwardMessage<BlockNumber>>) {
pub(crate) fn cache_dmq_contents(
&mut self,
key: (Hash, ParaId),
value: Vec<InboundDownwardMessage<BlockNumber>>,
) {
self.dmq_contents.insert(key, ResidentSizeOf(value));
}
pub(crate) fn inbound_hrmp_channels_contents(&mut self, key: (Hash, ParaId)) -> Option<&BTreeMap<ParaId, Vec<InboundHrmpMessage<BlockNumber>>>> {
pub(crate) fn inbound_hrmp_channels_contents(
&mut self,
key: (Hash, ParaId),
) -> Option<&BTreeMap<ParaId, Vec<InboundHrmpMessage<BlockNumber>>>> {
self.inbound_hrmp_channels_contents.get(&key).map(|v| &v.0)
}
pub(crate) fn cache_inbound_hrmp_channel_contents(&mut self, key: (Hash, ParaId), value: BTreeMap<ParaId, Vec<InboundHrmpMessage<BlockNumber>>>) {
pub(crate) fn cache_inbound_hrmp_channel_contents(
&mut self,
key: (Hash, ParaId),
value: BTreeMap<ParaId, Vec<InboundHrmpMessage<BlockNumber>>>,
) {
self.inbound_hrmp_channels_contents.insert(key, ResidentSizeOf(value));
}
@@ -246,6 +336,10 @@ pub(crate) enum RequestResult {
CandidateEvents(Hash, Vec<CandidateEvent>),
SessionInfo(Hash, SessionIndex, Option<SessionInfo>),
DmqContents(Hash, ParaId, Vec<InboundDownwardMessage<BlockNumber>>),
InboundHrmpChannelsContents(Hash, ParaId, BTreeMap<ParaId, Vec<InboundHrmpMessage<BlockNumber>>>),
InboundHrmpChannelsContents(
Hash,
ParaId,
BTreeMap<ParaId, Vec<InboundHrmpMessage<BlockNumber>>>,
),
CurrentBabeEpoch(Hash, Epoch),
}
polkadot/node/core/runtime-api/src/lib.rs
+67 -64
View File
@@ -22,27 +22,23 @@
#![deny(unused_crate_dependencies)]
#![warn(missing_docs)]
use polkadot_subsystem::{
SubsystemError, SubsystemResult,
FromOverseer, OverseerSignal, SpawnedSubsystem,
SubsystemContext,
errors::RuntimeApiError,
messages::{
RuntimeApiMessage, RuntimeApiRequest as Request,
},
overseer,
};
use polkadot_node_subsystem_util::metrics::{self, prometheus};
use polkadot_primitives::v1::{Block, BlockId, Hash, ParachainHost};
use polkadot_subsystem::{
errors::RuntimeApiError,
messages::{RuntimeApiMessage, RuntimeApiRequest as Request},
overseer, FromOverseer, OverseerSignal, SpawnedSubsystem, SubsystemContext, SubsystemError,
SubsystemResult,
};
use sp_api::ProvideRuntimeApi;
use sp_authority_discovery::AuthorityDiscoveryApi;
use sp_core::traits::SpawnNamed;
use sp_consensus_babe::BabeApi;
use sp_core::traits::SpawnNamed;
use futures::{prelude::*, stream::FuturesUnordered, channel::oneshot, select};
use std::{sync::Arc, collections::VecDeque, pin::Pin};
use cache::{RequestResult, RequestResultCache};
use futures::{channel::oneshot, prelude::*, select, stream::FuturesUnordered};
use std::{collections::VecDeque, pin::Pin, sync::Arc};
mod cache;
@@ -75,7 +71,11 @@ pub struct RuntimeApiSubsystem<Client> {
impl<Client> RuntimeApiSubsystem<Client> {
/// Create a new Runtime API subsystem wrapping the given client and metrics.
pub fn new(client: Arc<Client>, metrics: Metrics, spawn_handle: impl SpawnNamed + 'static) -> Self {
pub fn new(
client: Arc<Client>,
metrics: Metrics,
spawn_handle: impl SpawnNamed + 'static,
) -> Self {
RuntimeApiSubsystem {
client,
metrics,
@@ -87,21 +87,20 @@ impl<Client> RuntimeApiSubsystem<Client> {
}
}
impl<Client, Context> overseer::Subsystem<Context, SubsystemError> for RuntimeApiSubsystem<Client> where
impl<Client, Context> overseer::Subsystem<Context, SubsystemError> for RuntimeApiSubsystem<Client>
where
Client: ProvideRuntimeApi<Block> + Send + 'static + Sync,
Client::Api: ParachainHost<Block> + BabeApi<Block> + AuthorityDiscoveryApi<Block>,
Context: SubsystemContext<Message = RuntimeApiMessage>,
Context: overseer::SubsystemContext<Message = RuntimeApiMessage>,
{
fn start(self, ctx: Context) -> SpawnedSubsystem {
SpawnedSubsystem {
future: run(ctx, self).boxed(),
name: "runtime-api-subsystem",
}
SpawnedSubsystem { future: run(ctx, self).boxed(), name: "runtime-api-subsystem" }
}
}
impl<Client> RuntimeApiSubsystem<Client> where
impl<Client> RuntimeApiSubsystem<Client>
where
Client: ProvideRuntimeApi<Block> + Send + 'static + Sync,
Client::Api: ParachainHost<Block> + BabeApi<Block> + AuthorityDiscoveryApi<Block>,
{
@@ -117,26 +116,31 @@ impl<Client> RuntimeApiSubsystem<Client> where
self.requests_cache.cache_validator_groups(relay_parent, groups),
AvailabilityCores(relay_parent, cores) =>
self.requests_cache.cache_availability_cores(relay_parent, cores),
PersistedValidationData(relay_parent, para_id, assumption, data) =>
self.requests_cache.cache_persisted_validation_data((relay_parent, para_id, assumption), data),
CheckValidationOutputs(relay_parent, para_id, commitments, b) =>
self.requests_cache.cache_check_validation_outputs((relay_parent, para_id, commitments), b),
PersistedValidationData(relay_parent, para_id, assumption, data) => self
.requests_cache
.cache_persisted_validation_data((relay_parent, para_id, assumption), data),
CheckValidationOutputs(relay_parent, para_id, commitments, b) => self
.requests_cache
.cache_check_validation_outputs((relay_parent, para_id, commitments), b),
SessionIndexForChild(relay_parent, session_index) =>
self.requests_cache.cache_session_index_for_child(relay_parent, session_index),
ValidationCode(relay_parent, para_id, assumption, code) =>
self.requests_cache.cache_validation_code((relay_parent, para_id, assumption), code),
ValidationCode(relay_parent, para_id, assumption, code) => self
.requests_cache
.cache_validation_code((relay_parent, para_id, assumption), code),
ValidationCodeByHash(_relay_parent, validation_code_hash, code) =>
self.requests_cache.cache_validation_code_by_hash(validation_code_hash, code),
CandidatePendingAvailability(relay_parent, para_id, candidate) =>
self.requests_cache.cache_candidate_pending_availability((relay_parent, para_id), candidate),
CandidatePendingAvailability(relay_parent, para_id, candidate) => self
.requests_cache
.cache_candidate_pending_availability((relay_parent, para_id), candidate),
CandidateEvents(relay_parent, events) =>
self.requests_cache.cache_candidate_events(relay_parent, events),
SessionInfo(_relay_parent, session_index, info) =>
self.requests_cache.cache_session_info(session_index, info),
DmqContents(relay_parent, para_id, messages) =>
self.requests_cache.cache_dmq_contents((relay_parent, para_id), messages),
InboundHrmpChannelsContents(relay_parent, para_id, contents) =>
self.requests_cache.cache_inbound_hrmp_channel_contents((relay_parent, para_id), contents),
InboundHrmpChannelsContents(relay_parent, para_id, contents) => self
.requests_cache
.cache_inbound_hrmp_channel_contents((relay_parent, para_id), contents),
CurrentBabeEpoch(relay_parent, epoch) =>
self.requests_cache.cache_current_babe_epoch(relay_parent, epoch),
}
@@ -169,12 +173,12 @@ impl<Client> RuntimeApiSubsystem<Client> where
}
match request {
Request::Authorities(sender) => query!(authorities(), sender)
.map(|sender| Request::Authorities(sender)),
Request::Validators(sender) => query!(validators(), sender)
.map(|sender| Request::Validators(sender)),
Request::ValidatorGroups(sender) => query!(validator_groups(), sender)
.map(|sender| Request::ValidatorGroups(sender)),
Request::Authorities(sender) =>
query!(authorities(), sender).map(|sender| Request::Authorities(sender)),
Request::Validators(sender) =>
query!(validators(), sender).map(|sender| Request::Validators(sender)),
Request::ValidatorGroups(sender) =>
query!(validator_groups(), sender).map(|sender| Request::ValidatorGroups(sender)),
Request::AvailabilityCores(sender) => query!(availability_cores(), sender)
.map(|sender| Request::AvailabilityCores(sender)),
Request::PersistedValidationData(para, assumption, sender) =>
@@ -183,9 +187,8 @@ impl<Client> RuntimeApiSubsystem<Client> where
Request::CheckValidationOutputs(para, commitments, sender) =>
query!(check_validation_outputs(para, commitments), sender)
.map(|sender| Request::CheckValidationOutputs(para, commitments, sender)),
Request::SessionIndexForChild(sender) =>
query!(session_index_for_child(), sender)
.map(|sender| Request::SessionIndexForChild(sender)),
Request::SessionIndexForChild(sender) => query!(session_index_for_child(), sender)
.map(|sender| Request::SessionIndexForChild(sender)),
Request::ValidationCode(para, assumption, sender) =>
query!(validation_code(para, assumption), sender)
.map(|sender| Request::ValidationCode(para, assumption, sender)),
@@ -195,18 +198,17 @@ impl<Client> RuntimeApiSubsystem<Client> where
Request::CandidatePendingAvailability(para, sender) =>
query!(candidate_pending_availability(para), sender)
.map(|sender| Request::CandidatePendingAvailability(para, sender)),
Request::CandidateEvents(sender) => query!(candidate_events(), sender)
.map(|sender| Request::CandidateEvents(sender)),
Request::CandidateEvents(sender) =>
query!(candidate_events(), sender).map(|sender| Request::CandidateEvents(sender)),
Request::SessionInfo(index, sender) => query!(session_info(index), sender)
.map(|sender| Request::SessionInfo(index, sender)),
Request::DmqContents(id, sender) => query!(dmq_contents(id), sender)
.map(|sender| Request::DmqContents(id, sender)),
Request::DmqContents(id, sender) =>
query!(dmq_contents(id), sender).map(|sender| Request::DmqContents(id, sender)),
Request::InboundHrmpChannelsContents(id, sender) =>
query!(inbound_hrmp_channels_contents(id), sender)
.map(|sender| Request::InboundHrmpChannelsContents(id, sender)),
Request::CurrentBabeEpoch(sender) =>
query!(current_babe_epoch(), sender)
.map(|sender| Request::CurrentBabeEpoch(sender)),
query!(current_babe_epoch(), sender).map(|sender| Request::CurrentBabeEpoch(sender)),
}
}
@@ -224,14 +226,10 @@ impl<Client> RuntimeApiSubsystem<Client> where
};
let request = async move {
let result = make_runtime_api_request(
client,
metrics,
relay_parent,
request,
);
let result = make_runtime_api_request(client, metrics, relay_parent, request);
let _ = sender.send(result);
}.boxed();
}
.boxed();
if self.active_requests.len() >= MAX_PARALLEL_REQUESTS {
self.waiting_requests.push_back((request, receiver));
@@ -271,7 +269,8 @@ impl<Client> RuntimeApiSubsystem<Client> where
async fn run<Client, Context>(
mut ctx: Context,
mut subsystem: RuntimeApiSubsystem<Client>,
) -> SubsystemResult<()> where
) -> SubsystemResult<()>
where
Client: ProvideRuntimeApi<Block> + Send + Sync + 'static,
Client::Api: ParachainHost<Block> + BabeApi<Block> + AuthorityDiscoveryApi<Block>,
Context: SubsystemContext<Message = RuntimeApiMessage>,
@@ -323,12 +322,14 @@ where
Request::Authorities(sender) => query!(Authorities, authorities(), sender),
Request::Validators(sender) => query!(Validators, validators(), sender),
Request::ValidatorGroups(sender) => query!(ValidatorGroups, validator_groups(), sender),
Request::AvailabilityCores(sender) => query!(AvailabilityCores, availability_cores(), sender),
Request::AvailabilityCores(sender) =>
query!(AvailabilityCores, availability_cores(), sender),
Request::PersistedValidationData(para, assumption, sender) =>
query!(PersistedValidationData, persisted_validation_data(para, assumption), sender),
Request::CheckValidationOutputs(para, commitments, sender) =>
query!(CheckValidationOutputs, check_validation_outputs(para, commitments), sender),
Request::SessionIndexForChild(sender) => query!(SessionIndexForChild, session_index_for_child(), sender),
Request::SessionIndexForChild(sender) =>
query!(SessionIndexForChild, session_index_for_child(), sender),
Request::ValidationCode(para, assumption, sender) =>
query!(ValidationCode, validation_code(para, assumption), sender),
Request::ValidationCodeByHash(validation_code_hash, sender) =>
@@ -338,7 +339,8 @@ where
Request::CandidateEvents(sender) => query!(CandidateEvents, candidate_events(), sender),
Request::SessionInfo(index, sender) => query!(SessionInfo, session_info(index), sender),
Request::DmqContents(id, sender) => query!(DmqContents, dmq_contents(id), sender),
Request::InboundHrmpChannelsContents(id, sender) => query!(InboundHrmpChannelsContents, inbound_hrmp_channels_contents(id), sender),
Request::InboundHrmpChannelsContents(id, sender) =>
query!(InboundHrmpChannelsContents, inbound_hrmp_channels_contents(id), sender),
Request::CurrentBabeEpoch(sender) => query!(CurrentBabeEpoch, current_epoch(), sender),
}
}
@@ -365,12 +367,15 @@ impl Metrics {
}
fn on_cached_request(&self) {
self.0.as_ref()
self.0
.as_ref()
.map(|metrics| metrics.chain_api_requests.with_label_values(&["cached"]).inc());
}
/// Provide a timer for `make_runtime_api_request` which observes on drop.
fn time_make_runtime_api_request(&self) -> Option<metrics::prometheus::prometheus::HistogramTimer> {
fn time_make_runtime_api_request(
&self,
) -> Option<metrics::prometheus::prometheus::HistogramTimer> {
self.0.as_ref().map(|metrics| metrics.make_runtime_api_request.start_timer())
}
}
@@ -389,12 +394,10 @@ impl metrics::Metrics for Metrics {
registry,
)?,
make_runtime_api_request: prometheus::register(
prometheus::Histogram::with_opts(
prometheus::HistogramOpts::new(
"parachain_runtime_api_make_runtime_api_request",
"Time spent within `runtime_api::make_runtime_api_request`",
)
)?,
prometheus::Histogram::with_opts(prometheus::HistogramOpts::new(
"parachain_runtime_api_make_runtime_api_request",
"Time spent within `runtime_api::make_runtime_api_request`",
))?,
registry,
)?,
};
polkadot/node/core/runtime-api/src/tests.rs
+140 -138
View File
@@ -16,18 +16,19 @@
use super::*;
use polkadot_primitives::v1::{
ValidatorId, ValidatorIndex, GroupRotationInfo, CoreState, PersistedValidationData,
Id as ParaId, OccupiedCoreAssumption, SessionIndex, ValidationCode,
CommittedCandidateReceipt, CandidateEvent, InboundDownwardMessage,
InboundHrmpMessage, SessionInfo, AuthorityDiscoveryId, ValidationCodeHash,
};
use polkadot_node_subsystem_test_helpers as test_helpers;
use sp_core::testing::TaskExecutor;
use std::{collections::{HashMap, BTreeMap}, sync::{Arc, Mutex}};
use futures::channel::oneshot;
use polkadot_node_primitives::{
BabeEpoch, BabeEpochConfiguration, BabeAllowedSlots,
use polkadot_node_primitives::{BabeAllowedSlots, BabeEpoch, BabeEpochConfiguration};
use polkadot_node_subsystem_test_helpers as test_helpers;
use polkadot_primitives::v1::{
AuthorityDiscoveryId, CandidateEvent, CommittedCandidateReceipt, CoreState, GroupRotationInfo,
Id as ParaId, InboundDownwardMessage, InboundHrmpMessage, OccupiedCoreAssumption,
PersistedValidationData, SessionIndex, SessionInfo, ValidationCode, ValidationCodeHash,
ValidatorId, ValidatorIndex,
};
use sp_core::testing::TaskExecutor;
use std::{
collections::{BTreeMap, HashMap},
sync::{Arc, Mutex},
};
#[derive(Default, Clone)]
@@ -201,9 +202,11 @@ fn requests_authorities() {
let test_task = async move {
let (tx, rx) = oneshot::channel();
ctx_handle.send(FromOverseer::Communication {
msg: RuntimeApiMessage::Request(relay_parent, Request::Authorities(tx))
}).await;
ctx_handle
.send(FromOverseer::Communication {
msg: RuntimeApiMessage::Request(relay_parent, Request::Authorities(tx)),
})
.await;
assert_eq!(rx.await.unwrap().unwrap(), runtime_api.authorities);
@@ -225,9 +228,11 @@ fn requests_validators() {
let test_task = async move {
let (tx, rx) = oneshot::channel();
ctx_handle.send(FromOverseer::Communication {
msg: RuntimeApiMessage::Request(relay_parent, Request::Validators(tx))
}).await;
ctx_handle
.send(FromOverseer::Communication {
msg: RuntimeApiMessage::Request(relay_parent, Request::Validators(tx)),
})
.await;
assert_eq!(rx.await.unwrap().unwrap(), runtime_api.validators);
@@ -249,9 +254,11 @@ fn requests_validator_groups() {
let test_task = async move {
let (tx, rx) = oneshot::channel();
ctx_handle.send(FromOverseer::Communication {
msg: RuntimeApiMessage::Request(relay_parent, Request::ValidatorGroups(tx))
}).await;
ctx_handle
.send(FromOverseer::Communication {
msg: RuntimeApiMessage::Request(relay_parent, Request::ValidatorGroups(tx)),
})
.await;
assert_eq!(rx.await.unwrap().unwrap().0, runtime_api.validator_groups);
@@ -273,9 +280,11 @@ fn requests_availability_cores() {
let test_task = async move {
let (tx, rx) = oneshot::channel();
ctx_handle.send(FromOverseer::Communication {
msg: RuntimeApiMessage::Request(relay_parent, Request::AvailabilityCores(tx))
}).await;
ctx_handle
.send(FromOverseer::Communication {
msg: RuntimeApiMessage::Request(relay_parent, Request::AvailabilityCores(tx)),
})
.await;
assert_eq!(rx.await.unwrap().unwrap(), runtime_api.availability_cores);
@@ -302,22 +311,26 @@ fn requests_persisted_validation_data() {
let test_task = async move {
let (tx, rx) = oneshot::channel();
ctx_handle.send(FromOverseer::Communication {
msg: RuntimeApiMessage::Request(
relay_parent,
Request::PersistedValidationData(para_a, OccupiedCoreAssumption::Included, tx)
),
}).await;
ctx_handle
.send(FromOverseer::Communication {
msg: RuntimeApiMessage::Request(
relay_parent,
Request::PersistedValidationData(para_a, OccupiedCoreAssumption::Included, tx),
),
})
.await;
assert_eq!(rx.await.unwrap().unwrap(), Some(Default::default()));
let (tx, rx) = oneshot::channel();
ctx_handle.send(FromOverseer::Communication {
msg: RuntimeApiMessage::Request(
relay_parent,
Request::PersistedValidationData(para_b, OccupiedCoreAssumption::Included, tx)
),
}).await;
ctx_handle
.send(FromOverseer::Communication {
msg: RuntimeApiMessage::Request(
relay_parent,
Request::PersistedValidationData(para_b, OccupiedCoreAssumption::Included, tx),
),
})
.await;
assert_eq!(rx.await.unwrap().unwrap(), None);
@@ -347,36 +360,26 @@ fn requests_check_validation_outputs() {
let test_task = async move {
let (tx, rx) = oneshot::channel();
ctx_handle.send(FromOverseer::Communication {
msg: RuntimeApiMessage::Request(
relay_parent,
Request::CheckValidationOutputs(
para_a,
commitments.clone(),
tx,
ctx_handle
.send(FromOverseer::Communication {
msg: RuntimeApiMessage::Request(
relay_parent,
Request::CheckValidationOutputs(para_a, commitments.clone(), tx),
),
)
}).await;
assert_eq!(
rx.await.unwrap().unwrap(),
runtime_api.validation_outputs_results[&para_a],
);
})
.await;
assert_eq!(rx.await.unwrap().unwrap(), runtime_api.validation_outputs_results[&para_a],);
let (tx, rx) = oneshot::channel();
ctx_handle.send(FromOverseer::Communication {
msg: RuntimeApiMessage::Request(
relay_parent,
Request::CheckValidationOutputs(
para_b,
commitments,
tx,
ctx_handle
.send(FromOverseer::Communication {
msg: RuntimeApiMessage::Request(
relay_parent,
Request::CheckValidationOutputs(para_b, commitments, tx),
),
)
}).await;
assert_eq!(
rx.await.unwrap().unwrap(),
runtime_api.validation_outputs_results[&para_b],
);
})
.await;
assert_eq!(rx.await.unwrap().unwrap(), runtime_api.validation_outputs_results[&para_b],);
ctx_handle.send(FromOverseer::Signal(OverseerSignal::Conclude)).await;
};
@@ -396,9 +399,11 @@ fn requests_session_index_for_child() {
let test_task = async move {
let (tx, rx) = oneshot::channel();
ctx_handle.send(FromOverseer::Communication {
msg: RuntimeApiMessage::Request(relay_parent, Request::SessionIndexForChild(tx))
}).await;
ctx_handle
.send(FromOverseer::Communication {
msg: RuntimeApiMessage::Request(relay_parent, Request::SessionIndexForChild(tx)),
})
.await;
assert_eq!(rx.await.unwrap().unwrap(), runtime_api.session_index_for_child);
@@ -424,9 +429,14 @@ fn requests_session_info() {
let test_task = async move {
let (tx, rx) = oneshot::channel();
ctx_handle.send(FromOverseer::Communication {
msg: RuntimeApiMessage::Request(relay_parent, Request::SessionInfo(session_index, tx))
}).await;
ctx_handle
.send(FromOverseer::Communication {
msg: RuntimeApiMessage::Request(
relay_parent,
Request::SessionInfo(session_index, tx),
),
})
.await;
assert_eq!(rx.await.unwrap().unwrap(), Some(Default::default()));
@@ -454,22 +464,26 @@ fn requests_validation_code() {
let test_task = async move {
let (tx, rx) = oneshot::channel();
ctx_handle.send(FromOverseer::Communication {
msg: RuntimeApiMessage::Request(
relay_parent,
Request::ValidationCode(para_a, OccupiedCoreAssumption::Included, tx)
),
}).await;
ctx_handle
.send(FromOverseer::Communication {
msg: RuntimeApiMessage::Request(
relay_parent,
Request::ValidationCode(para_a, OccupiedCoreAssumption::Included, tx),
),
})
.await;
assert_eq!(rx.await.unwrap().unwrap(), Some(Default::default()));
let (tx, rx) = oneshot::channel();
ctx_handle.send(FromOverseer::Communication {
msg: RuntimeApiMessage::Request(
relay_parent,
Request::ValidationCode(para_b, OccupiedCoreAssumption::Included, tx)
),
}).await;
ctx_handle
.send(FromOverseer::Communication {
msg: RuntimeApiMessage::Request(
relay_parent,
Request::ValidationCode(para_b, OccupiedCoreAssumption::Included, tx),
),
})
.await;
assert_eq!(rx.await.unwrap().unwrap(), None);
@@ -496,23 +510,27 @@ fn requests_candidate_pending_availability() {
let test_task = async move {
let (tx, rx) = oneshot::channel();
ctx_handle.send(FromOverseer::Communication {
msg: RuntimeApiMessage::Request(
relay_parent,
Request::CandidatePendingAvailability(para_a, tx),
)
}).await;
ctx_handle
.send(FromOverseer::Communication {
msg: RuntimeApiMessage::Request(
relay_parent,
Request::CandidatePendingAvailability(para_a, tx),
),
})
.await;
assert_eq!(rx.await.unwrap().unwrap(), Some(Default::default()));
let (tx, rx) = oneshot::channel();
ctx_handle.send(FromOverseer::Communication {
msg: RuntimeApiMessage::Request(
relay_parent,
Request::CandidatePendingAvailability(para_b, tx),
)
}).await;
ctx_handle
.send(FromOverseer::Communication {
msg: RuntimeApiMessage::Request(
relay_parent,
Request::CandidatePendingAvailability(para_b, tx),
),
})
.await;
assert_eq!(rx.await.unwrap().unwrap(), None);
@@ -534,9 +552,11 @@ fn requests_candidate_events() {
let test_task = async move {
let (tx, rx) = oneshot::channel();
ctx_handle.send(FromOverseer::Communication {
msg: RuntimeApiMessage::Request(relay_parent, Request::CandidateEvents(tx))
}).await;
ctx_handle
.send(FromOverseer::Communication {
msg: RuntimeApiMessage::Request(relay_parent, Request::CandidateEvents(tx)),
})
.await;
assert_eq!(rx.await.unwrap().unwrap(), runtime_api.candidate_events);
@@ -561,10 +581,7 @@ fn requests_dmq_contents() {
runtime_api.dmq_contents.insert(para_a, vec![]);
runtime_api.dmq_contents.insert(
para_b,
vec![InboundDownwardMessage {
sent_at: 228,
msg: b"Novus Ordo Seclorum".to_vec(),
}],
vec![InboundDownwardMessage { sent_at: 228, msg: b"Novus Ordo Seclorum".to_vec() }],
);
runtime_api
@@ -589,15 +606,10 @@ fn requests_dmq_contents() {
.await;
assert_eq!(
rx.await.unwrap().unwrap(),
vec![InboundDownwardMessage {
sent_at: 228,
msg: b"Novus Ordo Seclorum".to_vec(),
}]
vec![InboundDownwardMessage { sent_at: 228, msg: b"Novus Ordo Seclorum".to_vec() }]
);
ctx_handle
.send(FromOverseer::Signal(OverseerSignal::Conclude))
.await;
ctx_handle.send(FromOverseer::Signal(OverseerSignal::Conclude)).await;
};
futures::executor::block_on(future::join(subsystem_task, test_task));
}
@@ -614,13 +626,7 @@ fn requests_inbound_hrmp_channels_contents() {
let para_b_inbound_channels = [
(para_a, vec![]),
(
para_c,
vec![InboundHrmpMessage {
sent_at: 1,
data: "𝙀=𝙈𝘾²".as_bytes().to_owned(),
}],
),
(para_c, vec![InboundHrmpMessage { sent_at: 1, data: "𝙀=𝙈𝘾²".as_bytes().to_owned() }]),
]
.iter()
.cloned()
@@ -630,9 +636,7 @@ fn requests_inbound_hrmp_channels_contents() {
let mut runtime_api = MockRuntimeApi::default();
runtime_api.hrmp_channels.insert(para_a, BTreeMap::new());
runtime_api
.hrmp_channels
.insert(para_b, para_b_inbound_channels.clone());
runtime_api.hrmp_channels.insert(para_b, para_b_inbound_channels.clone());
runtime_api
});
@@ -662,9 +666,7 @@ fn requests_inbound_hrmp_channels_contents() {
.await;
assert_eq!(rx.await.unwrap().unwrap(), para_b_inbound_channels,);
ctx_handle
.send(FromOverseer::Signal(OverseerSignal::Conclude))
.await;
ctx_handle.send(FromOverseer::Signal(OverseerSignal::Conclude)).await;
};
futures::executor::block_on(future::join(subsystem_task, test_task));
}
@@ -680,10 +682,7 @@ fn requests_validation_code_by_hash() {
for n in 0..5 {
let code = ValidationCode::from(vec![n; 32]);
runtime_api.validation_code_by_hash.insert(
code.hash(),
code.clone(),
);
runtime_api.validation_code_by_hash.insert(code.hash(), code.clone());
validation_code.push(code);
}
@@ -697,12 +696,14 @@ fn requests_validation_code_by_hash() {
let test_task = async move {
for code in validation_code {
let (tx, rx) = oneshot::channel();
ctx_handle.send(FromOverseer::Communication {
msg: RuntimeApiMessage::Request(
relay_parent,
Request::ValidationCodeByHash(code.hash(), tx),
)
}).await;
ctx_handle
.send(FromOverseer::Communication {
msg: RuntimeApiMessage::Request(
relay_parent,
Request::ValidationCodeByHash(code.hash(), tx),
),
})
.await;
assert_eq!(rx.await.unwrap().unwrap(), Some(code));
}
@@ -732,9 +733,11 @@ fn multiple_requests_in_parallel_are_working() {
for _ in 0..MAX_PARALLEL_REQUESTS * 10 {
let (tx, rx) = oneshot::channel();
ctx_handle.send(FromOverseer::Communication {
msg: RuntimeApiMessage::Request(relay_parent, Request::AvailabilityCores(tx))
}).await;
ctx_handle
.send(FromOverseer::Communication {
msg: RuntimeApiMessage::Request(relay_parent, Request::AvailabilityCores(tx)),
})
.await;
receivers.push(rx);
}
@@ -763,10 +766,7 @@ fn request_babe_epoch() {
duration: 10,
authorities: Vec::new(),
randomness: [1u8; 32],
config: BabeEpochConfiguration {
c: (1, 4),
allowed_slots: BabeAllowedSlots::PrimarySlots,
},
config: BabeEpochConfiguration { c: (1, 4), allowed_slots: BabeAllowedSlots::PrimarySlots },
};
runtime_api.babe_epoch = Some(epoch.clone());
let runtime_api = Arc::new(runtime_api);
@@ -778,9 +778,11 @@ fn request_babe_epoch() {
let test_task = async move {
let (tx, rx) = oneshot::channel();
ctx_handle.send(FromOverseer::Communication {
msg: RuntimeApiMessage::Request(relay_parent, Request::CurrentBabeEpoch(tx))
}).await;
ctx_handle
.send(FromOverseer::Communication {
msg: RuntimeApiMessage::Request(relay_parent, Request::CurrentBabeEpoch(tx)),
})
.await;
assert_eq!(rx.await.unwrap().unwrap(), epoch);
ctx_handle.send(FromOverseer::Signal(OverseerSignal::Conclude)).await;