mirror of
https://github.com/pezkuwichain/pezkuwi-subxt.git
synced 2026-04-28 12:07:57 +00:00
cargo +nightly fmt (#3540)
* cargo +nightly fmt * add cargo-fmt check to ci * update ci * fmt * fmt * skip macro * ignore bridges
This commit is contained in:
@@ -16,41 +16,36 @@
|
||||
|
||||
//! Implements a `AvailabilityStoreSubsystem`.
|
||||
|
||||
#![recursion_limit="256"]
|
||||
#![recursion_limit = "256"]
|
||||
#![warn(missing_docs)]
|
||||
|
||||
use std::collections::{HashMap, HashSet, BTreeSet};
|
||||
use std::io;
|
||||
use std::sync::Arc;
|
||||
use std::time::{Duration, SystemTime, SystemTimeError, UNIX_EPOCH};
|
||||
use std::{
|
||||
collections::{BTreeSet, HashMap, HashSet},
|
||||
io,
|
||||
sync::Arc,
|
||||
time::{Duration, SystemTime, SystemTimeError, UNIX_EPOCH},
|
||||
};
|
||||
|
||||
use parity_scale_codec::{Encode, Decode, Input, Error as CodecError};
|
||||
use futures::{select, channel::oneshot, future, FutureExt};
|
||||
use futures::{channel::oneshot, future, select, FutureExt};
|
||||
use futures_timer::Delay;
|
||||
use kvdb::{KeyValueDB, DBTransaction};
|
||||
use kvdb::{DBTransaction, KeyValueDB};
|
||||
use parity_scale_codec::{Decode, Encode, Error as CodecError, Input};
|
||||
|
||||
use polkadot_primitives::v1::{
|
||||
Hash, BlockNumber, CandidateEvent, ValidatorIndex, CandidateHash,
|
||||
CandidateReceipt, Header,
|
||||
};
|
||||
use polkadot_node_primitives::{
|
||||
ErasureChunk, AvailableData,
|
||||
};
|
||||
use polkadot_subsystem::{
|
||||
FromOverseer, OverseerSignal, SubsystemError,
|
||||
SubsystemContext, SpawnedSubsystem,
|
||||
overseer,
|
||||
ActiveLeavesUpdate,
|
||||
errors::{ChainApiError, RuntimeApiError},
|
||||
};
|
||||
use bitvec::{order::Lsb0 as BitOrderLsb0, vec::BitVec};
|
||||
use polkadot_node_primitives::{AvailableData, ErasureChunk};
|
||||
use polkadot_node_subsystem_util::{
|
||||
self as util,
|
||||
metrics::{self, prometheus},
|
||||
};
|
||||
use polkadot_subsystem::messages::{
|
||||
AvailabilityStoreMessage, ChainApiMessage,
|
||||
use polkadot_primitives::v1::{
|
||||
BlockNumber, CandidateEvent, CandidateHash, CandidateReceipt, Hash, Header, ValidatorIndex,
|
||||
};
|
||||
use polkadot_subsystem::{
|
||||
errors::{ChainApiError, RuntimeApiError},
|
||||
messages::{AvailabilityStoreMessage, ChainApiMessage},
|
||||
overseer, ActiveLeavesUpdate, FromOverseer, OverseerSignal, SpawnedSubsystem, SubsystemContext,
|
||||
SubsystemError,
|
||||
};
|
||||
use bitvec::{vec::BitVec, order::Lsb0 as BitOrderLsb0};
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests;
|
||||
@@ -126,7 +121,9 @@ impl Encode for BEBlockNumber {
|
||||
|
||||
impl Decode for BEBlockNumber {
|
||||
fn decode<I: Input>(value: &mut I) -> Result<Self, CodecError> {
|
||||
<[u8; std::mem::size_of::<BlockNumber>()]>::decode(value).map(BlockNumber::from_be_bytes).map(Self)
|
||||
<[u8; std::mem::size_of::<BlockNumber>()]>::decode(value)
|
||||
.map(BlockNumber::from_be_bytes)
|
||||
.map(Self)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -143,7 +140,7 @@ enum State {
|
||||
Unfinalized(BETimestamp, Vec<(BEBlockNumber, Hash)>),
|
||||
/// Candidate data has appeared in a finalized block and did so at the given time.
|
||||
#[codec(index = 2)]
|
||||
Finalized(BETimestamp)
|
||||
Finalized(BETimestamp),
|
||||
}
|
||||
|
||||
// Meta information about a candidate.
|
||||
@@ -163,12 +160,12 @@ fn query_inner<D: Decode>(
|
||||
Ok(Some(raw)) => {
|
||||
let res = D::decode(&mut &raw[..])?;
|
||||
Ok(Some(res))
|
||||
}
|
||||
},
|
||||
Ok(None) => Ok(None),
|
||||
Err(e) => {
|
||||
tracing::warn!(target: LOG_TARGET, err = ?e, "Error reading from the availability store");
|
||||
Err(e.into())
|
||||
}
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
@@ -193,11 +190,7 @@ fn load_available_data(
|
||||
query_inner(db, config.col_data, &key)
|
||||
}
|
||||
|
||||
fn delete_available_data(
|
||||
tx: &mut DBTransaction,
|
||||
config: &Config,
|
||||
hash: &CandidateHash,
|
||||
) {
|
||||
fn delete_available_data(tx: &mut DBTransaction, config: &Config, hash: &CandidateHash) {
|
||||
let key = (AVAILABLE_PREFIX, hash).encode();
|
||||
|
||||
tx.delete(config.col_data, &key[..])
|
||||
@@ -247,12 +240,7 @@ fn load_meta(
|
||||
query_inner(db, config.col_meta, &key)
|
||||
}
|
||||
|
||||
fn write_meta(
|
||||
tx: &mut DBTransaction,
|
||||
config: &Config,
|
||||
hash: &CandidateHash,
|
||||
meta: &CandidateMeta,
|
||||
) {
|
||||
fn write_meta(tx: &mut DBTransaction, config: &Config, hash: &CandidateHash, meta: &CandidateMeta) {
|
||||
let key = (META_PREFIX, hash).encode();
|
||||
|
||||
tx.put_vec(config.col_meta, &key, meta.encode());
|
||||
@@ -263,11 +251,7 @@ fn delete_meta(tx: &mut DBTransaction, config: &Config, hash: &CandidateHash) {
|
||||
tx.delete(config.col_meta, &key[..])
|
||||
}
|
||||
|
||||
fn delete_unfinalized_height(
|
||||
tx: &mut DBTransaction,
|
||||
config: &Config,
|
||||
block_number: BlockNumber,
|
||||
) {
|
||||
fn delete_unfinalized_height(tx: &mut DBTransaction, config: &Config, block_number: BlockNumber) {
|
||||
let prefix = (UNFINALIZED_PREFIX, BEBlockNumber(block_number)).encode();
|
||||
tx.delete_prefix(config.col_meta, &prefix);
|
||||
}
|
||||
@@ -279,12 +263,8 @@ fn delete_unfinalized_inclusion(
|
||||
block_hash: &Hash,
|
||||
candidate_hash: &CandidateHash,
|
||||
) {
|
||||
let key = (
|
||||
UNFINALIZED_PREFIX,
|
||||
BEBlockNumber(block_number),
|
||||
block_hash,
|
||||
candidate_hash,
|
||||
).encode();
|
||||
let key =
|
||||
(UNFINALIZED_PREFIX, BEBlockNumber(block_number), block_hash, candidate_hash).encode();
|
||||
|
||||
tx.delete(config.col_meta, &key[..]);
|
||||
}
|
||||
@@ -338,7 +318,7 @@ fn pruning_range(now: impl Into<BETimestamp>) -> (Vec<u8>, Vec<u8>) {
|
||||
|
||||
fn decode_unfinalized_key(s: &[u8]) -> Result<(BlockNumber, Hash, CandidateHash), CodecError> {
|
||||
if !s.starts_with(UNFINALIZED_PREFIX) {
|
||||
return Err("missing magic string".into());
|
||||
return Err("missing magic string".into())
|
||||
}
|
||||
|
||||
<(BEBlockNumber, Hash, CandidateHash)>::decode(&mut &s[UNFINALIZED_PREFIX.len()..])
|
||||
@@ -347,7 +327,7 @@ fn decode_unfinalized_key(s: &[u8]) -> Result<(BlockNumber, Hash, CandidateHash)
|
||||
|
||||
fn decode_pruning_key(s: &[u8]) -> Result<(Duration, CandidateHash), CodecError> {
|
||||
if !s.starts_with(PRUNE_BY_TIME_PREFIX) {
|
||||
return Err("missing magic string".into());
|
||||
return Err("missing magic string".into())
|
||||
}
|
||||
|
||||
<(BETimestamp, CandidateHash)>::decode(&mut &s[PRUNE_BY_TIME_PREFIX.len()..])
|
||||
@@ -389,8 +369,8 @@ impl Error {
|
||||
fn trace(&self) {
|
||||
match self {
|
||||
// don't spam the log with spurious errors
|
||||
Self::RuntimeApi(_) |
|
||||
Self::Oneshot(_) => tracing::debug!(target: LOG_TARGET, err = ?self),
|
||||
Self::RuntimeApi(_) | Self::Oneshot(_) =>
|
||||
tracing::debug!(target: LOG_TARGET, err = ?self),
|
||||
// it's worth reporting otherwise
|
||||
_ => tracing::warn!(target: LOG_TARGET, err = ?self),
|
||||
}
|
||||
@@ -457,11 +437,7 @@ pub struct AvailabilityStoreSubsystem {
|
||||
|
||||
impl AvailabilityStoreSubsystem {
|
||||
/// Create a new `AvailabilityStoreSubsystem` with a given config on disk.
|
||||
pub fn new(
|
||||
db: Arc<dyn KeyValueDB>,
|
||||
config: Config,
|
||||
metrics: Metrics,
|
||||
) -> Self {
|
||||
pub fn new(db: Arc<dyn KeyValueDB>, config: Config, metrics: Metrics) -> Self {
|
||||
Self::with_pruning_config_and_clock(
|
||||
db,
|
||||
config,
|
||||
@@ -530,14 +506,9 @@ where
|
||||
Context: overseer::SubsystemContext<Message = AvailabilityStoreMessage>,
|
||||
{
|
||||
fn start(self, ctx: Context) -> SpawnedSubsystem {
|
||||
let future = run(self, ctx)
|
||||
.map(|_| Ok(()))
|
||||
.boxed();
|
||||
let future = run(self, ctx).map(|_| Ok(())).boxed();
|
||||
|
||||
SpawnedSubsystem {
|
||||
name: "availability-store-subsystem",
|
||||
future,
|
||||
}
|
||||
SpawnedSubsystem { name: "availability-store-subsystem", future }
|
||||
}
|
||||
}
|
||||
|
||||
@@ -555,12 +526,12 @@ where
|
||||
e.trace();
|
||||
|
||||
if let Error::Subsystem(SubsystemError::Context(_)) = e {
|
||||
break;
|
||||
break
|
||||
}
|
||||
}
|
||||
},
|
||||
Ok(true) => {
|
||||
tracing::info!(target: LOG_TARGET, "received `Conclude` signal, exiting");
|
||||
break;
|
||||
break
|
||||
},
|
||||
Ok(false) => continue,
|
||||
}
|
||||
@@ -571,8 +542,7 @@ async fn run_iteration<Context>(
|
||||
ctx: &mut Context,
|
||||
subsystem: &mut AvailabilityStoreSubsystem,
|
||||
mut next_pruning: &mut future::Fuse<Delay>,
|
||||
)
|
||||
-> Result<bool, Error>
|
||||
) -> Result<bool, Error>
|
||||
where
|
||||
Context: SubsystemContext<Message = AvailabilityStoreMessage>,
|
||||
Context: overseer::SubsystemContext<Message = AvailabilityStoreMessage>,
|
||||
@@ -634,9 +604,7 @@ where
|
||||
let block_header = {
|
||||
let (tx, rx) = oneshot::channel();
|
||||
|
||||
ctx.send_message(
|
||||
ChainApiMessage::BlockHeader(activated, tx)
|
||||
).await;
|
||||
ctx.send_message(ChainApiMessage::BlockHeader(activated, tx)).await;
|
||||
|
||||
match rx.await?? {
|
||||
None => return Ok(()),
|
||||
@@ -647,13 +615,12 @@ where
|
||||
|
||||
let new_blocks = util::determine_new_blocks(
|
||||
ctx.sender(),
|
||||
|hash| -> Result<bool, Error> {
|
||||
Ok(subsystem.known_blocks.is_known(hash))
|
||||
},
|
||||
|hash| -> Result<bool, Error> { Ok(subsystem.known_blocks.is_known(hash)) },
|
||||
activated,
|
||||
&block_header,
|
||||
subsystem.finalized_number.unwrap_or(block_number.saturating_sub(1)),
|
||||
).await?;
|
||||
)
|
||||
.await?;
|
||||
|
||||
// determine_new_blocks is descending in block height
|
||||
for (hash, header) in new_blocks.into_iter().rev() {
|
||||
@@ -669,7 +636,8 @@ where
|
||||
now,
|
||||
hash,
|
||||
header,
|
||||
).await?;
|
||||
)
|
||||
.await?;
|
||||
subsystem.known_blocks.insert(hash, block_number);
|
||||
subsystem.db.write(tx)?;
|
||||
}
|
||||
@@ -691,18 +659,12 @@ where
|
||||
Context: SubsystemContext<Message = AvailabilityStoreMessage>,
|
||||
Context: overseer::SubsystemContext<Message = AvailabilityStoreMessage>,
|
||||
{
|
||||
|
||||
let candidate_events = util::request_candidate_events(
|
||||
hash,
|
||||
ctx.sender(),
|
||||
).await.await??;
|
||||
let candidate_events = util::request_candidate_events(hash, ctx.sender()).await.await??;
|
||||
|
||||
// We need to request the number of validators based on the parent state,
|
||||
// as that is the number of validators used to create this block.
|
||||
let n_validators = util::request_validators(
|
||||
header.parent_hash,
|
||||
ctx.sender(),
|
||||
).await.await??.len();
|
||||
let n_validators =
|
||||
util::request_validators(header.parent_hash, ctx.sender()).await.await??.len();
|
||||
|
||||
for event in candidate_events {
|
||||
match event {
|
||||
@@ -716,7 +678,7 @@ where
|
||||
n_validators,
|
||||
receipt,
|
||||
)?;
|
||||
}
|
||||
},
|
||||
CandidateEvent::CandidateIncluded(receipt, _head, _core_index, _group_index) => {
|
||||
note_block_included(
|
||||
db,
|
||||
@@ -726,8 +688,8 @@ where
|
||||
(header.number, hash),
|
||||
receipt,
|
||||
)?;
|
||||
}
|
||||
_ => {}
|
||||
},
|
||||
_ => {},
|
||||
}
|
||||
}
|
||||
|
||||
@@ -745,11 +707,7 @@ fn note_block_backed(
|
||||
) -> Result<(), Error> {
|
||||
let candidate_hash = candidate.hash();
|
||||
|
||||
tracing::debug!(
|
||||
target: LOG_TARGET,
|
||||
?candidate_hash,
|
||||
"Candidate backed",
|
||||
);
|
||||
tracing::debug!(target: LOG_TARGET, ?candidate_hash, "Candidate backed",);
|
||||
|
||||
if load_meta(db, config, &candidate_hash)?.is_none() {
|
||||
let meta = CandidateMeta {
|
||||
@@ -771,7 +729,7 @@ fn note_block_included(
|
||||
db: &Arc<dyn KeyValueDB>,
|
||||
db_transaction: &mut DBTransaction,
|
||||
config: &Config,
|
||||
pruning_config:&PruningConfig,
|
||||
pruning_config: &PruningConfig,
|
||||
block: (BlockNumber, Hash),
|
||||
candidate: CandidateReceipt,
|
||||
) -> Result<(), Error> {
|
||||
@@ -786,15 +744,11 @@ fn note_block_included(
|
||||
?candidate_hash,
|
||||
"Candidate included without being backed?",
|
||||
);
|
||||
}
|
||||
},
|
||||
Some(mut meta) => {
|
||||
let be_block = (BEBlockNumber(block.0), block.1);
|
||||
|
||||
tracing::debug!(
|
||||
target: LOG_TARGET,
|
||||
?candidate_hash,
|
||||
"Candidate included",
|
||||
);
|
||||
tracing::debug!(target: LOG_TARGET, ?candidate_hash, "Candidate included",);
|
||||
|
||||
meta.state = match meta.state {
|
||||
State::Unavailable(at) => {
|
||||
@@ -803,20 +757,20 @@ fn note_block_included(
|
||||
delete_pruning_key(db_transaction, config, prune_at, &candidate_hash);
|
||||
|
||||
State::Unfinalized(at, vec![be_block])
|
||||
}
|
||||
},
|
||||
State::Unfinalized(at, mut within) => {
|
||||
if let Err(i) = within.binary_search(&be_block) {
|
||||
within.insert(i, be_block);
|
||||
State::Unfinalized(at, within)
|
||||
} else {
|
||||
return Ok(());
|
||||
return Ok(())
|
||||
}
|
||||
}
|
||||
},
|
||||
State::Finalized(_at) => {
|
||||
// This should never happen as a candidate would have to be included after
|
||||
// finality.
|
||||
return Ok(())
|
||||
}
|
||||
},
|
||||
};
|
||||
|
||||
write_unfinalized_block_contains(
|
||||
@@ -827,7 +781,7 @@ fn note_block_included(
|
||||
&candidate_hash,
|
||||
);
|
||||
write_meta(db_transaction, config, &candidate_hash, &meta);
|
||||
}
|
||||
},
|
||||
}
|
||||
|
||||
Ok(())
|
||||
@@ -837,9 +791,9 @@ macro_rules! peek_num {
|
||||
($iter:ident) => {
|
||||
match $iter.peek() {
|
||||
Some((k, _)) => decode_unfinalized_key(&k[..]).ok().map(|(b, _, _)| b),
|
||||
None => None
|
||||
None => None,
|
||||
}
|
||||
}
|
||||
};
|
||||
}
|
||||
|
||||
async fn process_block_finalized<Context>(
|
||||
@@ -863,7 +817,9 @@ where
|
||||
// as it is not `Send`. That is why we create the iterator once within this loop, drop it,
|
||||
// do an asynchronous request, and then instantiate the exact same iterator again.
|
||||
let batch_num = {
|
||||
let mut iter = subsystem.db.iter_with_prefix(subsystem.config.col_meta, &start_prefix)
|
||||
let mut iter = subsystem
|
||||
.db
|
||||
.iter_with_prefix(subsystem.config.col_meta, &start_prefix)
|
||||
.take_while(|(k, _)| &k[..] < &end_prefix[..])
|
||||
.peekable();
|
||||
|
||||
@@ -873,7 +829,9 @@ where
|
||||
}
|
||||
};
|
||||
|
||||
if batch_num < next_possible_batch { continue } // sanity.
|
||||
if batch_num < next_possible_batch {
|
||||
continue
|
||||
} // sanity.
|
||||
next_possible_batch = batch_num + 1;
|
||||
|
||||
let batch_finalized_hash = if batch_num == finalized_number {
|
||||
@@ -884,19 +842,22 @@ where
|
||||
|
||||
match rx.await?? {
|
||||
None => {
|
||||
tracing::warn!(target: LOG_TARGET,
|
||||
tracing::warn!(
|
||||
target: LOG_TARGET,
|
||||
"Availability store was informed that block #{} is finalized, \
|
||||
but chain API has no finalized hash.",
|
||||
batch_num,
|
||||
);
|
||||
|
||||
break
|
||||
}
|
||||
},
|
||||
Some(h) => h,
|
||||
}
|
||||
};
|
||||
|
||||
let iter = subsystem.db.iter_with_prefix(subsystem.config.col_meta, &start_prefix)
|
||||
let iter = subsystem
|
||||
.db
|
||||
.iter_with_prefix(subsystem.config.col_meta, &start_prefix)
|
||||
.take_while(|(k, _)| &k[..] < &end_prefix[..])
|
||||
.peekable();
|
||||
|
||||
@@ -907,13 +868,7 @@ where
|
||||
|
||||
delete_unfinalized_height(&mut db_transaction, &subsystem.config, batch_num);
|
||||
|
||||
update_blocks_at_finalized_height(
|
||||
&subsystem,
|
||||
&mut db_transaction,
|
||||
batch,
|
||||
batch_num,
|
||||
now,
|
||||
)?;
|
||||
update_blocks_at_finalized_height(&subsystem, &mut db_transaction, batch, batch_num, now)?;
|
||||
|
||||
// We need to write at the end of the loop so the prefix iterator doesn't pick up the same values again
|
||||
// in the next iteration. Another unfortunate effect of having to re-initialize the iterator.
|
||||
@@ -936,14 +891,14 @@ fn load_all_at_finalized_height(
|
||||
// Load all candidates that were included at this height.
|
||||
loop {
|
||||
match peek_num!(iter) {
|
||||
None => break, // end of iterator.
|
||||
None => break, // end of iterator.
|
||||
Some(n) if n != block_number => break, // end of batch.
|
||||
_ => {}
|
||||
_ => {},
|
||||
}
|
||||
|
||||
let (k, _v) = iter.next().expect("`peek` used to check non-empty; qed");
|
||||
let (_, block_hash, candidate_hash) = decode_unfinalized_key(&k[..])
|
||||
.expect("`peek_num` checks validity of key; qed");
|
||||
let (_, block_hash, candidate_hash) =
|
||||
decode_unfinalized_key(&k[..]).expect("`peek_num` checks validity of key; qed");
|
||||
|
||||
if block_hash == finalized_hash {
|
||||
candidates.insert(candidate_hash, true);
|
||||
@@ -971,8 +926,8 @@ fn update_blocks_at_finalized_height(
|
||||
candidate_hash,
|
||||
);
|
||||
|
||||
continue;
|
||||
}
|
||||
continue
|
||||
},
|
||||
Some(c) => c,
|
||||
};
|
||||
|
||||
@@ -985,7 +940,7 @@ fn update_blocks_at_finalized_height(
|
||||
// iterating over the candidate here indicates that `State` should
|
||||
// be `Unfinalized`.
|
||||
delete_pruning_key(db_transaction, &subsystem.config, at, &candidate_hash);
|
||||
}
|
||||
},
|
||||
State::Unfinalized(_, blocks) => {
|
||||
for (block_num, block_hash) in blocks.iter().cloned() {
|
||||
// this exact height is all getting cleared out anyway.
|
||||
@@ -999,7 +954,7 @@ fn update_blocks_at_finalized_height(
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
},
|
||||
}
|
||||
|
||||
meta.state = State::Finalized(now.into());
|
||||
@@ -1014,7 +969,7 @@ fn update_blocks_at_finalized_height(
|
||||
);
|
||||
} else {
|
||||
meta.state = match meta.state {
|
||||
State::Finalized(_) => continue, // sanity.
|
||||
State::Finalized(_) => continue, // sanity.
|
||||
State::Unavailable(_) => continue, // sanity.
|
||||
State::Unfinalized(at, mut blocks) => {
|
||||
// Clear out everything at this height.
|
||||
@@ -1035,7 +990,7 @@ fn update_blocks_at_finalized_height(
|
||||
} else {
|
||||
State::Unfinalized(at, blocks)
|
||||
}
|
||||
}
|
||||
},
|
||||
};
|
||||
|
||||
// Update the meta entry.
|
||||
@@ -1053,20 +1008,22 @@ fn process_message(
|
||||
match msg {
|
||||
AvailabilityStoreMessage::QueryAvailableData(candidate, tx) => {
|
||||
let _ = tx.send(load_available_data(&subsystem.db, &subsystem.config, &candidate)?);
|
||||
}
|
||||
},
|
||||
AvailabilityStoreMessage::QueryDataAvailability(candidate, tx) => {
|
||||
let a = load_meta(&subsystem.db, &subsystem.config, &candidate)?.map_or(false, |m| m.data_available);
|
||||
let a = load_meta(&subsystem.db, &subsystem.config, &candidate)?
|
||||
.map_or(false, |m| m.data_available);
|
||||
let _ = tx.send(a);
|
||||
}
|
||||
},
|
||||
AvailabilityStoreMessage::QueryChunk(candidate, validator_index, tx) => {
|
||||
let _timer = subsystem.metrics.time_get_chunk();
|
||||
let _ = tx.send(load_chunk(&subsystem.db, &subsystem.config, &candidate, validator_index)?);
|
||||
}
|
||||
let _ =
|
||||
tx.send(load_chunk(&subsystem.db, &subsystem.config, &candidate, validator_index)?);
|
||||
},
|
||||
AvailabilityStoreMessage::QueryAllChunks(candidate, tx) => {
|
||||
match load_meta(&subsystem.db, &subsystem.config, &candidate)? {
|
||||
None => {
|
||||
let _ = tx.send(Vec::new());
|
||||
}
|
||||
},
|
||||
Some(meta) => {
|
||||
let mut chunks = Vec::new();
|
||||
|
||||
@@ -1086,64 +1043,61 @@ fn process_message(
|
||||
index,
|
||||
"No chunk found for set bit in meta"
|
||||
);
|
||||
}
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
let _ = tx.send(chunks);
|
||||
}
|
||||
},
|
||||
}
|
||||
}
|
||||
},
|
||||
AvailabilityStoreMessage::QueryChunkAvailability(candidate, validator_index, tx) => {
|
||||
let a = load_meta(&subsystem.db, &subsystem.config, &candidate)?
|
||||
.map_or(false, |m|
|
||||
*m.chunks_stored.get(validator_index.0 as usize).as_deref().unwrap_or(&false)
|
||||
);
|
||||
let a = load_meta(&subsystem.db, &subsystem.config, &candidate)?.map_or(false, |m| {
|
||||
*m.chunks_stored.get(validator_index.0 as usize).as_deref().unwrap_or(&false)
|
||||
});
|
||||
let _ = tx.send(a);
|
||||
}
|
||||
AvailabilityStoreMessage::StoreChunk {
|
||||
candidate_hash,
|
||||
chunk,
|
||||
tx,
|
||||
} => {
|
||||
},
|
||||
AvailabilityStoreMessage::StoreChunk { candidate_hash, chunk, tx } => {
|
||||
subsystem.metrics.on_chunks_received(1);
|
||||
let _timer = subsystem.metrics.time_store_chunk();
|
||||
|
||||
match store_chunk(&subsystem.db, &subsystem.config, candidate_hash, chunk) {
|
||||
Ok(true) => {
|
||||
let _ = tx.send(Ok(()));
|
||||
}
|
||||
},
|
||||
Ok(false) => {
|
||||
let _ = tx.send(Err(()));
|
||||
}
|
||||
},
|
||||
Err(e) => {
|
||||
let _ = tx.send(Err(()));
|
||||
return Err(e)
|
||||
}
|
||||
},
|
||||
}
|
||||
}
|
||||
AvailabilityStoreMessage::StoreAvailableData(candidate, _our_index, n_validators, available_data, tx) => {
|
||||
},
|
||||
AvailabilityStoreMessage::StoreAvailableData(
|
||||
candidate,
|
||||
_our_index,
|
||||
n_validators,
|
||||
available_data,
|
||||
tx,
|
||||
) => {
|
||||
subsystem.metrics.on_chunks_received(n_validators as _);
|
||||
|
||||
let _timer = subsystem.metrics.time_store_available_data();
|
||||
|
||||
let res = store_available_data(
|
||||
&subsystem,
|
||||
candidate,
|
||||
n_validators as _,
|
||||
available_data,
|
||||
);
|
||||
let res =
|
||||
store_available_data(&subsystem, candidate, n_validators as _, available_data);
|
||||
|
||||
match res {
|
||||
Ok(()) => {
|
||||
let _ = tx.send(Ok(()));
|
||||
}
|
||||
},
|
||||
Err(e) => {
|
||||
let _ = tx.send(Err(()));
|
||||
return Err(e)
|
||||
}
|
||||
},
|
||||
}
|
||||
}
|
||||
},
|
||||
}
|
||||
|
||||
Ok(())
|
||||
@@ -1170,7 +1124,7 @@ fn store_chunk(
|
||||
|
||||
write_chunk(&mut tx, config, &candidate_hash, chunk.index, &chunk);
|
||||
write_meta(&mut tx, config, &candidate_hash, &meta);
|
||||
}
|
||||
},
|
||||
None => return Ok(false), // out of bounds.
|
||||
}
|
||||
|
||||
@@ -1197,7 +1151,7 @@ fn store_available_data(
|
||||
let mut meta = match load_meta(&subsystem.db, &subsystem.config, &candidate_hash)? {
|
||||
Some(m) => {
|
||||
if m.data_available {
|
||||
return Ok(()); // already stored.
|
||||
return Ok(()) // already stored.
|
||||
}
|
||||
|
||||
m
|
||||
@@ -1214,20 +1168,19 @@ fn store_available_data(
|
||||
data_available: false,
|
||||
chunks_stored: BitVec::new(),
|
||||
}
|
||||
}
|
||||
},
|
||||
};
|
||||
|
||||
let chunks = erasure::obtain_chunks_v1(n_validators, &available_data)?;
|
||||
let branches = erasure::branches(chunks.as_ref());
|
||||
|
||||
let erasure_chunks = chunks.iter()
|
||||
.zip(branches.map(|(proof, _)| proof))
|
||||
.enumerate()
|
||||
.map(|(index, (chunk, proof))| ErasureChunk {
|
||||
let erasure_chunks = chunks.iter().zip(branches.map(|(proof, _)| proof)).enumerate().map(
|
||||
|(index, (chunk, proof))| ErasureChunk {
|
||||
chunk: chunk.clone(),
|
||||
proof,
|
||||
index: ValidatorIndex(index as u32),
|
||||
});
|
||||
},
|
||||
);
|
||||
|
||||
for chunk in erasure_chunks {
|
||||
write_chunk(&mut tx, &subsystem.config, &candidate_hash, chunk.index, &chunk);
|
||||
@@ -1236,16 +1189,12 @@ fn store_available_data(
|
||||
meta.data_available = true;
|
||||
meta.chunks_stored = bitvec::bitvec![BitOrderLsb0, u8; 1; n_validators];
|
||||
|
||||
write_meta(&mut tx, &subsystem.config, &candidate_hash, &meta);
|
||||
write_meta(&mut tx, &subsystem.config, &candidate_hash, &meta);
|
||||
write_available_data(&mut tx, &subsystem.config, &candidate_hash, &available_data);
|
||||
|
||||
subsystem.db.write(tx)?;
|
||||
|
||||
tracing::debug!(
|
||||
target: LOG_TARGET,
|
||||
?candidate_hash,
|
||||
"Stored data and chunks",
|
||||
);
|
||||
tracing::debug!(target: LOG_TARGET, ?candidate_hash, "Stored data and chunks",);
|
||||
|
||||
Ok(())
|
||||
}
|
||||
@@ -1255,7 +1204,8 @@ fn prune_all(db: &Arc<dyn KeyValueDB>, config: &Config, clock: &dyn Clock) -> Re
|
||||
let (range_start, range_end) = pruning_range(now);
|
||||
|
||||
let mut tx = DBTransaction::new();
|
||||
let iter = db.iter_with_prefix(config.col_meta, &range_start[..])
|
||||
let iter = db
|
||||
.iter_with_prefix(config.col_meta, &range_start[..])
|
||||
.take_while(|(k, _)| &k[..] < &range_end[..]);
|
||||
|
||||
for (k, _v) in iter {
|
||||
@@ -1334,7 +1284,9 @@ impl Metrics {
|
||||
}
|
||||
|
||||
/// Provide a timer for `process_block_finalized` which observes on drop.
|
||||
fn time_process_block_finalized(&self) -> Option<metrics::prometheus::prometheus::HistogramTimer> {
|
||||
fn time_process_block_finalized(
|
||||
&self,
|
||||
) -> Option<metrics::prometheus::prometheus::HistogramTimer> {
|
||||
self.0.as_ref().map(|metrics| metrics.process_block_finalized.start_timer())
|
||||
}
|
||||
|
||||
@@ -1375,66 +1327,52 @@ impl metrics::Metrics for Metrics {
|
||||
registry,
|
||||
)?,
|
||||
pruning: prometheus::register(
|
||||
prometheus::Histogram::with_opts(
|
||||
prometheus::HistogramOpts::new(
|
||||
"parachain_av_store_pruning",
|
||||
"Time spent within `av_store::prune_all`",
|
||||
)
|
||||
)?,
|
||||
prometheus::Histogram::with_opts(prometheus::HistogramOpts::new(
|
||||
"parachain_av_store_pruning",
|
||||
"Time spent within `av_store::prune_all`",
|
||||
))?,
|
||||
registry,
|
||||
)?,
|
||||
process_block_finalized: prometheus::register(
|
||||
prometheus::Histogram::with_opts(
|
||||
prometheus::HistogramOpts::new(
|
||||
"parachain_av_store_process_block_finalized",
|
||||
"Time spent within `av_store::process_block_finalized`",
|
||||
)
|
||||
)?,
|
||||
prometheus::Histogram::with_opts(prometheus::HistogramOpts::new(
|
||||
"parachain_av_store_process_block_finalized",
|
||||
"Time spent within `av_store::process_block_finalized`",
|
||||
))?,
|
||||
registry,
|
||||
)?,
|
||||
block_activated: prometheus::register(
|
||||
prometheus::Histogram::with_opts(
|
||||
prometheus::HistogramOpts::new(
|
||||
"parachain_av_store_block_activated",
|
||||
"Time spent within `av_store::process_block_activated`",
|
||||
)
|
||||
)?,
|
||||
prometheus::Histogram::with_opts(prometheus::HistogramOpts::new(
|
||||
"parachain_av_store_block_activated",
|
||||
"Time spent within `av_store::process_block_activated`",
|
||||
))?,
|
||||
registry,
|
||||
)?,
|
||||
process_message: prometheus::register(
|
||||
prometheus::Histogram::with_opts(
|
||||
prometheus::HistogramOpts::new(
|
||||
"parachain_av_store_process_message",
|
||||
"Time spent within `av_store::process_message`",
|
||||
)
|
||||
)?,
|
||||
prometheus::Histogram::with_opts(prometheus::HistogramOpts::new(
|
||||
"parachain_av_store_process_message",
|
||||
"Time spent within `av_store::process_message`",
|
||||
))?,
|
||||
registry,
|
||||
)?,
|
||||
store_available_data: prometheus::register(
|
||||
prometheus::Histogram::with_opts(
|
||||
prometheus::HistogramOpts::new(
|
||||
"parachain_av_store_store_available_data",
|
||||
"Time spent within `av_store::store_available_data`",
|
||||
)
|
||||
)?,
|
||||
prometheus::Histogram::with_opts(prometheus::HistogramOpts::new(
|
||||
"parachain_av_store_store_available_data",
|
||||
"Time spent within `av_store::store_available_data`",
|
||||
))?,
|
||||
registry,
|
||||
)?,
|
||||
store_chunk: prometheus::register(
|
||||
prometheus::Histogram::with_opts(
|
||||
prometheus::HistogramOpts::new(
|
||||
"parachain_av_store_store_chunk",
|
||||
"Time spent within `av_store::store_chunk`",
|
||||
)
|
||||
)?,
|
||||
prometheus::Histogram::with_opts(prometheus::HistogramOpts::new(
|
||||
"parachain_av_store_store_chunk",
|
||||
"Time spent within `av_store::store_chunk`",
|
||||
))?,
|
||||
registry,
|
||||
)?,
|
||||
get_chunk: prometheus::register(
|
||||
prometheus::Histogram::with_opts(
|
||||
prometheus::HistogramOpts::new(
|
||||
"parachain_av_store_get_chunk",
|
||||
"Time spent fetching requested chunks.`",
|
||||
)
|
||||
)?,
|
||||
prometheus::Histogram::with_opts(prometheus::HistogramOpts::new(
|
||||
"parachain_av_store_get_chunk",
|
||||
"Time spent fetching requested chunks.`",
|
||||
))?,
|
||||
registry,
|
||||
)?,
|
||||
};
|
||||
|
||||
@@ -17,27 +17,23 @@
|
||||
use super::*;
|
||||
|
||||
use assert_matches::assert_matches;
|
||||
use futures::{
|
||||
future,
|
||||
channel::oneshot,
|
||||
executor,
|
||||
Future,
|
||||
};
|
||||
use futures::{channel::oneshot, executor, future, Future};
|
||||
|
||||
use polkadot_primitives::v1::{
|
||||
CandidateDescriptor, CandidateReceipt, HeadData,
|
||||
PersistedValidationData, Id as ParaId, CandidateHash, Header, ValidatorId,
|
||||
CoreIndex, GroupIndex,
|
||||
};
|
||||
use polkadot_node_primitives::{AvailableData, BlockData, PoV};
|
||||
use polkadot_node_subsystem_util::TimeoutExt;
|
||||
use polkadot_subsystem::{
|
||||
ActiveLeavesUpdate, errors::RuntimeApiError, jaeger, ActivatedLeaf,
|
||||
LeafStatus, messages::{AllMessages, RuntimeApiMessage, RuntimeApiRequest},
|
||||
};
|
||||
use polkadot_node_subsystem_test_helpers as test_helpers;
|
||||
use sp_keyring::Sr25519Keyring;
|
||||
use parking_lot::Mutex;
|
||||
use polkadot_node_primitives::{AvailableData, BlockData, PoV};
|
||||
use polkadot_node_subsystem_test_helpers as test_helpers;
|
||||
use polkadot_node_subsystem_util::TimeoutExt;
|
||||
use polkadot_primitives::v1::{
|
||||
CandidateDescriptor, CandidateHash, CandidateReceipt, CoreIndex, GroupIndex, HeadData, Header,
|
||||
Id as ParaId, PersistedValidationData, ValidatorId,
|
||||
};
|
||||
use polkadot_subsystem::{
|
||||
errors::RuntimeApiError,
|
||||
jaeger,
|
||||
messages::{AllMessages, RuntimeApiMessage, RuntimeApiRequest},
|
||||
ActivatedLeaf, ActiveLeavesUpdate, LeafStatus,
|
||||
};
|
||||
use sp_keyring::Sr25519Keyring;
|
||||
|
||||
mod columns {
|
||||
pub const DATA: u32 = 0;
|
||||
@@ -45,10 +41,7 @@ mod columns {
|
||||
pub const NUM_COLUMNS: u32 = 2;
|
||||
}
|
||||
|
||||
const TEST_CONFIG: Config = Config {
|
||||
col_data: columns::DATA,
|
||||
col_meta: columns::META,
|
||||
};
|
||||
const TEST_CONFIG: Config = Config { col_data: columns::DATA, col_meta: columns::META };
|
||||
|
||||
type VirtualOverseer = test_helpers::TestSubsystemContextHandle<AvailabilityStoreMessage>;
|
||||
|
||||
@@ -95,7 +88,6 @@ impl Clock for TestClock {
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
#[derive(Clone)]
|
||||
struct TestState {
|
||||
persisted_validation_data: PersistedValidationData,
|
||||
@@ -126,34 +118,21 @@ impl Default for TestState {
|
||||
pruning_interval: Duration::from_millis(250),
|
||||
};
|
||||
|
||||
let clock = TestClock {
|
||||
inner: Arc::new(Mutex::new(Duration::from_secs(0))),
|
||||
};
|
||||
let clock = TestClock { inner: Arc::new(Mutex::new(Duration::from_secs(0))) };
|
||||
|
||||
Self {
|
||||
persisted_validation_data,
|
||||
pruning_config,
|
||||
clock,
|
||||
}
|
||||
Self { persisted_validation_data, pruning_config, clock }
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
fn test_harness<T: Future<Output=VirtualOverseer>>(
|
||||
fn test_harness<T: Future<Output = VirtualOverseer>>(
|
||||
state: TestState,
|
||||
store: Arc<dyn KeyValueDB>,
|
||||
test: impl FnOnce(VirtualOverseer) -> T,
|
||||
) {
|
||||
let _ = env_logger::builder()
|
||||
.is_test(true)
|
||||
.filter(
|
||||
Some("polkadot_node_core_av_store"),
|
||||
log::LevelFilter::Trace,
|
||||
)
|
||||
.filter(
|
||||
Some(LOG_TARGET),
|
||||
log::LevelFilter::Trace,
|
||||
)
|
||||
.filter(Some("polkadot_node_core_av_store"), log::LevelFilter::Trace)
|
||||
.filter(Some(LOG_TARGET), log::LevelFilter::Trace)
|
||||
.try_init();
|
||||
|
||||
let pool = sp_core::testing::TaskExecutor::new();
|
||||
@@ -174,21 +153,18 @@ fn test_harness<T: Future<Output=VirtualOverseer>>(
|
||||
futures::pin_mut!(test_fut);
|
||||
futures::pin_mut!(subsystem);
|
||||
|
||||
executor::block_on(future::join(async move {
|
||||
let mut overseer = test_fut.await;
|
||||
overseer_signal(
|
||||
&mut overseer,
|
||||
OverseerSignal::Conclude,
|
||||
).await;
|
||||
}, subsystem));
|
||||
executor::block_on(future::join(
|
||||
async move {
|
||||
let mut overseer = test_fut.await;
|
||||
overseer_signal(&mut overseer, OverseerSignal::Conclude).await;
|
||||
},
|
||||
subsystem,
|
||||
));
|
||||
}
|
||||
|
||||
const TIMEOUT: Duration = Duration::from_millis(100);
|
||||
|
||||
async fn overseer_send(
|
||||
overseer: &mut VirtualOverseer,
|
||||
msg: AvailabilityStoreMessage,
|
||||
) {
|
||||
async fn overseer_send(overseer: &mut VirtualOverseer, msg: AvailabilityStoreMessage) {
|
||||
tracing::trace!(meg = ?msg, "sending message");
|
||||
overseer
|
||||
.send(FromOverseer::Communication { msg })
|
||||
@@ -197,9 +173,7 @@ async fn overseer_send(
|
||||
.expect(&format!("{:?} is more than enough for sending messages.", TIMEOUT));
|
||||
}
|
||||
|
||||
async fn overseer_recv(
|
||||
overseer: &mut VirtualOverseer,
|
||||
) -> AllMessages {
|
||||
async fn overseer_recv(overseer: &mut VirtualOverseer) -> AllMessages {
|
||||
let msg = overseer_recv_with_timeout(overseer, TIMEOUT)
|
||||
.await
|
||||
.expect(&format!("{:?} is more than enough to receive messages", TIMEOUT));
|
||||
@@ -214,16 +188,10 @@ async fn overseer_recv_with_timeout(
|
||||
timeout: Duration,
|
||||
) -> Option<AllMessages> {
|
||||
tracing::trace!("waiting for message...");
|
||||
overseer
|
||||
.recv()
|
||||
.timeout(timeout)
|
||||
.await
|
||||
overseer.recv().timeout(timeout).await
|
||||
}
|
||||
|
||||
async fn overseer_signal(
|
||||
overseer: &mut VirtualOverseer,
|
||||
signal: OverseerSignal,
|
||||
) {
|
||||
async fn overseer_signal(overseer: &mut VirtualOverseer, signal: OverseerSignal) {
|
||||
overseer
|
||||
.send(FromOverseer::Signal(signal))
|
||||
.timeout(TIMEOUT)
|
||||
@@ -261,7 +229,8 @@ fn runtime_api_error_does_not_stop_the_subsystem() {
|
||||
status: LeafStatus::Fresh,
|
||||
span: Arc::new(jaeger::Span::Disabled),
|
||||
})),
|
||||
).await;
|
||||
)
|
||||
.await;
|
||||
|
||||
let header = Header {
|
||||
parent_hash: Hash::zero(),
|
||||
@@ -298,11 +267,7 @@ fn runtime_api_error_does_not_stop_the_subsystem() {
|
||||
let (tx, rx) = oneshot::channel();
|
||||
let candidate_hash = CandidateHash(Hash::repeat_byte(33));
|
||||
let validator_index = ValidatorIndex(5);
|
||||
let query_chunk = AvailabilityStoreMessage::QueryChunk(
|
||||
candidate_hash,
|
||||
validator_index,
|
||||
tx,
|
||||
);
|
||||
let query_chunk = AvailabilityStoreMessage::QueryChunk(candidate_hash, validator_index, tx);
|
||||
|
||||
overseer_send(&mut virtual_overseer, query_chunk.into()).await;
|
||||
|
||||
@@ -328,30 +293,28 @@ fn store_chunk_works() {
|
||||
// Ensure an entry already exists. In reality this would come from watching
|
||||
// chain events.
|
||||
with_tx(&store, |tx| {
|
||||
super::write_meta(tx, &TEST_CONFIG, &candidate_hash, &CandidateMeta {
|
||||
data_available: false,
|
||||
chunks_stored: bitvec::bitvec![BitOrderLsb0, u8; 0; n_validators],
|
||||
state: State::Unavailable(BETimestamp(0)),
|
||||
});
|
||||
super::write_meta(
|
||||
tx,
|
||||
&TEST_CONFIG,
|
||||
&candidate_hash,
|
||||
&CandidateMeta {
|
||||
data_available: false,
|
||||
chunks_stored: bitvec::bitvec![BitOrderLsb0, u8; 0; n_validators],
|
||||
state: State::Unavailable(BETimestamp(0)),
|
||||
},
|
||||
);
|
||||
});
|
||||
|
||||
let (tx, rx) = oneshot::channel();
|
||||
|
||||
let chunk_msg = AvailabilityStoreMessage::StoreChunk {
|
||||
candidate_hash,
|
||||
chunk: chunk.clone(),
|
||||
tx,
|
||||
};
|
||||
let chunk_msg =
|
||||
AvailabilityStoreMessage::StoreChunk { candidate_hash, chunk: chunk.clone(), tx };
|
||||
|
||||
overseer_send(&mut virtual_overseer, chunk_msg.into()).await;
|
||||
assert_eq!(rx.await.unwrap(), Ok(()));
|
||||
|
||||
let (tx, rx) = oneshot::channel();
|
||||
let query_chunk = AvailabilityStoreMessage::QueryChunk(
|
||||
candidate_hash,
|
||||
validator_index,
|
||||
tx,
|
||||
);
|
||||
let query_chunk = AvailabilityStoreMessage::QueryChunk(candidate_hash, validator_index, tx);
|
||||
|
||||
overseer_send(&mut virtual_overseer, query_chunk.into()).await;
|
||||
|
||||
@@ -360,7 +323,6 @@ fn store_chunk_works() {
|
||||
});
|
||||
}
|
||||
|
||||
|
||||
#[test]
|
||||
fn store_chunk_does_nothing_if_no_entry_already() {
|
||||
let store = Arc::new(kvdb_memorydb::create(columns::NUM_COLUMNS));
|
||||
@@ -376,21 +338,14 @@ fn store_chunk_does_nothing_if_no_entry_already() {
|
||||
|
||||
let (tx, rx) = oneshot::channel();
|
||||
|
||||
let chunk_msg = AvailabilityStoreMessage::StoreChunk {
|
||||
candidate_hash,
|
||||
chunk: chunk.clone(),
|
||||
tx,
|
||||
};
|
||||
let chunk_msg =
|
||||
AvailabilityStoreMessage::StoreChunk { candidate_hash, chunk: chunk.clone(), tx };
|
||||
|
||||
overseer_send(&mut virtual_overseer, chunk_msg.into()).await;
|
||||
assert_eq!(rx.await.unwrap(), Err(()));
|
||||
|
||||
let (tx, rx) = oneshot::channel();
|
||||
let query_chunk = AvailabilityStoreMessage::QueryChunk(
|
||||
candidate_hash,
|
||||
validator_index,
|
||||
tx,
|
||||
);
|
||||
let query_chunk = AvailabilityStoreMessage::QueryChunk(candidate_hash, validator_index, tx);
|
||||
|
||||
overseer_send(&mut virtual_overseer, query_chunk.into()).await;
|
||||
|
||||
@@ -410,23 +365,25 @@ fn query_chunk_checks_meta() {
|
||||
// Ensure an entry already exists. In reality this would come from watching
|
||||
// chain events.
|
||||
with_tx(&store, |tx| {
|
||||
super::write_meta(tx, &TEST_CONFIG, &candidate_hash, &CandidateMeta {
|
||||
data_available: false,
|
||||
chunks_stored: {
|
||||
let mut v = bitvec::bitvec![BitOrderLsb0, u8; 0; n_validators];
|
||||
v.set(validator_index.0 as usize, true);
|
||||
v
|
||||
super::write_meta(
|
||||
tx,
|
||||
&TEST_CONFIG,
|
||||
&candidate_hash,
|
||||
&CandidateMeta {
|
||||
data_available: false,
|
||||
chunks_stored: {
|
||||
let mut v = bitvec::bitvec![BitOrderLsb0, u8; 0; n_validators];
|
||||
v.set(validator_index.0 as usize, true);
|
||||
v
|
||||
},
|
||||
state: State::Unavailable(BETimestamp(0)),
|
||||
},
|
||||
state: State::Unavailable(BETimestamp(0)),
|
||||
});
|
||||
);
|
||||
});
|
||||
|
||||
let (tx, rx) = oneshot::channel();
|
||||
let query_chunk = AvailabilityStoreMessage::QueryChunkAvailability(
|
||||
candidate_hash,
|
||||
validator_index,
|
||||
tx,
|
||||
);
|
||||
let query_chunk =
|
||||
AvailabilityStoreMessage::QueryChunkAvailability(candidate_hash, validator_index, tx);
|
||||
|
||||
overseer_send(&mut virtual_overseer, query_chunk.into()).await;
|
||||
assert!(rx.await.unwrap());
|
||||
@@ -453,16 +410,13 @@ fn store_block_works() {
|
||||
let validator_index = ValidatorIndex(5);
|
||||
let n_validators = 10;
|
||||
|
||||
let pov = PoV {
|
||||
block_data: BlockData(vec![4, 5, 6]),
|
||||
};
|
||||
let pov = PoV { block_data: BlockData(vec![4, 5, 6]) };
|
||||
|
||||
let available_data = AvailableData {
|
||||
pov: Arc::new(pov),
|
||||
validation_data: test_state.persisted_validation_data.clone(),
|
||||
};
|
||||
|
||||
|
||||
let (tx, rx) = oneshot::channel();
|
||||
let block_msg = AvailabilityStoreMessage::StoreAvailableData(
|
||||
candidate_hash,
|
||||
@@ -472,24 +426,23 @@ fn store_block_works() {
|
||||
tx,
|
||||
);
|
||||
|
||||
virtual_overseer.send(FromOverseer::Communication{ msg: block_msg }).await;
|
||||
virtual_overseer.send(FromOverseer::Communication { msg: block_msg }).await;
|
||||
assert_eq!(rx.await.unwrap(), Ok(()));
|
||||
|
||||
let pov = query_available_data(&mut virtual_overseer, candidate_hash).await.unwrap();
|
||||
assert_eq!(pov, available_data);
|
||||
|
||||
let chunk = query_chunk(&mut virtual_overseer, candidate_hash, validator_index).await.unwrap();
|
||||
let chunk = query_chunk(&mut virtual_overseer, candidate_hash, validator_index)
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
let chunks = erasure::obtain_chunks_v1(10, &available_data).unwrap();
|
||||
|
||||
let mut branches = erasure::branches(chunks.as_ref());
|
||||
|
||||
let branch = branches.nth(5).unwrap();
|
||||
let expected_chunk = ErasureChunk {
|
||||
chunk: branch.1.to_vec(),
|
||||
index: ValidatorIndex(5),
|
||||
proof: branch.0,
|
||||
};
|
||||
let expected_chunk =
|
||||
ErasureChunk { chunk: branch.1.to_vec(), index: ValidatorIndex(5), proof: branch.0 };
|
||||
|
||||
assert_eq!(chunk, expected_chunk);
|
||||
virtual_overseer
|
||||
@@ -505,16 +458,15 @@ fn store_pov_and_query_chunk_works() {
|
||||
let candidate_hash = CandidateHash(Hash::repeat_byte(1));
|
||||
let n_validators = 10;
|
||||
|
||||
let pov = PoV {
|
||||
block_data: BlockData(vec![4, 5, 6]),
|
||||
};
|
||||
let pov = PoV { block_data: BlockData(vec![4, 5, 6]) };
|
||||
|
||||
let available_data = AvailableData {
|
||||
pov: Arc::new(pov),
|
||||
validation_data: test_state.persisted_validation_data.clone(),
|
||||
};
|
||||
|
||||
let chunks_expected = erasure::obtain_chunks_v1(n_validators as _, &available_data).unwrap();
|
||||
let chunks_expected =
|
||||
erasure::obtain_chunks_v1(n_validators as _, &available_data).unwrap();
|
||||
|
||||
let (tx, rx) = oneshot::channel();
|
||||
let block_msg = AvailabilityStoreMessage::StoreAvailableData(
|
||||
@@ -525,12 +477,14 @@ fn store_pov_and_query_chunk_works() {
|
||||
tx,
|
||||
);
|
||||
|
||||
virtual_overseer.send(FromOverseer::Communication{ msg: block_msg }).await;
|
||||
virtual_overseer.send(FromOverseer::Communication { msg: block_msg }).await;
|
||||
|
||||
assert_eq!(rx.await.unwrap(), Ok(()));
|
||||
|
||||
for i in 0..n_validators {
|
||||
let chunk = query_chunk(&mut virtual_overseer, candidate_hash, ValidatorIndex(i as _)).await.unwrap();
|
||||
let chunk = query_chunk(&mut virtual_overseer, candidate_hash, ValidatorIndex(i as _))
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
assert_eq!(chunk.chunk, chunks_expected[i as usize]);
|
||||
}
|
||||
@@ -553,9 +507,7 @@ fn query_all_chunks_works() {
|
||||
|
||||
let n_validators = 10;
|
||||
|
||||
let pov = PoV {
|
||||
block_data: BlockData(vec![4, 5, 6]),
|
||||
};
|
||||
let pov = PoV { block_data: BlockData(vec![4, 5, 6]) };
|
||||
|
||||
let available_data = AvailableData {
|
||||
pov: Arc::new(pov),
|
||||
@@ -578,11 +530,16 @@ fn query_all_chunks_works() {
|
||||
|
||||
{
|
||||
with_tx(&store, |tx| {
|
||||
super::write_meta(tx, &TEST_CONFIG, &candidate_hash_2, &CandidateMeta {
|
||||
data_available: false,
|
||||
chunks_stored: bitvec::bitvec![BitOrderLsb0, u8; 0; n_validators as _],
|
||||
state: State::Unavailable(BETimestamp(0)),
|
||||
});
|
||||
super::write_meta(
|
||||
tx,
|
||||
&TEST_CONFIG,
|
||||
&candidate_hash_2,
|
||||
&CandidateMeta {
|
||||
data_available: false,
|
||||
chunks_stored: bitvec::bitvec![BitOrderLsb0, u8; 0; n_validators as _],
|
||||
state: State::Unavailable(BETimestamp(0)),
|
||||
},
|
||||
);
|
||||
});
|
||||
|
||||
let chunk = ErasureChunk {
|
||||
@@ -598,7 +555,9 @@ fn query_all_chunks_works() {
|
||||
tx,
|
||||
};
|
||||
|
||||
virtual_overseer.send(FromOverseer::Communication { msg: store_chunk_msg }).await;
|
||||
virtual_overseer
|
||||
.send(FromOverseer::Communication { msg: store_chunk_msg })
|
||||
.await;
|
||||
assert_eq!(rx.await.unwrap(), Ok(()));
|
||||
}
|
||||
|
||||
@@ -638,9 +597,7 @@ fn stored_but_not_included_data_is_pruned() {
|
||||
let candidate_hash = CandidateHash(Hash::repeat_byte(1));
|
||||
let n_validators = 10;
|
||||
|
||||
let pov = PoV {
|
||||
block_data: BlockData(vec![4, 5, 6]),
|
||||
};
|
||||
let pov = PoV { block_data: BlockData(vec![4, 5, 6]) };
|
||||
|
||||
let available_data = AvailableData {
|
||||
pov: Arc::new(pov),
|
||||
@@ -656,7 +613,7 @@ fn stored_but_not_included_data_is_pruned() {
|
||||
tx,
|
||||
);
|
||||
|
||||
virtual_overseer.send(FromOverseer::Communication{ msg: block_msg }).await;
|
||||
virtual_overseer.send(FromOverseer::Communication { msg: block_msg }).await;
|
||||
|
||||
rx.await.unwrap().unwrap();
|
||||
|
||||
@@ -684,16 +641,11 @@ fn stored_data_kept_until_finalized() {
|
||||
test_harness(test_state.clone(), store.clone(), |mut virtual_overseer| async move {
|
||||
let n_validators = 10;
|
||||
|
||||
let pov = PoV {
|
||||
block_data: BlockData(vec![4, 5, 6]),
|
||||
};
|
||||
let pov = PoV { block_data: BlockData(vec![4, 5, 6]) };
|
||||
|
||||
let pov_hash = pov.hash();
|
||||
|
||||
let candidate = TestCandidateBuilder {
|
||||
pov_hash,
|
||||
..Default::default()
|
||||
}.build();
|
||||
let candidate = TestCandidateBuilder { pov_hash, ..Default::default() }.build();
|
||||
|
||||
let candidate_hash = candidate.hash();
|
||||
|
||||
@@ -714,7 +666,7 @@ fn stored_data_kept_until_finalized() {
|
||||
tx,
|
||||
);
|
||||
|
||||
virtual_overseer.send(FromOverseer::Communication{ msg: block_msg }).await;
|
||||
virtual_overseer.send(FromOverseer::Communication { msg: block_msg }).await;
|
||||
|
||||
rx.await.unwrap().unwrap();
|
||||
|
||||
@@ -730,7 +682,8 @@ fn stored_data_kept_until_finalized() {
|
||||
block_number,
|
||||
vec![candidate_included(candidate)],
|
||||
(0..n_validators).map(|_| Sr25519Keyring::Alice.public().into()).collect(),
|
||||
).await;
|
||||
)
|
||||
.await;
|
||||
|
||||
// Wait until unavailable data would definitely be pruned.
|
||||
test_state.clock.inc(test_state.pruning_config.keep_unavailable_for * 10);
|
||||
@@ -742,14 +695,13 @@ fn stored_data_kept_until_finalized() {
|
||||
available_data,
|
||||
);
|
||||
|
||||
assert!(
|
||||
has_all_chunks(&mut virtual_overseer, candidate_hash, n_validators, true).await
|
||||
);
|
||||
assert!(has_all_chunks(&mut virtual_overseer, candidate_hash, n_validators, true).await);
|
||||
|
||||
overseer_signal(
|
||||
&mut virtual_overseer,
|
||||
OverseerSignal::BlockFinalized(new_leaf, block_number)
|
||||
).await;
|
||||
OverseerSignal::BlockFinalized(new_leaf, block_number),
|
||||
)
|
||||
.await;
|
||||
|
||||
// Wait until unavailable data would definitely be pruned.
|
||||
test_state.clock.inc(test_state.pruning_config.keep_finalized_for / 2);
|
||||
@@ -761,22 +713,16 @@ fn stored_data_kept_until_finalized() {
|
||||
available_data,
|
||||
);
|
||||
|
||||
assert!(
|
||||
has_all_chunks(&mut virtual_overseer, candidate_hash, n_validators, true).await
|
||||
);
|
||||
assert!(has_all_chunks(&mut virtual_overseer, candidate_hash, n_validators, true).await);
|
||||
|
||||
// Wait until it definitely should be gone.
|
||||
test_state.clock.inc(test_state.pruning_config.keep_finalized_for);
|
||||
test_state.wait_for_pruning().await;
|
||||
|
||||
// At this point data should be gone from the store.
|
||||
assert!(
|
||||
query_available_data(&mut virtual_overseer, candidate_hash).await.is_none(),
|
||||
);
|
||||
assert!(query_available_data(&mut virtual_overseer, candidate_hash).await.is_none(),);
|
||||
|
||||
assert!(
|
||||
has_all_chunks(&mut virtual_overseer, candidate_hash, n_validators, false).await
|
||||
);
|
||||
assert!(has_all_chunks(&mut virtual_overseer, candidate_hash, n_validators, false).await);
|
||||
virtual_overseer
|
||||
});
|
||||
}
|
||||
@@ -787,10 +733,8 @@ fn we_dont_miss_anything_if_import_notifications_are_missed() {
|
||||
let test_state = TestState::default();
|
||||
|
||||
test_harness(test_state.clone(), store.clone(), |mut virtual_overseer| async move {
|
||||
overseer_signal(
|
||||
&mut virtual_overseer,
|
||||
OverseerSignal::BlockFinalized(Hash::zero(), 1)
|
||||
).await;
|
||||
overseer_signal(&mut virtual_overseer, OverseerSignal::BlockFinalized(Hash::zero(), 1))
|
||||
.await;
|
||||
|
||||
let header = Header {
|
||||
parent_hash: Hash::repeat_byte(3),
|
||||
@@ -809,7 +753,8 @@ fn we_dont_miss_anything_if_import_notifications_are_missed() {
|
||||
status: LeafStatus::Fresh,
|
||||
span: Arc::new(jaeger::Span::Disabled),
|
||||
})),
|
||||
).await;
|
||||
)
|
||||
.await;
|
||||
|
||||
assert_matches!(
|
||||
overseer_recv(&mut virtual_overseer).await,
|
||||
@@ -915,33 +860,26 @@ fn forkfullness_works() {
|
||||
let n_validators = 10;
|
||||
let block_number_1 = 5;
|
||||
let block_number_2 = 5;
|
||||
let validators: Vec<_> = (0..n_validators).map(|_| Sr25519Keyring::Alice.public().into()).collect();
|
||||
let validators: Vec<_> =
|
||||
(0..n_validators).map(|_| Sr25519Keyring::Alice.public().into()).collect();
|
||||
let parent_1 = Hash::repeat_byte(3);
|
||||
let parent_2 = Hash::repeat_byte(4);
|
||||
|
||||
let pov_1 = PoV {
|
||||
block_data: BlockData(vec![1, 2, 3]),
|
||||
};
|
||||
let pov_1 = PoV { block_data: BlockData(vec![1, 2, 3]) };
|
||||
|
||||
let pov_1_hash = pov_1.hash();
|
||||
|
||||
let pov_2 = PoV {
|
||||
block_data: BlockData(vec![4, 5, 6]),
|
||||
};
|
||||
let pov_2 = PoV { block_data: BlockData(vec![4, 5, 6]) };
|
||||
|
||||
let pov_2_hash = pov_2.hash();
|
||||
|
||||
let candidate_1 = TestCandidateBuilder {
|
||||
pov_hash: pov_1_hash,
|
||||
..Default::default()
|
||||
}.build();
|
||||
let candidate_1 =
|
||||
TestCandidateBuilder { pov_hash: pov_1_hash, ..Default::default() }.build();
|
||||
|
||||
let candidate_1_hash = candidate_1.hash();
|
||||
|
||||
let candidate_2 = TestCandidateBuilder {
|
||||
pov_hash: pov_2_hash,
|
||||
..Default::default()
|
||||
}.build();
|
||||
let candidate_2 =
|
||||
TestCandidateBuilder { pov_hash: pov_2_hash, ..Default::default() }.build();
|
||||
|
||||
let candidate_2_hash = candidate_2.hash();
|
||||
|
||||
@@ -964,7 +902,7 @@ fn forkfullness_works() {
|
||||
tx,
|
||||
);
|
||||
|
||||
virtual_overseer.send(FromOverseer::Communication{ msg }).await;
|
||||
virtual_overseer.send(FromOverseer::Communication { msg }).await;
|
||||
|
||||
rx.await.unwrap().unwrap();
|
||||
|
||||
@@ -977,7 +915,7 @@ fn forkfullness_works() {
|
||||
tx,
|
||||
);
|
||||
|
||||
virtual_overseer.send(FromOverseer::Communication{ msg }).await;
|
||||
virtual_overseer.send(FromOverseer::Communication { msg }).await;
|
||||
|
||||
rx.await.unwrap().unwrap();
|
||||
|
||||
@@ -997,7 +935,8 @@ fn forkfullness_works() {
|
||||
block_number_1,
|
||||
vec![candidate_included(candidate_1)],
|
||||
validators.clone(),
|
||||
).await;
|
||||
)
|
||||
.await;
|
||||
|
||||
let _new_leaf_2 = import_leaf(
|
||||
&mut virtual_overseer,
|
||||
@@ -1005,12 +944,14 @@ fn forkfullness_works() {
|
||||
block_number_2,
|
||||
vec![candidate_included(candidate_2)],
|
||||
validators.clone(),
|
||||
).await;
|
||||
)
|
||||
.await;
|
||||
|
||||
overseer_signal(
|
||||
&mut virtual_overseer,
|
||||
OverseerSignal::BlockFinalized(new_leaf_1, block_number_1)
|
||||
).await;
|
||||
OverseerSignal::BlockFinalized(new_leaf_1, block_number_1),
|
||||
)
|
||||
.await;
|
||||
|
||||
// Data of both candidates should be still present in the DB.
|
||||
assert_eq!(
|
||||
@@ -1023,13 +964,9 @@ fn forkfullness_works() {
|
||||
available_data_2,
|
||||
);
|
||||
|
||||
assert!(
|
||||
has_all_chunks(&mut virtual_overseer, candidate_1_hash, n_validators, true).await,
|
||||
);
|
||||
assert!(has_all_chunks(&mut virtual_overseer, candidate_1_hash, n_validators, true).await,);
|
||||
|
||||
assert!(
|
||||
has_all_chunks(&mut virtual_overseer, candidate_2_hash, n_validators, true).await,
|
||||
);
|
||||
assert!(has_all_chunks(&mut virtual_overseer, candidate_2_hash, n_validators, true).await,);
|
||||
|
||||
// Candidate 2 should now be considered unavailable and will be pruned.
|
||||
test_state.clock.inc(test_state.pruning_config.keep_unavailable_for);
|
||||
@@ -1040,38 +977,24 @@ fn forkfullness_works() {
|
||||
available_data_1,
|
||||
);
|
||||
|
||||
assert!(
|
||||
query_available_data(&mut virtual_overseer, candidate_2_hash).await.is_none(),
|
||||
);
|
||||
assert!(query_available_data(&mut virtual_overseer, candidate_2_hash).await.is_none(),);
|
||||
|
||||
assert!(
|
||||
has_all_chunks(&mut virtual_overseer, candidate_1_hash, n_validators, true).await,
|
||||
);
|
||||
assert!(has_all_chunks(&mut virtual_overseer, candidate_1_hash, n_validators, true).await,);
|
||||
|
||||
assert!(
|
||||
has_all_chunks(&mut virtual_overseer, candidate_2_hash, n_validators, false).await,
|
||||
);
|
||||
assert!(has_all_chunks(&mut virtual_overseer, candidate_2_hash, n_validators, false).await,);
|
||||
|
||||
// Wait for longer than finalized blocks should be kept for
|
||||
test_state.clock.inc(test_state.pruning_config.keep_finalized_for);
|
||||
test_state.wait_for_pruning().await;
|
||||
|
||||
// Everything should be pruned now.
|
||||
assert!(
|
||||
query_available_data(&mut virtual_overseer, candidate_1_hash).await.is_none(),
|
||||
);
|
||||
assert!(query_available_data(&mut virtual_overseer, candidate_1_hash).await.is_none(),);
|
||||
|
||||
assert!(
|
||||
query_available_data(&mut virtual_overseer, candidate_2_hash).await.is_none(),
|
||||
);
|
||||
assert!(query_available_data(&mut virtual_overseer, candidate_2_hash).await.is_none(),);
|
||||
|
||||
assert!(
|
||||
has_all_chunks(&mut virtual_overseer, candidate_1_hash, n_validators, false).await,
|
||||
);
|
||||
assert!(has_all_chunks(&mut virtual_overseer, candidate_1_hash, n_validators, false).await,);
|
||||
|
||||
assert!(
|
||||
has_all_chunks(&mut virtual_overseer, candidate_2_hash, n_validators, false).await,
|
||||
);
|
||||
assert!(has_all_chunks(&mut virtual_overseer, candidate_2_hash, n_validators, false).await,);
|
||||
virtual_overseer
|
||||
});
|
||||
}
|
||||
@@ -1083,7 +1006,7 @@ async fn query_available_data(
|
||||
let (tx, rx) = oneshot::channel();
|
||||
|
||||
let query = AvailabilityStoreMessage::QueryAvailableData(candidate_hash, tx);
|
||||
virtual_overseer.send(FromOverseer::Communication{ msg: query }).await;
|
||||
virtual_overseer.send(FromOverseer::Communication { msg: query }).await;
|
||||
|
||||
rx.await.unwrap()
|
||||
}
|
||||
@@ -1096,7 +1019,7 @@ async fn query_chunk(
|
||||
let (tx, rx) = oneshot::channel();
|
||||
|
||||
let query = AvailabilityStoreMessage::QueryChunk(candidate_hash, index, tx);
|
||||
virtual_overseer.send(FromOverseer::Communication{ msg: query }).await;
|
||||
virtual_overseer.send(FromOverseer::Communication { msg: query }).await;
|
||||
|
||||
rx.await.unwrap()
|
||||
}
|
||||
@@ -1108,7 +1031,9 @@ async fn has_all_chunks(
|
||||
expect_present: bool,
|
||||
) -> bool {
|
||||
for i in 0..n_validators {
|
||||
if query_chunk(virtual_overseer, candidate_hash, ValidatorIndex(i)).await.is_some() != expect_present {
|
||||
if query_chunk(virtual_overseer, candidate_hash, ValidatorIndex(i)).await.is_some() !=
|
||||
expect_present
|
||||
{
|
||||
return false
|
||||
}
|
||||
}
|
||||
@@ -1139,7 +1064,8 @@ async fn import_leaf(
|
||||
status: LeafStatus::Fresh,
|
||||
span: Arc::new(jaeger::Span::Disabled),
|
||||
})),
|
||||
).await;
|
||||
)
|
||||
.await;
|
||||
|
||||
assert_matches!(
|
||||
overseer_recv(virtual_overseer).await,
|
||||
@@ -1163,7 +1089,6 @@ async fn import_leaf(
|
||||
}
|
||||
);
|
||||
|
||||
|
||||
assert_matches!(
|
||||
overseer_recv(virtual_overseer).await,
|
||||
AllMessages::RuntimeApi(RuntimeApiMessage::Request(
|
||||
|
||||
Reference in New Issue
Block a user