cargo +nightly fmt (#3540)

* cargo +nightly fmt

* add cargo-fmt check to ci

* update ci

* fmt

* fmt

* skip macro

* ignore bridges
This commit is contained in:
Shawn Tabrizi
2021-08-02 12:47:33 +02:00
committed by GitHub
parent 30e3012270
commit ff5d56fb76
350 changed files with 20617 additions and 21266 deletions
@@ -16,15 +16,9 @@
//! A utility for fetching all unknown blocks based on a new chain-head hash.
use polkadot_node_subsystem::{
messages::ChainApiMessage,
};
use polkadot_node_subsystem::{
SubsystemSender,
};
use polkadot_primitives::v1::{Hash, Header, BlockNumber};
use futures::prelude::*;
use futures::channel::oneshot;
use futures::{channel::oneshot, prelude::*};
use polkadot_node_subsystem::{messages::ChainApiMessage, SubsystemSender};
use polkadot_primitives::v1::{BlockNumber, Hash, Header};
/// Given a new chain-head hash, this determines the hashes of all new blocks we should track
/// metadata for, given this head.
@@ -43,7 +37,8 @@ pub async fn determine_new_blocks<E, Sender>(
head: Hash,
header: &Header,
lower_bound_number: BlockNumber,
) -> Result<Vec<(Hash, Header)>, E> where
) -> Result<Vec<(Hash, Header)>, E>
where
Sender: SubsystemSender,
{
const ANCESTRY_STEP: usize = 4;
@@ -57,7 +52,7 @@ pub async fn determine_new_blocks<E, Sender>(
let before_relevant = header.number < min_block_needed;
if already_known || before_relevant {
return Ok(Vec::new());
return Ok(Vec::new())
}
}
@@ -66,11 +61,12 @@ pub async fn determine_new_blocks<E, Sender>(
// Early exit if the parent hash is in the DB or no further blocks
// are needed.
if is_known(&header.parent_hash)? || header.number == min_block_needed {
return Ok(ancestry);
return Ok(ancestry)
}
'outer: loop {
let &(ref last_hash, ref last_header) = ancestry.last()
let &(ref last_hash, ref last_header) = ancestry
.last()
.expect("ancestry has length 1 at initialization and is only added to; qed");
assert!(
@@ -84,19 +80,22 @@ pub async fn determine_new_blocks<E, Sender>(
// This is always non-zero as determined by the loop invariant
// above.
let ancestry_step = std::cmp::min(
ANCESTRY_STEP,
(last_header.number - min_block_needed) as usize,
);
let ancestry_step =
std::cmp::min(ANCESTRY_STEP, (last_header.number - min_block_needed) as usize);
let batch_hashes = if ancestry_step == 1 {
vec![last_header.parent_hash]
} else {
sender.send_message(ChainApiMessage::Ancestors {
hash: *last_hash,
k: ancestry_step,
response_channel: tx,
}.into()).await;
sender
.send_message(
ChainApiMessage::Ancestors {
hash: *last_hash,
k: ancestry_step,
response_channel: tx,
}
.into(),
)
.await;
// Continue past these errors.
match rx.await {
@@ -111,27 +110,31 @@ pub async fn determine_new_blocks<E, Sender>(
.unzip::<_, _, Vec<_>, Vec<_>>();
for (hash, batched_sender) in batch_hashes.iter().cloned().zip(batch_senders) {
sender.send_message(ChainApiMessage::BlockHeader(hash, batched_sender).into()).await;
sender
.send_message(ChainApiMessage::BlockHeader(hash, batched_sender).into())
.await;
}
let mut requests = futures::stream::FuturesOrdered::new();
batch_receivers.into_iter().map(|rx| async move {
match rx.await {
Err(_) | Ok(Err(_)) => None,
Ok(Ok(h)) => h,
}
})
batch_receivers
.into_iter()
.map(|rx| async move {
match rx.await {
Err(_) | Ok(Err(_)) => None,
Ok(Ok(h)) => h,
}
})
.for_each(|x| requests.push(x));
let batch_headers: Vec<_> = requests
.flat_map(|x: Option<Header>| stream::iter(x))
.collect()
.await;
let batch_headers: Vec<_> =
requests.flat_map(|x: Option<Header>| stream::iter(x)).collect().await;
// Any failed header fetch of the batch will yield a `None` result that will
// be skipped. Any failure at this stage means we'll just ignore those blocks
// as the chain DB has failed us.
if batch_headers.len() != batch_hashes.len() { break 'outer }
if batch_headers.len() != batch_hashes.len() {
break 'outer
}
batch_headers
};
@@ -159,11 +162,11 @@ pub async fn determine_new_blocks<E, Sender>(
#[cfg(test)]
mod tests {
use super::*;
use std::collections::{HashSet, HashMap};
use sp_core::testing::TaskExecutor;
use polkadot_overseer::{AllMessages, SubsystemContext};
use polkadot_node_subsystem_test_helpers::make_subsystem_context;
use assert_matches::assert_matches;
use polkadot_node_subsystem_test_helpers::make_subsystem_context;
use polkadot_overseer::{AllMessages, SubsystemContext};
use sp_core::testing::TaskExecutor;
use std::collections::{HashMap, HashSet};
#[derive(Default)]
struct TestKnownBlocks {
@@ -287,12 +290,11 @@ mod tests {
head_hash,
&head,
lower_bound_number,
).await.unwrap();
)
.await
.unwrap();
assert_eq!(
ancestry,
expected_ancestry,
);
assert_eq!(ancestry, expected_ancestry,);
});
let aux_fut = Box::pin(async move {
@@ -361,12 +363,11 @@ mod tests {
head_hash,
&head,
lower_bound_number,
).await.unwrap();
)
.await
.unwrap();
assert_eq!(
ancestry,
expected_ancestry,
);
assert_eq!(ancestry, expected_ancestry,);
});
let aux_fut = Box::pin(async move {
@@ -421,12 +422,11 @@ mod tests {
head_hash,
&head,
lower_bound_number,
).await.unwrap();
)
.await
.unwrap();
assert_eq!(
ancestry,
expected_ancestry,
);
assert_eq!(ancestry, expected_ancestry,);
});
futures::executor::block_on(test_fut);
@@ -458,12 +458,11 @@ mod tests {
head_hash,
&head,
lower_bound_number,
).await.unwrap();
)
.await
.unwrap();
assert_eq!(
ancestry,
expected_ancestry,
);
assert_eq!(ancestry, expected_ancestry,);
});
futures::executor::block_on(test_fut);
@@ -484,44 +483,26 @@ mod tests {
known.insert(parent_hash);
let test_fut = Box::pin(async move {
let after_finality = determine_new_blocks(
ctx.sender(),
|h| known.is_known(h),
head_hash,
&head,
17,
).await.unwrap();
let after_finality =
determine_new_blocks(ctx.sender(), |h| known.is_known(h), head_hash, &head, 17)
.await
.unwrap();
let at_finality = determine_new_blocks(
ctx.sender(),
|h| known.is_known(h),
head_hash,
&head,
18,
).await.unwrap();
let at_finality =
determine_new_blocks(ctx.sender(), |h| known.is_known(h), head_hash, &head, 18)
.await
.unwrap();
let before_finality = determine_new_blocks(
ctx.sender(),
|h| known.is_known(h),
head_hash,
&head,
19,
).await.unwrap();
let before_finality =
determine_new_blocks(ctx.sender(), |h| known.is_known(h), head_hash, &head, 19)
.await
.unwrap();
assert_eq!(
after_finality,
vec![(head_hash, head.clone())],
);
assert_eq!(after_finality, vec![(head_hash, head.clone())],);
assert_eq!(
at_finality,
Vec::new(),
);
assert_eq!(at_finality, Vec::new(),);
assert_eq!(
before_finality,
Vec::new(),
);
assert_eq!(before_finality, Vec::new(),);
});
futures::executor::block_on(test_fut);
@@ -544,13 +525,10 @@ mod tests {
.collect::<Vec<_>>();
let test_fut = Box::pin(async move {
let ancestry = determine_new_blocks(
ctx.sender(),
|h| known.is_known(h),
head_hash,
&head,
0,
).await.unwrap();
let ancestry =
determine_new_blocks(ctx.sender(), |h| known.is_known(h), head_hash, &head, 0)
.await
.unwrap();
assert_eq!(ancestry, expected_ancestry);
});
@@ -585,13 +563,10 @@ mod tests {
.collect::<Vec<_>>();
let test_fut = Box::pin(async move {
let ancestry = determine_new_blocks(
ctx.sender(),
|h| known.is_known(h),
head_hash,
&head,
0,
).await.unwrap();
let ancestry =
determine_new_blocks(ctx.sender(), |h| known.is_known(h), head_hash, &head, 0)
.await
.unwrap();
assert_eq!(ancestry, expected_ancestry);
});
@@ -64,14 +64,14 @@ use thiserror::Error;
/// Self(Fault::from_non_fatal(e))
/// }
/// }
///
///
/// // Make an Error from a `Fatal` one.
/// impl From<Fatal> for Error {
/// fn from(f: Fatal) -> Self {
/// Self(Fault::from_fatal(f))
/// }
/// }
///
///
/// // Easy conversion from sub error types from other modules:
/// impl From<runtime::Error> for Error {
/// fn from(o: runtime::Error) -> Self {
@@ -107,9 +107,10 @@ use thiserror::Error;
/// ```
#[derive(Debug, Error)]
pub enum Fault<E, F>
where
E: std::fmt::Debug + std::error::Error + 'static,
F: std::fmt::Debug + std::error::Error + 'static, {
where
E: std::fmt::Debug + std::error::Error + 'static,
F: std::fmt::Debug + std::error::Error + 'static,
{
/// Error is fatal and should be escalated up.
///
/// While we usually won't want to pattern match on those, a concrete descriptive enum might
@@ -126,9 +127,9 @@ pub enum Fault<E, F>
/// `From::from` implementations. So no auto conversions by default, a simple `Result::map_err` is
/// not too bad though.
impl<E, F> Fault<E, F>
where
E: std::fmt::Debug + std::error::Error + 'static,
F: std::fmt::Debug + std::error::Error + 'static,
where
E: std::fmt::Debug + std::error::Error + 'static,
F: std::fmt::Debug + std::error::Error + 'static,
{
/// Build an `Fault` from compatible fatal error.
pub fn from_fatal<F1: Into<F>>(f: F1) -> Self {
@@ -188,10 +189,10 @@ impl<E, F> Fault<E, F>
/// }
///
/// ```
pub fn unwrap_non_fatal<E,F>(result: Result<(), Fault<E,F>>) -> Result<Option<E>, F>
where
E: std::fmt::Debug + std::error::Error + 'static,
F: std::fmt::Debug + std::error::Error + Send + Sync + 'static
pub fn unwrap_non_fatal<E, F>(result: Result<(), Fault<E, F>>) -> Result<Option<E>, F>
where
E: std::fmt::Debug + std::error::Error + 'static,
F: std::fmt::Debug + std::error::Error + Send + Sync + 'static,
{
match result {
Ok(()) => Ok(None),
+95 -126
View File
@@ -25,53 +25,48 @@
#![warn(missing_docs)]
use polkadot_node_subsystem::{
overseer,
errors::RuntimeApiError,
errors::{RuntimeApiError, SubsystemError},
messages::{
AllMessages,
RuntimeApiMessage,
RuntimeApiRequest,
RuntimeApiSender,
BoundToRelayParent,
AllMessages, BoundToRelayParent, RuntimeApiMessage, RuntimeApiRequest, RuntimeApiSender,
},
ActiveLeavesUpdate, OverseerSignal,
overseer, ActiveLeavesUpdate, FromOverseer, OverseerSignal, SpawnedSubsystem, SubsystemContext,
SubsystemSender,
errors::{
SubsystemError,
},
SubsystemContext,
SpawnedSubsystem,
FromOverseer,
};
pub use overseer::{
Subsystem,
TimeoutExt,
gen::OverseerError,
gen::Timeout,
gen::{OverseerError, Timeout},
Subsystem, TimeoutExt,
};
pub use polkadot_node_metrics::{
Metronome,
metrics,
};
pub use polkadot_node_metrics::{metrics, Metronome};
use polkadot_node_jaeger as jaeger;
use futures::{channel::{mpsc, oneshot}, prelude::*, select, stream::{Stream, SelectAll}};
use futures::{
channel::{mpsc, oneshot},
prelude::*,
select,
stream::{SelectAll, Stream},
};
use parity_scale_codec::Encode;
use pin_project::pin_project;
use polkadot_node_jaeger as jaeger;
use polkadot_primitives::v1::{
CandidateEvent, CommittedCandidateReceipt, CoreState, EncodeAs, PersistedValidationData,
GroupRotationInfo, Hash, Id as ParaId, OccupiedCoreAssumption,
SessionIndex, Signed, SigningContext, ValidationCode, ValidatorId, ValidatorIndex, SessionInfo,
AuthorityDiscoveryId, GroupIndex,
AuthorityDiscoveryId, CandidateEvent, CommittedCandidateReceipt, CoreState, EncodeAs,
GroupIndex, GroupRotationInfo, Hash, Id as ParaId, OccupiedCoreAssumption,
PersistedValidationData, SessionIndex, SessionInfo, Signed, SigningContext, ValidationCode,
ValidatorId, ValidatorIndex,
};
use sp_core::{traits::SpawnNamed, Public};
use sp_application_crypto::AppKey;
use sp_keystore::{CryptoStore, SyncCryptoStorePtr, Error as KeystoreError};
use sp_core::{traits::SpawnNamed, Public};
use sp_keystore::{CryptoStore, Error as KeystoreError, SyncCryptoStorePtr};
use std::{
collections::{HashMap, hash_map::Entry}, convert::TryFrom, marker::Unpin, pin::Pin, task::{Poll, Context},
time::Duration, fmt, sync::Arc,
collections::{hash_map::Entry, HashMap},
convert::TryFrom,
fmt,
marker::Unpin,
pin::Pin,
sync::Arc,
task::{Context, Poll},
time::Duration,
};
use thiserror::Error;
@@ -80,22 +75,17 @@ pub use polkadot_node_network_protocol::MIN_GOSSIP_PEERS;
pub use determine_new_blocks::determine_new_blocks;
/// Error classification.
pub use error_handling::{Fault, unwrap_non_fatal};
pub use error_handling::{unwrap_non_fatal, Fault};
/// These reexports are required so that external crates can use the `delegated_subsystem` macro properly.
pub mod reexports {
pub use polkadot_overseer::gen::{
SpawnNamed,
SpawnedSubsystem,
Subsystem,
SubsystemContext,
};
pub use polkadot_overseer::gen::{SpawnNamed, SpawnedSubsystem, Subsystem, SubsystemContext};
}
/// Convenient and efficient runtime info access.
pub mod runtime;
/// A rolling session window cache.
pub mod rolling_session_window;
/// Convenient and efficient runtime info access.
pub mod runtime;
mod determine_new_blocks;
mod error_handling;
@@ -158,7 +148,9 @@ where
{
let (tx, rx) = oneshot::channel();
sender.send_message(RuntimeApiMessage::Request(parent, request_builder(tx)).into()).await;
sender
.send_message(RuntimeApiMessage::Request(parent, request_builder(tx)).into())
.await;
rx
}
@@ -225,39 +217,48 @@ specialize_requests! {
}
/// From the given set of validators, find the first key we can sign with, if any.
pub async fn signing_key(validators: &[ValidatorId], keystore: &SyncCryptoStorePtr)
-> Option<ValidatorId>
{
pub async fn signing_key(
validators: &[ValidatorId],
keystore: &SyncCryptoStorePtr,
) -> Option<ValidatorId> {
signing_key_and_index(validators, keystore).await.map(|(k, _)| k)
}
/// From the given set of validators, find the first key we can sign with, if any, and return it
/// along with the validator index.
pub async fn signing_key_and_index(validators: &[ValidatorId], keystore: &SyncCryptoStorePtr)
-> Option<(ValidatorId, ValidatorIndex)>
{
pub async fn signing_key_and_index(
validators: &[ValidatorId],
keystore: &SyncCryptoStorePtr,
) -> Option<(ValidatorId, ValidatorIndex)> {
for (i, v) in validators.iter().enumerate() {
if CryptoStore::has_keys(&**keystore, &[(v.to_raw_vec(), ValidatorId::ID)]).await {
return Some((v.clone(), ValidatorIndex(i as _)));
return Some((v.clone(), ValidatorIndex(i as _)))
}
}
None
}
/// Find the validator group the given validator index belongs to.
pub fn find_validator_group(groups: &[Vec<ValidatorIndex>], index: ValidatorIndex)
-> Option<GroupIndex>
{
groups.iter().enumerate().find_map(|(i, g)| if g.contains(&index) {
Some(GroupIndex(i as _))
} else {
None
pub fn find_validator_group(
groups: &[Vec<ValidatorIndex>],
index: ValidatorIndex,
) -> Option<GroupIndex> {
groups.iter().enumerate().find_map(|(i, g)| {
if g.contains(&index) {
Some(GroupIndex(i as _))
} else {
None
}
})
}
/// Choose a random subset of `min` elements.
/// But always include `is_priority` elements.
pub fn choose_random_subset<T, F: FnMut(&T) -> bool>(is_priority: F, mut v: Vec<T>, min: usize) -> Vec<T> {
pub fn choose_random_subset<T, F: FnMut(&T) -> bool>(
is_priority: F,
mut v: Vec<T>,
min: usize,
) -> Vec<T> {
use rand::seq::SliceRandom as _;
// partition the elements into priority first
@@ -266,7 +267,7 @@ pub fn choose_random_subset<T, F: FnMut(&T) -> bool>(is_priority: F, mut v: Vec<
if i >= min || v.len() <= i {
v.truncate(i);
return v;
return v
}
let mut rng = rand::thread_rng();
@@ -300,8 +301,7 @@ impl Validator {
parent: Hash,
keystore: SyncCryptoStorePtr,
sender: &mut impl SubsystemSender,
) -> Result<Self, Error>
{
) -> Result<Self, Error> {
// Note: request_validators and request_session_index_for_child do not and cannot
// run concurrently: they both have a mutable handle to the same sender.
// However, each of them returns a oneshot::Receiver, and those are resolved concurrently.
@@ -310,10 +310,7 @@ impl Validator {
request_session_index_for_child(parent, sender).await,
)?;
let signing_context = SigningContext {
session_index: session_index?,
parent_hash: parent,
};
let signing_context = SigningContext { session_index: session_index?, parent_hash: parent };
let validators = validators?;
@@ -328,15 +325,10 @@ impl Validator {
signing_context: SigningContext,
keystore: SyncCryptoStorePtr,
) -> Result<Self, Error> {
let (key, index) = signing_key_and_index(validators, &keystore)
.await
.ok_or(Error::NotAValidator)?;
let (key, index) =
signing_key_and_index(validators, &keystore).await.ok_or(Error::NotAValidator)?;
Ok(Validator {
signing_context,
key,
index,
})
Ok(Validator { signing_context, key, index })
}
/// Get this validator's id.
@@ -403,10 +395,7 @@ pub struct JobSender<S: SubsystemSender> {
// which `#[derive(Clone)]` requires.
impl<S: SubsystemSender> Clone for JobSender<S> {
fn clone(&self) -> Self {
Self {
sender: self.sender.clone(),
from_job: self.from_job.clone(),
}
Self { sender: self.sender.clone(), from_job: self.from_job.clone() }
}
}
@@ -424,13 +413,13 @@ impl<S: SubsystemSender> JobSender<S> {
/// Send multiple direct messages to other `Subsystem`s, routed based on message type.
pub async fn send_messages<T, M>(&mut self, msgs: T)
where
T: IntoIterator<Item = M> + Send, T::IntoIter: Send,
T: IntoIterator<Item = M> + Send,
T::IntoIter: Send,
M: Into<AllMessages>,
{
self.sender.send_messages(msgs.into_iter().map(|m| m.into())).await
}
/// Send a message onto the unbounded queue of some other `Subsystem`, routed based on message
/// type.
///
@@ -446,7 +435,6 @@ impl<S: SubsystemSender> JobSender<S> {
}
}
#[async_trait::async_trait]
impl<S, M> overseer::SubsystemSender<M> for JobSender<S>
where
@@ -458,7 +446,9 @@ where
}
async fn send_messages<T>(&mut self, msgs: T)
where T: IntoIterator<Item = M> + Send, T::IntoIter: Send
where
T: IntoIterator<Item = M> + Send,
T::IntoIter: Send,
{
self.sender.send_messages(msgs.into_iter().map(|m| m.into())).await
}
@@ -547,11 +537,7 @@ where
{
/// Create a new Jobs manager which handles spawning appropriate jobs.
pub fn new(spawner: Spawner) -> Self {
Self {
spawner,
running: HashMap::new(),
outgoing_msgs: SelectAll::new(),
}
Self { spawner, running: HashMap::new(), outgoing_msgs: SelectAll::new() }
}
/// Spawn a new job for this `parent_hash`, with whatever args are appropriate.
@@ -562,10 +548,9 @@ where
run_args: Job::RunArgs,
metrics: Job::Metrics,
sender: Sender,
)
where
Job: JobTrait<ToJob = ToJob>,
Sender: SubsystemSender,
) where
Job: JobTrait<ToJob = ToJob>,
Sender: SubsystemSender,
{
let (to_job_tx, to_job_rx) = mpsc::channel(JOB_CHANNEL_CAPACITY);
let (from_job_tx, from_job_rx) = mpsc::channel(JOB_CHANNEL_CAPACITY);
@@ -577,11 +562,10 @@ where
run_args,
metrics,
to_job_rx,
JobSender {
sender,
from_job: from_job_tx,
},
).await {
JobSender { sender, from_job: from_job_tx },
)
.await
{
tracing::error!(
job = Job::NAME,
parent_hash = %parent_hash,
@@ -589,7 +573,7 @@ where
"job finished with an error",
);
return Err(e);
return Err(e)
}
Ok(())
@@ -598,10 +582,7 @@ where
self.spawner.spawn(Job::NAME, future.map(drop).boxed());
self.outgoing_msgs.push(from_job_rx);
let handle = JobHandle {
_abort_handle: AbortOnDrop(abort_handle),
to_job: to_job_tx,
};
let handle = JobHandle { _abort_handle: AbortOnDrop(abort_handle), to_job: to_job_tx };
self.running.insert(parent_hash, handle);
}
@@ -672,34 +653,24 @@ impl<Job: JobTrait, Spawner> JobSubsystem<Job, Spawner> {
/// Create a new `JobSubsystem`.
pub fn new(spawner: Spawner, run_args: Job::RunArgs, metrics: Job::Metrics) -> Self {
JobSubsystem {
params: JobSubsystemParams {
spawner,
run_args,
metrics,
},
params: JobSubsystemParams { spawner, run_args, metrics },
_marker: std::marker::PhantomData,
}
}
/// Run the subsystem to completion.
pub async fn run<Context>(self, mut ctx: Context)
where
Spawner: SpawnNamed + Send + Clone + Unpin + 'static,
Context: SubsystemContext<Message=<Job as JobTrait>::ToJob, Signal=OverseerSignal>,
<Context as SubsystemContext>::Sender: SubsystemSender,
Job: 'static + JobTrait + Send,
<Job as JobTrait>::RunArgs: Clone + Sync,
<Job as JobTrait>::ToJob: Sync + From<<Context as polkadot_overseer::SubsystemContext>::Message>,
<Job as JobTrait>::Metrics: Sync,
where
Spawner: SpawnNamed + Send + Clone + Unpin + 'static,
Context: SubsystemContext<Message = <Job as JobTrait>::ToJob, Signal = OverseerSignal>,
<Context as SubsystemContext>::Sender: SubsystemSender,
Job: 'static + JobTrait + Send,
<Job as JobTrait>::RunArgs: Clone + Sync,
<Job as JobTrait>::ToJob:
Sync + From<<Context as polkadot_overseer::SubsystemContext>::Message>,
<Job as JobTrait>::Metrics: Sync,
{
let JobSubsystem {
params: JobSubsystemParams {
spawner,
run_args,
metrics,
},
..
} = self;
let JobSubsystem { params: JobSubsystemParams { spawner, run_args, metrics }, .. } = self;
let mut jobs = Jobs::<Spawner, Job::ToJob>::new(spawner);
@@ -768,10 +739,11 @@ impl<Job: JobTrait, Spawner> JobSubsystem<Job, Spawner> {
impl<Context, Job, Spawner> Subsystem<Context, SubsystemError> for JobSubsystem<Job, Spawner>
where
Spawner: SpawnNamed + Send + Clone + Unpin + 'static,
Context: SubsystemContext<Message=Job::ToJob,Signal=OverseerSignal>,
Context: SubsystemContext<Message = Job::ToJob, Signal = OverseerSignal>,
Job: 'static + JobTrait + Send,
Job::RunArgs: Clone + Sync,
<Job as JobTrait>::ToJob: Sync + From<<Context as polkadot_overseer::SubsystemContext>::Message>,
<Job as JobTrait>::ToJob:
Sync + From<<Context as polkadot_overseer::SubsystemContext>::Message>,
Job::Metrics: Sync,
{
fn start(self, ctx: Context) -> SpawnedSubsystem {
@@ -780,9 +752,6 @@ where
Ok(())
});
SpawnedSubsystem {
name: Job::NAME.strip_suffix("Job").unwrap_or(Job::NAME),
future,
}
SpawnedSubsystem { name: Job::NAME.strip_suffix("Job").unwrap_or(Job::NAME), future }
}
}
@@ -19,15 +19,14 @@
//! This is useful for consensus components which need to stay up-to-date about recent sessions but don't
//! care about the state of particular blocks.
use polkadot_primitives::v1::{Hash, Header, SessionInfo, SessionIndex};
use polkadot_primitives::v1::{Hash, Header, SessionIndex, SessionInfo};
use polkadot_node_subsystem::{
overseer,
messages::{RuntimeApiMessage, RuntimeApiRequest},
errors::RuntimeApiError,
SubsystemContext,
};
use futures::channel::oneshot;
use polkadot_node_subsystem::{
errors::RuntimeApiError,
messages::{RuntimeApiMessage, RuntimeApiRequest},
overseer, SubsystemContext,
};
/// Sessions unavailable in state to cache.
#[derive(Debug)]
@@ -96,11 +95,7 @@ pub struct RollingSessionWindow {
impl RollingSessionWindow {
/// Initialize a new session info cache with the given window size.
pub fn new(window_size: SessionIndex) -> Self {
RollingSessionWindow {
earliest_session: None,
session_info: Vec::new(),
window_size,
}
RollingSessionWindow { earliest_session: None, session_info: Vec::new(), window_size }
}
/// Initialize a new session info cache with the given window size and
@@ -110,11 +105,7 @@ impl RollingSessionWindow {
earliest_session: SessionIndex,
session_info: Vec<SessionInfo>,
) -> Self {
RollingSessionWindow {
earliest_session: Some(earliest_session),
session_info,
window_size,
}
RollingSessionWindow { earliest_session: Some(earliest_session), session_info, window_size }
}
/// Access the session info for the given session index, if stored within the window.
@@ -126,7 +117,6 @@ impl RollingSessionWindow {
self.session_info.get((index - earliest) as usize)
}
})
}
/// Access the index of the earliest session, if the window is not empty.
@@ -153,7 +143,9 @@ impl RollingSessionWindow {
block_hash: Hash,
block_header: &Header,
) -> Result<SessionWindowUpdate, SessionsUnavailable> {
if self.window_size == 0 { return Ok(SessionWindowUpdate::Unchanged) }
if self.window_size == 0 {
return Ok(SessionWindowUpdate::Unchanged)
}
let session_index = {
let (s_tx, s_rx) = oneshot::channel();
@@ -164,18 +156,21 @@ impl RollingSessionWindow {
ctx.send_message(RuntimeApiMessage::Request(
if block_header.number == 0 { block_hash } else { block_header.parent_hash },
RuntimeApiRequest::SessionIndexForChild(s_tx),
)).await;
))
.await;
match s_rx.await {
Ok(Ok(s)) => s,
Ok(Err(e)) => return Err(SessionsUnavailable {
kind: SessionsUnavailableKind::RuntimeApi(e),
info: None,
}),
Err(e) => return Err(SessionsUnavailable {
kind: SessionsUnavailableKind::RuntimeApiUnavailable(e),
info: None,
}),
Ok(Err(e)) =>
return Err(SessionsUnavailable {
kind: SessionsUnavailableKind::RuntimeApi(e),
info: None,
}),
Err(e) =>
return Err(SessionsUnavailable {
kind: SessionsUnavailableKind::RuntimeApiUnavailable(e),
info: None,
}),
}
};
@@ -186,16 +181,14 @@ impl RollingSessionWindow {
let window_start = session_index.saturating_sub(self.window_size - 1);
match load_all_sessions(ctx, block_hash, window_start, session_index).await {
Err(kind) => {
Err(SessionsUnavailable {
kind,
info: Some(SessionsUnavailableInfo {
window_start,
window_end: session_index,
block_hash,
}),
})
},
Err(kind) => Err(SessionsUnavailable {
kind,
info: Some(SessionsUnavailableInfo {
window_start,
window_end: session_index,
block_hash,
}),
}),
Ok(s) => {
let update = SessionWindowUpdate::Initialized {
window_start,
@@ -206,14 +199,17 @@ impl RollingSessionWindow {
self.session_info = s;
Ok(update)
}
},
}
}
},
Some(old_window_start) => {
let latest = self.latest_session().expect("latest always exists if earliest does; qed");
let latest =
self.latest_session().expect("latest always exists if earliest does; qed");
// Either cached or ancient.
if session_index <= latest { return Ok(SessionWindowUpdate::Unchanged) }
if session_index <= latest {
return Ok(SessionWindowUpdate::Unchanged)
}
let old_window_end = latest;
@@ -222,23 +218,17 @@ impl RollingSessionWindow {
// keep some of the old window, if applicable.
let overlap_start = window_start.saturating_sub(old_window_start);
let fresh_start = if latest < window_start {
window_start
} else {
latest + 1
};
let fresh_start = if latest < window_start { window_start } else { latest + 1 };
match load_all_sessions(ctx, block_hash, fresh_start, session_index).await {
Err(kind) => {
Err(SessionsUnavailable {
kind,
info: Some(SessionsUnavailableInfo {
window_start: fresh_start,
window_end: session_index,
block_hash,
}),
})
},
Err(kind) => Err(SessionsUnavailable {
kind,
info: Some(SessionsUnavailableInfo {
window_start: fresh_start,
window_end: session_index,
block_hash,
}),
}),
Ok(s) => {
let update = SessionWindowUpdate::Advanced {
prev_window_start: old_window_start,
@@ -247,7 +237,8 @@ impl RollingSessionWindow {
new_window_end: session_index,
};
let outdated = std::cmp::min(overlap_start as usize, self.session_info.len());
let outdated =
std::cmp::min(overlap_start as usize, self.session_info.len());
self.session_info.drain(..outdated);
self.session_info.extend(s);
// we need to account for this case:
@@ -257,9 +248,9 @@ impl RollingSessionWindow {
self.earliest_session = Some(new_earliest);
Ok(update)
}
},
}
}
},
}
}
}
@@ -272,17 +263,16 @@ async fn load_all_sessions(
) -> Result<Vec<SessionInfo>, SessionsUnavailableKind> {
let mut v = Vec::new();
for i in start..=end_inclusive {
let (tx, rx)= oneshot::channel();
let (tx, rx) = oneshot::channel();
ctx.send_message(RuntimeApiMessage::Request(
block_hash,
RuntimeApiRequest::SessionInfo(i, tx),
)).await;
))
.await;
let session_info = match rx.await {
Ok(Ok(Some(s))) => s,
Ok(Ok(None)) => {
return Err(SessionsUnavailableKind::Missing);
}
Ok(Ok(None)) => return Err(SessionsUnavailableKind::Missing),
Ok(Err(e)) => return Err(SessionsUnavailableKind::RuntimeApi(e)),
Err(canceled) => return Err(SessionsUnavailableKind::RuntimeApiUnavailable(canceled)),
};
@@ -296,10 +286,10 @@ async fn load_all_sessions(
#[cfg(test)]
mod tests {
use super::*;
use polkadot_node_subsystem_test_helpers::make_subsystem_context;
use polkadot_node_subsystem::messages::{AllMessages, AvailabilityRecoveryMessage};
use sp_core::testing::TaskExecutor;
use assert_matches::assert_matches;
use polkadot_node_subsystem::messages::{AllMessages, AvailabilityRecoveryMessage};
use polkadot_node_subsystem_test_helpers::make_subsystem_context;
use sp_core::testing::TaskExecutor;
const TEST_WINDOW_SIZE: SessionIndex = 6;
@@ -333,18 +323,15 @@ mod tests {
};
let pool = TaskExecutor::new();
let (mut ctx, mut handle) = make_subsystem_context::<AvailabilityRecoveryMessage, _>(pool.clone());
let (mut ctx, mut handle) =
make_subsystem_context::<AvailabilityRecoveryMessage, _>(pool.clone());
let hash = header.hash();
let test_fut = {
let header = header.clone();
Box::pin(async move {
window.cache_session_info_for_head(
&mut ctx,
hash,
&header,
).await.unwrap();
window.cache_session_info_for_head(&mut ctx, hash, &header).await.unwrap();
assert_eq!(window.earliest_session, Some(expected_start_session));
assert_eq!(
@@ -386,12 +373,7 @@ mod tests {
#[test]
fn cache_session_info_first_early() {
cache_session_info_test(
0,
1,
RollingSessionWindow::new(TEST_WINDOW_SIZE),
0,
);
cache_session_info_test(0, 1, RollingSessionWindow::new(TEST_WINDOW_SIZE), 0);
}
#[test]
@@ -402,12 +384,7 @@ mod tests {
window_size: TEST_WINDOW_SIZE,
};
cache_session_info_test(
1,
2,
window,
2,
);
cache_session_info_test(1, 2, window, 2);
}
#[test]
@@ -424,7 +401,11 @@ mod tests {
fn cache_session_info_jump() {
let window = RollingSessionWindow {
earliest_session: Some(50),
session_info: vec![dummy_session_info(50), dummy_session_info(51), dummy_session_info(52)],
session_info: vec![
dummy_session_info(50),
dummy_session_info(51),
dummy_session_info(52),
],
window_size: TEST_WINDOW_SIZE,
};
@@ -480,10 +461,7 @@ mod tests {
};
cache_session_info_test(
0,
2,
window,
2, // should only make one request.
0, 2, window, 2, // should only make one request.
);
}
@@ -496,12 +474,7 @@ mod tests {
window_size: TEST_WINDOW_SIZE,
};
cache_session_info_test(
0,
3,
window,
2,
);
cache_session_info_test(0, 3, window, 2);
}
#[test]
@@ -526,11 +499,7 @@ mod tests {
let test_fut = {
let header = header.clone();
Box::pin(async move {
let res = window.cache_session_info_for_head(
&mut ctx,
hash,
&header,
).await;
let res = window.cache_session_info_for_head(&mut ctx, hash, &header).await;
assert!(res.is_err());
})
@@ -592,17 +561,10 @@ mod tests {
let test_fut = {
let header = header.clone();
Box::pin(async move {
window.cache_session_info_for_head(
&mut ctx,
hash,
&header,
).await.unwrap();
window.cache_session_info_for_head(&mut ctx, hash, &header).await.unwrap();
assert_eq!(window.earliest_session, Some(session));
assert_eq!(
window.session_info,
vec![dummy_session_info(session)],
);
assert_eq!(window.session_info, vec![dummy_session_info(session)],);
})
};
@@ -17,8 +17,8 @@
//! Error handling related code and Error/Result definitions.
use thiserror::Error;
use futures::channel::oneshot;
use thiserror::Error;
use polkadot_node_subsystem::errors::RuntimeApiError;
use polkadot_primitives::v1::SessionIndex;
@@ -67,7 +67,8 @@ pub enum NonFatal {
pub(crate) async fn recv_runtime<V>(
r: oneshot::Receiver<std::result::Result<V, RuntimeApiError>>,
) -> Result<V> {
let result = r.await
let result = r
.await
.map_err(Fatal::RuntimeRequestCanceled)?
.map_err(NonFatal::RuntimeRequest)?;
Ok(result)
+41 -55
View File
@@ -25,13 +25,14 @@ use sp_application_crypto::AppKey;
use sp_core::crypto::Public;
use sp_keystore::{CryptoStore, SyncCryptoStorePtr};
use polkadot_primitives::v1::{CoreState, EncodeAs, GroupIndex, GroupRotationInfo, Hash, OccupiedCore, SessionIndex, SessionInfo, Signed, SigningContext, UncheckedSigned, ValidatorId, ValidatorIndex};
use polkadot_node_subsystem::{SubsystemSender, SubsystemContext};
use polkadot_node_subsystem::{SubsystemContext, SubsystemSender};
use polkadot_primitives::v1::{
CoreState, EncodeAs, GroupIndex, GroupRotationInfo, Hash, OccupiedCore, SessionIndex,
SessionInfo, Signed, SigningContext, UncheckedSigned, ValidatorId, ValidatorIndex,
};
use crate::{
request_session_index_for_child, request_session_info,
request_availability_cores,
request_availability_cores, request_session_index_for_child, request_session_info,
request_validator_groups,
};
@@ -39,7 +40,7 @@ use crate::{
mod error;
use error::{recv_runtime, Result};
pub use error::{Error, NonFatal, Fatal};
pub use error::{Error, Fatal, NonFatal};
/// Configuration for construction a `RuntimeInfo`.
pub struct Config {
@@ -100,10 +101,7 @@ impl Default for Config {
impl RuntimeInfo {
/// Create a new `RuntimeInfo` for convenient runtime fetches.
pub fn new(keystore: Option<SyncCryptoStorePtr>) -> Self {
Self::new_with_config(Config {
keystore,
..Default::default()
})
Self::new_with_config(Config { keystore, ..Default::default() })
}
/// Create with more elaborate configuration options.
@@ -128,11 +126,10 @@ impl RuntimeInfo {
Some(index) => Ok(*index),
None => {
let index =
recv_runtime(request_session_index_for_child(parent, sender).await)
.await?;
recv_runtime(request_session_index_for_child(parent, sender).await).await?;
self.session_index_cache.put(parent, index);
Ok(index)
}
},
}
}
@@ -170,17 +167,14 @@ impl RuntimeInfo {
.ok_or(NonFatal::NoSuchSession(session_index))?;
let validator_info = self.get_validator_info(&session_info).await?;
let full_info = ExtendedSessionInfo {
session_info,
validator_info,
};
let full_info = ExtendedSessionInfo { session_info, validator_info };
self.session_info_cache.put(session_index, full_info);
}
Ok(
self.session_info_cache.get(&session_index)
.expect("We just put the value there. qed.")
)
Ok(self
.session_info_cache
.get(&session_index)
.expect("We just put the value there. qed."))
}
/// Convenience function for checking the signature of something signed.
@@ -189,7 +183,9 @@ impl RuntimeInfo {
sender: &mut Sender,
parent: Hash,
signed: UncheckedSigned<Payload, RealPayload>,
) -> Result<std::result::Result<Signed<Payload, RealPayload>, UncheckedSigned<Payload, RealPayload>>>
) -> Result<
std::result::Result<Signed<Payload, RealPayload>, UncheckedSigned<Payload, RealPayload>>,
>
where
Sender: SubsystemSender,
Payload: EncodeAs<RealPayload> + Clone,
@@ -204,17 +200,11 @@ impl RuntimeInfo {
///
///
/// Returns: `None` if not a validator.
async fn get_validator_info(
&self,
session_info: &SessionInfo,
) -> Result<ValidatorInfo>
{
async fn get_validator_info(&self, session_info: &SessionInfo) -> Result<ValidatorInfo> {
if let Some(our_index) = self.get_our_index(&session_info.validators).await {
// Get our group index:
let our_group = session_info.validator_groups
.iter()
.enumerate()
.find_map(|(i, g)| {
let our_group =
session_info.validator_groups.iter().enumerate().find_map(|(i, g)| {
g.iter().find_map(|v| {
if *v == our_index {
Some(GroupIndex(i as u32))
@@ -222,12 +212,8 @@ impl RuntimeInfo {
None
}
})
}
);
let info = ValidatorInfo {
our_index: Some(our_index),
our_group,
};
});
let info = ValidatorInfo { our_index: Some(our_index), our_group };
return Ok(info)
}
return Ok(ValidatorInfo { our_index: None, our_group: None })
@@ -239,10 +225,8 @@ impl RuntimeInfo {
async fn get_our_index(&self, validators: &[ValidatorId]) -> Option<ValidatorIndex> {
let keystore = self.keystore.as_ref()?;
for (i, v) in validators.iter().enumerate() {
if CryptoStore::has_keys(&**keystore, &[(v.to_raw_vec(), ValidatorId::ID)])
.await
{
return Some(ValidatorIndex(i as u32));
if CryptoStore::has_keys(&**keystore, &[(v.to_raw_vec(), ValidatorId::ID)]).await {
return Some(ValidatorIndex(i as u32))
}
}
None
@@ -260,22 +244,22 @@ where
Payload: EncodeAs<RealPayload> + Clone,
RealPayload: Encode + Clone,
{
let signing_context = SigningContext {
session_index,
parent_hash: relay_parent,
};
let signing_context = SigningContext { session_index, parent_hash: relay_parent };
session_info.validators
session_info
.validators
.get(signed.unchecked_validator_index().0 as usize)
.ok_or_else(|| signed.clone())
.and_then(|v| signed.try_into_checked(&signing_context, v))
}
/// Request availability cores from the runtime.
pub async fn get_availability_cores<Context>(ctx: &mut Context, relay_parent: Hash)
-> Result<Vec<CoreState>>
where
Context: SubsystemContext,
pub async fn get_availability_cores<Context>(
ctx: &mut Context,
relay_parent: Hash,
) -> Result<Vec<CoreState>>
where
Context: SubsystemContext,
{
recv_runtime(request_availability_cores(relay_parent, ctx.sender()).await).await
}
@@ -299,18 +283,20 @@ where
None
}
})
.collect()
)
.collect())
}
/// Get group rotation info based on the given `relay_parent`.
pub async fn get_group_rotation_info<Context>(ctx: &mut Context, relay_parent: Hash)
-> Result<GroupRotationInfo>
pub async fn get_group_rotation_info<Context>(
ctx: &mut Context,
relay_parent: Hash,
) -> Result<GroupRotationInfo>
where
Context: SubsystemContext,
{
// We drop `groups` here as we don't need them, because of `RuntimeInfo`. Ideally we would not
// fetch them in the first place.
let (_, info) = recv_runtime(request_validator_groups(relay_parent, ctx.sender()).await).await?;
let (_, info) =
recv_runtime(request_validator_groups(relay_parent, ctx.sender()).await).await?;
Ok(info)
}
+42 -54
View File
@@ -15,18 +15,25 @@
// along with Polkadot. If not, see <http://www.gnu.org/licenses/>.
use super::*;
use assert_matches::assert_matches;
use executor::block_on;
use thiserror::Error;
use futures::{channel::mpsc, executor, future, Future, FutureExt, SinkExt, StreamExt};
use polkadot_node_jaeger as jaeger;
use polkadot_node_subsystem::{
messages::{AllMessages, CollatorProtocolMessage}, ActiveLeavesUpdate, FromOverseer, OverseerSignal,
SpawnedSubsystem, ActivatedLeaf, LeafStatus,
messages::{AllMessages, CollatorProtocolMessage},
ActivatedLeaf, ActiveLeavesUpdate, FromOverseer, LeafStatus, OverseerSignal, SpawnedSubsystem,
};
use assert_matches::assert_matches;
use futures::{channel::mpsc, executor, StreamExt, future, Future, FutureExt, SinkExt};
use polkadot_primitives::v1::Hash;
use polkadot_node_subsystem_test_helpers::{self as test_helpers, make_subsystem_context};
use std::{pin::Pin, sync::{Arc, atomic::{AtomicUsize, Ordering}}, time::Duration};
use polkadot_primitives::v1::Hash;
use std::{
pin::Pin,
sync::{
atomic::{AtomicUsize, Ordering},
Arc,
},
time::Duration,
};
use thiserror::Error;
// basic usage: in a nutshell, when you want to define a subsystem, just focus on what its jobs do;
// you can leave the subsystem itself to the job manager.
@@ -46,7 +53,7 @@ struct FakeCollatorProtocolJob {
#[derive(Debug, Error)]
enum Error {
#[error(transparent)]
Sending(#[from]mpsc::SendError),
Sending(#[from] mpsc::SendError),
}
impl JobTrait for FakeCollatorProtocolJob {
@@ -72,10 +79,12 @@ impl JobTrait for FakeCollatorProtocolJob {
let job = FakeCollatorProtocolJob { receiver };
if run_args {
sender.send_message(CollatorProtocolMessage::Invalid(
Default::default(),
Default::default(),
)).await;
sender
.send_message(CollatorProtocolMessage::Invalid(
Default::default(),
Default::default(),
))
.await;
}
// it isn't necessary to break run_loop into its own function,
@@ -92,7 +101,7 @@ impl FakeCollatorProtocolJob {
match self.receiver.next().await {
Some(_csm) => {
unimplemented!("we'd report the collator to the peer set manager here, but that's not implemented yet");
}
},
None => break,
}
}
@@ -102,32 +111,21 @@ impl FakeCollatorProtocolJob {
}
// with the job defined, it's straightforward to get a subsystem implementation.
type FakeCollatorProtocolSubsystem<Spawner> =
JobSubsystem<FakeCollatorProtocolJob, Spawner>;
type FakeCollatorProtocolSubsystem<Spawner> = JobSubsystem<FakeCollatorProtocolJob, Spawner>;
// this type lets us pretend to be the overseer
type OverseerHandle = test_helpers::TestSubsystemContextHandle<CollatorProtocolMessage>;
fn test_harness<T: Future<Output = ()>>(
run_args: bool,
test: impl FnOnce(OverseerHandle) -> T,
) {
fn test_harness<T: Future<Output = ()>>(run_args: bool, test: impl FnOnce(OverseerHandle) -> T) {
let _ = env_logger::builder()
.is_test(true)
.filter(
None,
log::LevelFilter::Trace,
)
.filter(None, log::LevelFilter::Trace)
.try_init();
let pool = sp_core::testing::TaskExecutor::new();
let (context, overseer_handle) = make_subsystem_context(pool.clone());
let subsystem = FakeCollatorProtocolSubsystem::new(
pool,
run_args,
(),
).run(context);
let subsystem = FakeCollatorProtocolSubsystem::new(pool, run_args, ()).run(context);
let test_future = test(overseer_handle);
futures::pin_mut!(subsystem, test_future);
@@ -155,19 +153,14 @@ fn starting_and_stopping_job_works() {
}),
)))
.await;
assert_matches!(
overseer_handle.recv().await,
AllMessages::CollatorProtocol(_)
);
assert_matches!(overseer_handle.recv().await, AllMessages::CollatorProtocol(_));
overseer_handle
.send(FromOverseer::Signal(OverseerSignal::ActiveLeaves(
ActiveLeavesUpdate::stop_work(relay_parent),
)))
.await;
overseer_handle
.send(FromOverseer::Signal(OverseerSignal::Conclude))
.await;
overseer_handle.send(FromOverseer::Signal(OverseerSignal::Conclude)).await;
});
}
@@ -189,20 +182,13 @@ fn sending_to_a_non_running_job_do_not_stop_the_subsystem() {
// send to a non running job
overseer_handle
.send(FromOverseer::Communication {
msg: Default::default(),
})
.send(FromOverseer::Communication { msg: Default::default() })
.await;
// the subsystem is still alive
assert_matches!(
overseer_handle.recv().await,
AllMessages::CollatorProtocol(_)
);
assert_matches!(overseer_handle.recv().await, AllMessages::CollatorProtocol(_));
overseer_handle
.send(FromOverseer::Signal(OverseerSignal::Conclude))
.await;
overseer_handle.send(FromOverseer::Signal(OverseerSignal::Conclude)).await;
});
}
@@ -216,7 +202,6 @@ fn test_subsystem_impl_and_name_derivation() {
assert_eq!(name, "FakeCollatorProtocol");
}
#[test]
fn tick_tack_metronome() {
let n = Arc::new(AtomicUsize::default());
@@ -226,13 +211,15 @@ fn tick_tack_metronome() {
let metronome = {
let n = n.clone();
let stream = Metronome::new(Duration::from_millis(137_u64));
stream.for_each(move |_res| {
let _ = n.fetch_add(1, Ordering::Relaxed);
let mut tick = tick.clone();
async move {
tick.send(()).await.expect("Test helper channel works. qed");
}
}).fuse()
stream
.for_each(move |_res| {
let _ = n.fetch_add(1, Ordering::Relaxed);
let mut tick = tick.clone();
async move {
tick.send(()).await.expect("Test helper channel works. qed");
}
})
.fuse()
};
let f2 = async move {
@@ -244,7 +231,8 @@ fn tick_tack_metronome() {
assert_eq!(n.load(Ordering::Relaxed), 3_usize);
block.next().await;
assert_eq!(n.load(Ordering::Relaxed), 4_usize);
}.fuse();
}
.fuse();
futures::pin_mut!(f2);
futures::pin_mut!(metronome);