cargo +nightly fmt (#3540)

* cargo +nightly fmt

* add cargo-fmt check to ci

* update ci

* fmt

* fmt

* skip macro

* ignore bridges
This commit is contained in:
Shawn Tabrizi
2021-08-02 12:47:33 +02:00
committed by GitHub
parent 30e3012270
commit ff5d56fb76
350 changed files with 20617 additions and 21266 deletions
@@ -16,28 +16,33 @@
use std::collections::HashSet;
use futures::channel::mpsc;
use futures::channel::oneshot;
use futures::future::select;
use futures::{FutureExt, SinkExt};
use futures::{
channel::{mpsc, oneshot},
future::select,
FutureExt, SinkExt,
};
use polkadot_erasure_coding::branch_hash;
use polkadot_node_network_protocol::request_response::{
request::{OutgoingRequest, RequestError, Requests, Recipient},
request::{OutgoingRequest, Recipient, RequestError, Requests},
v1::{ChunkFetchingRequest, ChunkFetchingResponse},
};
use polkadot_primitives::v1::{AuthorityDiscoveryId, BlakeTwo256, CandidateHash, GroupIndex, Hash, HashT, OccupiedCore, SessionIndex};
use polkadot_node_primitives::ErasureChunk;
use polkadot_subsystem::messages::{
AllMessages, AvailabilityStoreMessage, NetworkBridgeMessage, IfDisconnected,
use polkadot_primitives::v1::{
AuthorityDiscoveryId, BlakeTwo256, CandidateHash, GroupIndex, Hash, HashT, OccupiedCore,
SessionIndex,
};
use polkadot_subsystem::{
jaeger,
messages::{AllMessages, AvailabilityStoreMessage, IfDisconnected, NetworkBridgeMessage},
SubsystemContext,
};
use polkadot_subsystem::{SubsystemContext, jaeger};
use crate::{
error::{Fatal, Result},
metrics::{Metrics, FAILED, SUCCEEDED},
requester::session_cache::{BadValidators, SessionInfo},
LOG_TARGET,
metrics::{Metrics, SUCCEEDED, FAILED},
};
#[cfg(test)]
@@ -140,10 +145,7 @@ impl FetchTaskConfig {
// Don't run tasks for our backing group:
if session_info.our_group == Some(core.group_responsible) {
return FetchTaskConfig {
live_in,
prepared_running: None,
};
return FetchTaskConfig { live_in, prepared_running: None }
}
let span = jaeger::Span::new(core.candidate_hash, "availability-distribution")
@@ -165,10 +167,7 @@ impl FetchTaskConfig {
sender,
span,
};
FetchTaskConfig {
live_in,
prepared_running: Some(prepared_running),
}
FetchTaskConfig { live_in, prepared_running: Some(prepared_running) }
}
}
@@ -180,10 +179,7 @@ impl FetchTask {
where
Context: SubsystemContext,
{
let FetchTaskConfig {
prepared_running,
live_in,
} = config;
let FetchTaskConfig { prepared_running, live_in } = config;
if let Some(running) = prepared_running {
let (handle, kill) = oneshot::channel();
@@ -191,15 +187,9 @@ impl FetchTask {
ctx.spawn("chunk-fetcher", running.run(kill).boxed())
.map_err(|e| Fatal::SpawnTask(e))?;
Ok(FetchTask {
live_in,
state: FetchedState::Started(handle),
})
Ok(FetchTask { live_in, state: FetchedState::Started(handle) })
} else {
Ok(FetchTask {
live_in,
state: FetchedState::Canceled,
})
Ok(FetchTask { live_in, state: FetchedState::Canceled })
}
}
@@ -261,7 +251,9 @@ impl RunningTask {
let mut bad_validators = Vec::new();
let mut succeeded = false;
let mut count: u32 = 0;
let mut _span = self.span.child("fetch-task")
let mut _span = self
.span
.child("fetch-task")
.with_chunk_index(self.request.index.0)
.with_relay_parent(self.relay_parent);
// Try validators in reverse order:
@@ -271,7 +263,7 @@ impl RunningTask {
if count > 0 {
self.metrics.on_retry();
}
count +=1;
count += 1;
// Send request:
let resp = match self.do_request(&validator).await {
@@ -283,16 +275,14 @@ impl RunningTask {
);
self.metrics.on_fetch(FAILED);
return
}
},
Err(TaskError::PeerError) => {
bad_validators.push(validator);
continue
}
},
};
let chunk = match resp {
ChunkFetchingResponse::Chunk(resp) => {
resp.recombine_into_chunk(&self.request)
}
ChunkFetchingResponse::Chunk(resp) => resp.recombine_into_chunk(&self.request),
ChunkFetchingResponse::NoSuchChunk => {
tracing::debug!(
target: LOG_TARGET,
@@ -301,20 +291,20 @@ impl RunningTask {
);
bad_validators.push(validator);
continue
}
},
};
// Data genuine?
if !self.validate_chunk(&validator, &chunk) {
bad_validators.push(validator);
continue;
continue
}
// Ok, let's store it and be happy:
self.store_chunk(chunk).await;
succeeded = true;
_span.add_string_tag("success", "true");
break;
break
}
_span.add_int_tag("tries", count as _);
if succeeded {
@@ -337,7 +327,7 @@ impl RunningTask {
self.sender
.send(FromFetchTask::Message(AllMessages::NetworkBridge(
NetworkBridgeMessage::SendRequests(vec![requests], IfDisconnected::TryConnect)
NetworkBridgeMessage::SendRequests(vec![requests], IfDisconnected::TryConnect),
)))
.await
.map_err(|_| TaskError::ShuttingDown)?;
@@ -352,7 +342,7 @@ impl RunningTask {
"Peer sent us invalid erasure chunk data"
);
Err(TaskError::PeerError)
}
},
Err(RequestError::NetworkError(err)) => {
tracing::warn!(
target: LOG_TARGET,
@@ -361,13 +351,13 @@ impl RunningTask {
"Some network error occurred when fetching erasure chunk"
);
Err(TaskError::PeerError)
}
},
Err(RequestError::Canceled(oneshot::Canceled)) => {
tracing::warn!(target: LOG_TARGET,
origin= ?validator,
"Erasure chunk request got canceled");
Err(TaskError::PeerError)
}
},
}
}
@@ -383,13 +373,13 @@ impl RunningTask {
error = ?e,
"Failed to calculate chunk merkle proof",
);
return false;
}
return false
},
};
let erasure_chunk_hash = BlakeTwo256::hash(&chunk.chunk);
if anticipated_hash != erasure_chunk_hash {
tracing::warn!(target: LOG_TARGET, origin = ?validator, "Received chunk does not match merkle tree");
return false;
return false
}
true
}
@@ -437,12 +427,9 @@ impl RunningTask {
}
async fn conclude_fail(&mut self) {
if let Err(err) = self.sender.send(FromFetchTask::Failed(self.request.candidate_hash)).await {
tracing::warn!(
target: LOG_TARGET,
?err,
"Sending `Failed` message for task failed"
);
if let Err(err) = self.sender.send(FromFetchTask::Failed(self.request.candidate_hash)).await
{
tracing::warn!(target: LOG_TARGET, ?err, "Sending `Failed` message for task failed");
}
}
}
@@ -16,24 +16,24 @@
use std::collections::HashMap;
use parity_scale_codec::Encode;
use futures::channel::{mpsc, oneshot};
use futures::{executor, Future, FutureExt, StreamExt, select};
use futures::task::{Poll, Context, noop_waker};
use futures::{
channel::{mpsc, oneshot},
executor, select,
task::{noop_waker, Context, Poll},
Future, FutureExt, StreamExt,
};
use sc_network as network;
use sp_keyring::Sr25519Keyring;
use polkadot_primitives::v1::{CandidateHash, ValidatorIndex};
use polkadot_node_network_protocol::request_response::{v1, Recipient};
use polkadot_node_primitives::{BlockData, PoV};
use polkadot_node_network_protocol::request_response::v1;
use polkadot_node_network_protocol::request_response::Recipient;
use polkadot_primitives::v1::{CandidateHash, ValidatorIndex};
use crate::metrics::Metrics;
use crate::tests::mock::get_valid_chunk_data;
use super::*;
use crate::{metrics::Metrics, tests::mock::get_valid_chunk_data};
#[test]
fn task_can_be_canceled() {
@@ -54,16 +54,14 @@ fn task_does_not_accept_invalid_chunk() {
let validators = vec![Sr25519Keyring::Alice.public().into()];
task.group = validators;
let test = TestRun {
chunk_responses: {
chunk_responses: {
let mut m = HashMap::new();
m.insert(
Recipient::Authority(Sr25519Keyring::Alice.public().into()),
ChunkFetchingResponse::Chunk(
v1::ChunkResponse {
chunk: vec![1,2,3],
proof: vec![vec![9,8,2], vec![2,3,4]],
}
)
ChunkFetchingResponse::Chunk(v1::ChunkResponse {
chunk: vec![1, 2, 3],
proof: vec![vec![9, 8, 2], vec![2, 3, 4]],
}),
);
m
},
@@ -75,9 +73,7 @@ fn task_does_not_accept_invalid_chunk() {
#[test]
fn task_stores_valid_chunk() {
let (mut task, rx) = get_test_running_task();
let pov = PoV {
block_data: BlockData(vec![45, 46, 47]),
};
let pov = PoV { block_data: BlockData(vec![45, 46, 47]) };
let (root_hash, chunk) = get_valid_chunk_data(pov);
task.erasure_root = root_hash;
task.request.index = chunk.index;
@@ -86,16 +82,14 @@ fn task_stores_valid_chunk() {
task.group = validators;
let test = TestRun {
chunk_responses: {
chunk_responses: {
let mut m = HashMap::new();
m.insert(
Recipient::Authority(Sr25519Keyring::Alice.public().into()),
ChunkFetchingResponse::Chunk(
v1::ChunkResponse {
chunk: chunk.chunk.clone(),
proof: chunk.proof,
}
)
ChunkFetchingResponse::Chunk(v1::ChunkResponse {
chunk: chunk.chunk.clone(),
proof: chunk.proof,
}),
);
m
},
@@ -111,27 +105,23 @@ fn task_stores_valid_chunk() {
#[test]
fn task_does_not_accept_wrongly_indexed_chunk() {
let (mut task, rx) = get_test_running_task();
let pov = PoV {
block_data: BlockData(vec![45, 46, 47]),
};
let pov = PoV { block_data: BlockData(vec![45, 46, 47]) };
let (root_hash, chunk) = get_valid_chunk_data(pov);
task.erasure_root = root_hash;
task.request.index = ValidatorIndex(chunk.index.0+1);
task.request.index = ValidatorIndex(chunk.index.0 + 1);
let validators = vec![Sr25519Keyring::Alice.public().into()];
task.group = validators;
let test = TestRun {
chunk_responses: {
chunk_responses: {
let mut m = HashMap::new();
m.insert(
Recipient::Authority(Sr25519Keyring::Alice.public().into()),
ChunkFetchingResponse::Chunk(
v1::ChunkResponse {
chunk: chunk.chunk.clone(),
proof: chunk.proof,
}
)
ChunkFetchingResponse::Chunk(v1::ChunkResponse {
chunk: chunk.chunk.clone(),
proof: chunk.proof,
}),
);
m
},
@@ -144,46 +134,44 @@ fn task_does_not_accept_wrongly_indexed_chunk() {
#[test]
fn task_stores_valid_chunk_if_there_is_one() {
let (mut task, rx) = get_test_running_task();
let pov = PoV {
block_data: BlockData(vec![45, 46, 47]),
};
let pov = PoV { block_data: BlockData(vec![45, 46, 47]) };
let (root_hash, chunk) = get_valid_chunk_data(pov);
task.erasure_root = root_hash;
task.request.index = chunk.index;
let validators = [
// Only Alice has valid chunk - should succeed, even though she is tried last.
Sr25519Keyring::Alice,
Sr25519Keyring::Bob, Sr25519Keyring::Charlie,
Sr25519Keyring::Dave, Sr25519Keyring::Eve,
]
.iter().map(|v| v.public().into()).collect::<Vec<_>>();
// Only Alice has valid chunk - should succeed, even though she is tried last.
Sr25519Keyring::Alice,
Sr25519Keyring::Bob,
Sr25519Keyring::Charlie,
Sr25519Keyring::Dave,
Sr25519Keyring::Eve,
]
.iter()
.map(|v| v.public().into())
.collect::<Vec<_>>();
task.group = validators;
let test = TestRun {
chunk_responses: {
chunk_responses: {
let mut m = HashMap::new();
m.insert(
Recipient::Authority(Sr25519Keyring::Alice.public().into()),
ChunkFetchingResponse::Chunk(
v1::ChunkResponse {
chunk: chunk.chunk.clone(),
proof: chunk.proof,
}
)
ChunkFetchingResponse::Chunk(v1::ChunkResponse {
chunk: chunk.chunk.clone(),
proof: chunk.proof,
}),
);
m.insert(
Recipient::Authority(Sr25519Keyring::Bob.public().into()),
ChunkFetchingResponse::NoSuchChunk
ChunkFetchingResponse::NoSuchChunk,
);
m.insert(
Recipient::Authority(Sr25519Keyring::Charlie.public().into()),
ChunkFetchingResponse::Chunk(
v1::ChunkResponse {
chunk: vec![1,2,3],
proof: vec![vec![9,8,2], vec![2,3,4]],
}
)
ChunkFetchingResponse::Chunk(v1::ChunkResponse {
chunk: vec![1, 2, 3],
proof: vec![vec![9, 8, 2], vec![2, 3, 4]],
}),
);
m
@@ -205,7 +193,6 @@ struct TestRun {
valid_chunks: HashSet<Vec<u8>>,
}
impl TestRun {
fn run(self, task: RunningTask, rx: mpsc::Receiver<FromFetchTask>) {
sp_tracing::try_init_simple();
@@ -228,8 +215,7 @@ impl TestRun {
match msg {
FromFetchTask::Concluded(_) => break,
FromFetchTask::Failed(_) => break,
FromFetchTask::Message(msg) =>
end_ok = self.handle_message(msg).await,
FromFetchTask::Message(msg) => end_ok = self.handle_message(msg).await,
}
}
if !end_ok {
@@ -242,44 +228,50 @@ impl TestRun {
/// end.
async fn handle_message(&self, msg: AllMessages) -> bool {
match msg {
AllMessages::NetworkBridge(NetworkBridgeMessage::SendRequests(reqs, IfDisconnected::TryConnect)) => {
AllMessages::NetworkBridge(NetworkBridgeMessage::SendRequests(
reqs,
IfDisconnected::TryConnect,
)) => {
let mut valid_responses = 0;
for req in reqs {
let req = match req {
Requests::ChunkFetching(req) => req,
_ => panic!("Unexpected request"),
};
let response = self.chunk_responses.get(&req.peer)
.ok_or(network::RequestFailure::Refused);
let response =
self.chunk_responses.get(&req.peer).ok_or(network::RequestFailure::Refused);
if let Ok(ChunkFetchingResponse::Chunk(resp)) = &response {
if self.valid_chunks.contains(&resp.chunk) {
valid_responses += 1;
}
}
req.pending_response.send(response.map(Encode::encode))
req.pending_response
.send(response.map(Encode::encode))
.expect("Sending response should succeed");
}
return (valid_responses == 0) && self.valid_chunks.is_empty()
}
AllMessages::AvailabilityStore(
AvailabilityStoreMessage::StoreChunk { chunk, tx, .. }
) => {
},
AllMessages::AvailabilityStore(AvailabilityStoreMessage::StoreChunk {
chunk,
tx,
..
}) => {
assert!(self.valid_chunks.contains(&chunk.chunk));
tx.send(Ok(())).expect("Answering fetching task should work");
return true
}
},
_ => {
tracing::debug!(target: LOG_TARGET, "Unexpected message");
return false
}
},
}
}
}
/// Get a `RunningTask` filled with dummy values.
fn get_test_running_task() -> (RunningTask, mpsc::Receiver<FromFetchTask>) {
let (tx,rx) = mpsc::channel(0);
let (tx, rx) = mpsc::channel(0);
(
RunningTask {
@@ -287,7 +279,7 @@ fn get_test_running_task() -> (RunningTask, mpsc::Receiver<FromFetchTask>) {
group_index: GroupIndex(0),
group: Vec::new(),
request: ChunkFetchingRequest {
candidate_hash: CandidateHash([43u8;32].into()),
candidate_hash: CandidateHash([43u8; 32].into()),
index: ValidatorIndex(0),
},
erasure_root: Hash::repeat_byte(99),
@@ -296,6 +288,6 @@ fn get_test_running_task() -> (RunningTask, mpsc::Receiver<FromFetchTask>) {
metrics: Metrics::new_dummy(),
span: jaeger::Span::Disabled,
},
rx
rx,
)
}
@@ -17,12 +17,14 @@
//! Requester takes care of requesting erasure chunks for candidates that are pending
//! availability.
use std::collections::{
hash_map::{Entry, HashMap},
hash_set::HashSet,
use std::{
collections::{
hash_map::{Entry, HashMap},
hash_set::HashSet,
},
iter::IntoIterator,
pin::Pin,
};
use std::iter::IntoIterator;
use std::pin::Pin;
use futures::{
channel::mpsc,
@@ -30,20 +32,18 @@ use futures::{
Stream,
};
use polkadot_node_subsystem_util::runtime::{RuntimeInfo, get_occupied_cores};
use polkadot_node_subsystem_util::runtime::{get_occupied_cores, RuntimeInfo};
use polkadot_primitives::v1::{CandidateHash, Hash, OccupiedCore};
use polkadot_subsystem::{
messages::AllMessages,
ActiveLeavesUpdate, SubsystemContext, ActivatedLeaf,
messages::AllMessages, ActivatedLeaf, ActiveLeavesUpdate, SubsystemContext,
};
use super::{LOG_TARGET, Metrics};
use super::{Metrics, LOG_TARGET};
/// Cache for session information.
mod session_cache;
use session_cache::SessionCache;
/// A task fetching a particular chunk.
mod fetch_task;
use fetch_task::{FetchTask, FetchTaskConfig, FromFetchTask};
@@ -81,13 +81,7 @@ impl Requester {
/// by advancing the stream.
pub fn new(metrics: Metrics) -> Self {
let (tx, rx) = mpsc::channel(1);
Requester {
fetches: HashMap::new(),
session_cache: SessionCache::new(),
tx,
rx,
metrics,
}
Requester { fetches: HashMap::new(), session_cache: SessionCache::new(), tx, rx, metrics }
}
/// Update heads that need availability distribution.
///
@@ -101,15 +95,8 @@ impl Requester {
where
Context: SubsystemContext,
{
tracing::trace!(
target: LOG_TARGET,
?update,
"Update fetching heads"
);
let ActiveLeavesUpdate {
activated,
deactivated,
} = update;
tracing::trace!(target: LOG_TARGET, ?update, "Update fetching heads");
let ActiveLeavesUpdate { activated, deactivated } = update;
// Order important! We need to handle activated, prior to deactivated, otherwise we might
// cancel still needed jobs.
self.start_requesting_chunks(ctx, runtime, activated.into_iter()).await?;
@@ -194,7 +181,7 @@ impl Requester {
e.insert(FetchTask::start(task_cfg, ctx).await?);
}
// Not a validator, nothing to do.
}
},
}
}
Ok(())
@@ -204,28 +191,21 @@ impl Requester {
impl Stream for Requester {
type Item = AllMessages;
fn poll_next(
mut self: Pin<&mut Self>,
ctx: &mut Context,
) -> Poll<Option<AllMessages>> {
fn poll_next(mut self: Pin<&mut Self>, ctx: &mut Context) -> Poll<Option<AllMessages>> {
loop {
match Pin::new(&mut self.rx).poll_next(ctx) {
Poll::Ready(Some(FromFetchTask::Message(m))) =>
return Poll::Ready(Some(m)),
Poll::Ready(Some(FromFetchTask::Message(m))) => return Poll::Ready(Some(m)),
Poll::Ready(Some(FromFetchTask::Concluded(Some(bad_boys)))) => {
self.session_cache.report_bad_log(bad_boys);
continue
}
Poll::Ready(Some(FromFetchTask::Concluded(None))) =>
continue,
},
Poll::Ready(Some(FromFetchTask::Concluded(None))) => continue,
Poll::Ready(Some(FromFetchTask::Failed(candidate_hash))) => {
// Make sure we retry on next block still pending availability.
self.fetches.remove(&candidate_hash);
}
Poll::Ready(None) =>
return Poll::Ready(None),
Poll::Pending =>
return Poll::Pending,
},
Poll::Ready(None) => return Poll::Ready(None),
Poll::Pending => return Poll::Pending,
}
}
}
@@ -34,7 +34,6 @@ use crate::{
///
/// It should be ensured that a cached session stays live in the cache as long as we might need it.
pub struct SessionCache {
/// Look up cached sessions by `SessionIndex`.
///
/// Note: Performance of fetching is really secondary here, but we need to ensure we are going
@@ -110,12 +109,11 @@ impl SessionCache {
if let Some(o_info) = self.session_info_cache.get(&session_index) {
tracing::trace!(target: LOG_TARGET, session_index, "Got session from lru");
return Ok(Some(with_info(o_info)));
return Ok(Some(with_info(o_info)))
}
if let Some(info) = self
.query_info_from_runtime(ctx, runtime, parent, session_index)
.await?
if let Some(info) =
self.query_info_from_runtime(ctx, runtime, parent, session_index).await?
{
tracing::trace!(target: LOG_TARGET, session_index, "Calling `with_info`");
let r = with_info(&info);
@@ -132,7 +130,7 @@ impl SessionCache {
/// Not being able to report bad validators is not fatal, so we should not shutdown the
/// subsystem on this.
pub fn report_bad_log(&mut self, report: BadValidators) {
if let Err(err) = self.report_bad(report) {
if let Err(err) = self.report_bad(report) {
tracing::warn!(
target: LOG_TARGET,
err = ?err,
@@ -150,10 +148,9 @@ impl SessionCache {
.session_info_cache
.get_mut(&report.session_index)
.ok_or(NonFatal::NoSuchCachedSession)?;
let group = session
.validator_groups
.get_mut(report.group_index.0 as usize)
.expect("A bad validator report must contain a valid group for the reported session. qed.");
let group = session.validator_groups.get_mut(report.group_index.0 as usize).expect(
"A bad validator report must contain a valid group for the reported session. qed.",
);
let bad_set = report.bad_validators.iter().collect::<HashSet<_>>();
// Get rid of bad boys:
@@ -212,12 +209,7 @@ impl SessionCache {
})
.collect();
let info = SessionInfo {
validator_groups,
our_index,
session_index,
our_group,
};
let info = SessionInfo { validator_groups, our_index, session_index, our_group };
return Ok(Some(info))
}
return Ok(None)