cargo +nightly fmt (#3540)

* cargo +nightly fmt

* add cargo-fmt check to ci

* update ci

* fmt

* fmt

* skip macro

* ignore bridges
This commit is contained in:
Shawn Tabrizi
2021-08-02 12:47:33 +02:00
committed by GitHub
parent 30e3012270
commit ff5d56fb76
350 changed files with 20617 additions and 21266 deletions
+1 -1
View File
@@ -23,5 +23,5 @@
mod queue;
mod worker;
pub use queue::{ToQueue, start};
pub use queue::{start, ToQueue};
pub use worker::worker_entrypoint;
+31 -79
View File
@@ -16,31 +16,27 @@
//! A queue that handles requests for PVF execution.
use crate::{
worker_common::{IdleWorker, WorkerHandle},
host::ResultSender,
LOG_TARGET, InvalidCandidate, ValidationError,
};
use super::worker::Outcome;
use std::{collections::VecDeque, fmt, time::Duration};
use crate::{
host::ResultSender,
worker_common::{IdleWorker, WorkerHandle},
InvalidCandidate, ValidationError, LOG_TARGET,
};
use async_std::path::PathBuf;
use futures::{
Future, FutureExt,
channel::mpsc,
future::BoxFuture,
stream::{FuturesUnordered, StreamExt as _},
Future, FutureExt,
};
use async_std::path::PathBuf;
use slotmap::HopSlotMap;
use std::{collections::VecDeque, fmt, time::Duration};
slotmap::new_key_type! { struct Worker; }
#[derive(Debug)]
pub enum ToQueue {
Enqueue {
artifact_path: PathBuf,
params: Vec<u8>,
result_tx: ResultSender,
},
Enqueue { artifact_path: PathBuf, params: Vec<u8>, result_tx: ResultSender },
}
struct ExecuteJob {
@@ -86,11 +82,7 @@ impl Workers {
///
/// Returns `None` if either worker is not recognized or idle token is absent.
fn claim_idle(&mut self, worker: Worker) -> Option<IdleWorker> {
self
.running
.get_mut(worker)?
.idle
.take()
self.running.get_mut(worker)?.idle.take()
}
}
@@ -167,17 +159,9 @@ async fn purge_dead(workers: &mut Workers) {
}
fn handle_to_queue(queue: &mut Queue, to_queue: ToQueue) {
let ToQueue::Enqueue {
artifact_path,
params,
result_tx,
} = to_queue;
let ToQueue::Enqueue { artifact_path, params, result_tx } = to_queue;
let job = ExecuteJob {
artifact_path,
params,
result_tx,
};
let job = ExecuteJob { artifact_path, params, result_tx };
if let Some(available) = queue.workers.find_available() {
assign(queue, available, job);
@@ -194,18 +178,15 @@ async fn handle_mux(queue: &mut Queue, event: QueueEvent) {
QueueEvent::Spawn((idle, handle)) => {
queue.workers.spawn_inflight -= 1;
let worker = queue.workers.running.insert(WorkerData {
idle: Some(idle),
handle,
});
let worker = queue.workers.running.insert(WorkerData { idle: Some(idle), handle });
if let Some(job) = queue.queue.pop_front() {
assign(queue, worker, job);
}
}
},
QueueEvent::StartWork(worker, outcome, result_tx) => {
handle_job_finish(queue, worker, outcome, result_tx);
}
},
}
}
@@ -213,38 +194,22 @@ async fn handle_mux(queue: &mut Queue, event: QueueEvent) {
/// worker. Otherwise, puts back into the available workers list.
fn handle_job_finish(queue: &mut Queue, worker: Worker, outcome: Outcome, result_tx: ResultSender) {
let (idle_worker, result) = match outcome {
Outcome::Ok {
result_descriptor,
duration_ms,
idle_worker,
} => {
Outcome::Ok { result_descriptor, duration_ms, idle_worker } => {
// TODO: propagate the soft timeout
drop(duration_ms);
(Some(idle_worker), Ok(result_descriptor))
}
},
Outcome::InvalidCandidate { err, idle_worker } => (
Some(idle_worker),
Err(ValidationError::InvalidCandidate(
InvalidCandidate::WorkerReportedError(err),
)),
),
Outcome::InternalError { err, idle_worker } => (
Some(idle_worker),
Err(ValidationError::InternalError(err)),
),
Outcome::HardTimeout => (
None,
Err(ValidationError::InvalidCandidate(
InvalidCandidate::HardTimeout,
)),
),
Outcome::IoErr => (
None,
Err(ValidationError::InvalidCandidate(
InvalidCandidate::AmbigiousWorkerDeath,
)),
Err(ValidationError::InvalidCandidate(InvalidCandidate::WorkerReportedError(err))),
),
Outcome::InternalError { err, idle_worker } =>
(Some(idle_worker), Err(ValidationError::InternalError(err))),
Outcome::HardTimeout =>
(None, Err(ValidationError::InvalidCandidate(InvalidCandidate::HardTimeout))),
Outcome::IoErr =>
(None, Err(ValidationError::InvalidCandidate(InvalidCandidate::AmbigiousWorkerDeath))),
};
// First we send the result. It may fail due the other end of the channel being dropped, that's
@@ -293,15 +258,11 @@ async fn spawn_worker_task(program_path: PathBuf, spawn_timeout: Duration) -> Qu
match super::worker::spawn(&program_path, spawn_timeout).await {
Ok((idle, handle)) => break QueueEvent::Spawn((idle, handle)),
Err(err) => {
tracing::warn!(
target: LOG_TARGET,
"failed to spawn an execute worker: {:?}",
err,
);
tracing::warn!(target: LOG_TARGET, "failed to spawn an execute worker: {:?}", err,);
// Assume that the failure intermittent and retry after a delay.
Delay::new(Duration::from_secs(3)).await;
}
},
}
}
}
@@ -310,14 +271,11 @@ async fn spawn_worker_task(program_path: PathBuf, spawn_timeout: Duration) -> Qu
///
/// The worker must be running and idle.
fn assign(queue: &mut Queue, worker: Worker, job: ExecuteJob) {
let idle = queue
.workers
.claim_idle(worker)
.expect(
"this caller must supply a worker which is idle and running;
let idle = queue.workers.claim_idle(worker).expect(
"this caller must supply a worker which is idle and running;
thus claim_idle cannot return None;
qed."
);
qed.",
);
queue.mux.push(
async move {
let outcome = super::worker::start_work(idle, job.artifact_path, job.params).await;
@@ -333,12 +291,6 @@ pub fn start(
spawn_timeout: Duration,
) -> (mpsc::Sender<ToQueue>, impl Future<Output = ()>) {
let (to_queue_tx, to_queue_rx) = mpsc::channel(20);
let run = Queue::new(
program_path,
worker_capacity,
spawn_timeout,
to_queue_rx,
)
.run();
let run = Queue::new(program_path, worker_capacity, spawn_timeout, to_queue_rx).run();
(to_queue_tx, run)
}
+35 -84
View File
@@ -16,14 +16,13 @@
use crate::{
artifacts::Artifact,
LOG_TARGET,
executor_intf::TaskExecutor,
worker_common::{
IdleWorker, SpawnErr, WorkerHandle, bytes_to_path, framed_recv, framed_send, path_to_bytes,
spawn_with_program_path, worker_event_loop,
bytes_to_path, framed_recv, framed_send, path_to_bytes, spawn_with_program_path,
worker_event_loop, IdleWorker, SpawnErr, WorkerHandle,
},
LOG_TARGET,
};
use std::time::{Duration, Instant};
use async_std::{
io,
os::unix::net::UnixStream,
@@ -31,8 +30,9 @@ use async_std::{
};
use futures::FutureExt;
use futures_timer::Delay;
use parity_scale_codec::{Decode, Encode};
use polkadot_parachain::primitives::ValidationResult;
use parity_scale_codec::{Encode, Decode};
use std::time::{Duration, Instant};
const EXECUTION_TIMEOUT: Duration = Duration::from_secs(3);
@@ -43,36 +43,20 @@ pub async fn spawn(
program_path: &Path,
spawn_timeout: Duration,
) -> Result<(IdleWorker, WorkerHandle), SpawnErr> {
spawn_with_program_path(
"execute",
program_path,
&["execute-worker"],
spawn_timeout,
)
.await
spawn_with_program_path("execute", program_path, &["execute-worker"], spawn_timeout).await
}
/// Outcome of PVF execution.
pub enum Outcome {
/// PVF execution completed successfully and the result is returned. The worker is ready for
/// another job.
Ok {
result_descriptor: ValidationResult,
duration_ms: u64,
idle_worker: IdleWorker,
},
Ok { result_descriptor: ValidationResult, duration_ms: u64, idle_worker: IdleWorker },
/// The candidate validation failed. It may be for example because the preparation process
/// produced an error or the wasm execution triggered a trap.
InvalidCandidate {
err: String,
idle_worker: IdleWorker,
},
InvalidCandidate { err: String, idle_worker: IdleWorker },
/// An internal error happened during the validation. Such an error is most likely related to
/// some transient glitch.
InternalError {
err: String,
idle_worker: IdleWorker,
},
InternalError { err: String, idle_worker: IdleWorker },
/// The execution time exceeded the hard limit. The worker is terminated.
HardTimeout,
/// An I/O error happened during communication with the worker. This may mean that the worker
@@ -97,7 +81,7 @@ pub async fn start_work(
);
if send_request(&mut stream, &artifact_path, &validation_params).await.is_err() {
return Outcome::IoErr;
return Outcome::IoErr
}
let response = futures::select! {
@@ -111,22 +95,12 @@ pub async fn start_work(
};
match response {
Response::Ok {
result_descriptor,
duration_ms,
} => Outcome::Ok {
result_descriptor,
duration_ms,
idle_worker: IdleWorker { stream, pid },
},
Response::InvalidCandidate(err) => Outcome::InvalidCandidate {
err,
idle_worker: IdleWorker { stream, pid },
},
Response::InternalError(err) => Outcome::InternalError {
err,
idle_worker: IdleWorker { stream, pid },
},
Response::Ok { result_descriptor, duration_ms } =>
Outcome::Ok { result_descriptor, duration_ms, idle_worker: IdleWorker { stream, pid } },
Response::InvalidCandidate(err) =>
Outcome::InvalidCandidate { err, idle_worker: IdleWorker { stream, pid } },
Response::InternalError(err) =>
Outcome::InternalError { err, idle_worker: IdleWorker { stream, pid } },
}
}
@@ -167,10 +141,7 @@ async fn recv_response(stream: &mut UnixStream) -> io::Result<Response> {
#[derive(Encode, Decode)]
enum Response {
Ok {
result_descriptor: ValidationResult,
duration_ms: u64,
},
Ok { result_descriptor: ValidationResult, duration_ms: u64 },
InvalidCandidate(String),
InternalError(String),
}
@@ -190,10 +161,7 @@ impl Response {
pub fn worker_entrypoint(socket_path: &str) {
worker_event_loop("execute", socket_path, |mut stream| async move {
let executor = TaskExecutor::new().map_err(|e| {
io::Error::new(
io::ErrorKind::Other,
format!("cannot create task executor: {}", e),
)
io::Error::new(io::ErrorKind::Other, format!("cannot create task executor: {}", e))
})?;
loop {
let (artifact_path, params) = recv_request(&mut stream).await?;
@@ -215,13 +183,12 @@ async fn validate_using_artifact(
spawner: &TaskExecutor,
) -> Response {
let artifact_bytes = match async_std::fs::read(artifact_path).await {
Err(e) => {
Err(e) =>
return Response::InternalError(format!(
"failed to read the artifact at {}: {:?}",
artifact_path.display(),
e,
))
}
)),
Ok(b) => b,
};
@@ -231,47 +198,31 @@ async fn validate_using_artifact(
};
let compiled_artifact = match &artifact {
Artifact::PrevalidationErr(msg) => {
return Response::format_invalid("prevalidation", msg);
}
Artifact::PreparationErr(msg) => {
return Response::format_invalid("preparation", msg);
}
Artifact::DidntMakeIt => {
return Response::format_invalid("preparation timeout", "");
}
Artifact::PrevalidationErr(msg) => return Response::format_invalid("prevalidation", msg),
Artifact::PreparationErr(msg) => return Response::format_invalid("preparation", msg),
Artifact::DidntMakeIt => return Response::format_invalid("preparation timeout", ""),
Artifact::Compiled { compiled_artifact } => compiled_artifact,
};
let validation_started_at = Instant::now();
let descriptor_bytes =
match unsafe {
// SAFETY: this should be safe since the compiled artifact passed here comes from the
// file created by the prepare workers. These files are obtained by calling
// [`executor_intf::prepare`].
crate::executor_intf::execute(compiled_artifact, params, spawner.clone())
} {
Err(err) => {
return Response::format_invalid("execute", &err.to_string());
}
Ok(d) => d,
};
let descriptor_bytes = match unsafe {
// SAFETY: this should be safe since the compiled artifact passed here comes from the
// file created by the prepare workers. These files are obtained by calling
// [`executor_intf::prepare`].
crate::executor_intf::execute(compiled_artifact, params, spawner.clone())
} {
Err(err) => return Response::format_invalid("execute", &err.to_string()),
Ok(d) => d,
};
let duration_ms = validation_started_at.elapsed().as_millis() as u64;
let result_descriptor = match ValidationResult::decode(&mut &descriptor_bytes[..]) {
Err(err) => {
return Response::InvalidCandidate(format!(
"validation result decoding failed: {}",
err
))
}
Err(err) =>
return Response::InvalidCandidate(format!("validation result decoding failed: {}", err)),
Ok(r) => r,
};
Response::Ok {
result_descriptor,
duration_ms,
}
Response::Ok { result_descriptor, duration_ms }
}