mirror of
https://github.com/pezkuwichain/pezkuwi-subxt.git
synced 2026-07-20 17:25:41 +00:00
Use same fmt and clippy configs as in Substrate (#7611)
* Use same rustfmt.toml as Substrate Signed-off-by: Oliver Tale-Yazdi <oliver.tale-yazdi@parity.io> * format format file Signed-off-by: Oliver Tale-Yazdi <oliver.tale-yazdi@parity.io> * Format with new config Signed-off-by: Oliver Tale-Yazdi <oliver.tale-yazdi@parity.io> * Add Substrate Clippy config Signed-off-by: Oliver Tale-Yazdi <oliver.tale-yazdi@parity.io> * Print Clippy version in CI Otherwise its difficult to reproduce locally. Signed-off-by: Oliver Tale-Yazdi <oliver.tale-yazdi@parity.io> * Make fmt happy Signed-off-by: Oliver Tale-Yazdi <oliver.tale-yazdi@parity.io> * Update node/core/pvf/src/error.rs Co-authored-by: Tsvetomir Dimitrov <tsvetomir@parity.io> * Update node/core/pvf/src/error.rs Co-authored-by: Tsvetomir Dimitrov <tsvetomir@parity.io> --------- Signed-off-by: Oliver Tale-Yazdi <oliver.tale-yazdi@parity.io> Co-authored-by: Tsvetomir Dimitrov <tsvetomir@parity.io>
This commit is contained in:
committed by
GitHub
parent
ac435c96cf
commit
342d720573
@@ -18,8 +18,8 @@ use crate::prepare::PrepareStats;
|
||||
use parity_scale_codec::{Decode, Encode};
|
||||
use std::fmt;
|
||||
|
||||
/// Result of PVF preparation performed by the validation host. Contains stats about the preparation if
|
||||
/// successful
|
||||
/// Result of PVF preparation performed by the validation host. Contains stats about the preparation
|
||||
/// if successful
|
||||
pub type PrepareResult = Result<PrepareStats, PrepareError>;
|
||||
|
||||
/// An error that occurred during the prepare part of the PVF pipeline.
|
||||
@@ -35,13 +35,15 @@ pub enum PrepareError {
|
||||
Panic(String),
|
||||
/// Failed to prepare the PVF due to the time limit.
|
||||
TimedOut,
|
||||
/// An IO error occurred. This state is reported by either the validation host or by the worker.
|
||||
/// An IO error occurred. This state is reported by either the validation host or by the
|
||||
/// worker.
|
||||
IoErr(String),
|
||||
/// The temporary file for the artifact could not be created at the given cache path. This state is reported by the
|
||||
/// validation host (not by the worker).
|
||||
/// The temporary file for the artifact could not be created at the given cache path. This
|
||||
/// state is reported by the validation host (not by the worker).
|
||||
CreateTmpFileErr(String),
|
||||
/// The response from the worker is received, but the file cannot be renamed (moved) to the final destination
|
||||
/// location. This state is reported by the validation host (not by the worker).
|
||||
/// The response from the worker is received, but the file cannot be renamed (moved) to the
|
||||
/// final destination location. This state is reported by the validation host (not by the
|
||||
/// worker).
|
||||
RenameTmpFileErr(String),
|
||||
}
|
||||
|
||||
@@ -81,15 +83,16 @@ impl fmt::Display for PrepareError {
|
||||
|
||||
/// Some internal error occurred.
|
||||
///
|
||||
/// Should only ever be used for validation errors independent of the candidate and PVF, or for errors we ruled out
|
||||
/// during pre-checking (so preparation errors are fine).
|
||||
/// Should only ever be used for validation errors independent of the candidate and PVF, or for
|
||||
/// errors we ruled out during pre-checking (so preparation errors are fine).
|
||||
#[derive(Debug, Clone, Encode, Decode)]
|
||||
pub enum InternalValidationError {
|
||||
/// Some communication error occurred with the host.
|
||||
HostCommunication(String),
|
||||
/// Could not find or open compiled artifact file.
|
||||
CouldNotOpenFile(String),
|
||||
/// An error occurred in the CPU time monitor thread. Should be totally unrelated to validation.
|
||||
/// An error occurred in the CPU time monitor thread. Should be totally unrelated to
|
||||
/// validation.
|
||||
CpuTimeMonitorThread(String),
|
||||
/// Some non-deterministic preparation error occurred.
|
||||
NonDeterministicPrepareError(PrepareError),
|
||||
|
||||
@@ -35,10 +35,10 @@ use std::any::{Any, TypeId};
|
||||
// left for the stack; this is, of course, overridable at link time when compiling the runtime)
|
||||
// plus the number of pages specified in the `extra_heap_pages` passed to the executor.
|
||||
//
|
||||
// By default, rustc (or `lld` specifically) should allocate 1 MiB for the shadow stack, or 16 pages.
|
||||
// The data section for runtimes are typically rather small and can fit in a single digit number of
|
||||
// WASM pages, so let's say an extra 16 pages. Thus let's assume that 32 pages or 2 MiB are used for
|
||||
// these needs by default.
|
||||
// By default, rustc (or `lld` specifically) should allocate 1 MiB for the shadow stack, or 16
|
||||
// pages. The data section for runtimes are typically rather small and can fit in a single digit
|
||||
// number of WASM pages, so let's say an extra 16 pages. Thus let's assume that 32 pages or 2 MiB
|
||||
// are used for these needs by default.
|
||||
const DEFAULT_HEAP_PAGES_ESTIMATE: u32 = 32;
|
||||
const EXTRA_HEAP_PAGES: u32 = 2048;
|
||||
|
||||
@@ -65,9 +65,9 @@ pub const DEFAULT_CONFIG: Config = Config {
|
||||
//
|
||||
// Here is how the values below were chosen.
|
||||
//
|
||||
// At the moment of writing, the default native stack size limit is 1 MiB. Assuming a logical item
|
||||
// (see the docs about the field and the instrumentation algorithm) is 8 bytes, 1 MiB can
|
||||
// fit 2x 65536 logical items.
|
||||
// At the moment of writing, the default native stack size limit is 1 MiB. Assuming a
|
||||
// logical item (see the docs about the field and the instrumentation algorithm) is 8 bytes,
|
||||
// 1 MiB can fit 2x 65536 logical items.
|
||||
//
|
||||
// Since reaching the native stack limit is undesirable, we halve the logical item limit and
|
||||
// also increase the native 256x. This hopefully should preclude wasm code from reaching
|
||||
@@ -113,7 +113,7 @@ pub fn params_to_wasmtime_semantics(par: &ExecutorParams) -> Result<Semantics, S
|
||||
ExecutorParam::WasmExtBulkMemory => sem.wasm_bulk_memory = true,
|
||||
// TODO: Not implemented yet; <https://github.com/paritytech/polkadot/issues/6472>.
|
||||
ExecutorParam::PrecheckingMaxMemory(_) => (),
|
||||
ExecutorParam::PvfPrepTimeout(_, _) | ExecutorParam::PvfExecTimeout(_, _) => (), // Not used here
|
||||
ExecutorParam::PvfPrepTimeout(_, _) | ExecutorParam::PvfExecTimeout(_, _) => (), /* Not used here */
|
||||
}
|
||||
}
|
||||
sem.deterministic_stack_limit = Some(stack_limit);
|
||||
@@ -135,8 +135,8 @@ impl Executor {
|
||||
Ok(Self { config })
|
||||
}
|
||||
|
||||
/// Executes the given PVF in the form of a compiled artifact and returns the result of execution
|
||||
/// upon success.
|
||||
/// Executes the given PVF in the form of a compiled artifact and returns the result of
|
||||
/// execution upon success.
|
||||
///
|
||||
/// # Safety
|
||||
///
|
||||
|
||||
@@ -251,9 +251,9 @@ pub mod thread {
|
||||
Arc::new((Mutex::new(WaitOutcome::Pending), Condvar::new()))
|
||||
}
|
||||
|
||||
/// Runs a worker thread. Will first enable security features, and afterwards notify the threads waiting on the
|
||||
/// condvar. Catches panics during execution and resumes the panics after triggering the condvar, so that the
|
||||
/// waiting thread is notified on panics.
|
||||
/// Runs a worker thread. Will first enable security features, and afterwards notify the threads
|
||||
/// waiting on the condvar. Catches panics during execution and resumes the panics after
|
||||
/// triggering the condvar, so that the waiting thread is notified on panics.
|
||||
///
|
||||
/// # Returns
|
||||
///
|
||||
|
||||
@@ -239,7 +239,8 @@ pub fn worker_entrypoint(
|
||||
WaitOutcome::TimedOut => {
|
||||
match cpu_time_monitor_thread.join() {
|
||||
Ok(Some(cpu_time_elapsed)) => {
|
||||
// Log if we exceed the timeout and the other thread hasn't finished.
|
||||
// Log if we exceed the timeout and the other thread hasn't
|
||||
// finished.
|
||||
gum::warn!(
|
||||
target: LOG_TARGET,
|
||||
%worker_pid,
|
||||
|
||||
@@ -190,8 +190,9 @@ pub fn worker_entrypoint(
|
||||
|
||||
// If we are pre-checking, check for runtime construction errors.
|
||||
//
|
||||
// As pre-checking is more strict than just preparation in terms of memory and
|
||||
// time, it is okay to do extra checks here. This takes negligible time anyway.
|
||||
// As pre-checking is more strict than just preparation in terms of memory
|
||||
// and time, it is okay to do extra checks here. This takes negligible time
|
||||
// anyway.
|
||||
if let PrepareJobKind::Prechecking = prepare_job_kind {
|
||||
result = result.and_then(|output| {
|
||||
runtime_construction_check(output.0.as_ref(), executor_params)?;
|
||||
@@ -253,10 +254,11 @@ pub fn worker_entrypoint(
|
||||
|
||||
// Write the serialized artifact into a temp file.
|
||||
//
|
||||
// PVF host only keeps artifacts statuses in its memory, successfully
|
||||
// compiled code gets stored on the disk (and consequently deserialized
|
||||
// by execute-workers). The prepare worker is only required to send `Ok`
|
||||
// to the pool to indicate the success.
|
||||
// PVF host only keeps artifacts statuses in its memory,
|
||||
// successfully compiled code gets stored on the disk (and
|
||||
// consequently deserialized by execute-workers). The prepare worker
|
||||
// is only required to send `Ok` to the pool to indicate the
|
||||
// success.
|
||||
|
||||
gum::debug!(
|
||||
target: LOG_TARGET,
|
||||
@@ -275,7 +277,8 @@ pub fn worker_entrypoint(
|
||||
WaitOutcome::TimedOut => {
|
||||
match cpu_time_monitor_thread.join() {
|
||||
Ok(Some(cpu_time_elapsed)) => {
|
||||
// Log if we exceed the timeout and the other thread hasn't finished.
|
||||
// Log if we exceed the timeout and the other thread hasn't
|
||||
// finished.
|
||||
gum::warn!(
|
||||
target: LOG_TARGET,
|
||||
%worker_pid,
|
||||
|
||||
@@ -83,8 +83,8 @@ pub mod memory_tracker {
|
||||
///
|
||||
/// # Errors
|
||||
///
|
||||
/// For simplicity, any errors are returned as a string. As this is not a critical component, errors
|
||||
/// are used for informational purposes (logging) only.
|
||||
/// For simplicity, any errors are returned as a string. As this is not a critical component,
|
||||
/// errors are used for informational purposes (logging) only.
|
||||
pub fn memory_tracker_loop(condvar: thread::Cond) -> Result<MemoryAllocationStats, String> {
|
||||
// NOTE: This doesn't need to be too fine-grained since preparation currently takes 3-10s or
|
||||
// more. Apart from that, there is not really a science to this number.
|
||||
|
||||
@@ -224,7 +224,8 @@ impl Artifacts {
|
||||
.is_none());
|
||||
}
|
||||
|
||||
/// Remove and retrieve the artifacts from the table that are older than the supplied Time-To-Live.
|
||||
/// Remove and retrieve the artifacts from the table that are older than the supplied
|
||||
/// Time-To-Live.
|
||||
pub fn prune(&mut self, artifact_ttl: Duration) -> Vec<ArtifactId> {
|
||||
let now = SystemTime::now();
|
||||
|
||||
|
||||
@@ -38,29 +38,30 @@ pub enum InvalidCandidate {
|
||||
/// The worker has died during validation of a candidate. That may fall in one of the following
|
||||
/// categories, which we cannot distinguish programmatically:
|
||||
///
|
||||
/// (a) Some sort of transient glitch caused the worker process to abort. An example would be that
|
||||
/// the host machine ran out of free memory and the OOM killer started killing the processes,
|
||||
/// and in order to save the parent it will "sacrifice child" first.
|
||||
/// (a) Some sort of transient glitch caused the worker process to abort. An example would be
|
||||
/// that the host machine ran out of free memory and the OOM killer started killing the
|
||||
/// processes, and in order to save the parent it will "sacrifice child" first.
|
||||
///
|
||||
/// (b) The candidate triggered a code path that has lead to the process death. For example,
|
||||
/// the PVF found a way to consume unbounded amount of resources and then it either exceeded
|
||||
/// an `rlimit` (if set) or, again, invited OOM killer. Another possibility is a bug in
|
||||
/// wasmtime allowed the PVF to gain control over the execution worker.
|
||||
/// the PVF found a way to consume unbounded amount of resources and then it either
|
||||
/// exceeded an `rlimit` (if set) or, again, invited OOM killer. Another possibility is a
|
||||
/// bug in wasmtime allowed the PVF to gain control over the execution worker.
|
||||
///
|
||||
/// We attribute such an event to an *invalid candidate* in either case.
|
||||
///
|
||||
/// The rationale for this is that a glitch may lead to unfair rejecting candidate by a single
|
||||
/// validator. If the glitch is somewhat more persistent the validator will reject all candidate
|
||||
/// thrown at it and hopefully the operator notices it by decreased reward performance of the
|
||||
/// validator. On the other hand, if the worker died because of (b) we would have better chances
|
||||
/// to stop the attack.
|
||||
/// validator. If the glitch is somewhat more persistent the validator will reject all
|
||||
/// candidate thrown at it and hopefully the operator notices it by decreased reward
|
||||
/// performance of the validator. On the other hand, if the worker died because of (b) we would
|
||||
/// have better chances to stop the attack.
|
||||
AmbiguousWorkerDeath,
|
||||
/// PVF execution (compilation is not included) took more time than was allotted.
|
||||
HardTimeout,
|
||||
/// A panic occurred and we can't be sure whether the candidate is really invalid or some internal glitch occurred.
|
||||
/// Whenever we are unsure, we can never treat an error as internal as we would abstain from voting. This is bad
|
||||
/// because if the issue was due to the candidate, then all validators would abstain, stalling finality on the
|
||||
/// chain. So we will first retry the candidate, and if the issue persists we are forced to vote invalid.
|
||||
/// A panic occurred and we can't be sure whether the candidate is really invalid or some
|
||||
/// internal glitch occurred. Whenever we are unsure, we can never treat an error as internal
|
||||
/// as we would abstain from voting. This is bad because if the issue was due to the candidate,
|
||||
/// then all validators would abstain, stalling finality on the chain. So we will first retry
|
||||
/// the candidate, and if the issue persists we are forced to vote invalid.
|
||||
Panic(String),
|
||||
}
|
||||
|
||||
|
||||
@@ -419,7 +419,8 @@ fn spawn_extra_worker(queue: &mut Queue, job: ExecuteJob) {
|
||||
/// beforehand. In such a way, a race condition is avoided: during the worker being spawned,
|
||||
/// another job in the queue, with an incompatible execution environment, may become stale, and
|
||||
/// the queue would have to kill a newly started worker and spawn another one.
|
||||
/// Nevertheless, if the worker finishes executing the job, it becomes idle and may be used to execute other jobs with a compatible execution environment.
|
||||
/// Nevertheless, if the worker finishes executing the job, it becomes idle and may be used to
|
||||
/// execute other jobs with a compatible execution environment.
|
||||
async fn spawn_worker_task(
|
||||
program_path: PathBuf,
|
||||
job: ExecuteJob,
|
||||
|
||||
@@ -74,8 +74,9 @@ pub enum Outcome {
|
||||
/// PVF execution completed successfully and the result is returned. The worker is ready for
|
||||
/// another job.
|
||||
Ok { result_descriptor: ValidationResult, duration: Duration, idle_worker: IdleWorker },
|
||||
/// The candidate validation failed. It may be for example because the wasm execution triggered a trap.
|
||||
/// Errors related to the preparation process are not expected to be encountered by the execution workers.
|
||||
/// The candidate validation failed. It may be for example because the wasm execution triggered
|
||||
/// a trap. Errors related to the preparation process are not expected to be encountered by the
|
||||
/// execution workers.
|
||||
InvalidCandidate { err: String, idle_worker: IdleWorker },
|
||||
/// An internal error happened during the validation. Such an error is most likely related to
|
||||
/// some transient glitch.
|
||||
@@ -95,7 +96,8 @@ pub enum Outcome {
|
||||
/// Given the idle token of a worker and parameters of work, communicates with the worker and
|
||||
/// returns the outcome.
|
||||
///
|
||||
/// NOTE: Not returning the idle worker token in `Outcome` will trigger the child process being killed.
|
||||
/// NOTE: Not returning the idle worker token in `Outcome` will trigger the child process being
|
||||
/// killed.
|
||||
pub async fn start_work(
|
||||
worker: IdleWorker,
|
||||
artifact: ArtifactPathId,
|
||||
|
||||
@@ -455,8 +455,8 @@ async fn handle_precheck_pvf(
|
||||
ArtifactState::Preparing { waiting_for_response, num_failures: _ } =>
|
||||
waiting_for_response.push(result_sender),
|
||||
ArtifactState::FailedToProcess { error, .. } => {
|
||||
// Do not retry failed preparation if another pre-check request comes in. We do not retry pre-checking,
|
||||
// anyway.
|
||||
// Do not retry failed preparation if another pre-check request comes in. We do not
|
||||
// retry pre-checking, anyway.
|
||||
let _ = result_sender.send(PrepareResult::Err(error.clone()));
|
||||
},
|
||||
}
|
||||
@@ -470,8 +470,8 @@ async fn handle_precheck_pvf(
|
||||
|
||||
/// Handles PVF execution.
|
||||
///
|
||||
/// This will try to prepare the PVF, if a prepared artifact does not already exist. If there is already a
|
||||
/// preparation job, we coalesce the two preparation jobs.
|
||||
/// This will try to prepare the PVF, if a prepared artifact does not already exist. If there is
|
||||
/// already a preparation job, we coalesce the two preparation jobs.
|
||||
///
|
||||
/// If the prepare job succeeded previously, we will enqueue an execute job right away.
|
||||
///
|
||||
@@ -521,7 +521,8 @@ async fn handle_execute_pvf(
|
||||
"handle_execute_pvf: Re-queuing PVF preparation for prepared artifact with missing file."
|
||||
);
|
||||
|
||||
// The artifact has been prepared previously but the file is missing, prepare it again.
|
||||
// The artifact has been prepared previously but the file is missing, prepare it
|
||||
// again.
|
||||
*state = ArtifactState::Preparing {
|
||||
waiting_for_response: Vec::new(),
|
||||
num_failures: 0,
|
||||
@@ -721,8 +722,8 @@ async fn handle_prepare_done(
|
||||
pending_requests
|
||||
{
|
||||
if result_tx.is_canceled() {
|
||||
// Preparation could've taken quite a bit of time and the requester may be not interested
|
||||
// in execution anymore, in which case we just skip the request.
|
||||
// Preparation could've taken quite a bit of time and the requester may be not
|
||||
// interested in execution anymore, in which case we just skip the request.
|
||||
continue
|
||||
}
|
||||
|
||||
@@ -855,8 +856,8 @@ fn can_retry_prepare_after_failure(
|
||||
return false
|
||||
}
|
||||
|
||||
// Retry if the retry cooldown has elapsed and if we have already retried less than `NUM_PREPARE_RETRIES` times. IO
|
||||
// errors may resolve themselves.
|
||||
// Retry if the retry cooldown has elapsed and if we have already retried less than
|
||||
// `NUM_PREPARE_RETRIES` times. IO errors may resolve themselves.
|
||||
SystemTime::now() >= last_time_failed + PREPARE_FAILURE_COOLDOWN &&
|
||||
num_failures <= NUM_PREPARE_RETRIES
|
||||
}
|
||||
|
||||
@@ -32,26 +32,26 @@
|
||||
//! (a) PVF pre-checking. This takes the `Pvf` code and tries to prepare it (verify and
|
||||
//! compile) in order to pre-check its validity.
|
||||
//!
|
||||
//! (b) PVF execution. This accepts the PVF [`params`][`polkadot_parachain::primitives::ValidationParams`]
|
||||
//! and the `Pvf` code, prepares (verifies and compiles) the code, and then executes PVF
|
||||
//! with the `params`.
|
||||
//! (b) PVF execution. This accepts the PVF
|
||||
//! [`params`][`polkadot_parachain::primitives::ValidationParams`] and the `Pvf` code, prepares
|
||||
//! (verifies and compiles) the code, and then executes PVF with the `params`.
|
||||
//!
|
||||
//! (c) Heads up. This request allows to signal that the given PVF may be needed soon and that it
|
||||
//! should be prepared for execution.
|
||||
//!
|
||||
//! The preparation results are cached for some time after they either used or was signaled in heads up.
|
||||
//! All requests that depends on preparation of the same PVF are bundled together and will be executed
|
||||
//! as soon as the artifact is prepared.
|
||||
//! The preparation results are cached for some time after they either used or was signaled in heads
|
||||
//! up. All requests that depends on preparation of the same PVF are bundled together and will be
|
||||
//! executed as soon as the artifact is prepared.
|
||||
//!
|
||||
//! # Priority
|
||||
//!
|
||||
//! PVF execution requests can specify the [priority][`Priority`] with which the given request should
|
||||
//! be handled. Different priority levels have different effects. This is discussed below.
|
||||
//! PVF execution requests can specify the [priority][`Priority`] with which the given request
|
||||
//! should be handled. Different priority levels have different effects. This is discussed below.
|
||||
//!
|
||||
//! Preparation started by a heads up signal always starts with the background priority. If there
|
||||
//! is already a request for that PVF preparation under way the priority is inherited. If after heads
|
||||
//! up, a new PVF execution request comes in with a higher priority, then the original task's priority
|
||||
//! will be adjusted to match the new one if it's larger.
|
||||
//! is already a request for that PVF preparation under way the priority is inherited. If after
|
||||
//! heads up, a new PVF execution request comes in with a higher priority, then the original task's
|
||||
//! priority will be adjusted to match the new one if it's larger.
|
||||
//!
|
||||
//! Priority can never go down, only up.
|
||||
//!
|
||||
@@ -63,11 +63,11 @@
|
||||
//! dissimilar to actors. Each of such "processes" is a future task that contains an event loop that
|
||||
//! processes incoming messages, potentially delegating sub-tasks to other "processes".
|
||||
//!
|
||||
//! Two of these processes are queues. The first one is for preparation jobs and the second one is for
|
||||
//! execution. Both of the queues are backed by separate pools of workers of different kind.
|
||||
//! Two of these processes are queues. The first one is for preparation jobs and the second one is
|
||||
//! for execution. Both of the queues are backed by separate pools of workers of different kind.
|
||||
//!
|
||||
//! Preparation workers handle preparation requests by prevalidating and instrumenting PVF wasm code,
|
||||
//! and then passing it into the compiler, to prepare the artifact.
|
||||
//! Preparation workers handle preparation requests by prevalidating and instrumenting PVF wasm
|
||||
//! code, and then passing it into the compiler, to prepare the artifact.
|
||||
//!
|
||||
//! ## Artifacts
|
||||
//!
|
||||
|
||||
@@ -85,7 +85,8 @@ impl Metrics {
|
||||
|
||||
#[cfg(any(target_os = "linux", feature = "jemalloc-allocator"))]
|
||||
if let Some(tracker_stats) = memory_stats.memory_tracker_stats {
|
||||
// We convert these stats from B to KB to match the unit of `ru_maxrss` from `getrusage`.
|
||||
// We convert these stats from B to KB to match the unit of `ru_maxrss` from
|
||||
// `getrusage`.
|
||||
let max_resident_kb = (tracker_stats.resident / 1024) as f64;
|
||||
let max_allocated_kb = (tracker_stats.allocated / 1024) as f64;
|
||||
|
||||
|
||||
@@ -61,9 +61,9 @@ pub enum ToPool {
|
||||
|
||||
/// Request the given worker to start working on the given code.
|
||||
///
|
||||
/// Once the job either succeeded or failed, a [`FromPool::Concluded`] message will be sent back.
|
||||
/// It's also possible that the worker dies before handling the message in which case [`FromPool::Rip`]
|
||||
/// will be sent back.
|
||||
/// Once the job either succeeded or failed, a [`FromPool::Concluded`] message will be sent
|
||||
/// back. It's also possible that the worker dies before handling the message in which case
|
||||
/// [`FromPool::Rip`] will be sent back.
|
||||
///
|
||||
/// In either case, the worker is considered busy and no further `StartWork` messages should be
|
||||
/// sent until either `Concluded` or `Rip` message is received.
|
||||
@@ -237,8 +237,8 @@ fn handle_to_pool(
|
||||
);
|
||||
} else {
|
||||
// idle token is present after spawn and after a job is concluded;
|
||||
// the precondition for `StartWork` is it should be sent only if all previous work
|
||||
// items concluded;
|
||||
// the precondition for `StartWork` is it should be sent only if all previous
|
||||
// work items concluded;
|
||||
// thus idle token is Some;
|
||||
// qed.
|
||||
never!("unexpected absence of the idle token in prepare pool");
|
||||
@@ -311,7 +311,8 @@ fn handle_mux(
|
||||
match outcome {
|
||||
Outcome::Concluded { worker: idle, result } =>
|
||||
handle_concluded_no_rip(from_pool, spawned, worker, idle, result),
|
||||
// Return `Concluded`, but do not kill the worker since the error was on the host side.
|
||||
// Return `Concluded`, but do not kill the worker since the error was on the host
|
||||
// side.
|
||||
Outcome::CreateTmpFileErr { worker: idle, err } => handle_concluded_no_rip(
|
||||
from_pool,
|
||||
spawned,
|
||||
@@ -319,7 +320,8 @@ fn handle_mux(
|
||||
idle,
|
||||
Err(PrepareError::CreateTmpFileErr(err)),
|
||||
),
|
||||
// Return `Concluded`, but do not kill the worker since the error was on the host side.
|
||||
// Return `Concluded`, but do not kill the worker since the error was on the host
|
||||
// side.
|
||||
Outcome::RenameTmpFileErr { worker: idle, result: _, err } =>
|
||||
handle_concluded_no_rip(
|
||||
from_pool,
|
||||
|
||||
@@ -96,8 +96,9 @@ impl WorkerData {
|
||||
}
|
||||
}
|
||||
|
||||
/// A queue structured like this is prone to starving, however, we don't care that much since we expect
|
||||
/// there is going to be a limited number of critical jobs and we don't really care if background starve.
|
||||
/// A queue structured like this is prone to starving, however, we don't care that much since we
|
||||
/// expect there is going to be a limited number of critical jobs and we don't really care if
|
||||
/// background starve.
|
||||
#[derive(Default)]
|
||||
struct Unscheduled {
|
||||
normal: VecDeque<Job>,
|
||||
|
||||
@@ -247,8 +247,8 @@ where
|
||||
|
||||
let outcome = f(tmp_file.clone(), stream).await;
|
||||
|
||||
// The function called above is expected to move `tmp_file` to a new location upon success. However,
|
||||
// the function may as well fail and in that case we should remove the tmp file here.
|
||||
// The function called above is expected to move `tmp_file` to a new location upon success.
|
||||
// However, the function may as well fail and in that case we should remove the tmp file here.
|
||||
//
|
||||
// In any case, we try to remove the file here so that there are no leftovers. We only report
|
||||
// errors that are different from the `NotFound`.
|
||||
|
||||
@@ -196,13 +196,15 @@ pub enum SpawnErr {
|
||||
Handshake,
|
||||
}
|
||||
|
||||
/// This is a representation of a potentially running worker. Drop it and the process will be killed.
|
||||
/// This is a representation of a potentially running worker. Drop it and the process will be
|
||||
/// killed.
|
||||
///
|
||||
/// A worker's handle is also a future that resolves when it's detected that the worker's process
|
||||
/// has been terminated. Since the worker is running in another process it is obviously not
|
||||
/// necessary to poll this future to make the worker run, it's only for termination detection.
|
||||
///
|
||||
/// This future relies on the fact that a child process's stdout `fd` is closed upon it's termination.
|
||||
/// This future relies on the fact that a child process's stdout `fd` is closed upon it's
|
||||
/// termination.
|
||||
#[pin_project]
|
||||
pub struct WorkerHandle {
|
||||
child: process::Child,
|
||||
@@ -240,15 +242,15 @@ impl WorkerHandle {
|
||||
child_id,
|
||||
stdout,
|
||||
program: program.as_ref().to_path_buf(),
|
||||
// We don't expect the bytes to be ever read. But in case we do, we should not use a buffer
|
||||
// of a small size, because otherwise if the child process does return any data we will end up
|
||||
// issuing a syscall for each byte. We also prefer not to do allocate that on the stack, since
|
||||
// each poll the buffer will be allocated and initialized (and that's due `poll_read` takes &mut [u8]
|
||||
// and there are no guarantees that a `poll_read` won't ever read from there even though that's
|
||||
// unlikely).
|
||||
// We don't expect the bytes to be ever read. But in case we do, we should not use a
|
||||
// buffer of a small size, because otherwise if the child process does return any data
|
||||
// we will end up issuing a syscall for each byte. We also prefer not to do allocate
|
||||
// that on the stack, since each poll the buffer will be allocated and initialized (and
|
||||
// that's due `poll_read` takes &mut [u8] and there are no guarantees that a `poll_read`
|
||||
// won't ever read from there even though that's unlikely).
|
||||
//
|
||||
// OTOH, we also don't want to be super smart here and we could just afford to allocate a buffer
|
||||
// for that here.
|
||||
// OTOH, we also don't want to be super smart here and we could just afford to allocate
|
||||
// a buffer for that here.
|
||||
drop_box: vec![0; 8192].into_boxed_slice(),
|
||||
})
|
||||
}
|
||||
@@ -280,8 +282,8 @@ impl futures::Future for WorkerHandle {
|
||||
}
|
||||
},
|
||||
Err(err) => {
|
||||
// The implementation is guaranteed to not to return `WouldBlock` and Interrupted. This
|
||||
// leaves us with legit errors which we suppose were due to termination.
|
||||
// The implementation is guaranteed to not to return `WouldBlock` and Interrupted.
|
||||
// This leaves us with legit errors which we suppose were due to termination.
|
||||
|
||||
// Log the status code.
|
||||
gum::debug!(
|
||||
|
||||
Reference in New Issue
Block a user