diff --git a/polkadot/node/core/pvf/src/execute/queue.rs b/polkadot/node/core/pvf/src/execute/queue.rs index af147a2ba2..bb00a5a652 100644 --- a/polkadot/node/core/pvf/src/execute/queue.rs +++ b/polkadot/node/core/pvf/src/execute/queue.rs @@ -562,6 +562,9 @@ fn assign(queue: &mut Queue, worker: Worker, job: ExecuteJob) { thus claim_idle cannot return None; qed.", ); + queue + .metrics + .observe_execution_queued_time(job.waiting_since.elapsed().as_millis() as u32); let execution_timer = queue.metrics.time_execution(); queue.mux.push( async move { diff --git a/polkadot/node/core/pvf/src/metrics.rs b/polkadot/node/core/pvf/src/metrics.rs index 7fd876cf17..bc8d300037 100644 --- a/polkadot/node/core/pvf/src/metrics.rs +++ b/polkadot/node/core/pvf/src/metrics.rs @@ -74,6 +74,12 @@ impl Metrics { self.0.as_ref().map(|metrics| metrics.execution_time.start_timer()) } + pub(crate) fn observe_execution_queued_time(&self, queued_for_millis: u32) { + self.0.as_ref().map(|metrics| { + metrics.execution_queued_time.observe(queued_for_millis as f64 / 1000 as f64) + }); + } + /// Observe memory stats for preparation. #[allow(unused_variables)] pub(crate) fn observe_preparation_memory_metrics(&self, memory_stats: MemoryStats) { @@ -112,6 +118,7 @@ struct MetricsInner { execute_finished: prometheus::Counter, preparation_time: prometheus::Histogram, execution_time: prometheus::Histogram, + execution_queued_time: prometheus::Histogram, #[cfg(target_os = "linux")] preparation_max_rss: prometheus::Histogram, // Max. allocated memory, tracked by Jemallocator, polling-based @@ -240,6 +247,31 @@ impl metrics::Metrics for Metrics { )?, registry, )?, + execution_queued_time: prometheus::register( + prometheus::Histogram::with_opts( + prometheus::HistogramOpts::new( + "polkadot_pvf_execution_queued_time", + "Time spent in queue waiting for PVFs execution job to be assigned", + ).buckets(vec![ + 0.01, + 0.025, + 0.05, + 0.1, + 0.25, + 0.5, + 1.0, + 2.0, + 3.0, + 4.0, + 5.0, + 6.0, + 12.0, + 24.0, + 48.0, + ]), + )?, + registry, + )?, #[cfg(target_os = "linux")] preparation_max_rss: prometheus::register( prometheus::Histogram::with_opts(