track tof for inter subsystem messages (#5135)

polkadot/node/overseer/src/lib.rs

@@ -555,7 +555,7 @@ where
 		// We combine the amount of messages from subsystems to the overseer
 		// as well as the amount of messages from external sources to the overseer
 		// into one `to_overseer` value.
-		metronome_metrics.channel_fill_level_snapshot(
+		metronome_metrics.channel_metrics_snapshot(
 			subsystem_meters
 				.iter()
 				.cloned()

polkadot/node/overseer/src/metrics.rs

@@ -27,10 +27,15 @@ struct MetricsInner {
 	activated_heads_total: prometheus::Counter<prometheus::U64>,
 	deactivated_heads_total: prometheus::Counter<prometheus::U64>,
 	messages_relayed_total: prometheus::Counter<prometheus::U64>,
+
+	to_subsystem_bounded_tof: prometheus::HistogramVec,
 	to_subsystem_bounded_sent: prometheus::GaugeVec<prometheus::U64>,
 	to_subsystem_bounded_received: prometheus::GaugeVec<prometheus::U64>,
+
+	to_subsystem_unbounded_tof: prometheus::HistogramVec,
 	to_subsystem_unbounded_sent: prometheus::GaugeVec<prometheus::U64>,
 	to_subsystem_unbounded_received: prometheus::GaugeVec<prometheus::U64>,
+
 	signals_sent: prometheus::GaugeVec<prometheus::U64>,
 	signals_received: prometheus::GaugeVec<prometheus::U64>,
 
@@ -68,7 +73,7 @@ impl Metrics {
 		}
 	}
 
-	pub(crate) fn channel_fill_level_snapshot(
+	pub(crate) fn channel_metrics_snapshot(
 		&self,
 		collection: impl IntoIterator<Item = (&'static str, SubsystemMeterReadouts)>,
 	) {
@@ -105,6 +110,17 @@ impl Metrics {
 						.signals_received
 						.with_label_values(&[name])
 						.set(readouts.signals.received as u64);
+
+					let hist_bounded = metrics.to_subsystem_bounded_tof.with_label_values(&[name]);
+					for tof in readouts.bounded.tof {
+						hist_bounded.observe(tof.as_f64());
+					}
+
+					let hist_unbounded =
+						metrics.to_subsystem_unbounded_tof.with_label_values(&[name]);
+					for tof in readouts.unbounded.tof {
+						hist_unbounded.observe(tof.as_f64());
+					}
 				});
 		}
 	}
@@ -134,6 +150,16 @@ impl MetricsTrait for Metrics {
 				)?,
 				registry,
 			)?,
+			to_subsystem_bounded_tof: prometheus::register(
+				prometheus::HistogramVec::new(
+					prometheus::HistogramOpts::new(
+						"polkadot_parachain_subsystem_bounded_tof",
+						"Duration spent in a particular channel from entrance to removal",
+					),
+					&["subsystem_name"],
+				)?,
+				registry,
+			)?,
 			to_subsystem_bounded_sent: prometheus::register(
 				prometheus::GaugeVec::<prometheus::U64>::new(
 					prometheus::Opts::new(
@@ -154,6 +180,16 @@ impl MetricsTrait for Metrics {
 				)?,
 				registry,
 			)?,
+			to_subsystem_unbounded_tof: prometheus::register(
+				prometheus::HistogramVec::new(
+					prometheus::HistogramOpts::new(
+						"polkadot_parachain_subsystem_unbounded_tof",
+						"Duration spent in a particular channel from entrance to removal",
+					),
+					&["subsystem_name"],
+				)?,
+				registry,
+			)?,
 			to_subsystem_unbounded_sent: prometheus::register(
 				prometheus::GaugeVec::<prometheus::U64>::new(
 					prometheus::Opts::new(