Add support for wasm runtime metrics try #2 (#4483)

* Add runtime metrics provider Signed-off-by: Andrei Sandu <andrei-mihail@parity.io> * Runner changes Signed-off-by: Andrei Sandu <andrei-mihail@parity.io> * Some sample metrics in paras_inherent Signed-off-by: Andrei Sandu <andrei-mihail@parity.io> * update cargo toml Signed-off-by: Andrei Sandu <andrei-mihail@parity.io> * fmt Signed-off-by: Andrei Sandu <andrei-mihail@parity.io> * bug Signed-off-by: Andrei Sandu <andrei-mihail@parity.io> * more fmt after merge Signed-off-by: Andrei Sandu <andrei-mihail@parity.io> * Refactor metric prefix override Signed-off-by: Andrei Sandu <andrei-mihail@parity.io> * fmt Signed-off-by: Andrei Sandu <andrei-mihail@parity.io> * remove bug comment Signed-off-by: Andrei Sandu <andrei-mihail@parity.io> * Add runtime metric primitives Signed-off-by: Andrei Sandu <andrei-mihail@parity.io> * Impl trace event parsing Signed-off-by: Andrei Sandu <andrei-mihail@parity.io> * Update metrics Signed-off-by: Andrei Sandu <andrei-mihail@parity.io> * cargo lock Signed-off-by: Andrei Sandu <andrei-mihail@parity.io> * fmt Signed-off-by: Andrei Sandu <andrei-mihail@parity.io> * Fix target check Signed-off-by: Andrei Sandu <andrei-mihail@parity.io> * Runtime metrics primitives Signed-off-by: Andrei Sandu <andrei-mihail@parity.io> * Review feedback Signed-off-by: Andrei Sandu <andrei-mihail@parity.io> * Runtime metrics crate Signed-off-by: Andrei Sandu <andrei-mihail@parity.io> * Node side runtime metric changes Signed-off-by: Andrei Sandu <andrei-mihail@parity.io> * use runtime CounterVec instead of macro Signed-off-by: Andrei Sandu <andrei-mihail@parity.io> * fmt nice Signed-off-by: Andrei Sandu <andrei-mihail@parity.io> * remove dead code Signed-off-by: Andrei Sandu <andrei-mihail@parity.io> * base58 decoding Signed-off-by: Andrei Sandu <andrei-mihail@parity.io> * base58 encoding Signed-off-by: Andrei Sandu <andrei-mihail@parity.io> * fix warn Signed-off-by: Andrei Sandu <andrei-mihail@parity.io> * typo Signed-off-by: Andrei Sandu <andrei-mihail@parity.io> * Review feedback Signed-off-by: Andrei Sandu <andrei-mihail@parity.io> * Finish label support Signed-off-by: Andrei Sandu <andrei-mihail@parity.io> * fmt Signed-off-by: Andrei Sandu <andrei-mihail@parity.io> * please compile Signed-off-by: Andrei Sandu <andrei-mihail@parity.io> * add feature gate Signed-off-by: Andrei Sandu <andrei-mihail@parity.io> * fmt Signed-off-by: Andrei Sandu <andrei-mihail@parity.io> * Comment cargo toml Signed-off-by: Andrei Sandu <andrei-mihail@parity.io> * Fix cargo toml description Signed-off-by: Andrei Sandu <andrei-mihail@parity.io> * Update doc. Signed-off-by: Andrei Sandu <andrei-mihail@parity.io> * switch to `runtime-metrics` feature Signed-off-by: Andrei Sandu <andrei-mihail@parity.io> * fmt Signed-off-by: Andrei Sandu <andrei-mihail@parity.io> * cargo toml Signed-off-by: Andrei Sandu <andrei-mihail@parity.io> * fix tests Signed-off-by: Andrei Sandu <andrei-mihail@parity.io> * fixes Signed-off-by: Andrei Sandu <andrei-mihail@parity.io> * better ux Signed-off-by: Andrei Sandu <andrei-mihail@parity.io> * from_utf8_unchecked is safe Signed-off-by: Andrei Sandu <andrei-mihail@parity.io> * fmt Signed-off-by: Andrei Sandu <andrei-mihail@parity.io> * Add Counter and refactor Signed-off-by: Andrei Sandu <andrei-mihail@parity.io> * Fixes Signed-off-by: Andrei Sandu <andrei-mihail@parity.io> * review fixes Signed-off-by: Andrei Sandu <andrei-mihail@parity.io> * more fixes Signed-off-by: Andrei Sandu <andrei-mihail@parity.io> * add integration test Signed-off-by: Andrei Sandu <andrei-mihail@parity.io> * dev deps Signed-off-by: Andrei Sandu <andrei-mihail@parity.io> * gitlab script update Signed-off-by: Andrei Sandu <andrei-mihail@parity.io> * review fixes Signed-off-by: Andrei Sandu <andrei-mihail@parity.io> * fix merge damage Signed-off-by: Andrei Sandu <andrei-mihail@parity.io> * Run tests twice Signed-off-by: Andrei Sandu <andrei-mihail@parity.io> * small fix Signed-off-by: Andrei Sandu <andrei-mihail@parity.io> * typo Signed-off-by: Andrei Sandu <andrei-mihail@parity.io> * cargo lock Signed-off-by: Andrei Sandu <andrei-mihail@parity.io> * tests Signed-off-by: Andrei Sandu <andrei-mihail@parity.io> * spellcheck happy ? Signed-off-by: Andrei Sandu <andrei-mihail@parity.io> * more fixes Signed-off-by: Andrei Sandu <andrei-mihail@parity.io> * Guard tracing init Signed-off-by: Andrei Sandu <andrei-mihail@parity.io> * missing copyright Signed-off-by: Andrei Sandu <andrei-mihail@parity.io> * update lockfile for substrate Co-authored-by: parity-processbot <>
2026-06-09 20:11:09 +00:00 · 2021-12-16 13:56:25 +02:00
parent 6f2f9fe381
commit e75ad88ae8
23 changed files with 1014 additions and 185 deletions
@@ -4,7 +4,7 @@ path = "src/main.rs"

 [package]
 name = "polkadot"
-description = "Implementation of a https://polkadot.network node in Rust based on the Substrate framework."
+description = "Implementation of a `https://polkadot.network` node in Rust based on the Substrate framework."
 license = "GPL-3.0-only"
 version = "0.9.13"
 authors = ["Parity Technologies <admin@parity.io>"]
@@ -118,6 +118,7 @@ panic = "unwind"
 runtime-benchmarks= [ "polkadot-cli/runtime-benchmarks" ]
 try-runtime = [ "polkadot-cli/try-runtime" ]
 disputes = [ "polkadot-cli/disputes" ]
+runtime-metrics = [ "polkadot-cli/runtime-metrics" ]

 # Configuration for building a .deb package - for use with `cargo-deb`
 [package.metadata.deb]
@@ -28,6 +28,7 @@ frame-benchmarking-cli = { git = "https://github.com/paritytech/substrate", bran
 try-runtime-cli = { git = "https://github.com/paritytech/substrate", branch = "master", optional = true }
 sc-cli = { git = "https://github.com/paritytech/substrate", branch = "master", optional = true }
 sc-service = { git = "https://github.com/paritytech/substrate", branch = "master", optional = true }
+polkadot-node-metrics = { path = "../node/metrics" }
 sc-tracing = { git = "https://github.com/paritytech/substrate", branch = "master", optional = true }

 # this crate is used only to enable `trie-memory-tracker` feature
@@ -66,3 +67,4 @@ rococo-native = [ "service/rococo-native" ]

 malus = [ "full-node", "service/malus" ]
 disputes = [ "service/disputes" ]
+runtime-metrics = ["service/runtime-metrics", "polkadot-node-metrics/runtime-metrics"]
@@ -234,11 +234,20 @@ fn host_perf_check() -> Result<()> {
 /// for integration tests as needed.
 #[cfg(feature = "malus")]
 pub fn run_node(run: Cli, overseer_gen: impl service::OverseerGen) -> Result<()> {
-	run_node_inner(run, overseer_gen)
+	run_node_inner(run, overseer_gen, |_logger_builder, _config| {})
 }

-fn run_node_inner(cli: Cli, overseer_gen: impl service::OverseerGen) -> Result<()> {
-	let runner = cli.create_runner(&cli.run.base).map_err(Error::from)?;
+fn run_node_inner<F>(
+	cli: Cli,
+	overseer_gen: impl service::OverseerGen,
+	logger_hook: F,
+) -> Result<()>
+where
+	F: FnOnce(&mut sc_cli::LoggerBuilder, &sc_service::Configuration),
+{
+	let runner = cli
+		.create_runner_with_logger_hook::<sc_cli::RunCmd, F>(&cli.run.base, logger_hook)
+		.map_err(Error::from)?;
 	let chain_spec = &runner.config().chain_spec;

 	set_default_ss58_version(chain_spec);
@@ -281,10 +290,10 @@ fn run_node_inner(cli: Cli, overseer_gen: impl service::OverseerGen) -> Result<(

 /// Parses polkadot specific CLI arguments and run the service.
 pub fn run() -> Result<()> {
-	let cli = Cli::from_args();
+	let cli: Cli = Cli::from_args();

 	match &cli.subcommand {
-		None => run_node_inner(cli, service::RealOverseerGen),
+		None => run_node_inner(cli, service::RealOverseerGen, polkadot_node_metrics::logger_hook()),
 		Some(Subcommand::BuildSpec(cmd)) => {
 			let runner = cli.create_runner(cmd)?;
 			Ok(runner.sync_run(|config| cmd.run(config.chain_spec, config.network))?)
@@ -45,3 +45,9 @@ polkadot = ["polkadot-runtime"]
 kusama = ["kusama-runtime"]
 rococo = ["rococo-runtime"]
 westend = ["westend-runtime"]
+runtime-metrics = [
+    "rococo-runtime/runtime-metrics",
+    "kusama-runtime/runtime-metrics",
+    "westend-runtime/runtime-metrics",
+    "polkadot-runtime/runtime-metrics",
+]
@@ -3,15 +3,33 @@ name = "polkadot-node-metrics"
 version = "0.9.13"
 authors = ["Parity Technologies <admin@parity.io>"]
 edition = "2018"
-description = "Subsystem traits and message definitions"
+description = "Subsystem metric helpers"

 [dependencies]
 futures = "0.3.17"
 futures-timer = "3.0.2"
+tracing = "0.1.29"

 metered-channel = { path = "../metered-channel" }

+# Both `sc-service` and `sc-cli` are required by runtime metrics `logger_hook()`.
+sc-service = { git = "https://github.com/paritytech/substrate", branch = "master" }
+sc-cli = { git = "https://github.com/paritytech/substrate", branch = "master" }
+
 substrate-prometheus-endpoint = { git = "https://github.com/paritytech/substrate", branch = "master" }
+sc-tracing = { git = "https://github.com/paritytech/substrate", branch = "master" }
+codec = { package = "parity-scale-codec", version = "2.2.0" }
+primitives = { package = "polkadot-primitives", path = "../../primitives/" }
+bs58 = { version = "0.4.0", features = ["alloc"] }
+log = "0.4.13"
+
+[dev-dependencies]
+assert_cmd = "2.0.2"
+nix = "0.23.0"
+tempfile = "3.2.0"
+hyper = { version = "0.14.14", default-features = false, features = ["http1", "tcp"] }
+tokio = "1.13"

 [features]
 default = []
+runtime-metrics = []
@@ -30,6 +30,17 @@ pub use metered_channel as metered;
 pub mod metronome;
 pub use self::metronome::Metronome;

+#[cfg(feature = "runtime-metrics")]
+pub mod runtime;
+#[cfg(feature = "runtime-metrics")]
+pub use self::runtime::logger_hook;
+
+/// Export a dummy logger hook when `wasm tracing` is not enabled.
+#[cfg(not(feature = "runtime-metrics"))]
+pub fn logger_hook() -> impl FnOnce(&mut sc_cli::LoggerBuilder, &sc_service::Configuration) -> () {
+	|_logger_builder, _config| {}
+}
+
 /// This module reexports Prometheus types and defines the [`Metrics`] trait.
 pub mod metrics {
 	/// Reexport Substrate Prometheus types.
@@ -69,3 +80,6 @@ pub mod metrics {
 		}
 	}
 }
+
+#[cfg(test)]
+mod tests;
@@ -0,0 +1,210 @@
+// Copyright 2021 Parity Technologies (UK) Ltd.
+// This file is part of Polkadot.
+
+// Polkadot is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// Polkadot is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with Polkadot.  If not, see <http://www.gnu.org/licenses/>.
+
+//! Runtime Metrics helpers.
+//!
+//! A runtime metric provider implementation that builds on top of Substrate wasm
+//! tracing support. This requires that the custom profiler (`TraceHandler`) to be
+//! registered in substrate via a `logger_hook()`. Events emitted from runtime are
+//! then captured/processed by the `TraceHandler` implementation.
+#![cfg(feature = "runtime-metrics")]
+
+use codec::Decode;
+use primitives::v1::{
+	RuntimeMetricLabelValues, RuntimeMetricOp, RuntimeMetricRegisterParams, RuntimeMetricUpdate,
+};
+use std::{
+	collections::hash_map::HashMap,
+	sync::{Arc, Mutex, MutexGuard},
+};
+use substrate_prometheus_endpoint::{
+	register, Counter, CounterVec, Opts, PrometheusError, Registry, U64,
+};
+
+const LOG_TARGET: &'static str = "metrics::runtime";
+const METRIC_PREFIX: &'static str = "polkadot";
+
+/// Holds the registered Prometheus metric collections.
+#[derive(Clone, Default)]
+pub struct Metrics {
+	counter_vecs: Arc<Mutex<HashMap<String, CounterVec<U64>>>>,
+	counters: Arc<Mutex<HashMap<String, Counter<U64>>>>,
+}
+
+/// Runtime metrics wrapper.
+#[derive(Clone)]
+pub struct RuntimeMetricsProvider(Registry, Metrics);
+
+impl RuntimeMetricsProvider {
+	/// Creates new instance.
+	pub fn new(metrics_registry: Registry) -> Self {
+		Self(metrics_registry, Metrics::default())
+	}
+
+	/// Register a counter vec metric.
+	pub fn register_countervec(&self, metric_name: &str, params: &RuntimeMetricRegisterParams) {
+		self.with_counter_vecs_lock_held(|mut hashmap| {
+			hashmap.entry(metric_name.to_owned()).or_insert(register(
+				CounterVec::new(
+					Opts::new(metric_name, params.description()),
+					&params.labels().unwrap_or_default(),
+				)?,
+				&self.0,
+			)?);
+			Ok(())
+		})
+	}
+
+	/// Register a counter metric.
+	pub fn register_counter(&self, metric_name: &str, params: &RuntimeMetricRegisterParams) {
+		self.with_counters_lock_held(|mut hashmap| {
+			hashmap
+				.entry(metric_name.to_owned())
+				.or_insert(register(Counter::new(metric_name, params.description())?, &self.0)?);
+			return Ok(())
+		})
+	}
+
+	/// Increment a counter with labels by a value.
+	pub fn inc_counter_vec_by(&self, name: &str, value: u64, labels: &RuntimeMetricLabelValues) {
+		self.with_counter_vecs_lock_held(|mut hashmap| {
+			hashmap.entry(name.to_owned()).and_modify(|counter_vec| {
+				counter_vec.with_label_values(&labels.as_str_vec()).inc_by(value)
+			});
+
+			Ok(())
+		});
+	}
+
+	/// Increment a counter by a value.
+	pub fn inc_counter_by(&self, name: &str, value: u64) {
+		self.with_counters_lock_held(|mut hashmap| {
+			hashmap
+				.entry(name.to_owned())
+				.and_modify(|counter_vec| counter_vec.inc_by(value));
+			Ok(())
+		})
+	}
+
+	fn with_counters_lock_held<F>(&self, do_something: F)
+	where
+		F: FnOnce(MutexGuard<'_, HashMap<String, Counter<U64>>>) -> Result<(), PrometheusError>,
+	{
+		let _ = self.1.counters.lock().map(do_something).or_else(|error| {
+			tracing::error!(
+				target: LOG_TARGET,
+				"Cannot acquire the counter hashmap lock: {:?}",
+				error
+			);
+			Err(error)
+		});
+	}
+
+	fn with_counter_vecs_lock_held<F>(&self, do_something: F)
+	where
+		F: FnOnce(MutexGuard<'_, HashMap<String, CounterVec<U64>>>) -> Result<(), PrometheusError>,
+	{
+		let _ = self.1.counter_vecs.lock().map(do_something).or_else(|error| {
+			tracing::error!(
+				target: LOG_TARGET,
+				"Cannot acquire the countervec hashmap lock: {:?}",
+				error
+			);
+			Err(error)
+		});
+	}
+}
+
+impl sc_tracing::TraceHandler for RuntimeMetricsProvider {
+	fn handle_span(&self, _span: &sc_tracing::SpanDatum) {}
+	fn handle_event(&self, event: &sc_tracing::TraceEvent) {
+		if event
+			.values
+			.string_values
+			.get("target")
+			.unwrap_or(&String::default())
+			.ne("metrics")
+		{
+			return
+		}
+
+		if let Some(update_op_bs58) = event.values.string_values.get("params") {
+			// Deserialize the metric update struct.
+			match RuntimeMetricUpdate::decode(
+				&mut RuntimeMetricsProvider::parse_event_params(&update_op_bs58)
+					.unwrap_or_default()
+					.as_slice(),
+			) {
+				Ok(update_op) => {
+					self.parse_metric_update(update_op);
+				},
+				Err(e) => {
+					tracing::error!(target: LOG_TARGET, "TraceEvent decode failed: {:?}", e);
+				},
+			}
+		}
+	}
+}
+
+impl RuntimeMetricsProvider {
+	// Parse end execute the update operation.
+	fn parse_metric_update(&self, update: RuntimeMetricUpdate) {
+		let metric_name = &format!("{}_{}", METRIC_PREFIX, update.metric_name());
+
+		match update.op {
+			RuntimeMetricOp::Register(ref params) =>
+				if params.labels.is_none() {
+					self.register_counter(metric_name, &params);
+				} else {
+					self.register_countervec(metric_name, &params);
+				},
+			RuntimeMetricOp::IncrementCounterVec(value, ref labels) =>
+				self.inc_counter_vec_by(metric_name, value, labels),
+			RuntimeMetricOp::IncrementCounter(value) => self.inc_counter_by(metric_name, value),
+		}
+	}
+
+	// Returns the `bs58` encoded metric update operation.
+	fn parse_event_params(event_params: &str) -> Option<Vec<u8>> {
+		// Shave " }" suffix.
+		let new_len = event_params.len().saturating_sub(2);
+		let event_params = &event_params[..new_len];
+
+		// Shave " { update_op: " prefix.
+		const SKIP_CHARS: &'static str = " { update_op: ";
+		if SKIP_CHARS.len() < event_params.len() {
+			if SKIP_CHARS.eq_ignore_ascii_case(&event_params[..SKIP_CHARS.len()]) {
+				return bs58::decode(&event_params[SKIP_CHARS.len()..].as_bytes()).into_vec().ok()
+			}
+		}
+
+		// No event was parsed
+		None
+	}
+}
+
+/// Returns the custom profiling closure that we'll apply to the `LoggerBuilder`.
+pub fn logger_hook() -> impl FnOnce(&mut sc_cli::LoggerBuilder, &sc_service::Configuration) -> () {
+	|logger_builder, config| {
+		if config.prometheus_registry().is_none() {
+			tracing::debug!(target: LOG_TARGET, "Prometheus registry is not configured.",);
+			return
+		}
+		let registry = config.prometheus_registry().cloned().unwrap();
+		let metrics_provider = RuntimeMetricsProvider::new(registry);
+		logger_builder.with_custom_profiling(Box::new(metrics_provider));
+	}
+}
@@ -0,0 +1,70 @@
+// Copyright 2021 Parity Technologies (UK) Ltd.
+// This file is part of Substrate.
+
+// Substrate is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// Substrate is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with Substrate.  If not, see <http://www.gnu.org/licenses/>.
+#![cfg(feature = "runtime-metrics")]
+use assert_cmd::cargo::cargo_bin;
+use std::{convert::TryInto, process::Command, thread, time::Duration};
+use tempfile::tempdir;
+
+#[test]
+#[cfg(unix)]
+fn runtime_can_publish_metrics() {
+	use hyper::{Client, Uri};
+	use nix::{
+		sys::signal::{kill, Signal::SIGINT},
+		unistd::Pid,
+	};
+	use std::convert::TryFrom;
+
+	const RUNTIME_METRIC_NAME: &str = "polkadot_parachain_inherent_data_bitfields_processed";
+	const DEFAULT_PROMETHEUS_PORT: u16 = 9615;
+	let metrics_uri = format!("http://localhost:{}/metrics", DEFAULT_PROMETHEUS_PORT);
+
+	// Start the node with tracing enabled and forced wasm runtime execution.
+	let cmd = Command::new(cargo_bin("polkadot"))
+		// Runtime metrics require this trace target.
+		.args(&["--tracing-targets", "wasm_tracing=trace"])
+		.args(&["--execution", "wasm"])
+		.args(&["--dev", "-d"])
+		.arg(tempdir().expect("failed to create temp dir.").path())
+		.spawn()
+		.expect("failed to start the node process");
+
+	// Enough time to author one block.
+	thread::sleep(Duration::from_secs(10));
+
+	let runtime = tokio::runtime::Runtime::new().expect("failed to create tokio runtime");
+
+	runtime.block_on(async {
+		let client = Client::new();
+
+		let res = client
+			.get(Uri::try_from(&metrics_uri).expect("bad URI"))
+			.await
+			.expect("get request failed");
+
+		let body = String::from_utf8(
+			hyper::body::to_bytes(res).await.expect("can't get body as bytes").to_vec(),
+		)
+		.expect("body is not an UTF8 string");
+
+		// Time to die.
+		kill(Pid::from_raw(cmd.id().try_into().unwrap()), SIGINT)
+			.expect("failed to kill the node process");
+
+		// If the node has authored at least 1 block this should pass.
+		assert!(body.contains(&RUNTIME_METRIC_NAME));
+	});
+}
@@ -179,3 +179,11 @@ try-runtime = [
 ]
 malus = ["full-node"]
 disputes = ["polkadot-node-core-dispute-coordinator/disputes"]
+runtime-metrics = [
+	"polkadot-client/runtime-metrics", 
+	"rococo-runtime/runtime-metrics",
+	"westend-runtime/runtime-metrics",
+	"kusama-runtime/runtime-metrics",
+	"polkadot-runtime/runtime-metrics",
+	"polkadot-runtime-parachains/runtime-metrics"
+]
@@ -0,0 +1,139 @@
+// Copyright 2021 Parity Technologies (UK) Ltd.
+// This file is part of Polkadot.
+
+// Polkadot is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// Polkadot is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with Polkadot.  If not, see <http://www.gnu.org/licenses/>.
+
+//! Runtime metric primitives.
+
+use parity_scale_codec::{Decode, Encode};
+use sp_std::prelude::*;
+
+/// Metric registration parameters.
+#[derive(Encode, Decode)]
+#[cfg_attr(feature = "std", derive(Debug))]
+pub struct RuntimeMetricRegisterParams {
+	/// Metric description.
+	description: Vec<u8>,
+	/// Only for counter vec.
+	pub labels: Option<RuntimeMetricLabels>,
+}
+
+/// Runtime metric operations.
+#[derive(Encode, Decode)]
+#[cfg_attr(feature = "std", derive(Debug))]
+pub enum RuntimeMetricOp {
+	/// Register a new metric.
+	Register(RuntimeMetricRegisterParams),
+	/// Increment a counter metric with labels by value.
+	IncrementCounterVec(u64, RuntimeMetricLabelValues),
+	/// Increment a counter metric by value.
+	IncrementCounter(u64),
+}
+
+impl RuntimeMetricRegisterParams {
+	/// Create new metric registration params.
+	pub fn new(description: Vec<u8>, labels: Option<RuntimeMetricLabels>) -> Self {
+		Self { description, labels }
+	}
+}
+
+/// Runtime metric update event.
+#[derive(Encode, Decode)]
+#[cfg_attr(feature = "std", derive(Debug))]
+pub struct RuntimeMetricUpdate {
+	/// The name of the metric.
+	pub metric_name: Vec<u8>,
+	/// The operation applied to the metric.
+	pub op: RuntimeMetricOp,
+}
+
+fn vec_to_str<'a>(v: &'a Vec<u8>, default: &'static str) -> &'a str {
+	return sp_std::str::from_utf8(v).unwrap_or(default)
+}
+
+impl RuntimeMetricRegisterParams {
+	/// Returns the metric description.
+	pub fn description(&self) -> &str {
+		vec_to_str(&self.description, "No description provided.")
+	}
+
+	/// Returns a label names as an `Option` of `Vec<&str>`.
+	pub fn labels(&self) -> Option<Vec<&str>> {
+		self.labels.as_ref().map(|labels| labels.as_str_vec())
+	}
+}
+
+impl RuntimeMetricLabels {
+	/// Returns a labels as `Vec<&str>`.
+	pub fn as_str_vec(&self) -> Vec<&str> {
+		self.0
+			.iter()
+			.map(|label_vec| vec_to_str(&label_vec.0, "invalid_label"))
+			.collect()
+	}
+
+	/// Return the inner values as vec.
+	pub fn clear(&mut self) {
+		self.0.clear();
+	}
+}
+
+impl From<&[&'static str]> for RuntimeMetricLabels {
+	fn from(v: &[&'static str]) -> RuntimeMetricLabels {
+		RuntimeMetricLabels(
+			v.iter().map(|label| RuntimeMetricLabel(label.as_bytes().to_vec())).collect(),
+		)
+	}
+}
+
+impl RuntimeMetricUpdate {
+	/// Returns the metric name.
+	pub fn metric_name(&self) -> &str {
+		vec_to_str(&self.metric_name, "invalid_metric_name")
+	}
+}
+
+/// A set of metric labels.
+#[derive(Clone, Default, Encode, Decode)]
+#[cfg_attr(feature = "std", derive(Debug))]
+pub struct RuntimeMetricLabels(Vec<RuntimeMetricLabel>);
+
+/// A metric label.
+#[derive(Clone, Default, Encode, Decode)]
+#[cfg_attr(feature = "std", derive(Debug))]
+pub struct RuntimeMetricLabel(Vec<u8>);
+
+/// A metric label value.
+pub type RuntimeMetricLabelValue = RuntimeMetricLabel;
+
+/// A set of metric label values.
+pub type RuntimeMetricLabelValues = RuntimeMetricLabels;
+
+/// Trait for converting Vec<u8> to `&str`.
+pub trait AsStr {
+	/// Return a str reference.
+	fn as_str(&self) -> Option<&str>;
+}
+
+impl AsStr for RuntimeMetricLabel {
+	fn as_str(&self) -> Option<&str> {
+		sp_std::str::from_utf8(&self.0).ok()
+	}
+}
+
+impl From<&'static str> for RuntimeMetricLabel {
+	fn from(s: &'static str) -> Self {
+		Self(s.as_bytes().to_vec())
+	}
+}
@@ -63,6 +63,12 @@ pub use sp_staking::SessionIndex;
 mod signed;
 pub use signed::{EncodeAs, Signed, UncheckedSigned};

+mod metrics;
+pub use metrics::{
+	RuntimeMetricLabel, RuntimeMetricLabelValue, RuntimeMetricLabelValues, RuntimeMetricLabels,
+	RuntimeMetricOp, RuntimeMetricRegisterParams, RuntimeMetricUpdate,
+};
+
 /// A declarations of storage keys where an external observer can find some interesting data.
 pub mod well_known_keys {
 	use super::{HrmpChannelId, Id};
@@ -270,3 +270,4 @@ disable-runtime-api = []
 on-chain-release-build = [
 	"sp-api/disable-logging",
 ]
+runtime-metrics = ["runtime-parachains/runtime-metrics", "sp-io/with-tracing"]
@@ -0,0 +1,24 @@
+[package]
+name = "polkadot-runtime-metrics"
+version = "0.9.13"
+authors = ["Parity Technologies <admin@parity.io>"]
+edition = "2018"
+
+[dependencies]
+sp-std = { package = "sp-std", git = "https://github.com/paritytech/substrate", branch = "master", default-features = false}
+sp-tracing = { git = "https://github.com/paritytech/substrate", branch = "master", default-features = false }
+parity-scale-codec = { version = "2.3.1", default-features = false }
+primitives = { package = "polkadot-primitives", path = "../../primitives", default-features = false }
+
+bs58 = { version = "0.4.0", default-features = false, features = ["alloc"] }
+
+[features]
+default = ["std"]
+std = [
+	"sp-std/std",
+	"sp-tracing/std",
+    "parity-scale-codec/std",
+    "primitives/std",
+    "bs58/std"
+]
+runtime-metrics = ["sp-tracing/with-tracing"]
@@ -0,0 +1,33 @@
+// Copyright 2021 Parity Technologies (UK) Ltd.
+// This file is part of Polkadot.
+
+// Polkadot is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// Polkadot is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with Polkadot.  If not, see <http://www.gnu.org/licenses/>.
+
+//! Runtime metric interface similar to native Prometheus metrics.
+//!
+//! This is intended to be used only for testing and debugging and **must never
+//! be used in production**. It requires the Substrate wasm tracing support
+//! and command line configuration: `--tracing-targets wasm_tracing=trace`.
+
+#![cfg_attr(not(feature = "std"), no_std)]
+
+#[cfg(feature = "runtime-metrics")]
+mod with_runtime_metrics;
+#[cfg(feature = "runtime-metrics")]
+pub use crate::with_runtime_metrics::*;
+
+#[cfg(not(feature = "runtime-metrics"))]
+mod without_runtime_metrics;
+#[cfg(not(feature = "runtime-metrics"))]
+pub use crate::without_runtime_metrics::*;
@@ -0,0 +1,126 @@
+// Copyright 2021 Parity Technologies (UK) Ltd.
+// This file is part of Polkadot.
+
+// Polkadot is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// Polkadot is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with Polkadot.  If not, see <http://www.gnu.org/licenses/>.
+
+//! This module provides an implementation for the runtime metrics types: `Counter`
+//! and `CounterVec`. These types expose a Prometheus like interface and same functionality.
+//! Each instance of a runtime metric is mapped to a Prometheus metric on the node side.
+
+const TRACING_TARGET: &'static str = "metrics";
+
+use parity_scale_codec::Encode;
+use primitives::v1::{
+	RuntimeMetricLabelValues, RuntimeMetricOp, RuntimeMetricRegisterParams, RuntimeMetricUpdate,
+};
+use sp_std::prelude::*;
+
+/// Holds a set of counters that have different values for their labels,
+/// like Prometheus CounterVec.
+pub struct CounterVec {
+	name: &'static str,
+	label_values: Option<RuntimeMetricLabelValues>,
+}
+
+/// A counter metric.
+pub struct Counter {
+	name: &'static str,
+}
+
+/// Convenience trait implemented for all metric types.
+trait MetricEmitter {
+	fn emit(metric_op: &RuntimeMetricUpdate) {
+		sp_tracing::event!(
+			target: TRACING_TARGET,
+			sp_tracing::Level::TRACE,
+			update_op = bs58::encode(&metric_op.encode()).into_string().as_str()
+		);
+	}
+}
+
+impl MetricEmitter for CounterVec {}
+impl MetricEmitter for Counter {}
+
+impl CounterVec {
+	/// Create a new metric with specified `name`, `description` and `labels`.
+	pub fn new(name: &'static str, description: &'static str, labels: &[&'static str]) -> Self {
+		// Send a register metric operation to node side.
+		let metric_update = RuntimeMetricUpdate {
+			metric_name: Vec::from(name),
+			op: RuntimeMetricOp::Register(RuntimeMetricRegisterParams::new(
+				Vec::from(description),
+				Some(labels.into()),
+			)),
+		};
+
+		Self::emit(&metric_update);
+
+		CounterVec { name, label_values: None }
+	}
+
+	/// Set the label values. Must be called before each increment operation.
+	pub fn with_label_values(&mut self, label_values: &[&'static str]) -> &mut Self {
+		self.label_values = Some(label_values.into());
+		self
+	}
+
+	/// Increment the counter by `value`.
+	pub fn inc_by(&mut self, value: u64) {
+		self.label_values.take().map(|label_values| {
+			let metric_update = RuntimeMetricUpdate {
+				metric_name: Vec::from(self.name),
+				op: RuntimeMetricOp::IncrementCounterVec(value, label_values),
+			};
+
+			Self::emit(&metric_update);
+		});
+	}
+
+	/// Increment the counter value.
+	pub fn inc(&mut self) {
+		self.inc_by(1);
+	}
+}
+
+impl Counter {
+	/// Create a new counter metric with specified `name`, `description`.
+	pub fn new(name: &'static str, description: &'static str) -> Self {
+		// Send a register metric operation to node side.
+		let metric_update = RuntimeMetricUpdate {
+			metric_name: Vec::from(name),
+			op: RuntimeMetricOp::Register(RuntimeMetricRegisterParams::new(
+				Vec::from(description),
+				None,
+			)),
+		};
+
+		Self::emit(&metric_update);
+		Counter { name }
+	}
+
+	/// Increment counter by `value`.
+	pub fn inc_by(&mut self, value: u64) {
+		let metric_update = RuntimeMetricUpdate {
+			metric_name: Vec::from(self.name),
+			op: RuntimeMetricOp::IncrementCounter(value),
+		};
+
+		Self::emit(&metric_update);
+	}
+
+	/// Increment counter.
+	pub fn inc(&mut self) {
+		self.inc_by(1);
+	}
+}
@@ -0,0 +1,52 @@
+// Copyright 2021 Parity Technologies (UK) Ltd.
+// This file is part of Polkadot.
+
+// Polkadot is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// Polkadot is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with Polkadot.  If not, see <http://www.gnu.org/licenses/>.
+
+//! Runtime metrics are usable from the wasm runtime only. The purpose of this module is to
+//! provide a dummy implementation for the native runtime to avoid cluttering the runtime code
+//! with `#[cfg(feature = "runtime-metrics")]`.
+
+/// A dummy Counter.
+pub struct Counter;
+/// A dummy CounterVec.
+pub struct CounterVec;
+
+/// Dummy implementation.
+impl CounterVec {
+	/// Constructor.
+	pub fn new(_name: &'static str, _description: &'static str, _labels: &[&'static str]) -> Self {
+		CounterVec
+	}
+	/// Sets label values, implementation is a `no op`.
+	pub fn with_label_values(&mut self, _label_values: &[&'static str]) -> &mut Self {
+		self
+	}
+	/// Increment counter by value, implementation is a `no op`.
+	pub fn inc_by(&mut self, _: u64) {}
+	/// Increment counter, implementation is a `no op`.
+	pub fn inc(&mut self) {}
+}
+
+/// Dummy implementation.
+impl Counter {
+	/// Constructor.
+	pub fn new(_name: &'static str, _description: &'static str) -> Self {
+		Counter
+	}
+	/// Increment counter by value, implementation is a `no op`.
+	pub fn inc_by(&mut self, _: u64) {}
+	/// Increment counter, implementation is a `no op`.
+	pub fn inc(&mut self) {}
+}
@@ -23,6 +23,7 @@ sp-session = { git = "https://github.com/paritytech/substrate", branch = "master
 sp-staking = { git = "https://github.com/paritytech/substrate", branch = "master", default-features = false }
 sp-core = { git = "https://github.com/paritytech/substrate", branch = "master", default-features = false }
 sp-keystore = { git = "https://github.com/paritytech/substrate", branch = "master", optional = true }
+sp-tracing = { version = "4.0.0-dev", branch = "master", git = "https://github.com/paritytech/substrate", default-features = false, optional = true }

 pallet-authority-discovery = { git = "https://github.com/paritytech/substrate", branch = "master", default-features = false }
 pallet-authorship = { git = "https://github.com/paritytech/substrate", branch = "master", default-features = false }
@@ -42,6 +43,7 @@ primitives = { package = "polkadot-primitives", path = "../../primitives", defau

 rand = { version = "0.8.3", default-features = false }
 rand_chacha = { version = "0.3.1", default-features = false }
+polkadot-runtime-metrics = { path = "../metrics", default-features = false}

 [dev-dependencies]
 futures = "0.3.17"
@@ -82,6 +84,7 @@ std = [
 	"xcm/std",
 	"xcm-executor/std",
 	"log/std",
+	"polkadot-runtime-metrics/std",
 ]
 runtime-benchmarks = [
 	"frame-benchmarking/runtime-benchmarks",
@@ -98,3 +101,4 @@ try-runtime = [
 	"pallet-timestamp/try-runtime",
 	"pallet-vesting/try-runtime",
 ]
+runtime-metrics = ["sp-tracing/with-tracing", "polkadot-runtime-metrics/runtime-metrics"]
@@ -55,6 +55,7 @@ use sp_std::{
 	vec::Vec,
 };

+use polkadot_runtime_metrics::{Counter, CounterVec};
 mod misc;
 mod weights;

@@ -260,13 +261,41 @@ impl<T: Config> Pallet<T> {
 			parent_header,
 			mut disputes,
 		} = data;
+		#[cfg(feature = "runtime-metrics")]
+		sp_io::init_tracing();

-		let parent_header_hash = parent_header.hash();
+		let mut weight_metric = CounterVec::new(
+			"parachain_inherent_data_weight",
+			"Inherent data weight before and after filtering",
+			&["when"],
+		);
+
+		let mut bitfields_processed_metric = Counter::new(
+			"parachain_inherent_data_bitfields_processed",
+			"Counts the number of bitfields processed in `enter_inner`.",
+		);
+
+		let mut candidates_processed_metric = CounterVec::new(
+			"parachain_inherent_data_candidates_processed",
+			"Counts the number of parachain block candidates processed in `enter_inner`.",
+			&["category"],
+		);
+
+		let mut dispute_sets_processed_metric = CounterVec::new(
+			"parachain_inherent_data_dispute_sets_processed",
+			"Counts the number of dispute statements sets processed in `enter_inner`.",
+			&["category"],
+		);
+
+		let mut disputes_included_metric = Counter::new(
+			"parachain_inherent_data_disputes_included",
+			"Counts the number of dispute statements sets included in a block in `enter_inner`.",
+		);

 		log::debug!(
 			target: LOG_TARGET,
 			"[enter_inner] parent_header={:?} bitfields.len(): {}, backed_candidates.len(): {}, disputes.len(): {}",
-			parent_header_hash,
+			parent_header.hash(),
 			signed_bitfields.len(),
 			backed_candidates.len(),
 			disputes.len()
@@ -275,7 +304,7 @@ impl<T: Config> Pallet<T> {
 		// Check that the submitted parent header indeed corresponds to the previous block hash.
 		let parent_hash = <frame_system::Pallet<T>>::parent_hash();
 		ensure!(
-			parent_header_hash.as_ref() == parent_hash.as_ref(),
+			parent_header.hash().as_ref() == parent_hash.as_ref(),
 			Error::<T>::InvalidParentHeader,
 		);

@@ -287,6 +316,10 @@ impl<T: Config> Pallet<T> {

 		let max_block_weight = <T as frame_system::Config>::BlockWeights::get().max_block;

+		weight_metric
+			.with_label_values(&["before-filter"])
+			.inc_by(candidate_weight + bitfields_weight + disputes_weight);
+
 		// Potentially trim inherent data to ensure processing will be within weight limits
 		let total_weight = {
 			if candidate_weight
@@ -333,11 +366,23 @@ impl<T: Config> Pallet<T> {
 			// Note that `provide_multi_dispute_data` will iterate, verify, and import each
 			// dispute; so the input here must be reasonably bounded.
 			let _ = T::DisputesHandler::provide_multi_dispute_data(disputes.clone())?;
+			dispute_sets_processed_metric
+				.with_label_values(&["imported"])
+				.inc_by(disputes.len() as u64);
+
 			if T::DisputesHandler::is_frozen() {
+				// Relay chain freeze, at this point we will not include any parachain blocks.
+				dispute_sets_processed_metric.with_label_values(&["frozen"]).inc();
+
 				// The relay chain we are currently on is invalid. Proceed no further on parachains.
 				return Ok(Some(dispute_statements_weight::<T>(&disputes)).into())
 			}

+			// Process the dispute sets of the current session.
+			dispute_sets_processed_metric
+				.with_label_values(&["current"])
+				.inc_by(new_current_dispute_sets.len() as u64);
+
 			let mut freed_disputed = if !new_current_dispute_sets.is_empty() {
 				let concluded_invalid_disputes = new_current_dispute_sets
 					.iter()
@@ -347,11 +392,17 @@ impl<T: Config> Pallet<T> {
 					.map(|(_, candidate)| *candidate)
 					.collect::<BTreeSet<CandidateHash>>();

-				let freed_disputed =
+				// Count invalid dispute sets.
+				dispute_sets_processed_metric
+					.with_label_values(&["concluded_invalid"])
+					.inc_by(concluded_invalid_disputes.len() as u64);
+
+				let freed_disputed: Vec<_> =
 					<inclusion::Pallet<T>>::collect_disputed(&concluded_invalid_disputes)
 						.into_iter()
 						.map(|core| (core, FreedReason::Concluded))
 						.collect();
+
 				freed_disputed
 			} else {
 				Vec::new()
@@ -374,6 +425,7 @@ impl<T: Config> Pallet<T> {
 			disputed_bitfield
 		};

+		bitfields_processed_metric.inc_by(signed_bitfields.len() as u64);
 		// Process new availability bitfields, yielding any availability cores whose
 		// work has now concluded.
 		let freed_concluded = <inclusion::Pallet<T>>::process_bitfields(
@@ -388,11 +440,17 @@ impl<T: Config> Pallet<T> {
 			T::DisputesHandler::note_included(current_session, *candidate_hash, now);
 		}

+		candidates_processed_metric
+			.with_label_values(&["included"])
+			.inc_by(freed_concluded.len() as u64);
 		let freed = collect_all_freed_cores::<T, _>(freed_concluded.iter().cloned());

 		<scheduler::Pallet<T>>::clear();
 		<scheduler::Pallet<T>>::schedule(freed, now);

+		candidates_processed_metric
+			.with_label_values(&["total"])
+			.inc_by(backed_candidates.len() as u64);
 		let scheduled = <scheduler::Pallet<T>>::scheduled();
 		let backed_candidates = sanitize_backed_candidates::<T, _>(
 			parent_hash,
@@ -403,6 +461,9 @@ impl<T: Config> Pallet<T> {
 			},
 			&scheduled[..],
 		);
+		candidates_processed_metric
+			.with_label_values(&["sanitized"])
+			.inc_by(backed_candidates.len() as u64);

 		// Process backed candidates according to scheduled cores.
 		let parent_storage_root = parent_header.state_root().clone();
@@ -417,6 +478,8 @@ impl<T: Config> Pallet<T> {
 			full_check,
 		)?;

+		disputes_included_metric.inc_by(disputes.len() as u64);
+
 		// The number of disputes included in a block is
 		// limited by the weight as well as the number of candidate blocks.
 		OnChainVotes::<T>::put(ScrapedOnChainVotes::<<T::Header as HeaderT>::Hash> {
@@ -432,6 +495,8 @@ impl<T: Config> Pallet<T> {
 		// this is max config.ump_service_total_weight
 		let _ump_weight = <ump::Pallet<T>>::process_pending_upward_messages();

+		weight_metric.with_label_values(&vec!["after-filter"]).inc_by(total_weight);
+
 		Ok(Some(total_weight).into())
 	}
 }
@@ -800,6 +865,12 @@ pub(crate) fn sanitize_bitfields<T: crate::inclusion::Config>(
 	validators: &[ValidatorId],
 	full_check: FullCheck,
 ) -> UncheckedSignedAvailabilityBitfields {
+	let mut bitfields_signature_checks_metric = CounterVec::new(
+		"create_inherent_bitfields_signature_checks",
+		"Counts the number of bitfields signature checked in `enter_inner`.",
+		&["validity"],
+	);
+
 	let mut bitfields = Vec::with_capacity(unchecked_bitfields.len());

 	let mut last_index: Option<ValidatorIndex> = None;
@@ -865,12 +936,15 @@ pub(crate) fn sanitize_bitfields<T: crate::inclusion::Config>(
 		let validator_public = &validators[validator_index.0 as usize];

 		if let FullCheck::Yes = full_check {
+			// Validate bitfield signature.
 			if let Ok(signed_bitfield) =
 				unchecked_bitfield.try_into_checked(&signing_context, validator_public)
 			{
 				bitfields.push(signed_bitfield.into_unchecked());
+				bitfields_signature_checks_metric.with_label_values(&["valid"]).inc();
 			} else {
 				log::warn!(target: LOG_TARGET, "Invalid bitfield signature");
+				bitfields_signature_checks_metric.with_label_values(&["invalid"]).inc();
 			};
 		} else {
 			bitfields.push(unchecked_bitfield);
@@ -257,3 +257,4 @@ disable-runtime-api = []
 on-chain-release-build = [
 	"sp-api/disable-logging",
 ]
+runtime-metrics = ["runtime-parachains/runtime-metrics", "sp-io/with-tracing"]
@@ -204,3 +204,5 @@ try-runtime = [
 	"runtime-common/try-runtime",
 	"pallet-multisig/try-runtime",
 ]
+
+runtime-metrics = ["runtime-parachains/runtime-metrics", "sp-io/with-tracing"]
@@ -254,3 +254,4 @@ try-runtime = [
 # runtime without clashing with the runtime API exported functions
 # in WASM.
 disable-runtime-api = []
+runtime-metrics = ["runtime-parachains/runtime-metrics", "sp-io/with-tracing"]
@@ -1,8 +1,9 @@
 #!/usr/bin/env bash
-set -e
+set -eux

 #shellcheck source=../common/lib.sh
 source "$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )/../common/lib.sh"

 time cargo test --release --locked -p polkadot-node-core-dispute-coordinator --features disputes
-time cargo test --workspace --release --verbose --locked --features=runtime-benchmarks
+# Builds with the runtime benchmarks/metrics features are only to be used for testing.
+time cargo test --workspace --release --verbose --locked --features=runtime-benchmarks,runtime-metrics