Add Prometheus timers to the subsystems (#1923)

* reexport prometheus-super for ease of use of other subsystems

* add some prometheus timers for collation generation subsystem

* add timing metrics to av-store

* add metrics to candidate backing

* add timing metric to bitfield signing

* add timing metrics to candidate selection

* add timing metrics to candidate-validation

* add timing metrics to chain-api

* add timing metrics to provisioner

* add timing metrics to runtime-api

* add timing metrics to availability-distribution

* add timing metrics to bitfield-distribution

* add timing metrics to collator protocol: collator side

* add timing metrics to collator protocol: validator side

* fix candidate validation test failures

* add timing metrics to pov distribution

* add timing metrics to statement-distribution

* use substrate_prometheus_endpoint prometheus reexport instead of prometheus_super

* don't include JOB_DELAY in bitfield-signing metrics

* give adder-collator ability to easily export its genesis-state and validation code

* wip: adder-collator pushbutton script

* don't attempt to register the adder-collator automatically

Instead, get these values with

```sh
target/release/adder-collator export-genesis-state
target/release/adder-collator export-genesis-wasm
```

And then register the parachain on https://polkadot.js.org/apps/?rpc=ws%3A%2F%2F127.0.0.1%3A9944#/explorer

To collect prometheus data, after running the script, create `prometheus.yml` per the instructions
at https://www.notion.so/paritytechnologies/Setting-up-Prometheus-locally-835cb3a9df7541a781c381006252b5ff
and then run:

```sh
docker run -v `pwd`/prometheus.yml:/etc/prometheus/prometheus.yml:z --network host prom/prometheus
```

Demonstrates that data makes it across to prometheus, though it is likely to be useful in the future
to tweak the buckets.

* Update parachain/test-parachains/adder/collator/src/cli.rs

Co-authored-by: Andronik Ordian <write@reusable.software>

* use the grandpa-pause parameter

* skip metrics in tracing instrumentation

* remove unnecessary grandpa_pause cli param

Co-authored-by: Andronik Ordian <write@reusable.software>

polkadot/node/core/av-store/src/lib.rs

@@ -313,6 +313,8 @@ impl AvailabilityStoreSubsystem {
 	// Perform pruning of PoVs
 	#[tracing::instrument(level = "trace", skip(self), fields(subsystem = LOG_TARGET))]
 	fn prune_povs(&self) -> Result<(), Error> {
+		let _timer = self.metrics.time_prune_povs();
+
 		let mut tx = DBTransaction::new();
 		let mut pov_pruning = pov_pruning(&self.inner).unwrap_or_default();
 		let now = PruningDelay::now()?;
@@ -338,6 +340,8 @@ impl AvailabilityStoreSubsystem {
 	// Perform pruning of chunks.
 	#[tracing::instrument(level = "trace", skip(self), fields(subsystem = LOG_TARGET))]
 	fn prune_chunks(&self) -> Result<(), Error> {
+		let _timer = self.metrics.time_prune_chunks();
+
 		let mut tx = DBTransaction::new();
 		let mut chunk_pruning = chunk_pruning(&self.inner).unwrap_or_default();
 		let now = PruningDelay::now()?;
@@ -522,7 +526,7 @@ where
 					ActiveLeavesUpdate { activated, .. })
 				) => {
 					for activated in activated.into_iter() {
-						process_block_activated(ctx, &subsystem.inner, activated).await?;
+						process_block_activated(ctx, &subsystem.inner, activated, &subsystem.metrics).await?;
 					}
 				}
 				FromOverseer::Signal(OverseerSignal::BlockFinalized(hash)) => {
@@ -561,6 +565,8 @@ async fn process_block_finalized<Context>(
 where
 	Context: SubsystemContext<Message=AvailabilityStoreMessage>
 {
+	let _timer = subsystem.metrics.time_process_block_finalized();
+
 	let block_number = get_block_number(ctx, hash).await?;
 
 	if let Some(mut pov_pruning) = pov_pruning(db) {
@@ -606,15 +612,18 @@ where
 	Ok(())
 }
 
-#[tracing::instrument(level = "trace", skip(ctx, db), fields(subsystem = LOG_TARGET))]
+#[tracing::instrument(level = "trace", skip(ctx, db, metrics), fields(subsystem = LOG_TARGET))]
 async fn process_block_activated<Context>(
 	ctx: &mut Context,
 	db: &Arc<dyn KeyValueDB>,
 	hash: Hash,
+	metrics: &Metrics,
 ) -> Result<(), Error>
 where
 	Context: SubsystemContext<Message=AvailabilityStoreMessage>
 {
+	let _timer = metrics.time_block_activated();
+
 	let events = match request_candidate_events(ctx, hash).await {
 		Ok(events) => events,
 		Err(err) => {
@@ -697,6 +706,8 @@ where
 {
 	use AvailabilityStoreMessage::*;
 
+	let _timer = subsystem.metrics.time_process_message();
+
 	match msg {
 		QueryAvailableData(hash, tx) => {
 			tx.send(available_data(&subsystem.inner, &hash).map(|d| d.data))
@@ -860,6 +871,8 @@ fn store_available_data(
 	n_validators: u32,
 	available_data: AvailableData,
 ) -> Result<(), Error> {
+	let _timer = subsystem.metrics.time_store_available_data();
+
 	let mut tx = DBTransaction::new();
 
 	let block_number = available_data.validation_data.block_number;
@@ -927,6 +940,8 @@ fn store_chunk(
 	chunk: ErasureChunk,
 	block_number: BlockNumber,
 ) -> Result<(), Error> {
+	let _timer = subsystem.metrics.time_store_chunk();
+
 	let mut tx = DBTransaction::new();
 
 	let dbkey = erasure_chunk_key(candidate_hash, chunk.index);
@@ -977,6 +992,8 @@ fn get_chunk(
 	candidate_hash: &CandidateHash,
 	index: u32,
 ) -> Result<Option<ErasureChunk>, Error> {
+	let _timer = subsystem.metrics.time_get_chunk();
+
 	if let Some(chunk) = query_inner(
 		&subsystem.inner,
 		columns::DATA,
@@ -1059,6 +1076,14 @@ fn get_chunks(data: &AvailableData, n_validators: usize, metrics: &Metrics) -> R
 #[derive(Clone)]
 struct MetricsInner {
 	received_availability_chunks_total: prometheus::Counter<prometheus::U64>,
+	prune_povs: prometheus::Histogram,
+	prune_chunks: prometheus::Histogram,
+	process_block_finalized: prometheus::Histogram,
+	block_activated: prometheus::Histogram,
+	process_message: prometheus::Histogram,
+	store_available_data: prometheus::Histogram,
+	store_chunk: prometheus::Histogram,
+	get_chunk: prometheus::Histogram,
 }
 
 /// Availability metrics.
@@ -1074,6 +1099,46 @@ impl Metrics {
 			metrics.received_availability_chunks_total.inc_by(by);
 		}
 	}
+
+	/// Provide a timer for `prune_povs` which observes on drop.
+	fn time_prune_povs(&self) -> Option<metrics::prometheus::prometheus::HistogramTimer> {
+		self.0.as_ref().map(|metrics| metrics.prune_povs.start_timer())
+	}
+
+	/// Provide a timer for `prune_chunks` which observes on drop.
+	fn time_prune_chunks(&self) -> Option<metrics::prometheus::prometheus::HistogramTimer> {
+		self.0.as_ref().map(|metrics| metrics.prune_chunks.start_timer())
+	}
+
+	/// Provide a timer for `process_block_finalized` which observes on drop.
+	fn time_process_block_finalized(&self) -> Option<metrics::prometheus::prometheus::HistogramTimer> {
+		self.0.as_ref().map(|metrics| metrics.process_block_finalized.start_timer())
+	}
+
+	/// Provide a timer for `block_activated` which observes on drop.
+	fn time_block_activated(&self) -> Option<metrics::prometheus::prometheus::HistogramTimer> {
+		self.0.as_ref().map(|metrics| metrics.block_activated.start_timer())
+	}
+
+	/// Provide a timer for `process_message` which observes on drop.
+	fn time_process_message(&self) -> Option<metrics::prometheus::prometheus::HistogramTimer> {
+		self.0.as_ref().map(|metrics| metrics.process_message.start_timer())
+	}
+
+	/// Provide a timer for `store_available_data` which observes on drop.
+	fn time_store_available_data(&self) -> Option<metrics::prometheus::prometheus::HistogramTimer> {
+		self.0.as_ref().map(|metrics| metrics.store_available_data.start_timer())
+	}
+
+	/// Provide a timer for `store_chunk` which observes on drop.
+	fn time_store_chunk(&self) -> Option<metrics::prometheus::prometheus::HistogramTimer> {
+		self.0.as_ref().map(|metrics| metrics.store_chunk.start_timer())
+	}
+
+	/// Provide a timer for `get_chunk` which observes on drop.
+	fn time_get_chunk(&self) -> Option<metrics::prometheus::prometheus::HistogramTimer> {
+		self.0.as_ref().map(|metrics| metrics.get_chunk.start_timer())
+	}
 }
 
 impl metrics::Metrics for Metrics {
@@ -1086,6 +1151,78 @@ impl metrics::Metrics for Metrics {
 				)?,
 				registry,
 			)?,
+			prune_povs: prometheus::register(
+				prometheus::Histogram::with_opts(
+					prometheus::HistogramOpts::new(
+						"parachain_av_store_prune_povs",
+						"Time spent within `av_store::prune_povs`",
+					)
+				)?,
+				registry,
+			)?,
+			prune_chunks: prometheus::register(
+				prometheus::Histogram::with_opts(
+					prometheus::HistogramOpts::new(
+						"parachain_av_store_prune_chunks",
+						"Time spent within `av_store::prune_chunks`",
+					)
+				)?,
+				registry,
+			)?,
+			process_block_finalized: prometheus::register(
+				prometheus::Histogram::with_opts(
+					prometheus::HistogramOpts::new(
+						"parachain_av_store_process_block_finalized",
+						"Time spent within `av_store::block_finalized`",
+					)
+				)?,
+				registry,
+			)?,
+			block_activated: prometheus::register(
+				prometheus::Histogram::with_opts(
+					prometheus::HistogramOpts::new(
+						"parachain_av_store_block_activated",
+						"Time spent within `av_store::block_activated`",
+					)
+				)?,
+				registry,
+			)?,
+			process_message: prometheus::register(
+				prometheus::Histogram::with_opts(
+					prometheus::HistogramOpts::new(
+						"parachain_av_store_process_message",
+						"Time spent within `av_store::process_message`",
+					)
+				)?,
+				registry,
+			)?,
+			store_available_data: prometheus::register(
+				prometheus::Histogram::with_opts(
+					prometheus::HistogramOpts::new(
+						"parachain_av_store_store_available_data",
+						"Time spent within `av_store::store_available_data`",
+					)
+				)?,
+				registry,
+			)?,
+			store_chunk: prometheus::register(
+				prometheus::Histogram::with_opts(
+					prometheus::HistogramOpts::new(
+						"parachain_av_store_store_chunk",
+						"Time spent within `av_store::store_chunk`",
+					)
+				)?,
+				registry,
+			)?,
+			get_chunk: prometheus::register(
+				prometheus::Histogram::with_opts(
+					prometheus::HistogramOpts::new(
+						"parachain_av_store_get_chunk",
+						"Time spent within `av_store::get_chunk`",
+					)
+				)?,
+				registry,
+			)?,
 		};
 		Ok(Metrics(Some(metrics)))
 	}