Add Prometheus timers to the subsystems (#1923)

* reexport prometheus-super for ease of use of other subsystems

* add some prometheus timers for collation generation subsystem

* add timing metrics to av-store

* add metrics to candidate backing

* add timing metric to bitfield signing

* add timing metrics to candidate selection

* add timing metrics to candidate-validation

* add timing metrics to chain-api

* add timing metrics to provisioner

* add timing metrics to runtime-api

* add timing metrics to availability-distribution

* add timing metrics to bitfield-distribution

* add timing metrics to collator protocol: collator side

* add timing metrics to collator protocol: validator side

* fix candidate validation test failures

* add timing metrics to pov distribution

* add timing metrics to statement-distribution

* use substrate_prometheus_endpoint prometheus reexport instead of prometheus_super

* don't include JOB_DELAY in bitfield-signing metrics

* give adder-collator ability to easily export its genesis-state and validation code

* wip: adder-collator pushbutton script

* don't attempt to register the adder-collator automatically

Instead, get these values with

```sh
target/release/adder-collator export-genesis-state
target/release/adder-collator export-genesis-wasm
```

And then register the parachain on https://polkadot.js.org/apps/?rpc=ws%3A%2F%2F127.0.0.1%3A9944#/explorer

To collect prometheus data, after running the script, create `prometheus.yml` per the instructions
at https://www.notion.so/paritytechnologies/Setting-up-Prometheus-locally-835cb3a9df7541a781c381006252b5ff
and then run:

```sh
docker run -v `pwd`/prometheus.yml:/etc/prometheus/prometheus.yml:z --network host prom/prometheus
```

Demonstrates that data makes it across to prometheus, though it is likely to be useful in the future
to tweak the buckets.

* Update parachain/test-parachains/adder/collator/src/cli.rs

Co-authored-by: Andronik Ordian <write@reusable.software>

* use the grandpa-pause parameter

* skip metrics in tracing instrumentation

* remove unnecessary grandpa_pause cli param

Co-authored-by: Andronik Ordian <write@reusable.software>

polkadot/node/collation-generation/src/lib.rs

@@ -189,7 +189,11 @@ async fn handle_new_activations<Context: SubsystemContext>(
 	// follow the procedure from the guide:
 	// https://w3f.github.io/parachain-implementers-guide/node/collators/collation-generation.html
 
+	let _overall_timer = metrics.time_new_activations();
+
 	for relay_parent in activated.iter().copied() {
+		let _relay_parent_timer = metrics.time_new_activations_relay_parent();
+
 		// double-future magic happens here: the first layer of requests takes a mutable borrow of the context, and
 		// returns a receiver. The second layer of requests actually polls those receivers to completion.
 		let (availability_cores, validators) = join!(
@@ -201,6 +205,8 @@ async fn handle_new_activations<Context: SubsystemContext>(
 		let n_validators = validators??.len();
 
 		for core in availability_cores {
+			let _availability_core_timer = metrics.time_new_activations_availability_core();
+
 			let (scheduled_core, assumption) = match core {
 				CoreState::Scheduled(scheduled_core) => {
 					(scheduled_core, OccupiedCoreAssumption::Free)
@@ -335,6 +341,9 @@ fn erasure_root(
 #[derive(Clone)]
 struct MetricsInner {
 	collations_generated_total: prometheus::Counter<prometheus::U64>,
+	new_activations_overall: prometheus::Histogram,
+	new_activations_per_relay_parent: prometheus::Histogram,
+	new_activations_per_availability_core: prometheus::Histogram,
 }
 
 /// CollationGenerationSubsystem metrics.
@@ -347,6 +356,21 @@ impl Metrics {
 			metrics.collations_generated_total.inc();
 		}
 	}
+
+	/// Provide a timer for new activations which updates on drop.
+	fn time_new_activations(&self) -> Option<metrics::prometheus::prometheus::HistogramTimer> {
+		self.0.as_ref().map(|metrics| metrics.new_activations_overall.start_timer())
+	}
+
+	/// Provide a timer per relay parents which updates on drop.
+	fn time_new_activations_relay_parent(&self) -> Option<metrics::prometheus::prometheus::HistogramTimer> {
+		self.0.as_ref().map(|metrics| metrics.new_activations_per_relay_parent.start_timer())
+	}
+
+	/// Provide a timer per availability core which updates on drop.
+	fn time_new_activations_availability_core(&self) -> Option<metrics::prometheus::prometheus::HistogramTimer> {
+		self.0.as_ref().map(|metrics| metrics.new_activations_per_availability_core.start_timer())
+	}
 }
 
 impl metrics::Metrics for Metrics {
@@ -359,6 +383,33 @@ impl metrics::Metrics for Metrics {
 				)?,
 				registry,
 			)?,
+			new_activations_overall: prometheus::register(
+				prometheus::Histogram::with_opts(
+					prometheus::HistogramOpts::new(
+						"parachain_collation_generation_new_activations",
+						"Time spent within fn handle_new_activations",
+					)
+				)?,
+				registry,
+			)?,
+			new_activations_per_relay_parent: prometheus::register(
+				prometheus::Histogram::with_opts(
+					prometheus::HistogramOpts::new(
+						"parachain_collation_generation_per_relay_parent",
+						"Time spent handling a particular relay parent within fn handle_new_activations"
+					)
+				)?,
+				registry,
+			)?,
+			new_activations_per_availability_core: prometheus::register(
+				prometheus::Histogram::with_opts(
+					prometheus::HistogramOpts::new(
+						"parachain_collation_generation_per_availability_core",
+						"Time spent handling a particular availability core for a relay parent in fn handle_new_activations",
+					)
+				)?,
+				registry,
+			)?,
 		};
 		Ok(Metrics(Some(metrics)))
 	}