Add Prometheus timers to the subsystems (#1923)

* reexport prometheus-super for ease of use of other subsystems

* add some prometheus timers for collation generation subsystem

* add timing metrics to av-store

* add metrics to candidate backing

* add timing metric to bitfield signing

* add timing metrics to candidate selection

* add timing metrics to candidate-validation

* add timing metrics to chain-api

* add timing metrics to provisioner

* add timing metrics to runtime-api

* add timing metrics to availability-distribution

* add timing metrics to bitfield-distribution

* add timing metrics to collator protocol: collator side

* add timing metrics to collator protocol: validator side

* fix candidate validation test failures

* add timing metrics to pov distribution

* add timing metrics to statement-distribution

* use substrate_prometheus_endpoint prometheus reexport instead of prometheus_super

* don't include JOB_DELAY in bitfield-signing metrics

* give adder-collator ability to easily export its genesis-state and validation code

* wip: adder-collator pushbutton script

* don't attempt to register the adder-collator automatically

Instead, get these values with

```sh
target/release/adder-collator export-genesis-state
target/release/adder-collator export-genesis-wasm
```

And then register the parachain on https://polkadot.js.org/apps/?rpc=ws%3A%2F%2F127.0.0.1%3A9944#/explorer

To collect prometheus data, after running the script, create `prometheus.yml` per the instructions
at https://www.notion.so/paritytechnologies/Setting-up-Prometheus-locally-835cb3a9df7541a781c381006252b5ff
and then run:

```sh
docker run -v `pwd`/prometheus.yml:/etc/prometheus/prometheus.yml:z --network host prom/prometheus
```

Demonstrates that data makes it across to prometheus, though it is likely to be useful in the future
to tweak the buckets.

* Update parachain/test-parachains/adder/collator/src/cli.rs

Co-authored-by: Andronik Ordian <write@reusable.software>

* use the grandpa-pause parameter

* skip metrics in tracing instrumentation

* remove unnecessary grandpa_pause cli param

Co-authored-by: Andronik Ordian <write@reusable.software>

polkadot/node/core/backing/src/lib.rs

@@ -481,8 +481,11 @@ impl CandidateBackingJob {
 
 	#[tracing::instrument(level = "trace", skip(self), fields(subsystem = LOG_TARGET))]
 	async fn process_msg(&mut self, msg: CandidateBackingMessage) -> Result<(), Error> {
+
 		match msg {
 			CandidateBackingMessage::Second(_, candidate, pov) => {
+				let _timer = self.metrics.time_process_second();
+
 				// Sanity check that candidate is from our assignment.
 				if candidate.descriptor().para_id != self.assignment {
 					return Ok(());
@@ -509,6 +512,8 @@ impl CandidateBackingJob {
 				}
 			}
 			CandidateBackingMessage::Statement(_, statement) => {
+				let _timer = self.metrics.time_process_statement();
+
 				self.check_statement_signature(&statement)?;
 				match self.maybe_validate_and_import(statement).await {
 					Err(Error::ValidationFailed(_)) => return Ok(()),
@@ -517,6 +522,8 @@ impl CandidateBackingJob {
 				}
 			}
 			CandidateBackingMessage::GetBackedCandidates(_, tx) => {
+				let _timer = self.metrics.time_get_backed_candidates();
+
 				let backed = self.get_backed();
 
 				tx.send(backed).map_err(|data| Error::Send(data))?;
@@ -898,7 +905,10 @@ impl util::JobTrait for CandidateBackingJob {
 #[derive(Clone)]
 struct MetricsInner {
 	signed_statements_total: prometheus::Counter<prometheus::U64>,
-	candidates_seconded_total: prometheus::Counter<prometheus::U64>
+	candidates_seconded_total: prometheus::Counter<prometheus::U64>,
+	process_second: prometheus::Histogram,
+	process_statement: prometheus::Histogram,
+	get_backed_candidates: prometheus::Histogram,
 }
 
 /// Candidate backing metrics.
@@ -917,6 +927,21 @@ impl Metrics {
 			metrics.candidates_seconded_total.inc();
 		}
 	}
+
+	/// Provide a timer for handling `CandidateBackingMessage:Second` which observes on drop.
+	fn time_process_second(&self) -> Option<metrics::prometheus::prometheus::HistogramTimer> {
+		self.0.as_ref().map(|metrics| metrics.process_second.start_timer())
+	}
+
+	/// Provide a timer for handling `CandidateBackingMessage::Statement` which observes on drop.
+	fn time_process_statement(&self) -> Option<metrics::prometheus::prometheus::HistogramTimer> {
+		self.0.as_ref().map(|metrics| metrics.process_statement.start_timer())
+	}
+
+	/// Provide a timer for handling `CandidateBackingMessage::GetBackedCandidates` which observes on drop.
+	fn time_get_backed_candidates(&self) -> Option<metrics::prometheus::prometheus::HistogramTimer> {
+		self.0.as_ref().map(|metrics| metrics.get_backed_candidates.start_timer())
+	}
 }
 
 impl metrics::Metrics for Metrics {
@@ -924,18 +949,45 @@ impl metrics::Metrics for Metrics {
 		let metrics = MetricsInner {
 			signed_statements_total: prometheus::register(
 				prometheus::Counter::new(
-					"parachain_signed_statements_total",
+					"parachain_candidate_backing_signed_statements_total",
 					"Number of statements signed.",
 				)?,
 				registry,
 			)?,
 			candidates_seconded_total: prometheus::register(
 				prometheus::Counter::new(
-					"parachain_candidates_seconded_total",
+					"parachain_candidate_backing_candidates_seconded_total",
 					"Number of candidates seconded.",
 				)?,
 				registry,
 			)?,
+			process_second: prometheus::register(
+				prometheus::Histogram::with_opts(
+					prometheus::HistogramOpts::new(
+						"parachain_candidate_backing_process_second",
+						"Time spent within `candidate_backing::process_second`",
+					)
+				)?,
+				registry,
+			)?,
+			process_statement: prometheus::register(
+				prometheus::Histogram::with_opts(
+					prometheus::HistogramOpts::new(
+						"parachain_candidate_backing_process_statement",
+						"Time spent within `candidate_backing::process_statement`",
+					)
+				)?,
+				registry,
+			)?,
+			get_backed_candidates: prometheus::register(
+				prometheus::Histogram::with_opts(
+					prometheus::HistogramOpts::new(
+						"parachain_candidate_backing_get_backed_candidates",
+						"Time spent within `candidate_backing::get_backed_candidates`",
+					)
+				)?,
+				registry,
+			)?,
 		};
 		Ok(Metrics(Some(metrics)))
 	}