Add Prometheus timers to the subsystems (#1923)

* reexport prometheus-super for ease of use of other subsystems

* add some prometheus timers for collation generation subsystem

* add timing metrics to av-store

* add metrics to candidate backing

* add timing metric to bitfield signing

* add timing metrics to candidate selection

* add timing metrics to candidate-validation

* add timing metrics to chain-api

* add timing metrics to provisioner

* add timing metrics to runtime-api

* add timing metrics to availability-distribution

* add timing metrics to bitfield-distribution

* add timing metrics to collator protocol: collator side

* add timing metrics to collator protocol: validator side

* fix candidate validation test failures

* add timing metrics to pov distribution

* add timing metrics to statement-distribution

* use substrate_prometheus_endpoint prometheus reexport instead of prometheus_super

* don't include JOB_DELAY in bitfield-signing metrics

* give adder-collator ability to easily export its genesis-state and validation code

* wip: adder-collator pushbutton script

* don't attempt to register the adder-collator automatically

Instead, get these values with

```sh
target/release/adder-collator export-genesis-state
target/release/adder-collator export-genesis-wasm
```

And then register the parachain on https://polkadot.js.org/apps/?rpc=ws%3A%2F%2F127.0.0.1%3A9944#/explorer

To collect prometheus data, after running the script, create `prometheus.yml` per the instructions
at https://www.notion.so/paritytechnologies/Setting-up-Prometheus-locally-835cb3a9df7541a781c381006252b5ff
and then run:

```sh
docker run -v `pwd`/prometheus.yml:/etc/prometheus/prometheus.yml:z --network host prom/prometheus
```

Demonstrates that data makes it across to prometheus, though it is likely to be useful in the future
to tweak the buckets.

* Update parachain/test-parachains/adder/collator/src/cli.rs

Co-authored-by: Andronik Ordian <write@reusable.software>

* use the grandpa-pause parameter

* skip metrics in tracing instrumentation

* remove unnecessary grandpa_pause cli param

Co-authored-by: Andronik Ordian <write@reusable.software>

polkadot/node/core/chain-api/src/lib.rs

@@ -92,11 +92,13 @@ where
 			FromOverseer::Signal(OverseerSignal::BlockFinalized(_)) => {},
 			FromOverseer::Communication { msg } => match msg {
 				ChainApiMessage::BlockNumber(hash, response_channel) => {
+					let _timer = subsystem.metrics.time_block_number();
 					let result = subsystem.client.number(hash).map_err(|e| e.to_string().into());
 					subsystem.metrics.on_request(result.is_ok());
 					let _ = response_channel.send(result);
 				},
 				ChainApiMessage::BlockHeader(hash, response_channel) => {
+					let _timer = subsystem.metrics.time_block_header();
 					let result = subsystem.client
 						.header(BlockId::Hash(hash))
 						.map_err(|e| e.to_string().into());
@@ -104,19 +106,23 @@ where
 					let _ = response_channel.send(result);
 				},
 				ChainApiMessage::FinalizedBlockHash(number, response_channel) => {
+					let _timer = subsystem.metrics.time_finalized_block_hash();
 					// Note: we don't verify it's finalized
 					let result = subsystem.client.hash(number).map_err(|e| e.to_string().into());
 					subsystem.metrics.on_request(result.is_ok());
 					let _ = response_channel.send(result);
 				},
 				ChainApiMessage::FinalizedBlockNumber(response_channel) => {
+					let _timer = subsystem.metrics.time_finalized_block_number();
 					let result = subsystem.client.info().finalized_number;
 					// always succeeds
 					subsystem.metrics.on_request(true);
 					let _ = response_channel.send(Ok(result));
 				},
 				ChainApiMessage::Ancestors { hash, k, response_channel } => {
+					let _timer = subsystem.metrics.time_ancestors();
 					tracing::span!(tracing::Level::TRACE, "ChainApiMessage::Ancestors", subsystem=LOG_TARGET, hash=%hash, k=k);
+
 					let mut hash = hash;
 
 					let next_parent = core::iter::from_fn(|| {
@@ -153,6 +159,11 @@ where
 #[derive(Clone)]
 struct MetricsInner {
 	chain_api_requests: prometheus::CounterVec<prometheus::U64>,
+	block_number: prometheus::Histogram,
+	block_header: prometheus::Histogram,
+	finalized_block_hash: prometheus::Histogram,
+	finalized_block_number: prometheus::Histogram,
+	ancestors: prometheus::Histogram,
 }
 
 /// Chain API metrics.
@@ -169,6 +180,31 @@ impl Metrics {
 			}
 		}
 	}
+
+	/// Provide a timer for `block_number` which observes on drop.
+	fn time_block_number(&self) -> Option<metrics::prometheus::prometheus::HistogramTimer> {
+		self.0.as_ref().map(|metrics| metrics.block_number.start_timer())
+	}
+
+	/// Provide a timer for `block_header` which observes on drop.
+	fn time_block_header(&self) -> Option<metrics::prometheus::prometheus::HistogramTimer> {
+		self.0.as_ref().map(|metrics| metrics.block_header.start_timer())
+	}
+
+	/// Provide a timer for `finalized_block_hash` which observes on drop.
+	fn time_finalized_block_hash(&self) -> Option<metrics::prometheus::prometheus::HistogramTimer> {
+		self.0.as_ref().map(|metrics| metrics.finalized_block_hash.start_timer())
+	}
+
+	/// Provide a timer for `finalized_block_number` which observes on drop.
+	fn time_finalized_block_number(&self) -> Option<metrics::prometheus::prometheus::HistogramTimer> {
+		self.0.as_ref().map(|metrics| metrics.finalized_block_number.start_timer())
+	}
+
+	/// Provide a timer for `ancestors` which observes on drop.
+	fn time_ancestors(&self) -> Option<metrics::prometheus::prometheus::HistogramTimer> {
+		self.0.as_ref().map(|metrics| metrics.ancestors.start_timer())
+	}
 }
 
 impl metrics::Metrics for Metrics {
@@ -184,6 +220,51 @@ impl metrics::Metrics for Metrics {
 				)?,
 				registry,
 			)?,
+			block_number: prometheus::register(
+				prometheus::Histogram::with_opts(
+					prometheus::HistogramOpts::new(
+						"parachain_chain_api_block_number",
+						"Time spent within `chain_api::block_number`",
+					)
+				)?,
+				registry,
+			)?,
+			block_header: prometheus::register(
+				prometheus::Histogram::with_opts(
+					prometheus::HistogramOpts::new(
+						"parachain_chain_api_block_headers",
+						"Time spent within `chain_api::block_headers`",
+					)
+				)?,
+				registry,
+			)?,
+			finalized_block_hash: prometheus::register(
+				prometheus::Histogram::with_opts(
+					prometheus::HistogramOpts::new(
+						"parachain_chain_api_finalized_block_hash",
+						"Time spent within `chain_api::finalized_block_hash`",
+					)
+				)?,
+				registry,
+			)?,
+			finalized_block_number: prometheus::register(
+				prometheus::Histogram::with_opts(
+					prometheus::HistogramOpts::new(
+						"parachain_chain_api_finalized_block_number",
+						"Time spent within `chain_api::finalized_block_number`",
+					)
+				)?,
+				registry,
+			)?,
+			ancestors: prometheus::register(
+				prometheus::Histogram::with_opts(
+					prometheus::HistogramOpts::new(
+						"parachain_chain_api_ancestors",
+						"Time spent within `chain_api::ancestors`",
+					)
+				)?,
+				registry,
+			)?,
 		};
 		Ok(Metrics(Some(metrics)))
 	}