client/finality-grandpa: Instrument until-imported queue (#5438)

* client/finality-grandpa: Instrument until-imported queue

The `UntilImported` queue takes as input finality grandpa messages that
depend on blocks that are not yet imported and holds them back until
those blocks are imported.

This commit adds a basic metric, the amount of messages waiting in the
queue, to the module. For now this metric is only available for the
global `UntilImported` queue awaiting blocks for commit and catch-up
messages.

* client/finality-grandpa/src/until_imported: Update metric help text

Co-Authored-By: Ashley <ashley.ruglys@gmail.com>

Co-authored-by: Ashley <ashley.ruglys@gmail.com>

substrate/client/finality-grandpa/src/lib.rs

@@ -63,6 +63,7 @@ use sc_client_api::{
 };
 use sp_blockchain::{HeaderBackend, Error as ClientError, HeaderMetadata};
 use parity_scale_codec::{Decode, Encode};
+use prometheus_endpoint::{PrometheusError, Registry};
 use sp_runtime::generic::BlockId;
 use sp_runtime::traits::{NumberFor, Block as BlockT, DigestFor, Zero};
 use sc_keystore::KeyStorePtr;
@@ -104,7 +105,7 @@ pub use voting_rule::{
 };
 
 use aux_schema::PersistentData;
-use environment::{Environment, VoterSetState, Metrics};
+use environment::{Environment, VoterSetState};
 use import::GrandpaBlockImport;
 use until_imported::UntilGlobalMessageBlocksImported;
 use communication::{NetworkBridge, Network as NetworkT};
@@ -519,6 +520,7 @@ fn global_communication<BE, Block: BlockT, C, N>(
 	client: Arc<C>,
 	network: &NetworkBridge<Block, N>,
 	keystore: &Option<KeyStorePtr>,
+	metrics: Option<until_imported::Metrics>,
 ) -> (
 	impl Stream<
 		Item = Result<CommunicationInH<Block, Block::Hash>, CommandOrError<Block::Hash, NumberFor<Block>>>,
@@ -549,6 +551,7 @@ fn global_communication<BE, Block: BlockT, C, N>(
 		client.clone(),
 		global_in,
 		"global",
+		metrics,
 	);
 
 	let global_in = global_in.map_err(CommandOrError::from);
@@ -696,6 +699,20 @@ pub fn run_grandpa_voter<Block: BlockT, BE: 'static, C, N, SC, VR>(
 	Ok(future::select(voter_work, telemetry_task).map(drop))
 }
 
+struct Metrics {
+	environment: environment::Metrics,
+	until_imported: until_imported::Metrics,
+}
+
+impl Metrics {
+	fn register(registry: &Registry) -> Result<Self, PrometheusError> {
+		Ok(Metrics {
+			environment: environment::Metrics::register(registry)?,
+			until_imported: until_imported::Metrics::register(registry)?,
+		})
+	}
+}
+
 /// Future that powers the voter.
 #[must_use]
 struct VoterWork<B, Block: BlockT, C, N: NetworkT<Block>, SC, VR> {
@@ -703,6 +720,9 @@ struct VoterWork<B, Block: BlockT, C, N: NetworkT<Block>, SC, VR> {
 	env: Arc<Environment<B, Block, C, N, SC, VR>>,
 	voter_commands_rx: mpsc::UnboundedReceiver<VoterCommand<Block::Hash, NumberFor<Block>>>,
 	network: NetworkBridge<Block, N>,
+
+	/// Prometheus metrics.
+	metrics: Option<Metrics>,
 }
 
 impl<B, Block, C, N, SC, VR> VoterWork<B, Block, C, N, SC, VR>
@@ -725,6 +745,14 @@ where
 		voter_commands_rx: mpsc::UnboundedReceiver<VoterCommand<Block::Hash, NumberFor<Block>>>,
 		prometheus_registry: Option<prometheus_endpoint::Registry>,
 	) -> Self {
+		let metrics = match prometheus_registry.as_ref().map(Metrics::register) {
+			Some(Ok(metrics)) => Some(metrics),
+			Some(Err(e)) => {
+				debug!(target: "afg", "Failed to register metrics: {:?}", e);
+				None
+			}
+			None => None,
+		};
 
 		let voters = persistent_data.authority_set.current_authorities();
 		let env = Arc::new(Environment {
@@ -738,10 +766,7 @@ where
 			authority_set: persistent_data.authority_set.clone(),
 			consensus_changes: persistent_data.consensus_changes.clone(),
 			voter_set_state: persistent_data.set_state.clone(),
-			metrics: prometheus_registry.map(|registry| {
-				Metrics::register(&registry)
-					.expect("Other metrics would have failed to register before these; qed")
-			}),
+			metrics: metrics.as_ref().map(|m| m.environment.clone()),
 			_phantom: PhantomData,
 		});
 
@@ -752,6 +777,7 @@ where
 			env,
 			voter_commands_rx,
 			network,
+			metrics,
 		};
 		work.rebuild_voter();
 		work
@@ -800,6 +826,7 @@ where
 					self.env.client.clone(),
 					&self.env.network,
 					&self.env.config.keystore,
+					self.metrics.as_ref().map(|m| m.until_imported.clone()),
 				);
 
 				let last_completed_round = completed_rounds.last();