client/beefy: add more metrics for production visibility (#12910)

* few beefy metrics

* more beefy metrics

* some beefy metrics

* some beefy metrics

* more metrics

* other metrics

* fix tests

* merge changes

* Apply suggestions from code review

* client/beefy: fix metrics

* client/beefy: separate metrics per component, avoid double registering

* client/beefy: deduplicate metrics registration code

* remove unused metric

* impl review suggestions

---------

Co-authored-by: Adrian Catangiu <adrian@parity.io>

substrate/client/beefy/src/communication/request_response/incoming_requests_handler.rs

@@ -29,8 +29,13 @@ use sc_network_common::protocol::ProtocolName;
 use sp_runtime::traits::Block;
 use std::{marker::PhantomData, sync::Arc};
 
-use crate::communication::request_response::{
-	on_demand_justifications_protocol_config, Error, JustificationRequest, BEEFY_SYNC_LOG_TARGET,
+use crate::{
+	communication::request_response::{
+		on_demand_justifications_protocol_config, Error, JustificationRequest,
+		BEEFY_SYNC_LOG_TARGET,
+	},
+	metric_inc,
+	metrics::{register_metrics, OnDemandIncomingRequestsMetrics},
 };
 
 /// A request coming in, including a sender for sending responses.
@@ -119,6 +124,7 @@ pub struct BeefyJustifsRequestHandler<B, Client> {
 	pub(crate) request_receiver: IncomingRequestReceiver,
 	pub(crate) justif_protocol_name: ProtocolName,
 	pub(crate) client: Arc<Client>,
+	pub(crate) metrics: Option<OnDemandIncomingRequestsMetrics>,
 	pub(crate) _block: PhantomData<B>,
 }
 
@@ -132,12 +138,16 @@ where
 		genesis_hash: Hash,
 		fork_id: Option<&str>,
 		client: Arc<Client>,
+		prometheus_registry: Option<prometheus::Registry>,
 	) -> (Self, RequestResponseConfig) {
 		let (request_receiver, config) =
 			on_demand_justifications_protocol_config(genesis_hash, fork_id);
 		let justif_protocol_name = config.name.clone();
-
-		(Self { request_receiver, justif_protocol_name, client, _block: PhantomData }, config)
+		let metrics = register_metrics(prometheus_registry);
+		(
+			Self { request_receiver, justif_protocol_name, client, metrics, _block: PhantomData },
+			config,
+		)
 	}
 
 	/// Network request-response protocol name used by this handler.
@@ -180,12 +190,14 @@ where
 			let peer = request.peer;
 			match self.handle_request(request) {
 				Ok(()) => {
+					metric_inc!(self, beefy_successful_justification_responses);
 					debug!(
 						target: BEEFY_SYNC_LOG_TARGET,
 						"🥩 Handled BEEFY justification request from {:?}.", peer
 					)
 				},
 				Err(e) => {
+					metric_inc!(self, beefy_failed_justification_responses);
 					// TODO (issue #12293): apply reputation changes here based on error type.
 					debug!(
 						target: BEEFY_SYNC_LOG_TARGET,

substrate/client/beefy/src/communication/request_response/outgoing_requests_engine.rs

@@ -34,6 +34,8 @@ use std::{collections::VecDeque, result::Result, sync::Arc};
 use crate::{
 	communication::request_response::{Error, JustificationRequest, BEEFY_SYNC_LOG_TARGET},
 	justification::{decode_and_verify_finality_proof, BeefyVersionedFinalityProof},
+	metric_inc,
+	metrics::{register_metrics, OnDemandOutgoingRequestsMetrics},
 	KnownPeers,
 };
 
@@ -61,6 +63,7 @@ pub struct OnDemandJustificationsEngine<B: Block> {
 	peers_cache: VecDeque<PeerId>,
 
 	state: State<B>,
+	metrics: Option<OnDemandOutgoingRequestsMetrics>,
 }
 
 impl<B: Block> OnDemandJustificationsEngine<B> {
@@ -68,13 +71,16 @@ impl<B: Block> OnDemandJustificationsEngine<B> {
 		network: Arc<dyn NetworkRequest + Send + Sync>,
 		protocol_name: ProtocolName,
 		live_peers: Arc<Mutex<KnownPeers<B>>>,
+		prometheus_registry: Option<prometheus::Registry>,
 	) -> Self {
+		let metrics = register_metrics(prometheus_registry);
 		Self {
 			network,
 			protocol_name,
 			live_peers,
 			peers_cache: VecDeque::new(),
 			state: State::Idle,
+			metrics,
 		}
 	}
 
@@ -130,6 +136,7 @@ impl<B: Block> OnDemandJustificationsEngine<B> {
 		if let Some(peer) = self.try_next_peer() {
 			self.request_from_peer(peer, RequestInfo { block, active_set });
 		} else {
+			metric_inc!(self, beefy_on_demand_justification_no_peer_to_request_from);
 			debug!(
 				target: BEEFY_SYNC_LOG_TARGET,
 				"🥩 no good peers to request justif #{:?} from", block
@@ -159,6 +166,7 @@ impl<B: Block> OnDemandJustificationsEngine<B> {
 	) -> Result<BeefyVersionedFinalityProof<B>, Error> {
 		response
 			.map_err(|e| {
+				metric_inc!(self, beefy_on_demand_justification_peer_hang_up);
 				debug!(
 					target: BEEFY_SYNC_LOG_TARGET,
 					"🥩 for on demand justification #{:?}, peer {:?} hung up: {:?}",
@@ -169,6 +177,7 @@ impl<B: Block> OnDemandJustificationsEngine<B> {
 				Error::InvalidResponse
 			})?
 			.map_err(|e| {
+				metric_inc!(self, beefy_on_demand_justification_peer_error);
 				debug!(
 					target: BEEFY_SYNC_LOG_TARGET,
 					"🥩 for on demand justification #{:?}, peer {:?} error: {:?}",
@@ -185,6 +194,7 @@ impl<B: Block> OnDemandJustificationsEngine<B> {
 					&req_info.active_set,
 				)
 				.map_err(|e| {
+					metric_inc!(self, beefy_on_demand_justification_invalid_proof);
 					debug!(
 						target: BEEFY_SYNC_LOG_TARGET,
 						"🥩 for on demand justification #{:?}, peer {:?} responded with invalid proof: {:?}",
@@ -224,6 +234,7 @@ impl<B: Block> OnDemandJustificationsEngine<B> {
 				}
 			})
 			.map(|proof| {
+				metric_inc!(self, beefy_on_demand_justification_good_proof);
 				debug!(
 					target: BEEFY_SYNC_LOG_TARGET,
 					"🥩 received valid on-demand justif #{:?} from {:?}", block, peer

substrate/client/beefy/src/import.rs

@@ -35,6 +35,8 @@ use sc_consensus::{BlockCheckParams, BlockImport, BlockImportParams, ImportResul
 use crate::{
 	communication::notification::BeefyVersionedFinalityProofSender,
 	justification::{decode_and_verify_finality_proof, BeefyVersionedFinalityProof},
+	metric_inc,
+	metrics::BlockImportMetrics,
 	LOG_TARGET,
 };
 
@@ -49,6 +51,7 @@ pub struct BeefyBlockImport<Block: BlockT, Backend, RuntimeApi, I> {
 	runtime: Arc<RuntimeApi>,
 	inner: I,
 	justification_sender: BeefyVersionedFinalityProofSender<Block>,
+	metrics: Option<BlockImportMetrics>,
 }
 
 impl<Block: BlockT, BE, Runtime, I: Clone> Clone for BeefyBlockImport<Block, BE, Runtime, I> {
@@ -58,6 +61,7 @@ impl<Block: BlockT, BE, Runtime, I: Clone> Clone for BeefyBlockImport<Block, BE,
 			runtime: self.runtime.clone(),
 			inner: self.inner.clone(),
 			justification_sender: self.justification_sender.clone(),
+			metrics: self.metrics.clone(),
 		}
 	}
 }
@@ -69,8 +73,9 @@ impl<Block: BlockT, BE, Runtime, I> BeefyBlockImport<Block, BE, Runtime, I> {
 		runtime: Arc<Runtime>,
 		inner: I,
 		justification_sender: BeefyVersionedFinalityProofSender<Block>,
+		metrics: Option<BlockImportMetrics>,
 	) -> BeefyBlockImport<Block, BE, Runtime, I> {
-		BeefyBlockImport { backend, runtime, inner, justification_sender }
+		BeefyBlockImport { backend, runtime, inner, justification_sender, metrics }
 	}
 }
 
@@ -147,6 +152,8 @@ where
 					self.justification_sender
 						.notify(|| Ok::<_, ()>(proof))
 						.expect("forwards closure result; the closure always returns Ok; qed.");
+
+					metric_inc!(self, beefy_good_justification_imports);
 				} else {
 					debug!(
 						target: LOG_TARGET,
@@ -154,6 +161,7 @@ where
 						encoded,
 						number,
 					);
+					metric_inc!(self, beefy_bad_justification_imports);
 				}
 			},
 			_ => (),

substrate/client/beefy/src/lib.rs

@@ -28,6 +28,7 @@ use crate::{
 		},
 	},
 	import::BeefyBlockImport,
+	metrics::register_metrics,
 	round::Rounds,
 	worker::PersistedState,
 };
@@ -36,7 +37,7 @@ use beefy_primitives::{
 	GENESIS_AUTHORITY_SET_ID,
 };
 use futures::{stream::Fuse, StreamExt};
-use log::{debug, error, info};
+use log::{error, info};
 use parking_lot::Mutex;
 use prometheus::Registry;
 use sc_client_api::{Backend, BlockBackend, BlockchainEvents, FinalityNotifications, Finalizer};
@@ -133,6 +134,7 @@ pub fn beefy_block_import_and_links<B, BE, RuntimeApi, I>(
 	wrapped_block_import: I,
 	backend: Arc<BE>,
 	runtime: Arc<RuntimeApi>,
+	prometheus_registry: Option<Registry>,
 ) -> (BeefyBlockImport<B, BE, RuntimeApi, I>, BeefyVoterLinks<B>, BeefyRPCLinks<B>)
 where
 	B: Block,
@@ -152,10 +154,16 @@ where
 	// BlockImport -> Voter links
 	let (to_voter_justif_sender, from_block_import_justif_stream) =
 		BeefyVersionedFinalityProofStream::<B>::channel();
+	let metrics = register_metrics(prometheus_registry);
 
 	// BlockImport
-	let import =
-		BeefyBlockImport::new(backend, runtime, wrapped_block_import, to_voter_justif_sender);
+	let import = BeefyBlockImport::new(
+		backend,
+		runtime,
+		wrapped_block_import,
+		to_voter_justif_sender,
+		metrics,
+	);
 	let voter_links = BeefyVoterLinks {
 		from_block_import_justif_stream,
 		to_rpc_justif_sender,
@@ -242,28 +250,16 @@ where
 		gossip_validator.clone(),
 		None,
 	);
+	let metrics = register_metrics(prometheus_registry.clone());
 
 	// The `GossipValidator` adds and removes known peers based on valid votes and network events.
 	let on_demand_justifications = OnDemandJustificationsEngine::new(
 		network.clone(),
 		justifications_protocol_name,
 		known_peers,
+		prometheus_registry.clone(),
 	);
 
-	let metrics =
-		prometheus_registry.as_ref().map(metrics::Metrics::register).and_then(
-			|result| match result {
-				Ok(metrics) => {
-					debug!(target: LOG_TARGET, "🥩 Registered metrics");
-					Some(metrics)
-				},
-				Err(err) => {
-					debug!(target: LOG_TARGET, "🥩 Failed to register metrics: {:?}", err);
-					None
-				},
-			},
-		);
-
 	// Subscribe to finality notifications and justifications before waiting for runtime pallet and
 	// reuse the streams, so we don't miss notifications while waiting for pallet to be available.
 	let mut finality_notifications = client.finality_notification_stream().fuse();

substrate/client/beefy/src/metrics.rs

@@ -18,16 +18,22 @@
 
 //! BEEFY Prometheus metrics definition
 
+use log::debug;
 use prometheus::{register, Counter, Gauge, PrometheusError, Registry, U64};
 
-/// BEEFY metrics exposed through Prometheus
-pub(crate) struct Metrics {
+/// Helper trait for registering BEEFY metrics to Prometheus registry.
+pub(crate) trait PrometheusRegister<T: Sized = Self>: Sized {
+	const DESCRIPTION: &'static str;
+	fn register(registry: &Registry) -> Result<Self, PrometheusError>;
+}
+
+/// BEEFY voting-related metrics exposed through Prometheus
+#[derive(Clone, Debug)]
+pub struct VoterMetrics {
 	/// Current active validator set id
 	pub beefy_validator_set_id: Gauge<U64>,
 	/// Total number of votes sent by this node
 	pub beefy_votes_sent: Counter<U64>,
-	/// Most recent concluded voting round
-	pub beefy_round_concluded: Gauge<U64>,
 	/// Best block finalized by BEEFY
 	pub beefy_best_block: Gauge<U64>,
 	/// Best block BEEFY voted on
@@ -36,10 +42,31 @@ pub(crate) struct Metrics {
 	pub beefy_should_vote_on: Gauge<U64>,
 	/// Number of sessions with lagging signed commitment on mandatory block
 	pub beefy_lagging_sessions: Counter<U64>,
+	/// Number of times no Authority public key found in store
+	pub beefy_no_authority_found_in_store: Counter<U64>,
+	/// Number of currently buffered votes
+	pub beefy_buffered_votes: Gauge<U64>,
+	/// Number of valid but stale votes received
+	pub beefy_stale_votes: Counter<U64>,
+	/// Number of votes dropped due to full buffers
+	pub beefy_buffered_votes_dropped: Counter<U64>,
+	/// Number of currently buffered justifications
+	pub beefy_buffered_justifications: Gauge<U64>,
+	/// Number of valid but stale justifications received
+	pub beefy_stale_justifications: Counter<U64>,
+	/// Number of valid justifications successfully imported
+	pub beefy_imported_justifications: Counter<U64>,
+	/// Number of justifications dropped due to full buffers
+	pub beefy_buffered_justifications_dropped: Counter<U64>,
+	/// Trying to set Best Beefy block to old block
+	pub beefy_best_block_set_last_failure: Gauge<U64>,
+	/// Number of Successful handled votes
+	pub beefy_successful_handled_votes: Counter<U64>,
 }
 
-impl Metrics {
-	pub(crate) fn register(registry: &Registry) -> Result<Self, PrometheusError> {
+impl PrometheusRegister for VoterMetrics {
+	const DESCRIPTION: &'static str = "voter";
+	fn register(registry: &Registry) -> Result<Self, PrometheusError> {
 		Ok(Self {
 			beefy_validator_set_id: register(
 				Gauge::new(
@@ -52,13 +79,6 @@ impl Metrics {
 				Counter::new("substrate_beefy_votes_sent", "Number of votes sent by this node")?,
 				registry,
 			)?,
-			beefy_round_concluded: register(
-				Gauge::new(
-					"substrate_beefy_round_concluded",
-					"Voting round, that has been concluded",
-				)?,
-				registry,
-			)?,
 			beefy_best_block: register(
 				Gauge::new("substrate_beefy_best_block", "Best block finalized by BEEFY")?,
 				registry,
@@ -78,10 +98,212 @@ impl Metrics {
 				)?,
 				registry,
 			)?,
+			beefy_no_authority_found_in_store: register(
+				Counter::new(
+					"substrate_beefy_no_authority_found_in_store",
+					"Number of times no Authority public key found in store",
+				)?,
+				registry,
+			)?,
+			beefy_buffered_votes: register(
+				Gauge::new("substrate_beefy_buffered_votes", "Number of currently buffered votes")?,
+				registry,
+			)?,
+			beefy_stale_votes: register(
+				Counter::new(
+					"substrate_beefy_stale_votes",
+					"Number of valid but stale votes received",
+				)?,
+				registry,
+			)?,
+			beefy_buffered_votes_dropped: register(
+				Counter::new(
+					"substrate_beefy_buffered_votes_dropped",
+					"Number of votes dropped due to full buffers",
+				)?,
+				registry,
+			)?,
+			beefy_buffered_justifications: register(
+				Gauge::new(
+					"substrate_beefy_buffered_justifications",
+					"Number of currently buffered justifications",
+				)?,
+				registry,
+			)?,
+			beefy_stale_justifications: register(
+				Counter::new(
+					"substrate_beefy_stale_justifications",
+					"Number of valid but stale justifications received",
+				)?,
+				registry,
+			)?,
+			beefy_imported_justifications: register(
+				Counter::new(
+					"substrate_beefy_imported_justifications",
+					"Number of valid justifications successfully imported",
+				)?,
+				registry,
+			)?,
+			beefy_buffered_justifications_dropped: register(
+				Counter::new(
+					"substrate_beefy_buffered_justifications_dropped",
+					"Number of justifications dropped due to full buffers",
+				)?,
+				registry,
+			)?,
+			beefy_best_block_set_last_failure: register(
+				Gauge::new(
+					"substrate_beefy_best_block_to_old_block",
+					"Trying to set Best Beefy block to old block",
+				)?,
+				registry,
+			)?,
+			beefy_successful_handled_votes: register(
+				Counter::new(
+					"substrate_beefy_successful_handled_votes",
+					"Number of Successful handled votes",
+				)?,
+				registry,
+			)?,
 		})
 	}
 }
 
+/// BEEFY block-import-related metrics exposed through Prometheus
+#[derive(Clone, Debug)]
+pub struct BlockImportMetrics {
+	/// Number of Good Justification imports
+	pub beefy_good_justification_imports: Counter<U64>,
+	/// Number of Bad Justification imports
+	pub beefy_bad_justification_imports: Counter<U64>,
+}
+
+impl PrometheusRegister for BlockImportMetrics {
+	const DESCRIPTION: &'static str = "block-import";
+	fn register(registry: &Registry) -> Result<Self, PrometheusError> {
+		Ok(Self {
+			beefy_good_justification_imports: register(
+				Counter::new(
+					"substrate_beefy_good_justification_imports",
+					"Number of Good Justification imports",
+				)?,
+				registry,
+			)?,
+			beefy_bad_justification_imports: register(
+				Counter::new(
+					"substrate_beefy_bad_justification_imports",
+					"Number of Bad Justification imports",
+				)?,
+				registry,
+			)?,
+		})
+	}
+}
+
+/// BEEFY on-demand-justifications-related metrics exposed through Prometheus
+#[derive(Clone, Debug)]
+pub struct OnDemandIncomingRequestsMetrics {
+	/// Number of Successful Justification responses
+	pub beefy_successful_justification_responses: Counter<U64>,
+	/// Number of Failed Justification responses
+	pub beefy_failed_justification_responses: Counter<U64>,
+}
+
+impl PrometheusRegister for OnDemandIncomingRequestsMetrics {
+	const DESCRIPTION: &'static str = "on-demand incoming justification requests";
+	fn register(registry: &Registry) -> Result<Self, PrometheusError> {
+		Ok(Self {
+			beefy_successful_justification_responses: register(
+				Counter::new(
+					"substrate_beefy_successful_justification_responses",
+					"Number of Successful Justification responses",
+				)?,
+				registry,
+			)?,
+			beefy_failed_justification_responses: register(
+				Counter::new(
+					"substrate_beefy_failed_justification_responses",
+					"Number of Failed Justification responses",
+				)?,
+				registry,
+			)?,
+		})
+	}
+}
+
+/// BEEFY on-demand-justifications-related metrics exposed through Prometheus
+#[derive(Clone, Debug)]
+pub struct OnDemandOutgoingRequestsMetrics {
+	/// Number of times there was no good peer to request justification from
+	pub beefy_on_demand_justification_no_peer_to_request_from: Counter<U64>,
+	/// Number of on-demand justification peer hang up
+	pub beefy_on_demand_justification_peer_hang_up: Counter<U64>,
+	/// Number of on-demand justification peer error
+	pub beefy_on_demand_justification_peer_error: Counter<U64>,
+	/// Number of on-demand justification invalid proof
+	pub beefy_on_demand_justification_invalid_proof: Counter<U64>,
+	/// Number of on-demand justification good proof
+	pub beefy_on_demand_justification_good_proof: Counter<U64>,
+}
+
+impl PrometheusRegister for OnDemandOutgoingRequestsMetrics {
+	const DESCRIPTION: &'static str = "on-demand outgoing justification requests";
+	fn register(registry: &Registry) -> Result<Self, PrometheusError> {
+		Ok(Self {
+			beefy_on_demand_justification_no_peer_to_request_from: register(
+				Counter::new(
+					"substrate_beefy_on_demand_justification_no_peer_to_request_from",
+					"Number of times there was no good peer to request justification from",
+				)?,
+				registry,
+			)?,
+			beefy_on_demand_justification_peer_hang_up: register(
+				Counter::new(
+					"substrate_beefy_on_demand_justification_peer_hang_up",
+					"Number of on-demand justification peer hang up",
+				)?,
+				registry,
+			)?,
+			beefy_on_demand_justification_peer_error: register(
+				Counter::new(
+					"substrate_beefy_on_demand_justification_peer_error",
+					"Number of on-demand justification peer error",
+				)?,
+				registry,
+			)?,
+			beefy_on_demand_justification_invalid_proof: register(
+				Counter::new(
+					"substrate_beefy_on_demand_justification_invalid_proof",
+					"Number of on-demand justification invalid proof",
+				)?,
+				registry,
+			)?,
+			beefy_on_demand_justification_good_proof: register(
+				Counter::new(
+					"substrate_beefy_on_demand_justification_good_proof",
+					"Number of on-demand justification good proof",
+				)?,
+				registry,
+			)?,
+		})
+	}
+}
+
+pub(crate) fn register_metrics<T: PrometheusRegister>(
+	prometheus_registry: Option<prometheus::Registry>,
+) -> Option<T> {
+	prometheus_registry.as_ref().map(T::register).and_then(|result| match result {
+		Ok(metrics) => {
+			debug!(target: "beefy", "🥩 Registered {} metrics", T::DESCRIPTION);
+			Some(metrics)
+		},
+		Err(err) => {
+			debug!(target: "beefy", "🥩 Failed to register {} metrics: {:?}", T::DESCRIPTION, err);
+			None
+		},
+	})
+}
+
 // Note: we use the `format` macro to convert an expr into a `u64`. This will fail,
 // if expr does not derive `Display`.
 #[macro_export]
@@ -104,7 +326,6 @@ macro_rules! metric_inc {
 	}};
 }
 
-#[cfg(test)]
 #[macro_export]
 macro_rules! metric_get {
 	($self:ident, $m:ident) => {{

substrate/client/beefy/src/tests.rs

@@ -124,6 +124,7 @@ impl BeefyTestNet {
 				justif_protocol_name,
 				client,
 				_block: PhantomData,
+				metrics: None,
 			};
 			*net.peers[i].data.beefy_justif_req_handler.lock() = Some(justif_handler);
 		}
@@ -203,7 +204,7 @@ impl TestNetFactory for BeefyTestNet {
 		let api = Arc::new(TestApi::with_validator_set(&validator_set));
 		let inner = BlockImportAdapter::new(client.clone());
 		let (block_import, voter_links, rpc_links) =
-			beefy_block_import_and_links(inner, client.as_backend(), api);
+			beefy_block_import_and_links(inner, client.as_backend(), api, None);
 		let peer_data = PeerData {
 			beefy_rpc_links: Mutex::new(Some(rpc_links)),
 			beefy_voter_links: Mutex::new(Some(voter_links)),

substrate/client/beefy/src/worker.rs

@@ -24,8 +24,8 @@ use crate::{
 	error::Error,
 	justification::BeefyVersionedFinalityProof,
 	keystore::BeefyKeystore,
-	metric_inc, metric_set,
-	metrics::Metrics,
+	metric_get, metric_inc, metric_set,
+	metrics::VoterMetrics,
 	round::{Rounds, VoteImportResult},
 	BeefyVoterLinks, LOG_TARGET,
 };
@@ -252,7 +252,7 @@ pub(crate) struct WorkerParams<B: Block, BE, P, N> {
 	pub gossip_validator: Arc<GossipValidator<B>>,
 	pub on_demand_justifications: OnDemandJustificationsEngine<B>,
 	pub links: BeefyVoterLinks<B>,
-	pub metrics: Option<Metrics>,
+	pub metrics: Option<VoterMetrics>,
 	pub persisted_state: PersistedState<B>,
 }
 
@@ -312,7 +312,7 @@ pub(crate) struct BeefyWorker<B: Block, BE, P, N> {
 
 	// voter state
 	/// BEEFY client metrics.
-	metrics: Option<Metrics>,
+	metrics: Option<VoterMetrics>,
 	/// Buffer holding votes for future processing.
 	pending_votes: BTreeMap<
 		NumberFor<B>,
@@ -407,6 +407,7 @@ where
 		if store.intersection(&active).count() == 0 {
 			let msg = "no authority public key found in store".to_string();
 			debug!(target: LOG_TARGET, "🥩 for block {:?} {}", block, msg);
+			metric_inc!(self, beefy_no_authority_found_in_store);
 			Err(Error::Keystore(msg))
 		} else {
 			Ok(())
@@ -494,17 +495,21 @@ where
 				debug!(target: LOG_TARGET, "🥩 Buffer vote for round: {:?}.", block_num);
 				if self.pending_votes.len() < MAX_BUFFERED_VOTE_ROUNDS {
 					let votes_vec = self.pending_votes.entry(block_num).or_default();
-					if votes_vec.try_push(vote).is_err() {
+					if votes_vec.try_push(vote).is_ok() {
+						metric_inc!(self, beefy_buffered_votes);
+					} else {
 						warn!(
 							target: LOG_TARGET,
 							"🥩 Buffer vote dropped for round: {:?}", block_num
-						)
+						);
+						metric_inc!(self, beefy_buffered_votes_dropped);
 					}
 				} else {
 					warn!(target: LOG_TARGET, "🥩 Buffer vote dropped for round: {:?}.", block_num);
+					metric_inc!(self, beefy_buffered_votes_dropped);
 				}
 			},
-			RoundAction::Drop => (),
+			RoundAction::Drop => metric_inc!(self, beefy_stale_votes),
 		};
 		Ok(())
 	}
@@ -524,20 +529,23 @@ where
 		match self.voting_oracle().triage_round(block_num, best_grandpa)? {
 			RoundAction::Process => {
 				debug!(target: LOG_TARGET, "🥩 Process justification for round: {:?}.", block_num);
+				metric_inc!(self, beefy_imported_justifications);
 				self.finalize(justification)?
 			},
 			RoundAction::Enqueue => {
 				debug!(target: LOG_TARGET, "🥩 Buffer justification for round: {:?}.", block_num);
 				if self.pending_justifications.len() < MAX_BUFFERED_JUSTIFICATIONS {
 					self.pending_justifications.entry(block_num).or_insert(justification);
+					metric_inc!(self, beefy_buffered_justifications);
 				} else {
+					metric_inc!(self, beefy_buffered_justifications_dropped);
 					warn!(
 						target: LOG_TARGET,
 						"🥩 Buffer justification dropped for round: {:?}.", block_num
 					);
 				}
 			},
-			RoundAction::Drop => (),
+			RoundAction::Drop => metric_inc!(self, beefy_stale_justifications),
 		};
 		Ok(())
 	}
@@ -555,8 +563,6 @@ where
 		let block_number = vote.commitment.block_number;
 		match rounds.add_vote(vote) {
 			VoteImportResult::RoundConcluded(signed_commitment) => {
-				metric_set!(self, beefy_round_concluded, block_number);
-
 				let finality_proof = VersionedFinalityProof::V1(signed_commitment);
 				info!(
 					target: LOG_TARGET,
@@ -584,6 +590,7 @@ where
 			},
 			VoteImportResult::Invalid | VoteImportResult::Stale => (),
 		};
+		metric_inc!(self, beefy_successful_handled_votes);
 		Ok(())
 	}
 
@@ -637,7 +644,8 @@ where
 				.notify(|| Ok::<_, ()>(finality_proof))
 				.expect("forwards closure result; the closure always returns Ok; qed.");
 		} else {
-			debug!(target: LOG_TARGET, "🥩 Can't set best beefy to older: {}", block_num);
+			debug!(target: LOG_TARGET, "🥩 Can't set best beefy to old: {}", block_num);
+			metric_set!(self, beefy_best_block_set_last_failure, block_num);
 		}
 		Ok(())
 	}
@@ -669,25 +677,33 @@ where
 			let justifs_to_handle = to_process_for(&mut self.pending_justifications, interval, _ph);
 			for (num, justification) in justifs_to_handle.into_iter() {
 				debug!(target: LOG_TARGET, "🥩 Handle buffered justification for: {:?}.", num);
+				metric_inc!(self, beefy_imported_justifications);
 				if let Err(err) = self.finalize(justification) {
 					error!(target: LOG_TARGET, "🥩 Error finalizing block: {}", err);
 				}
 			}
+			metric_set!(self, beefy_buffered_justifications, self.pending_justifications.len());
 			// Possibly new interval after processing justifications.
 			interval = self.voting_oracle().accepted_interval(best_grandpa)?;
 		}
 
 		// Process pending votes.
 		if !self.pending_votes.is_empty() {
+			let mut processed = 0u64;
 			let votes_to_handle = to_process_for(&mut self.pending_votes, interval, _ph);
 			for (num, votes) in votes_to_handle.into_iter() {
 				debug!(target: LOG_TARGET, "🥩 Handle buffered votes for: {:?}.", num);
+				processed += votes.len() as u64;
 				for v in votes.into_iter() {
 					if let Err(err) = self.handle_vote(v) {
 						error!(target: LOG_TARGET, "🥩 Error handling buffered vote: {}", err);
 					};
 				}
 			}
+			if let Some(previous) = metric_get!(self, beefy_buffered_votes) {
+				previous.sub(processed);
+				metric_set!(self, beefy_buffered_votes, previous.get());
+			}
 		}
 		Ok(())
 	}
@@ -1053,10 +1069,12 @@ pub(crate) mod tests {
 		let gossip_validator = Arc::new(GossipValidator::new(known_peers.clone()));
 		let gossip_engine =
 			GossipEngine::new(network.clone(), "/beefy/1", gossip_validator.clone(), None);
+		let metrics = None;
 		let on_demand_justifications = OnDemandJustificationsEngine::new(
 			network.clone(),
 			"/beefy/justifs/1".into(),
 			known_peers,
+			None,
 		);
 		let genesis_header = backend
 			.blockchain()
@@ -1077,7 +1095,7 @@ pub(crate) mod tests {
 			links,
 			gossip_engine,
 			gossip_validator,
-			metrics: None,
+			metrics,
 			network,
 			on_demand_justifications,
 			persisted_state,