Add block construction prometheus metrics (#6030)

* Add basic authorship metrics

* fixes

* no arc

* move to crate

* Update client/proposer-metrics/Cargo.toml

Co-authored-by: Max Inden <mail@max-inden.de>

* remove prefix

* use HistogramTimer

* Update client/proposer-metrics/src/lib.rs

Co-authored-by: Bastian Köcher <bkchr@users.noreply.github.com>

* Update client/basic-authorship/src/basic_authorship.rs

Co-authored-by: Bastian Köcher <bkchr@users.noreply.github.com>

* Update client/basic-authorship/src/basic_authorship.rs

Co-authored-by: Bastian Köcher <bkchr@users.noreply.github.com>

* Update client/basic-authorship/src/basic_authorship.rs

Co-authored-by: Bastian Köcher <bkchr@users.noreply.github.com>

* Update client/basic-authorship/src/basic_authorship.rs

Co-authored-by: Bastian Köcher <bkchr@users.noreply.github.com>

* Update client/basic-authorship/src/basic_authorship.rs

Co-authored-by: Bastian Köcher <bkchr@users.noreply.github.com>

* Update client/proposer-metrics/src/lib.rs

Co-authored-by: Bastian Köcher <bkchr@users.noreply.github.com>

* Update client/proposer-metrics/src/lib.rs

Co-authored-by: Bastian Köcher <bkchr@users.noreply.github.com>

Co-authored-by: Max Inden <mail@max-inden.de>
Co-authored-by: Bastian Köcher <bkchr@users.noreply.github.com>

substrate/Cargo.lock

@@ -5885,6 +5885,7 @@ dependencies = [
  "parking_lot 0.10.2",
  "sc-block-builder",
  "sc-client-api",
+ "sc-proposer-metrics",
  "sc-telemetry",
  "sc-transaction-pool",
  "sp-api",
@@ -5894,6 +5895,7 @@ dependencies = [
  "sp-inherents",
  "sp-runtime",
  "sp-transaction-pool",
+ "substrate-prometheus-endpoint",
  "substrate-test-runtime-client",
  "tokio-executor 0.2.0-alpha.6",
 ]
@@ -6601,6 +6603,14 @@ dependencies = [
  "wasm-timer",
 ]
 
+[[package]]
+name = "sc-proposer-metrics"
+version = "0.8.0-dev"
+dependencies = [
+ "log",
+ "substrate-prometheus-endpoint",
+]
+
 [[package]]
 name = "sc-rpc"
 version = "2.0.0-dev"

substrate/Cargo.toml

@@ -45,6 +45,7 @@ members = [
 	"client/network-gossip",
 	"client/offchain",
 	"client/peerset",
+	"client/proposer-metrics",
 	"client/rpc-servers",
 	"client/rpc",
 	"client/rpc-api",

substrate/bin/node-template/node/src/service.rs

@@ -106,8 +106,11 @@ pub fn new_full(config: Configuration) -> Result<impl AbstractService, ServiceEr
 		.build()?;
 
 	if role.is_authority() {
-		let proposer =
-			sc_basic_authorship::ProposerFactory::new(service.client(), service.transaction_pool());
+		let proposer = sc_basic_authorship::ProposerFactory::new(
+			service.client(),
+			service.transaction_pool(),
+			service.prometheus_registry().as_ref(),
+		);
 
 		let client = service.client();
 		let select_chain = service.select_chain()

substrate/bin/node/cli/src/service.rs

@@ -173,7 +173,8 @@ macro_rules! new_full {
 		if let sc_service::config::Role::Authority { .. } = &role {
 			let proposer = sc_basic_authorship::ProposerFactory::new(
 				service.client(),
-				service.transaction_pool()
+				service.transaction_pool(),
+				service.prometheus_registry().as_ref(),
 			);
 
 			let client = service.client();
@@ -552,7 +553,8 @@ mod tests {
 
 				let mut proposer_factory = sc_basic_authorship::ProposerFactory::new(
 					service.client(),
-					service.transaction_pool()
+					service.transaction_pool(),
+					None,
 				);
 
 				let epoch_descriptor = babe_link.epoch_changes().lock().epoch_descriptor_for_child_of(

substrate/client/basic-authorship/Cargo.toml

@@ -12,9 +12,11 @@ description = "Basic implementation of block-authoring logic."
 targets = ["x86_64-unknown-linux-gnu"]
 
 [dependencies]
-log = "0.4.8"
-futures = "0.3.4"
 codec = { package = "parity-scale-codec", version = "1.3.0" }
+futures = "0.3.4"
+futures-timer = "3.0.1"
+log = "0.4.8"
+prometheus-endpoint = { package = "substrate-prometheus-endpoint", path = "../../utils/prometheus", version = "0.8.0-dev"}
 sp-api = { version = "2.0.0-dev", path = "../../primitives/api" }
 sp-runtime = { version = "2.0.0-dev", path = "../../primitives/runtime" }
 sp-core = { version = "2.0.0-dev", path = "../../primitives/core" }
@@ -25,8 +27,8 @@ sp-inherents = { version = "2.0.0-dev", path = "../../primitives/inherents" }
 sc-telemetry = { version = "2.0.0-dev", path = "../telemetry" }
 sp-transaction-pool = { version = "2.0.0-dev", path = "../../primitives/transaction-pool" }
 sc-block-builder = { version = "0.8.0-dev", path = "../block-builder" }
+sc-proposer-metrics = { version = "0.8.0-dev", path = "../proposer-metrics" }
 tokio-executor = { version = "0.2.0-alpha.6", features = ["blocking"] }
-futures-timer = "3.0.1"
 
 [dev-dependencies]
 sc-transaction-pool = { version = "2.0.0-dev", path = "../../client/transaction-pool" }

substrate/client/basic-authorship/src/basic_authorship.rs

@@ -15,6 +15,7 @@
 
 // You should have received a copy of the GNU General Public License
 // along with this program. If not, see <https://www.gnu.org/licenses/>.
+
 //! A consensus proposer for "basic" chains which use the primitive inherent-data.
 
 // FIXME #1021 move this into sp-consensus
@@ -38,21 +39,31 @@ use futures::{executor, future, future::Either};
 use sp_blockchain::{HeaderBackend, ApplyExtrinsicFailed::Validity, Error::ApplyExtrinsicFailed};
 use std::marker::PhantomData;
 
+use prometheus_endpoint::Registry as PrometheusRegistry;
+use sc_proposer_metrics::MetricsLink as PrometheusMetrics;
+
 /// Proposer factory.
 pub struct ProposerFactory<A, B, C> {
 	/// The client instance.
 	client: Arc<C>,
 	/// The transaction pool.
 	transaction_pool: Arc<A>,
+	/// Prometheus Link,
+	metrics: PrometheusMetrics,
 	/// phantom member to pin the `Backend` type.
 	_phantom: PhantomData<B>,
 }
 
 impl<A, B, C> ProposerFactory<A, B, C> {
-	pub fn new(client: Arc<C>, transaction_pool: Arc<A>) -> Self {
+	pub fn new(
+		client: Arc<C>,
+		transaction_pool: Arc<A>,
+		prometheus: Option<&PrometheusRegistry>,
+	) -> Self {
 		ProposerFactory {
 			client,
 			transaction_pool,
+			metrics: PrometheusMetrics::new(prometheus),
 			_phantom: PhantomData,
 		}
 	}
@@ -87,6 +98,7 @@ impl<B, Block, C, A> ProposerFactory<A, B, C>
 				parent_number: *parent_header.number(),
 				transaction_pool: self.transaction_pool.clone(),
 				now,
+				metrics: self.metrics.clone(),
 				_phantom: PhantomData,
 			}),
 		};
@@ -131,6 +143,7 @@ struct ProposerInner<B, Block: BlockT, C, A: TransactionPool> {
 	parent_number: <<Block as BlockT>::Header as HeaderT>::Number,
 	transaction_pool: Arc<A>,
 	now: Box<dyn Fn() -> time::Instant + Send + Sync>,
+	metrics: PrometheusMetrics,
 	_phantom: PhantomData<B>,
 }
 
@@ -219,6 +232,7 @@ impl<A, B, Block, C> ProposerInner<B, Block, C, A>
 		}
 
 		// proceed with transactions
+		let block_timer = self.metrics.report(|metrics| metrics.block_constructed.start_timer());
 		let mut is_first = true;
 		let mut skipped = 0;
 		let mut unqueue_invalid = Vec::new();
@@ -289,6 +303,9 @@ impl<A, B, Block, C> ProposerInner<B, Block, C, A>
 
 		let (block, storage_changes, proof) = block_builder.build()?.into_inner();
 
+		drop(block_timer);
+		self.metrics.report(|metrics| metrics.number_of_transactions.set(block.extrinsics().len() as u64));
+
 		info!("🎁 Prepared block for proposing at {} [hash: {:?}; parent_hash: {}; extrinsics ({}): [{}]]",
 			block.header().number(),
 			<Block as BlockT>::Hash::from(block.header().hash()),
@@ -379,7 +396,7 @@ mod tests {
 			))
 		);
 
-		let mut proposer_factory = ProposerFactory::new(client.clone(), txpool.clone());
+		let mut proposer_factory = ProposerFactory::new(client.clone(), txpool.clone(), None);
 
 		let cell = Mutex::new((false, time::Instant::now()));
 		let mut proposer = proposer_factory.init_with_now(
@@ -420,7 +437,7 @@ mod tests {
 			).0
 		);
 
-		let mut proposer_factory = ProposerFactory::new(client.clone(), txpool.clone());
+		let mut proposer_factory = ProposerFactory::new(client.clone(), txpool.clone(), None);
 
 		let cell = Mutex::new((false, time::Instant::now()));
 		let mut proposer = proposer_factory.init_with_now(
@@ -470,7 +487,7 @@ mod tests {
 			))
 		);
 
-		let mut proposer_factory = ProposerFactory::new(client.clone(), txpool.clone());
+		let mut proposer_factory = ProposerFactory::new(client.clone(), txpool.clone(), None);
 
 		let mut proposer = proposer_factory.init_with_now(
 			&client.header(&block_id).unwrap().unwrap(),
@@ -536,7 +553,7 @@ mod tests {
 			])
 		).unwrap();
 
-		let mut proposer_factory = ProposerFactory::new(client.clone(), txpool.clone());
+		let mut proposer_factory = ProposerFactory::new(client.clone(), txpool.clone(), None);
 		let mut propose_block = |
 			client: &TestClient,
 			number,

substrate/client/basic-authorship/src/lib.rs

@@ -29,7 +29,7 @@
 //! # let client = Arc::new(substrate_test_runtime_client::new());
 //! # let txpool = Arc::new(BasicPool::new(Default::default(), Arc::new(FullChainApi::new(client.clone())), None).0);
 //! // The first step is to create a `ProposerFactory`.
-//! let mut proposer_factory = ProposerFactory::new(client.clone(), txpool.clone());
+//! let mut proposer_factory = ProposerFactory::new(client.clone(), txpool.clone(), None);
 //!
 //! // From this factory, we create a `Proposer`.
 //! let proposer = proposer_factory.init(

substrate/client/consensus/manual-seal/src/lib.rs

@@ -225,7 +225,8 @@ mod tests {
 		let pool = Arc::new(BasicPool::new(Options::default(), api(), None).0);
 		let env = ProposerFactory::new(
 			client.clone(),
-			pool.clone()
+			pool.clone(),
+			None,
 		);
 		// this test checks that blocks are created as soon as transactions are imported into the pool.
 		let (sender, receiver) = futures::channel::oneshot::channel();
@@ -289,7 +290,8 @@ mod tests {
 		let pool = Arc::new(BasicPool::new(Options::default(), api(), None).0);
 		let env = ProposerFactory::new(
 			client.clone(),
-			pool.clone()
+			pool.clone(),
+			None,
 		);
 		// this test checks that blocks are created as soon as an engine command is sent over the stream.
 		let (mut sink, stream) = futures::channel::mpsc::channel(1024);
@@ -358,6 +360,7 @@ mod tests {
 		let env = ProposerFactory::new(
 			client.clone(),
 			pool.clone(),
+			None,
 		);
 		// this test checks that blocks are created as soon as an engine command is sent over the stream.
 		let (mut sink, stream) = futures::channel::mpsc::channel(1024);

substrate/client/proposer-metrics/Cargo.toml

@@ -0,0 +1,16 @@
+[package]
+name = "sc-proposer-metrics"
+version = "0.8.0-dev"
+authors = ["Parity Technologies <admin@parity.io>"]
+edition = "2018"
+license = "GPL-3.0-or-later WITH Classpath-exception-2.0"
+homepage = "https://substrate.dev"
+repository = "https://github.com/paritytech/substrate/"
+description = "Basic metrics for block production."
+
+[package.metadata.docs.rs]
+targets = ["x86_64-unknown-linux-gnu"]
+
+[dependencies]
+log = "0.4.8"
+prometheus-endpoint = { package = "substrate-prometheus-endpoint", path = "../../utils/prometheus", version = "0.8.0-dev"}

substrate/client/proposer-metrics/src/lib.rs

@@ -0,0 +1,67 @@
+// Copyright 2020 Parity Technologies (UK) Ltd.
+// This file is part of Substrate.
+
+// Substrate is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// Substrate is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with Substrate.  If not, see <http://www.gnu.org/licenses/>.
+
+//! Prometheus basic proposer metrics.
+
+use prometheus_endpoint::{register, PrometheusError, Registry, Histogram, HistogramOpts, Gauge, U64};
+
+/// Optional shareable link to basic authorship metrics.
+#[derive(Clone, Default)]
+pub struct MetricsLink(Option<Metrics>);
+
+impl MetricsLink {
+	pub fn new(registry: Option<&Registry>) -> Self {
+		Self(
+			registry.and_then(|registry|
+				Metrics::register(registry)
+					.map_err(|err| log::warn!("Failed to register proposer prometheus metrics: {}", err))
+					.ok()
+			)
+		)
+	}
+
+	pub fn report<O>(&self, do_this: impl FnOnce(&Metrics) -> O) -> Option<O> {
+		Some(do_this(self.0.as_ref()?))
+	}
+}
+
+/// Authorship metrics.
+#[derive(Clone)]
+pub struct Metrics {
+    pub block_constructed: Histogram,
+    pub number_of_transactions: Gauge<U64>,
+}
+
+impl Metrics {
+	pub fn register(registry: &Registry) -> Result<Self, PrometheusError> {
+		Ok(Self {
+			block_constructed: register(
+				Histogram::with_opts(HistogramOpts::new(
+					"proposer_block_constructed",
+					"Histogram of time taken to construct new block",
+				))?,
+				registry,
+            )?,
+            number_of_transactions: register(
+                Gauge::new(
+                    "proposer_number_of_transactions",
+                    "Number of transactions included in block",
+                )?,
+                registry,
+            )?,
+		})
+    }
+}

substrate/primitives/consensus/common/src/lib.rs

@@ -72,7 +72,9 @@ pub enum BlockStatus {
 	Unknown,
 }
 
-/// Environment producer for a Consensus instance. Creates proposer instance and communication streams.
+/// Environment for a Consensus instance.
+///
+/// Creates proposer instance.
 pub trait Environment<B: BlockT> {
 	/// The proposer type this creates.
 	type Proposer: Proposer<B> + Send + 'static;