Rework telemetry to replace the use of tracing with an object we pass around (#8143)

polkadot companion: paritytech/polkadot#2535

substrate/client/telemetry/src/error.rs

@@ -0,0 +1,31 @@
+// This file is part of Substrate.
+
+// Copyright (C) 2021 Parity Technologies (UK) Ltd.
+// SPDX-License-Identifier: GPL-3.0-or-later WITH Classpath-exception-2.0
+
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with this program. If not, see <https://www.gnu.org/licenses/>.
+
+#[allow(missing_docs)]
+#[derive(Debug, thiserror::Error)]
+pub enum Error {
+	#[error("IO Error")]
+	IoError(#[from] std::io::Error),
+	#[error("This telemetry instance has already been initialized!")]
+	TelemetryAlreadyInitialized,
+	#[error("The telemetry worker has been dropped already.")]
+	TelemetryWorkerDropped,
+}
+
+#[allow(missing_docs)]
+pub type Result<T> = std::result::Result<T, Error>;

substrate/client/telemetry/src/layer.rs

@@ -1,149 +0,0 @@
-// This file is part of Substrate.
-
-// Copyright (C) 2021 Parity Technologies (UK) Ltd.
-// SPDX-License-Identifier: GPL-3.0-or-later WITH Classpath-exception-2.0
-
-// This program is free software: you can redistribute it and/or modify
-// it under the terms of the GNU General Public License as published by
-// the Free Software Foundation, either version 3 of the License, or
-// (at your option) any later version.
-
-// This program is distributed in the hope that it will be useful,
-// but WITHOUT ANY WARRANTY; without even the implied warranty of
-// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-// GNU General Public License for more details.
-
-// You should have received a copy of the GNU General Public License
-// along with this program. If not, see <https://www.gnu.org/licenses/>.
-
-use crate::{initialize_transport, TelemetryWorker};
-use futures::channel::mpsc;
-use libp2p::wasm_ext::ExtTransport;
-use parking_lot::Mutex;
-use std::convert::TryInto;
-use std::io;
-use tracing::{Event, Id, Subscriber};
-use tracing_subscriber::{layer::Context, registry::LookupSpan, Layer};
-
-/// Span name used to report the telemetry.
-pub const TELEMETRY_LOG_SPAN: &str = "telemetry-logger";
-
-/// `Layer` that handles the logs for telemetries.
-#[derive(Debug)]
-pub struct TelemetryLayer(Mutex<mpsc::Sender<(Id, u8, String)>>);
-
-impl TelemetryLayer {
-	/// Create a new [`TelemetryLayer`] and [`TelemetryWorker`].
-	///
-	/// The `buffer_size` defaults to 16.
-	///
-	/// The [`ExtTransport`] is used in WASM contexts where we need some binding between the
-	/// networking provided by the operating system or environment and libp2p.
-	///
-	/// > **Important**: Each individual call to `write` corresponds to one message. There is no
-	/// >                internal buffering going on. In the context of WebSockets, each `write`
-	/// >                must be one individual WebSockets frame.
-	pub fn new(
-		buffer_size: Option<usize>,
-		telemetry_external_transport: Option<ExtTransport>,
-	) -> io::Result<(Self, TelemetryWorker)> {
-		let transport = initialize_transport(telemetry_external_transport)?;
-		let worker = TelemetryWorker::new(buffer_size.unwrap_or(16), transport);
-		let sender = worker.message_sender();
-		Ok((Self(Mutex::new(sender)), worker))
-	}
-}
-
-impl<S> Layer<S> for TelemetryLayer
-where
-	S: Subscriber + for<'a> LookupSpan<'a>,
-{
-	fn on_event(&self, event: &Event<'_>, ctx: Context<S>) {
-		if event.metadata().target() != TELEMETRY_LOG_SPAN {
-			return;
-		}
-
-		if let Some(span) = ctx.lookup_current() {
-			let parents = span.parents();
-
-			if let Some(span) = std::iter::once(span)
-				.chain(parents)
-				.find(|x| x.name() == TELEMETRY_LOG_SPAN)
-			{
-				let id = span.id();
-				let mut attrs = TelemetryAttrs::new(id.clone());
-				let mut vis = TelemetryAttrsVisitor(&mut attrs);
-				event.record(&mut vis);
-
-				if let TelemetryAttrs {
-					verbosity: Some(verbosity),
-					json: Some(json),
-					..
-				} = attrs
-				{
-					match self.0.lock().try_send((
-						id,
-						verbosity
-							.try_into()
-							.expect("telemetry log message verbosity are u8; qed"),
-						json,
-					)) {
-						Err(err) if err.is_full() => eprintln!("Telemetry buffer overflowed!"),
-						_ => {}
-					}
-				} else {
-					// NOTE: logging in this function doesn't work
-					eprintln!(
-						"missing fields in telemetry log: {:?}. This can happen if \
-						`tracing::info_span!` is (mis-)used with the telemetry target \
-						directly; you should use the `telemetry!` macro.",
-						event,
-					);
-				}
-			}
-		}
-	}
-}
-
-#[derive(Debug)]
-struct TelemetryAttrs {
-	verbosity: Option<u64>,
-	json: Option<String>,
-	id: Id,
-}
-
-impl TelemetryAttrs {
-	fn new(id: Id) -> Self {
-		Self {
-			verbosity: None,
-			json: None,
-			id,
-		}
-	}
-}
-
-#[derive(Debug)]
-struct TelemetryAttrsVisitor<'a>(&'a mut TelemetryAttrs);
-
-impl<'a> tracing::field::Visit for TelemetryAttrsVisitor<'a> {
-	fn record_debug(&mut self, _field: &tracing::field::Field, _value: &dyn std::fmt::Debug) {
-		// noop
-	}
-
-	fn record_u64(&mut self, field: &tracing::field::Field, value: u64) {
-		if field.name() == "verbosity" {
-			(*self.0).verbosity = Some(value);
-		}
-	}
-
-	fn record_str(&mut self, field: &tracing::field::Field, value: &str) {
-		if field.name() == "json" {
-			(*self.0).json = Some(format!(
-				r#"{{"id":{},"ts":{:?},"payload":{}}}"#,
-				self.0.id.into_u64(),
-				chrono::Local::now().to_rfc3339().to_string(),
-				value,
-			));
-		}
-	}
-}

substrate/client/telemetry/src/lib.rs

@@ -29,7 +29,7 @@
 //! identify which substrate node is reporting the telemetry. Every task spawned using sc-service's
 //! `TaskManager` automatically inherit this span.
 //!
-//! Substrate's nodes initialize/register with the [`TelemetryWorker`] using a [`TelemetryHandle`].
+//! Substrate's nodes initialize/register with the [`TelemetryWorker`] using a [`TelemetryWorkerHandle`].
 //! This handle can be cloned and passed around. It uses an asynchronous channel to communicate with
 //! the running [`TelemetryWorker`] dedicated to registration. Registering can happen at any point
 //! in time during the process execution.
@@ -39,61 +39,45 @@
 use futures::{channel::mpsc, prelude::*};
 use libp2p::Multiaddr;
 use log::{error, warn};
+use parking_lot::Mutex;
 use serde::Serialize;
-use sp_utils::mpsc::{tracing_unbounded, TracingUnboundedReceiver};
 use std::collections::HashMap;
-use tracing::Id;
+use std::sync::{atomic, Arc};
 
 pub use libp2p::wasm_ext::ExtTransport;
+pub use log;
 pub use serde_json;
-pub use tracing;
 
 mod endpoints;
-mod layer;
+mod error;
 mod node;
 mod transport;
 
 pub use endpoints::*;
-pub use layer::*;
+pub use error::*;
 use node::*;
 use transport::*;
 
 /// Substrate DEBUG log level.
-pub const SUBSTRATE_DEBUG: u8 = 9;
+pub const SUBSTRATE_DEBUG: VerbosityLevel = 9;
 /// Substrate INFO log level.
-pub const SUBSTRATE_INFO: u8 = 0;
+pub const SUBSTRATE_INFO: VerbosityLevel = 0;
 
 /// Consensus TRACE log level.
-pub const CONSENSUS_TRACE: u8 = 9;
+pub const CONSENSUS_TRACE: VerbosityLevel = 9;
 /// Consensus DEBUG log level.
-pub const CONSENSUS_DEBUG: u8 = 5;
+pub const CONSENSUS_DEBUG: VerbosityLevel = 5;
 /// Consensus WARN log level.
-pub const CONSENSUS_WARN: u8 = 4;
+pub const CONSENSUS_WARN: VerbosityLevel = 4;
 /// Consensus INFO log level.
-pub const CONSENSUS_INFO: u8 = 1;
+pub const CONSENSUS_INFO: VerbosityLevel = 1;
 
-pub(crate) type TelemetryMessage = (Id, u8, String);
+/// Telemetry message verbosity.
+pub type VerbosityLevel = u8;
 
-/// A handle representing a telemetry span, with the capability to enter the span if it exists.
-#[derive(Debug, Clone)]
-pub struct TelemetrySpan(tracing::Span);
-
-impl TelemetrySpan {
-	/// Enters this span, returning a guard that will exit the span when dropped.
-	pub fn enter(&self) -> tracing::span::Entered {
-		self.0.enter()
-	}
-
-	/// Constructs a new [`TelemetrySpan`].
-	pub fn new() -> Self {
-		Self(tracing::error_span!(TELEMETRY_LOG_SPAN))
-	}
-
-	/// Return a clone of the underlying `tracing::Span` instance.
-	pub fn span(&self) -> tracing::Span {
-		self.0.clone()
-	}
-}
+pub(crate) type Id = u64;
+pub(crate) type TelemetryPayload = serde_json::Map<String, serde_json::Value>;
+pub(crate) type TelemetryMessage = (Id, VerbosityLevel, TelemetryPayload);
 
 /// Message sent when the connection (re-)establishes.
 #[derive(Debug, Serialize)]
@@ -129,64 +113,79 @@ pub struct TelemetryWorker {
 	message_sender: mpsc::Sender<TelemetryMessage>,
 	register_receiver: mpsc::UnboundedReceiver<Register>,
 	register_sender: mpsc::UnboundedSender<Register>,
+	id_counter: Arc<atomic::AtomicU64>,
 	transport: WsTrans,
 }
 
 impl TelemetryWorker {
-	pub(crate) fn new(buffer_size: usize, transport: WsTrans) -> Self {
+	/// Instantiate a new [`TelemetryWorker`] which can run in background.
+	///
+	/// Only one is needed per process.
+	pub fn new(buffer_size: usize) -> Result<Self> {
+		let transport = initialize_transport(None)?;
 		let (message_sender, message_receiver) = mpsc::channel(buffer_size);
 		let (register_sender, register_receiver) = mpsc::unbounded();
 
-		Self {
+		Ok(Self {
 			message_receiver,
 			message_sender,
 			register_receiver,
 			register_sender,
+			id_counter: Arc::new(atomic::AtomicU64::new(1)),
 			transport,
-		}
+		})
 	}
 
-	/// Get a new [`TelemetryHandle`].
+	/// Instantiate a new [`TelemetryWorker`] which can run in background.
+	///
+	/// Only one is needed per process.
+	pub fn with_transport(buffer_size: usize, transport: Option<ExtTransport>) -> Result<Self> {
+		let transport = initialize_transport(transport)?;
+		let (message_sender, message_receiver) = mpsc::channel(buffer_size);
+		let (register_sender, register_receiver) = mpsc::unbounded();
+
+		Ok(Self {
+			message_receiver,
+			message_sender,
+			register_receiver,
+			register_sender,
+			id_counter: Arc::new(atomic::AtomicU64::new(1)),
+			transport,
+		})
+	}
+
+	/// Get a new [`TelemetryWorkerHandle`].
 	///
 	/// This is used when you want to register with the [`TelemetryWorker`].
-	pub fn handle(&self) -> TelemetryHandle {
-		TelemetryHandle {
-			message_sender: self.register_sender.clone(),
+	pub fn handle(&self) -> TelemetryWorkerHandle {
+		TelemetryWorkerHandle {
+			message_sender: self.message_sender.clone(),
+			register_sender: self.register_sender.clone(),
+			id_counter: self.id_counter.clone(),
 		}
 	}
 
-	/// Get a clone of the channel's `Sender` used to send telemetry events.
-	pub(crate) fn message_sender(&self) -> mpsc::Sender<TelemetryMessage> {
-		self.message_sender.clone()
-	}
-
 	/// Run the telemetry worker.
 	///
 	/// This should be run in a background task.
-	pub async fn run(self) {
-		let Self {
-			mut message_receiver,
-			message_sender: _,
-			mut register_receiver,
-			register_sender: _,
-			transport,
-		} = self;
-
-		let mut node_map: HashMap<Id, Vec<(u8, Multiaddr)>> = HashMap::new();
+	pub async fn run(mut self) {
+		let mut node_map: HashMap<Id, Vec<(VerbosityLevel, Multiaddr)>> = HashMap::new();
 		let mut node_pool: HashMap<Multiaddr, _> = HashMap::new();
+		let mut pending_connection_notifications: Vec<_> = Vec::new();
 
 		loop {
 			futures::select! {
-				message = message_receiver.next() => Self::process_message(
+				message = self.message_receiver.next() => Self::process_message(
 					message,
 					&mut node_pool,
 					&node_map,
 				).await,
-				init_payload = register_receiver.next() => Self::process_register(
+				init_payload = self.register_receiver.next() => Self::process_register(
 					init_payload,
 					&mut node_pool,
 					&mut node_map,
-					transport.clone(),
+					&mut pending_connection_notifications,
+					self.transport.clone(),
 				).await,
 			}
 		}
@@ -195,7 +194,8 @@ impl TelemetryWorker {
 	async fn process_register(
 		input: Option<Register>,
 		node_pool: &mut HashMap<Multiaddr, Node<WsTrans>>,
-		node_map: &mut HashMap<Id, Vec<(u8, Multiaddr)>>,
+		node_map: &mut HashMap<Id, Vec<(VerbosityLevel, Multiaddr)>>,
+		pending_connection_notifications: &mut Vec<(Multiaddr, ConnectionNotifierSender)>,
 		transport: WsTrans,
 	) {
 		let input = input.expect("the stream is never closed; qed");
@@ -212,7 +212,7 @@ impl TelemetryWorker {
 					Ok(serde_json::Value::Object(mut value)) => {
 						value.insert("msg".into(), "system.connected".into());
 						let mut obj = serde_json::Map::new();
-						obj.insert("id".to_string(), id.into_u64().into());
+						obj.insert("id".to_string(), id.into());
 						obj.insert("payload".to_string(), value.into());
 						Some(obj)
 					}
@@ -245,6 +245,16 @@ impl TelemetryWorker {
 					});
 
 					node.connection_messages.extend(connection_message.clone());
+
+					pending_connection_notifications.retain(|(addr_b, connection_message)| {
+						if *addr_b == addr {
+							node.telemetry_connection_notifier
+								.push(connection_message.clone());
+							false
+						} else {
+							true
+						}
+					});
 				}
 			}
 			Register::Notifier {
@@ -252,15 +262,15 @@ impl TelemetryWorker {
 				connection_notifier,
 			} => {
 				for addr in addresses {
+					// If the Node has been initialized, we directly push the connection_notifier.
+					// Otherwise we push it to a queue that will be consumed when the connection
+					// initializes, thus ensuring that the connection notifier will be sent to the
+					// Node when it becomes available.
 					if let Some(node) = node_pool.get_mut(&addr) {
 						node.telemetry_connection_notifier
 							.push(connection_notifier.clone());
 					} else {
-						log::error!(
-							target: "telemetry",
-							"Received connection notifier for unknown node ({}). This is a bug.",
-							addr,
-						);
+						pending_connection_notifications.push((addr, connection_notifier.clone()));
 					}
 				}
 			}
@@ -271,21 +281,31 @@ impl TelemetryWorker {
 	async fn process_message(
 		input: Option<TelemetryMessage>,
 		node_pool: &mut HashMap<Multiaddr, Node<WsTrans>>,
-		node_map: &HashMap<Id, Vec<(u8, Multiaddr)>>,
+		node_map: &HashMap<Id, Vec<(VerbosityLevel, Multiaddr)>>,
 	) {
-		let (id, verbosity, message) = input.expect("the stream is never closed; qed");
+		let (id, verbosity, payload) = input.expect("the stream is never closed; qed");
+
+		let ts = chrono::Local::now().to_rfc3339().to_string();
+		let mut message = serde_json::Map::new();
+		message.insert("id".into(), id.into());
+		message.insert("ts".into(), ts.into());
+		message.insert("payload".into(), payload.into());
 
 		let nodes = if let Some(nodes) = node_map.get(&id) {
 			nodes
 		} else {
-			// This is a normal error because the telemetry span is entered before the telemetry
-			// is initialized so it is possible that some messages in the beginning don't get
-			// through.
+			// This is a normal error because the telemetry ID exists before the telemetry is
+			// initialized.
 			log::trace!(
 				target: "telemetry",
 				"Received telemetry log for unknown id ({:?}): {}",
 				id,
-				message,
+				serde_json::to_string(&message)
+					.unwrap_or_else(|err| format!(
+						"could not be serialized ({}): {:?}",
+						err,
+						message,
+					)),
 			);
 			return;
 		};
@@ -304,12 +324,17 @@ impl TelemetryWorker {
 			if let Some(node) = node_pool.get_mut(&addr) {
 				let _ = node.send(message.clone()).await;
 			} else {
-				log::error!(
+				log::debug!(
 					target: "telemetry",
 					"Received message for unknown node ({}). This is a bug. \
 					Message sent: {}",
 					addr,
-					message,
+					serde_json::to_string(&message)
+						.unwrap_or_else(|err| format!(
+							"could not be serialized ({}): {:?}",
+							err,
+							message,
+						)),
 				);
 			}
 		}
@@ -318,11 +343,41 @@ impl TelemetryWorker {
 
 /// Handle to the [`TelemetryWorker`] thats allows initializing the telemetry for a Substrate node.
 #[derive(Debug, Clone)]
-pub struct TelemetryHandle {
-	message_sender: mpsc::UnboundedSender<Register>,
+pub struct TelemetryWorkerHandle {
+	message_sender: mpsc::Sender<TelemetryMessage>,
+	register_sender: mpsc::UnboundedSender<Register>,
+	id_counter: Arc<atomic::AtomicU64>,
 }
 
-impl TelemetryHandle {
+impl TelemetryWorkerHandle {
+	/// Instantiate a new [`Telemetry`] object.
+	pub fn new_telemetry(&mut self, endpoints: TelemetryEndpoints) -> Telemetry {
+		let addresses = endpoints.0.iter().map(|(addr, _)| addr.clone()).collect();
+
+		Telemetry {
+			message_sender: self.message_sender.clone(),
+			register_sender: self.register_sender.clone(),
+			id: self.id_counter.fetch_add(1, atomic::Ordering::Relaxed),
+			connection_notifier: TelemetryConnectionNotifier {
+				register_sender: self.register_sender.clone(),
+				addresses,
+			},
+			endpoints: Some(endpoints),
+		}
+	}
+}
+
+/// A telemetry instance that can be used to send telemetry messages.
+#[derive(Debug)]
+pub struct Telemetry {
+	message_sender: mpsc::Sender<TelemetryMessage>,
+	register_sender: mpsc::UnboundedSender<Register>,
+	id: Id,
+	connection_notifier: TelemetryConnectionNotifier,
+	endpoints: Option<TelemetryEndpoints>,
+}
+
+impl Telemetry {
 	/// Initialize the telemetry with the endpoints provided in argument for the current substrate
 	/// node.
 	///
@@ -333,42 +388,67 @@ impl TelemetryHandle {
 	///
 	/// The `connection_message` argument is a JSON object that is sent every time the connection
 	/// (re-)establishes.
-	pub fn start_telemetry(
-		&mut self,
-		span: TelemetrySpan,
-		endpoints: TelemetryEndpoints,
-		connection_message: ConnectionMessage,
-	) -> TelemetryConnectionNotifier {
-		let Self { message_sender } = self;
-
-		let connection_notifier = TelemetryConnectionNotifier {
-			message_sender: message_sender.clone(),
-			addresses: endpoints.0.iter().map(|(addr, _)| addr.clone()).collect(),
+	pub fn start_telemetry(&mut self, connection_message: ConnectionMessage) -> Result<()> {
+		let endpoints = match self.endpoints.take() {
+			Some(x) => x,
+			None => return Err(Error::TelemetryAlreadyInitialized),
 		};
 
-		match span.0.id() {
-			Some(id) => {
-				match message_sender.unbounded_send(Register::Telemetry {
-					id,
-					endpoints,
-					connection_message,
-				}) {
-					Ok(()) => {}
-					Err(err) => error!(
-						target: "telemetry",
-						"Could not initialize telemetry: \
-						the telemetry is probably already running: {}",
-						err,
-					),
-				}
-			}
-			None => error!(
+		self.register_sender
+			.unbounded_send(Register::Telemetry {
+				id: self.id,
+				endpoints,
+				connection_message,
+			})
+			.map_err(|_| Error::TelemetryWorkerDropped)
+	}
+
+	/// Make a new clonable handle to this [`Telemetry`]. This is used for reporting telemetries.
+	pub fn handle(&self) -> TelemetryHandle {
+		TelemetryHandle {
+			message_sender: Arc::new(Mutex::new(self.message_sender.clone())),
+			id: self.id,
+			connection_notifier: self.connection_notifier.clone(),
+		}
+	}
+}
+
+/// Handle to a [`Telemetry`].
+///
+/// Used to report telemetry messages.
+#[derive(Debug, Clone)]
+pub struct TelemetryHandle {
+	message_sender: Arc<Mutex<mpsc::Sender<TelemetryMessage>>>,
+	id: Id,
+	connection_notifier: TelemetryConnectionNotifier,
+}
+
+impl TelemetryHandle {
+	/// Send telemetry messages.
+	pub fn send_telemetry(&self, verbosity: VerbosityLevel, payload: TelemetryPayload) {
+		match self
+			.message_sender
+			.lock()
+			.try_send((self.id, verbosity, payload))
+		{
+			Ok(()) => {}
+			Err(err) if err.is_full() => log::trace!(
 				target: "telemetry",
-				"Could not initialize telemetry: the span could not be entered",
+				"Telemetry channel full.",
+			),
+			Err(_) => log::trace!(
+				target: "telemetry",
+				"Telemetry channel closed.",
 			),
 		}
+	}
 
-		connection_notifier
+	/// Get event stream for telemetry connection established events.
+	///
+	/// This function will return an error if the telemetry has already been started by
+	/// [`Telemetry::start_telemetry`].
+	pub fn on_connect_stream(&self) -> ConnectionNotifierReceiver {
+		self.connection_notifier.on_connect_stream()
 	}
 }
 
@@ -376,18 +456,14 @@ impl TelemetryHandle {
 /// (re-)establishes.
 #[derive(Clone, Debug)]
 pub struct TelemetryConnectionNotifier {
-	message_sender: mpsc::UnboundedSender<Register>,
+	register_sender: mpsc::UnboundedSender<Register>,
 	addresses: Vec<Multiaddr>,
 }
 
 impl TelemetryConnectionNotifier {
-	/// Get event stream for telemetry connection established events.
-	///
-	/// This function will return an error if the telemetry has already been started by
-	/// [`TelemetryHandle::start_telemetry`].
-	pub fn on_connect_stream(&self) -> TracingUnboundedReceiver<()> {
-		let (message_sender, message_receiver) = tracing_unbounded("mpsc_telemetry_on_connect");
-		if let Err(err) = self.message_sender.unbounded_send(Register::Notifier {
+	fn on_connect_stream(&self) -> ConnectionNotifierReceiver {
+		let (message_sender, message_receiver) = connection_notifier_channel();
+		if let Err(err) = self.register_sender.unbounded_send(Register::Notifier {
 			addresses: self.addresses.clone(),
 			connection_notifier: message_sender,
 		}) {
@@ -428,34 +504,34 @@ enum Register {
 /// # let authority_id = 42_u64;
 /// # let set_id = (43_u64, 44_u64);
 /// # let authorities = vec![45_u64];
-/// telemetry!(CONSENSUS_INFO; "afg.authority_set";
-/// 	"authority_id" => authority_id.to_string(),
-/// 	"authority_set_id" => ?set_id,
-/// 	"authorities" => authorities,
+/// # let telemetry: Option<TelemetryHandle> = None;
+/// telemetry!(
+///     telemetry;      // an `Option<TelemetryHandle>`
+///     CONSENSUS_INFO;
+///     "afg.authority_set";
+///     "authority_id" => authority_id.to_string(),
+///     "authority_set_id" => ?set_id,
+///     "authorities" => authorities,
 /// );
 /// ```
 #[macro_export(local_inner_macros)]
 macro_rules! telemetry {
-	( $verbosity:expr; $msg:expr; $( $t:tt )* ) => {{
-		let verbosity: u8 = $verbosity;
-		match format_fields_to_json!($($t)*) {
-			Err(err) => {
-				$crate::tracing::error!(
-					target: "telemetry",
-					"Could not serialize value for telemetry: {}",
-					err,
-				);
-			},
-			Ok(mut json) => {
-				// NOTE: the span id will be added later in the JSON for the greater good
-				json.insert("msg".into(), $msg.into());
-				let serialized_json = $crate::serde_json::to_string(&json)
-					.expect("contains only string keys; qed");
-				$crate::tracing::info!(target: $crate::TELEMETRY_LOG_SPAN,
-					verbosity,
-					json = serialized_json.as_str(),
-				);
-			},
+	( $telemetry:expr; $verbosity:expr; $msg:expr; $( $t:tt )* ) => {{
+		if let Some(telemetry) = $telemetry.as_ref() {
+			let verbosity: $crate::VerbosityLevel = $verbosity;
+			match format_fields_to_json!($($t)*) {
+				Err(err) => {
+					$crate::log::debug!(
+						target: "telemetry",
+						"Could not serialize value for telemetry: {}",
+						err,
+					);
+				},
+				Ok(mut json) => {
+					json.insert("msg".into(), $msg.into());
+					telemetry.send_telemetry(verbosity, json);
+				},
+			}
 		}
 	}};
 }

substrate/client/telemetry/src/node.rs

@@ -16,6 +16,8 @@
 // You should have received a copy of the GNU General Public License
 // along with this program. If not, see <https://www.gnu.org/licenses/>.
 
+use crate::TelemetryPayload;
+use futures::channel::mpsc;
 use futures::prelude::*;
 use libp2p::core::transport::Transport;
 use libp2p::Multiaddr;
@@ -23,7 +25,13 @@ use rand::Rng as _;
 use std::{fmt, mem, pin::Pin, task::Context, task::Poll, time::Duration};
 use wasm_timer::Delay;
 
-pub(crate) type ConnectionNotifierSender = sp_utils::mpsc::TracingUnboundedSender<()>;
+pub(crate) type ConnectionNotifierSender = mpsc::Sender<()>;
+pub(crate) type ConnectionNotifierReceiver = mpsc::Receiver<()>;
+
+pub(crate) fn connection_notifier_channel() -> (ConnectionNotifierSender, ConnectionNotifierReceiver)
+{
+	mpsc::channel(0)
+}
 
 /// Handler for a single telemetry node.
 ///
@@ -45,7 +53,7 @@ pub(crate) struct Node<TTrans: Transport> {
 	/// Transport used to establish new connections.
 	transport: TTrans,
 	/// Messages that are sent when the connection (re-)establishes.
-	pub(crate) connection_messages: Vec<serde_json::Map<String, serde_json::Value>>,
+	pub(crate) connection_messages: Vec<TelemetryPayload>,
 	/// Notifier for when the connection (re-)establishes.
 	pub(crate) telemetry_connection_notifier: Vec<ConnectionNotifierSender>,
 }
@@ -123,7 +131,7 @@ where
 
 pub(crate) enum Infallible {}
 
-impl<TTrans: Transport, TSinkErr> Sink<String> for Node<TTrans>
+impl<TTrans: Transport, TSinkErr> Sink<TelemetryPayload> for Node<TTrans>
 where
 	TTrans: Clone + Unpin,
 	TTrans::Dial: Unpin,
@@ -234,16 +242,28 @@ where
 		Poll::Ready(Ok(()))
 	}
 
-	fn start_send(mut self: Pin<&mut Self>, item: String) -> Result<(), Self::Error> {
+	fn start_send(mut self: Pin<&mut Self>, item: TelemetryPayload) -> Result<(), Self::Error> {
 		match &mut self.socket {
-			NodeSocket::Connected(conn) => {
-				let _ = conn.sink.start_send_unpin(item.into()).expect("boo");
-			}
+			NodeSocket::Connected(conn) => match serde_json::to_vec(&item) {
+				Ok(data) => {
+					let _ = conn.sink.start_send_unpin(data);
+				}
+				Err(err) => log::debug!(
+					target: "telemetry",
+					"Could not serialize payload: {}",
+					err,
+				),
+			},
 			_socket => {
 				log::trace!(
 					target: "telemetry",
 					"Message has been discarded: {}",
-					item,
+					serde_json::to_string(&item)
+						.unwrap_or_else(|err| format!(
+							"could not be serialized ({}): {:?}",
+							err,
+							item,
+						)),
 				);
 			}
 		}