observability: tracing gum, automatically cross ref traceID (#5079)

* add some gum

* bump expander

* gum

* fix all remaining issues

* last fixup

* Update node/gum/proc-macro/src/lib.rs

Co-authored-by: Bastian Köcher <bkchr@users.noreply.github.com>

* change

* netowrk

* fixins

* chore

* allow optional fmt str + args, prep for expr as kv field

* tracing -> gum rename fallout

* restrict further

* allow multiple levels of field accesses

* another round of docs and a slip of the pen

* update ADR

* fixup lock fiel

* use target: instead of target=

* minors

* fix

* chore

* Update node/gum/README.md

Co-authored-by: Andrei Sandu <54316454+sandreim@users.noreply.github.com>

Co-authored-by: Bastian Köcher <bkchr@users.noreply.github.com>
Co-authored-by: Andrei Sandu <54316454+sandreim@users.noreply.github.com>

polkadot/node/network/availability-distribution/src/error.rs

@@ -100,12 +100,11 @@ pub fn log_error(result: Result<()>, ctx: &'static str) -> std::result::Result<(
 				JfyiError::InvalidValidatorIndex |
 				JfyiError::NoSuchCachedSession { .. } |
 				JfyiError::QueryAvailableDataResponseChannel(_) |
-				JfyiError::QueryChunkResponseChannel(_) =>
-					tracing::warn!(target: LOG_TARGET, error = %jfyi, ctx),
+				JfyiError::QueryChunkResponseChannel(_) => gum::warn!(target: LOG_TARGET, error = %jfyi, ctx),
 				JfyiError::FetchPoV(_) |
 				JfyiError::SendResponse |
 				JfyiError::NoSuchPoV |
-				JfyiError::Runtime(_) => tracing::debug!(target: LOG_TARGET, error = ?jfyi, ctx),
+				JfyiError::Runtime(_) => gum::debug!(target: LOG_TARGET, error = ?jfyi, ctx),
 			}
 			Ok(())
 		},

polkadot/node/network/availability-distribution/src/pov_requester/mod.rs

@@ -91,7 +91,7 @@ async fn fetch_pov_job(
 	metrics: Metrics,
 ) {
 	if let Err(err) = do_fetch_pov(pov_hash, pending_response, span, tx, metrics).await {
-		tracing::warn!(target: LOG_TARGET, ?err, ?pov_hash, ?authority_id, "fetch_pov_job");
+		gum::warn!(target: LOG_TARGET, ?err, ?pov_hash, ?authority_id, "fetch_pov_job");
 	}
 }
 
@@ -207,7 +207,7 @@ mod tests {
 							.unwrap();
 						break
 					},
-					msg => tracing::debug!(target: LOG_TARGET, msg = ?msg, "Received msg"),
+					msg => gum::debug!(target: LOG_TARGET, msg = ?msg, "Received msg"),
 				}
 			}
 			if pov.hash() == pov_hash {

polkadot/node/network/availability-distribution/src/requester/fetch_task/mod.rs

@@ -271,7 +271,7 @@ impl RunningTask {
 			let resp = match self.do_request(&validator).await {
 				Ok(resp) => resp,
 				Err(TaskError::ShuttingDown) => {
-					tracing::info!(
+					gum::info!(
 						target: LOG_TARGET,
 						"Node seems to be shutting down, canceling fetch task"
 					);
@@ -286,7 +286,7 @@ impl RunningTask {
 			let chunk = match resp {
 				ChunkFetchingResponse::Chunk(resp) => resp.recombine_into_chunk(&self.request),
 				ChunkFetchingResponse::NoSuchChunk => {
-					tracing::debug!(
+					gum::debug!(
 						target: LOG_TARGET,
 						validator = ?validator,
 						"Validator did not have our chunk"
@@ -337,7 +337,7 @@ impl RunningTask {
 		match response_recv.await {
 			Ok(resp) => Ok(resp),
 			Err(RequestError::InvalidResponse(err)) => {
-				tracing::warn!(
+				gum::warn!(
 					target: LOG_TARGET,
 					origin= ?validator,
 					err= ?err,
@@ -346,7 +346,7 @@ impl RunningTask {
 				Err(TaskError::PeerError)
 			},
 			Err(RequestError::NetworkError(err)) => {
-				tracing::debug!(
+				gum::debug!(
 					target: LOG_TARGET,
 					origin= ?validator,
 					err= ?err,
@@ -355,7 +355,7 @@ impl RunningTask {
 				Err(TaskError::PeerError)
 			},
 			Err(RequestError::Canceled(oneshot::Canceled)) => {
-				tracing::debug!(target: LOG_TARGET,
+				gum::debug!(target: LOG_TARGET,
 							   origin= ?validator,
 							   "Erasure chunk request got canceled");
 				Err(TaskError::PeerError)
@@ -368,7 +368,7 @@ impl RunningTask {
 			match branch_hash(&self.erasure_root, chunk.proof(), chunk.index.0 as usize) {
 				Ok(hash) => hash,
 				Err(e) => {
-					tracing::warn!(
+					gum::warn!(
 						target: LOG_TARGET,
 						candidate_hash = ?self.request.candidate_hash,
 						origin = ?validator,
@@ -380,7 +380,7 @@ impl RunningTask {
 			};
 		let erasure_chunk_hash = BlakeTwo256::hash(&chunk.chunk);
 		if anticipated_hash != erasure_chunk_hash {
-			tracing::warn!(target: LOG_TARGET, origin = ?validator,  "Received chunk does not match merkle tree");
+			gum::warn!(target: LOG_TARGET, origin = ?validator,  "Received chunk does not match merkle tree");
 			return false
 		}
 		true
@@ -400,11 +400,11 @@ impl RunningTask {
 			)))
 			.await;
 		if let Err(err) = r {
-			tracing::error!(target: LOG_TARGET, err= ?err, "Storing erasure chunk failed, system shutting down?");
+			gum::error!(target: LOG_TARGET, err= ?err, "Storing erasure chunk failed, system shutting down?");
 		}
 
 		if let Err(oneshot::Canceled) = rx.await {
-			tracing::error!(target: LOG_TARGET, "Storing erasure chunk failed");
+			gum::error!(target: LOG_TARGET, "Storing erasure chunk failed");
 		}
 	}
 
@@ -420,7 +420,7 @@ impl RunningTask {
 			})
 		};
 		if let Err(err) = self.sender.send(FromFetchTask::Concluded(payload)).await {
-			tracing::warn!(
+			gum::warn!(
 				target: LOG_TARGET,
 				err= ?err,
 				"Sending concluded message for task failed"
@@ -431,7 +431,7 @@ impl RunningTask {
 	async fn conclude_fail(&mut self) {
 		if let Err(err) = self.sender.send(FromFetchTask::Failed(self.request.candidate_hash)).await
 		{
-			tracing::warn!(target: LOG_TARGET, ?err, "Sending `Failed` message for task failed");
+			gum::warn!(target: LOG_TARGET, ?err, "Sending `Failed` message for task failed");
 		}
 	}
 }

polkadot/node/network/availability-distribution/src/requester/fetch_task/tests.rs

@@ -262,7 +262,7 @@ impl TestRun {
 				return true
 			},
 			_ => {
-				tracing::debug!(target: LOG_TARGET, "Unexpected message");
+				gum::debug!(target: LOG_TARGET, "Unexpected message");
 				return false
 			},
 		}

polkadot/node/network/availability-distribution/src/requester/mod.rs

@@ -103,7 +103,7 @@ impl Requester {
 	where
 		Context: SubsystemContext,
 	{
-		tracing::trace!(target: LOG_TARGET, ?update, "Update fetching heads");
+		gum::trace!(target: LOG_TARGET, ?update, "Update fetching heads");
 		let ActiveLeavesUpdate { activated, deactivated } = update;
 		// Stale leaves happen after a reversion - we don't want to re-run availability there.
 		if let Some(leaf) = activated.filter(|leaf| leaf.status == LeafStatus::Fresh) {
@@ -140,7 +140,7 @@ impl Requester {
 		// Also spawn or bump tasks for candidates in ancestry in the same session.
 		for hash in std::iter::once(leaf).chain(ancestors_in_session) {
 			let cores = get_occupied_cores(ctx, hash).await?;
-			tracing::trace!(
+			gum::trace!(
 				target: LOG_TARGET,
 				occupied_cores = ?cores,
 				"Query occupied core"
@@ -212,7 +212,7 @@ impl Requester {
 						)
 						.await
 						.map_err(|err| {
-							tracing::warn!(
+							gum::warn!(
 								target: LOG_TARGET,
 								error = ?err,
 								"Failed to spawn a fetch task"

polkadot/node/network/availability-distribution/src/requester/session_cache.rs

@@ -107,16 +107,16 @@ impl SessionCache {
 		F: FnOnce(&SessionInfo) -> R,
 	{
 		if let Some(o_info) = self.session_info_cache.get(&session_index) {
-			tracing::trace!(target: LOG_TARGET, session_index, "Got session from lru");
+			gum::trace!(target: LOG_TARGET, session_index, "Got session from lru");
 			return Ok(Some(with_info(o_info)))
 		}
 
 		if let Some(info) =
 			self.query_info_from_runtime(ctx, runtime, parent, session_index).await?
 		{
-			tracing::trace!(target: LOG_TARGET, session_index, "Calling `with_info`");
+			gum::trace!(target: LOG_TARGET, session_index, "Calling `with_info`");
 			let r = with_info(&info);
-			tracing::trace!(target: LOG_TARGET, session_index, "Storing session info in lru!");
+			gum::trace!(target: LOG_TARGET, session_index, "Storing session info in lru!");
 			self.session_info_cache.put(session_index, info);
 			Ok(Some(r))
 		} else {
@@ -130,7 +130,7 @@ impl SessionCache {
 	/// subsystem on this.
 	pub fn report_bad_log(&mut self, report: BadValidators) {
 		if let Err(err) = self.report_bad(report) {
-			tracing::warn!(
+			gum::warn!(
 				target: LOG_TARGET,
 				err = ?err,
 				"Reporting bad validators failed with error"

polkadot/node/network/availability-distribution/src/responder.rs

@@ -51,7 +51,7 @@ pub async fn run_pov_receiver<Sender>(
 				answer_pov_request_log(&mut sender, msg, &metrics).await;
 			},
 			Err(fatal) => {
-				tracing::debug!(
+				gum::debug!(
 					target: LOG_TARGET,
 					error = ?fatal,
 					"Shutting down POV receiver."
@@ -59,7 +59,7 @@ pub async fn run_pov_receiver<Sender>(
 				return
 			},
 			Ok(Err(jfyi)) => {
-				tracing::debug!(target: LOG_TARGET, error = ?jfyi, "Error decoding incoming PoV request.");
+				gum::debug!(target: LOG_TARGET, error = ?jfyi, "Error decoding incoming PoV request.");
 			},
 		}
 	}
@@ -79,7 +79,7 @@ pub async fn run_chunk_receiver<Sender>(
 				answer_chunk_request_log(&mut sender, msg, &metrics).await;
 			},
 			Err(fatal) => {
-				tracing::debug!(
+				gum::debug!(
 					target: LOG_TARGET,
 					error = ?fatal,
 					"Shutting down chunk receiver."
@@ -87,7 +87,7 @@ pub async fn run_chunk_receiver<Sender>(
 				return
 			},
 			Ok(Err(jfyi)) => {
-				tracing::debug!(
+				gum::debug!(
 					target: LOG_TARGET,
 					error = ?jfyi,
 					"Error decoding incoming chunk request."
@@ -111,7 +111,7 @@ pub async fn answer_pov_request_log<Sender>(
 	match res {
 		Ok(result) => metrics.on_served_pov(if result { SUCCEEDED } else { NOT_FOUND }),
 		Err(err) => {
-			tracing::warn!(
+			gum::warn!(
 				target: LOG_TARGET,
 				err= ?err,
 				"Serving PoV failed with error"
@@ -136,7 +136,7 @@ where
 	match res {
 		Ok(result) => metrics.on_served_chunk(if result { SUCCEEDED } else { NOT_FOUND }),
 		Err(err) => {
-			tracing::warn!(
+			gum::warn!(
 				target: LOG_TARGET,
 				err= ?err,
 				"Serving chunk failed with error"
@@ -192,7 +192,7 @@ where
 
 	let result = chunk.is_some();
 
-	tracing::trace!(
+	gum::trace!(
 		target: LOG_TARGET,
 		hash = ?req.payload.candidate_hash,
 		index = ?req.payload.index,
@@ -227,7 +227,7 @@ where
 		.await;
 
 	let result = rx.await.map_err(|e| {
-		tracing::trace!(
+		gum::trace!(
 			target: LOG_TARGET,
 			?validator_index,
 			?candidate_hash,

polkadot/node/network/availability-distribution/src/tests/state.rs

@@ -211,7 +211,7 @@ impl TestState {
 		);
 
 		while remaining_stores > 0 {
-			tracing::trace!(target: LOG_TARGET, remaining_stores, "Stores left to go");
+			gum::trace!(target: LOG_TARGET, remaining_stores, "Stores left to go");
 			let msg = overseer_recv(&mut rx).await;
 			match msg {
 				AllMessages::NetworkBridge(NetworkBridgeMessage::SendRequests(
@@ -255,7 +255,7 @@ impl TestState {
 						"Only valid chunks should ever get stored."
 					);
 					tx.send(Ok(())).expect("Receiver is expected to be alive");
-					tracing::trace!(target: LOG_TARGET, "'Stored' fetched chunk.");
+					gum::trace!(target: LOG_TARGET, "'Stored' fetched chunk.");
 					remaining_stores -= 1;
 				},
 				AllMessages::RuntimeApi(RuntimeApiMessage::Request(hash, req)) => {
@@ -269,7 +269,7 @@ impl TestState {
 								.expect("Receiver should be alive.");
 						},
 						RuntimeApiRequest::AvailabilityCores(tx) => {
-							tracing::trace!(target: LOG_TARGET, cores= ?self.cores[&hash], hash = ?hash, "Sending out cores for hash");
+							gum::trace!(target: LOG_TARGET, cores= ?self.cores[&hash], hash = ?hash, "Sending out cores for hash");
 							tx.send(Ok(self.cores[&hash].clone()))
 								.expect("Receiver should still be alive");
 						},
@@ -299,12 +299,12 @@ async fn overseer_signal(
 	msg: impl Into<OverseerSignal>,
 ) {
 	let msg = msg.into();
-	tracing::trace!(target: LOG_TARGET, msg = ?msg, "sending message");
+	gum::trace!(target: LOG_TARGET, msg = ?msg, "sending message");
 	tx.send(FromOverseer::Signal(msg)).await.expect("Test subsystem no longer live");
 }
 
 async fn overseer_recv(rx: &mut mpsc::UnboundedReceiver<AllMessages>) -> AllMessages {
-	tracing::trace!(target: LOG_TARGET, "waiting for message ...");
+	gum::trace!(target: LOG_TARGET, "waiting for message ...");
 	rx.next().await.expect("Test subsystem no longer live")
 }