Add tracing support to node (#1940)

* drop in tracing to replace log

* add structured logging to trace messages

* add structured logging to debug messages

* add structured logging to info messages

* add structured logging to warn messages

* add structured logging to error messages

* normalize spacing and Display vs Debug

* add instrumentation to the various 'fn run'

* use explicit tracing module throughout

* fix availability distribution test

* don't double-print errors

* remove further redundancy from logs

* fix test errors

* fix more test errors

* remove unused kv_log_macro

* fix unused variable

* add tracing spans to collation generation

* add tracing spans to av-store

* add tracing spans to backing

* add tracing spans to bitfield-signing

* add tracing spans to candidate-selection

* add tracing spans to candidate-validation

* add tracing spans to chain-api

* add tracing spans to provisioner

* add tracing spans to runtime-api

* add tracing spans to availability-distribution

* add tracing spans to bitfield-distribution

* add tracing spans to network-bridge

* add tracing spans to collator-protocol

* add tracing spans to pov-distribution

* add tracing spans to statement-distribution

* add tracing spans to overseer

* cleanup

polkadot/node/core/av-store/Cargo.toml

@@ -9,8 +9,9 @@ futures = "0.3.8"
 futures-timer = "3.0.2"
 kvdb = "0.7.0"
 kvdb-rocksdb = "0.9.1"
-log = "0.4.11"
 thiserror = "1.0.22"
+tracing = "0.1.21"
+tracing-futures = "0.2.4"
 
 parity-scale-codec = { version = "1.3.5", features = ["derive"] }
 erasure = { package = "polkadot-erasure-coding", path = "../../../erasure-coding" }
@@ -22,6 +23,7 @@ polkadot-primitives = { path = "../../../primitives" }
 sc-service = { git = "https://github.com/paritytech/substrate", branch = "master", default-features = false }
 
 [dev-dependencies]
+log = "0.4.11"
 env_logger = "0.8.2"
 assert_matches = "1.4.0"
 smallvec = "1.5.0"

polkadot/node/core/av-store/src/lib.rs

@@ -73,13 +73,13 @@ enum Error {
 }
 
 impl Error {
-	fn severity(&self) -> log::Level {
+	fn trace(&self) {
 		match self {
 			// don't spam the log with spurious errors
 			Self::RuntimeApi(_) |
-			Self::Oneshot(_) => log::Level::Debug,
+			Self::Oneshot(_) => tracing::debug!(target: LOG_TARGET, err = ?self),
 			// it's worth reporting otherwise
-			_ => log::Level::Warn,
+			_ => tracing::warn!(target: LOG_TARGET, err = ?self),
 		}
 	}
 }
@@ -311,18 +311,19 @@ pub struct AvailabilityStoreSubsystem {
 
 impl AvailabilityStoreSubsystem {
 	// Perform pruning of PoVs
+	#[tracing::instrument(level = "trace", skip(self), fields(subsystem = LOG_TARGET))]
 	fn prune_povs(&self) -> Result<(), Error> {
 		let mut tx = DBTransaction::new();
 		let mut pov_pruning = pov_pruning(&self.inner).unwrap_or_default();
 		let now = PruningDelay::now()?;
 
-		log::trace!(target: LOG_TARGET, "Pruning PoVs");
+		tracing::trace!(target: LOG_TARGET, "Pruning PoVs");
 		let outdated_records_count = pov_pruning.iter()
 			.take_while(|r| r.prune_at <= now)
 			.count();
 
 		for record in pov_pruning.drain(..outdated_records_count) {
-			log::trace!(target: LOG_TARGET, "Removing record {:?}", record);
+			tracing::trace!(target: LOG_TARGET, record = ?record, "Removing record");
 			tx.delete(
 				columns::DATA,
 				available_data_key(&record.candidate_hash).as_slice(),
@@ -335,18 +336,19 @@ impl AvailabilityStoreSubsystem {
 	}
 
 	// Perform pruning of chunks.
+	#[tracing::instrument(level = "trace", skip(self), fields(subsystem = LOG_TARGET))]
 	fn prune_chunks(&self) -> Result<(), Error> {
 		let mut tx = DBTransaction::new();
 		let mut chunk_pruning = chunk_pruning(&self.inner).unwrap_or_default();
 		let now = PruningDelay::now()?;
 
-		log::trace!(target: LOG_TARGET, "Pruning Chunks");
+		tracing::trace!(target: LOG_TARGET, "Pruning Chunks");
 		let outdated_records_count = chunk_pruning.iter()
 			.take_while(|r| r.prune_at <= now)
 			.count();
 
 		for record in chunk_pruning.drain(..outdated_records_count) {
-			log::trace!(target: LOG_TARGET, "Removing record {:?}", record);
+			tracing::trace!(target: LOG_TARGET, record = ?record, "Removing record");
 			tx.delete(
 				columns::DATA,
 				erasure_chunk_key(&record.candidate_hash, record.chunk_index).as_slice(),
@@ -361,6 +363,7 @@ impl AvailabilityStoreSubsystem {
 	// Return a `Future` that either resolves when another PoV pruning has to happen
 	// or is indefinitely `pending` in case no pruning has to be done.
 	// Just a helper to `select` over multiple things at once.
+	#[tracing::instrument(level = "trace", skip(self), fields(subsystem = LOG_TARGET))]
 	fn maybe_prune_povs(&self) -> Result<impl Future<Output = ()>, Error> {
 		let future = match get_next_pov_pruning_time(&self.inner) {
 			Some(pruning) => {
@@ -375,6 +378,7 @@ impl AvailabilityStoreSubsystem {
 	// Return a `Future` that either resolves when another chunk pruning has to happen
 	// or is indefinitely `pending` in case no pruning has to be done.
 	// Just a helper to `select` over multiple things at once.
+	#[tracing::instrument(level = "trace", skip(self), fields(subsystem = LOG_TARGET))]
 	fn maybe_prune_chunks(&self) -> Result<impl Future<Output = ()>, Error> {
 		let future = match get_next_chunk_pruning_time(&self.inner) {
 			Some(pruning) => {
@@ -473,6 +477,7 @@ fn get_next_chunk_pruning_time(db: &Arc<dyn KeyValueDB>) -> Option<NextChunkPrun
 	query_inner(db, columns::META, &NEXT_CHUNK_PRUNING)
 }
 
+#[tracing::instrument(skip(subsystem, ctx), fields(subsystem = LOG_TARGET))]
 async fn run<Context>(mut subsystem: AvailabilityStoreSubsystem, mut ctx: Context)
 where
 	Context: SubsystemContext<Message=AvailabilityStoreMessage>,
@@ -481,10 +486,10 @@ where
 		let res = run_iteration(&mut subsystem, &mut ctx).await;
 		match res {
 			Err(e) => {
-				log::log!(target: LOG_TARGET, e.severity(), "{}", e);
+				e.trace();
 			}
 			Ok(true) => {
-				log::info!(target: LOG_TARGET, "received `Conclude` signal, exiting");
+				tracing::info!(target: LOG_TARGET, "received `Conclude` signal, exiting");
 				break;
 			},
 			Ok(false) => continue,
@@ -492,6 +497,7 @@ where
 	}
 }
 
+#[tracing::instrument(level = "trace", skip(subsystem, ctx), fields(subsystem = LOG_TARGET))]
 async fn run_iteration<Context>(subsystem: &mut AvailabilityStoreSubsystem, ctx: &mut Context)
 	-> Result<bool, Error>
 where
@@ -545,6 +551,7 @@ where
 /// The state of data has to be changed from
 /// `CandidateState::Included` to `CandidateState::Finalized` and their pruning times have
 /// to be updated to `now` + keep_finalized_{block, chunk}_for`.
+#[tracing::instrument(level = "trace", skip(subsystem, ctx, db), fields(subsystem = LOG_TARGET))]
 async fn process_block_finalized<Context>(
 	subsystem: &AvailabilityStoreSubsystem,
 	ctx: &mut Context,
@@ -561,10 +568,10 @@ where
 		// numbers we have to iterate through the whole collection here.
 		for record in pov_pruning.iter_mut() {
 			if record.block_number <= block_number {
-				log::trace!(
+				tracing::trace!(
 					target: LOG_TARGET,
-					"Updating pruning record for finalized block {}",
-					record.block_number,
+					block_number = %record.block_number,
+					"Updating pruning record for finalized block",
 				);
 
 				record.prune_at = PruningDelay::into_the_future(
@@ -580,10 +587,10 @@ where
 	if let Some(mut chunk_pruning) = chunk_pruning(db) {
 		for record in chunk_pruning.iter_mut() {
 			if record.block_number <= block_number {
-				log::trace!(
+				tracing::trace!(
 					target: LOG_TARGET,
-					"Updating chunk pruning record for finalized block {}",
-					record.block_number,
+					block_number = %record.block_number,
+					"Updating chunk pruning record for finalized block",
 				);
 
 				record.prune_at = PruningDelay::into_the_future(
@@ -599,6 +606,7 @@ where
 	Ok(())
 }
 
+#[tracing::instrument(level = "trace", skip(ctx, db), fields(subsystem = LOG_TARGET))]
 async fn process_block_activated<Context>(
 	ctx: &mut Context,
 	db: &Arc<dyn KeyValueDB>,
@@ -610,17 +618,21 @@ where
 	let events = match request_candidate_events(ctx, hash).await {
 		Ok(events) => events,
 		Err(err) => {
-			log::debug!(target: LOG_TARGET, "requesting candidate events failed due to {}", err);
+			tracing::debug!(target: LOG_TARGET, err = ?err, "requesting candidate events failed");
 			return Ok(());
 		}
 	};
 
-	log::trace!(target: LOG_TARGET, "block activated {}", hash);
+	tracing::trace!(target: LOG_TARGET, hash = %hash, "block activated");
 	let mut included = HashSet::new();
 
 	for event in events.into_iter() {
 		if let CandidateEvent::CandidateIncluded(receipt, _) = event {
-			log::trace!(target: LOG_TARGET, "Candidate {:?} was included", receipt.hash());
+			tracing::trace!(
+				target: LOG_TARGET,
+				hash = %receipt.hash(),
+				"Candidate {:?} was included", receipt.hash(),
+			);
 			included.insert(receipt.hash());
 		}
 	}
@@ -654,6 +666,7 @@ where
 	Ok(())
 }
 
+#[tracing::instrument(level = "trace", skip(ctx), fields(subsystem = LOG_TARGET))]
 async fn request_candidate_events<Context>(
 	ctx: &mut Context,
 	hash: Hash,
@@ -673,6 +686,7 @@ where
 	Ok(rx.await??)
 }
 
+#[tracing::instrument(level = "trace", skip(subsystem, ctx), fields(subsystem = LOG_TARGET))]
 async fn process_message<Context>(
 	subsystem: &mut AvailabilityStoreSubsystem,
 	ctx: &mut Context,
@@ -744,6 +758,7 @@ fn chunk_pruning(db: &Arc<dyn KeyValueDB>) -> Option<Vec<ChunkPruningRecord>> {
 	query_inner(db, columns::META, &CHUNK_PRUNING_KEY)
 }
 
+#[tracing::instrument(level = "trace", skip(db, tx), fields(subsystem = LOG_TARGET))]
 fn put_pov_pruning(
 	db: &Arc<dyn KeyValueDB>,
 	tx: Option<DBTransaction>,
@@ -784,6 +799,7 @@ fn put_pov_pruning(
 	Ok(())
 }
 
+#[tracing::instrument(level = "trace", skip(db, tx), fields(subsystem = LOG_TARGET))]
 fn put_chunk_pruning(
 	db: &Arc<dyn KeyValueDB>,
 	tx: Option<DBTransaction>,
@@ -836,6 +852,7 @@ where
 	Ok(rx.await??.map(|number| number).unwrap_or_default())
 }
 
+#[tracing::instrument(level = "trace", skip(subsystem, available_data), fields(subsystem = LOG_TARGET))]
 fn store_available_data(
 	subsystem: &mut AvailabilityStoreSubsystem,
 	candidate_hash: &CandidateHash,
@@ -902,6 +919,7 @@ fn store_available_data(
 	Ok(())
 }
 
+#[tracing::instrument(level = "trace", skip(subsystem), fields(subsystem = LOG_TARGET))]
 fn store_chunk(
 	subsystem: &mut AvailabilityStoreSubsystem,
 	candidate_hash: &CandidateHash,
@@ -953,6 +971,7 @@ fn store_chunk(
 	Ok(())
 }
 
+#[tracing::instrument(level = "trace", skip(subsystem), fields(subsystem = LOG_TARGET))]
 fn get_chunk(
 	subsystem: &mut AvailabilityStoreSubsystem,
 	candidate_hash: &CandidateHash,
@@ -996,7 +1015,7 @@ fn query_inner<D: Decode>(
 		}
 		Ok(None) => None,
 		Err(e) => {
-			log::warn!(target: LOG_TARGET, "Error reading from the availability store: {:?}", e);
+			tracing::warn!(target: LOG_TARGET, err = ?e, "Error reading from the availability store");
 			None
 		}
 	}
@@ -1018,6 +1037,7 @@ where
 	}
 }
 
+#[tracing::instrument(level = "trace", skip(metrics), fields(subsystem = LOG_TARGET))]
 fn get_chunks(data: &AvailableData, n_validators: usize, metrics: &Metrics) -> Result<Vec<ErasureChunk>, Error> {
 	let chunks = erasure::obtain_chunks_v1(n_validators, data)?;
 	metrics.on_chunks_received(chunks.len());

polkadot/node/core/av-store/src/tests.rs

@@ -128,7 +128,7 @@ async fn overseer_send(
 	overseer: &mut test_helpers::TestSubsystemContextHandle<AvailabilityStoreMessage>,
 	msg: AvailabilityStoreMessage,
 ) {
-	log::trace!("Sending message:\n{:?}", &msg);
+	tracing::trace!(meg = ?msg, "sending message");
 	overseer
 		.send(FromOverseer::Communication { msg })
 		.timeout(TIMEOUT)
@@ -143,7 +143,7 @@ async fn overseer_recv(
 		.await
 		.expect(&format!("{:?} is more than enough to receive messages", TIMEOUT));
 
-	log::trace!("Received message:\n{:?}", &msg);
+	tracing::trace!(msg = ?msg, "received message");
 
 	msg
 }
@@ -152,7 +152,7 @@ async fn overseer_recv_with_timeout(
 	overseer: &mut test_helpers::TestSubsystemContextHandle<AvailabilityStoreMessage>,
 	timeout: Duration,
 ) -> Option<AllMessages> {
-	log::trace!("Waiting for message...");
+	tracing::trace!("waiting for message...");
 	overseer
 		.recv()
 		.timeout(timeout)

polkadot/node/core/backing/Cargo.toml

@@ -14,7 +14,8 @@ polkadot-node-subsystem-util = { path = "../../subsystem-util" }
 erasure-coding = { package = "polkadot-erasure-coding", path = "../../../erasure-coding" }
 statement-table = { package = "polkadot-statement-table", path = "../../../statement-table" }
 bitvec = { version = "0.17.4", default-features = false, features = ["alloc"] }
-log = "0.4.11"
+tracing = "0.1.21"
+tracing-futures = "0.2.4"
 thiserror = "1.0.22"
 
 [dev-dependencies]

polkadot/node/core/backing/src/lib.rs

@@ -244,6 +244,7 @@ fn primitive_statement_to_table(s: &SignedFullStatement) -> TableSignedStatement
 	}
 }
 
+#[tracing::instrument(level = "trace", skip(attested, table_context), fields(subsystem = LOG_TARGET))]
 fn table_attested_to_backed(
 	attested: TableAttestedCandidate<
 		ParaId,
@@ -308,6 +309,7 @@ impl CandidateBackingJob {
 	/// Validate the candidate that is requested to be `Second`ed and distribute validation result.
 	///
 	/// Returns `Ok(true)` if we issued a `Seconded` statement about this candidate.
+	#[tracing::instrument(level = "trace", skip(self), fields(subsystem = LOG_TARGET))]
 	async fn validate_and_second(
 		&mut self,
 		candidate: &CandidateReceipt,
@@ -390,6 +392,7 @@ impl CandidateBackingJob {
 		Ok(())
 	}
 
+	#[tracing::instrument(level = "trace", skip(self), fields(subsystem = LOG_TARGET))]
 	fn get_backed(&self) -> Vec<NewBackedCandidate> {
 		let proposed = self.table.proposed_candidates(&self.table_context);
 		let mut res = Vec::with_capacity(proposed.len());
@@ -407,6 +410,7 @@ impl CandidateBackingJob {
 	/// Check if there have happened any new misbehaviors and issue necessary messages.
 	///
 	/// TODO: Report multiple misbehaviors (https://github.com/paritytech/polkadot/issues/1387)
+	#[tracing::instrument(level = "trace", skip(self), fields(subsystem = LOG_TARGET))]
 	async fn issue_new_misbehaviors(&mut self) -> Result<(), Error> {
 		let mut reports = Vec::new();
 
@@ -440,6 +444,7 @@ impl CandidateBackingJob {
 	}
 
 	/// Import a statement into the statement table and return the summary of the import.
+	#[tracing::instrument(level = "trace", skip(self), fields(subsystem = LOG_TARGET))]
 	async fn import_statement(
 		&mut self,
 		statement: &SignedFullStatement,
@@ -474,6 +479,7 @@ impl CandidateBackingJob {
 		Ok(summary)
 	}
 
+	#[tracing::instrument(level = "trace", skip(self), fields(subsystem = LOG_TARGET))]
 	async fn process_msg(&mut self, msg: CandidateBackingMessage) -> Result<(), Error> {
 		match msg {
 			CandidateBackingMessage::Second(_, candidate, pov) => {
@@ -521,6 +527,7 @@ impl CandidateBackingJob {
 	}
 
 	/// Kick off validation work and distribute the result as a signed statement.
+	#[tracing::instrument(level = "trace", skip(self), fields(subsystem = LOG_TARGET))]
 	async fn kick_off_validation_work(
 		&mut self,
 		summary: TableSummary,
@@ -585,6 +592,7 @@ impl CandidateBackingJob {
 	}
 
 	/// Import the statement and kick off validation work if it is a part of our assignment.
+	#[tracing::instrument(level = "trace", skip(self), fields(subsystem = LOG_TARGET))]
 	async fn maybe_validate_and_import(
 		&mut self,
 		statement: SignedFullStatement,
@@ -600,6 +608,7 @@ impl CandidateBackingJob {
 		Ok(())
 	}
 
+	#[tracing::instrument(level = "trace", skip(self), fields(subsystem = LOG_TARGET))]
 	async fn sign_statement(&self, statement: Statement) -> Option<SignedFullStatement> {
 		let signed = self.table_context
 			.validator
@@ -611,6 +620,7 @@ impl CandidateBackingJob {
 		Some(signed)
 	}
 
+	#[tracing::instrument(level = "trace", skip(self), fields(subsystem = LOG_TARGET))]
 	fn check_statement_signature(&self, statement: &SignedFullStatement) -> Result<(), Error> {
 		let idx = statement.validator_index() as usize;
 
@@ -703,6 +713,7 @@ impl CandidateBackingJob {
 	// This calls an inspection function before making the PoV available for any last checks
 	// that need to be done. If the inspection function returns an error, this function returns
 	// early without making the PoV available.
+	#[tracing::instrument(level = "trace", skip(self, pov, with_commitments), fields(subsystem = LOG_TARGET))]
 	async fn make_pov_available<T, E>(
 		&mut self,
 		pov: Arc<PoV>,
@@ -767,6 +778,7 @@ impl util::JobTrait for CandidateBackingJob {
 
 	const NAME: &'static str = "CandidateBackingJob";
 
+	#[tracing::instrument(skip(keystore, metrics, rx_to, tx_from), fields(subsystem = LOG_TARGET))]
 	fn run(
 		parent: Hash,
 		keystore: SyncCryptoStorePtr,
@@ -780,10 +792,10 @@ impl util::JobTrait for CandidateBackingJob {
 					match $x {
 						Ok(x) => x,
 						Err(e) => {
-							log::warn!(
+							tracing::warn!(
 								target: LOG_TARGET,
-								"Failed to fetch runtime API data for job: {:?}",
-								e,
+								err = ?e,
+								"Failed to fetch runtime API data for job",
 							);
 
 							// We can't do candidate validation work if we don't have the
@@ -820,10 +832,10 @@ impl util::JobTrait for CandidateBackingJob {
 				Ok(v) => v,
 				Err(util::Error::NotAValidator) => { return Ok(()) },
 				Err(e) => {
-					log::warn!(
+					tracing::warn!(
 						target: LOG_TARGET,
-						"Cannot participate in candidate backing: {:?}",
-						e
+						err = ?e,
+						"Cannot participate in candidate backing",
 					);
 
 					return Ok(())

polkadot/node/core/bitfield-signing/Cargo.toml

@@ -6,7 +6,8 @@ edition = "2018"
 
 [dependencies]
 futures = "0.3.8"
-log = "0.4.11"
+tracing = "0.1.21"
+tracing-futures = "0.2.4"
 polkadot-primitives = { path = "../../../primitives" }
 polkadot-node-subsystem = { path = "../../subsystem" }
 polkadot-node-subsystem-util = { path = "../../subsystem-util" }

polkadot/node/core/bitfield-signing/src/lib.rs

@@ -140,6 +140,7 @@ pub enum Error {
 
 /// If there is a candidate pending availability, query the Availability Store
 /// for whether we have the availability chunk for our validator index.
+#[tracing::instrument(level = "trace", skip(sender), fields(subsystem = LOG_TARGET))]
 async fn get_core_availability(
 	relay_parent: Hash,
 	core: CoreState,
@@ -164,7 +165,7 @@ async fn get_core_availability(
 			Ok(None) => return Ok(false),
 			Err(e) => {
 				// Don't take down the node on runtime API errors.
-				log::warn!(target: LOG_TARGET, "Encountered a runtime API error: {:?}", e);
+				tracing::warn!(target: LOG_TARGET, err = ?e, "Encountered a runtime API error");
 				return Ok(false);
 			}
 		};
@@ -201,6 +202,7 @@ async fn get_availability_cores(relay_parent: Hash, sender: &mut mpsc::Sender<Fr
 /// - for each core, concurrently determine chunk availability (see `get_core_availability`)
 /// - return the bitfield if there were no errors at any point in this process
 ///   (otherwise, it's prone to false negatives)
+#[tracing::instrument(level = "trace", skip(sender), fields(subsystem = LOG_TARGET))]
 async fn construct_availability_bitfield(
 	relay_parent: Hash,
 	validator_idx: ValidatorIndex,
@@ -267,6 +269,7 @@ impl JobTrait for BitfieldSigningJob {
 	const NAME: &'static str = "BitfieldSigningJob";
 
 	/// Run a job for the parent block indicated
+	#[tracing::instrument(skip(keystore, metrics, _receiver, sender), fields(subsystem = LOG_TARGET))]
 	fn run(
 		relay_parent: Hash,
 		keystore: Self::RunArgs,
@@ -293,7 +296,7 @@ impl JobTrait for BitfieldSigningJob {
 			{
 				Err(Error::Runtime(runtime_err)) => {
 					// Don't take down the node on runtime API errors.
-					log::warn!(target: LOG_TARGET, "Encountered a runtime API error: {:?}", runtime_err);
+					tracing::warn!(target: LOG_TARGET, err = ?runtime_err, "Encountered a runtime API error");
 					return Ok(());
 				}
 				Err(err) => return Err(err),

polkadot/node/core/candidate-selection/Cargo.toml

@@ -6,7 +6,8 @@ edition = "2018"
 
 [dependencies]
 futures = "0.3.8"
-log = "0.4.11"
+tracing = "0.1.21"
+tracing-futures = "0.2.4"
 thiserror = "1.0.22"
 polkadot-primitives = { path = "../../../primitives" }
 polkadot-node-subsystem = { path = "../../subsystem" }

polkadot/node/core/candidate-selection/src/lib.rs

@@ -37,7 +37,7 @@ use polkadot_primitives::v1::{CandidateReceipt, CollatorId, Hash, Id as ParaId,
 use std::{convert::TryFrom, pin::Pin};
 use thiserror::Error;
 
-const TARGET: &'static str = "candidate_selection";
+const LOG_TARGET: &'static str = "candidate_selection";
 
 struct CandidateSelectionJob {
 	sender: mpsc::Sender<FromJob>,
@@ -134,6 +134,7 @@ impl JobTrait for CandidateSelectionJob {
 	/// Run a job for the parent block indicated
 	//
 	// this function is in charge of creating and executing the job's main loop
+	#[tracing::instrument(skip(_relay_parent, _run_args, metrics, receiver, sender), fields(subsystem = LOG_TARGET))]
 	fn run(
 		_relay_parent: Hash,
 		_run_args: Self::RunArgs,
@@ -196,6 +197,7 @@ impl CandidateSelectionJob {
 		Ok(())
 	}
 
+	#[tracing::instrument(level = "trace", skip(self), fields(subsystem = LOG_TARGET))]
 	async fn handle_collation(
 		&mut self,
 		relay_parent: Hash,
@@ -212,10 +214,10 @@ impl CandidateSelectionJob {
 				).await {
 					Ok(response) => response,
 					Err(err) => {
-						log::warn!(
-							target: TARGET,
-							"failed to get collation from collator protocol subsystem: {:?}",
-							err
+						tracing::warn!(
+							target: LOG_TARGET,
+							err = ?err,
+							"failed to get collation from collator protocol subsystem",
 						);
 						return;
 					}
@@ -230,35 +232,36 @@ impl CandidateSelectionJob {
 			)
 			.await
 			{
-				Err(err) => log::warn!(target: TARGET, "failed to second a candidate: {:?}", err),
+				Err(err) => tracing::warn!(target: LOG_TARGET, err = ?err, "failed to second a candidate"),
 				Ok(()) => self.seconded_candidate = Some(collator_id),
 			}
 		}
 	}
 
+	#[tracing::instrument(level = "trace", skip(self), fields(subsystem = LOG_TARGET))]
 	async fn handle_invalid(&mut self, candidate_receipt: CandidateReceipt) {
 		let received_from = match &self.seconded_candidate {
 			Some(peer) => peer,
 			None => {
-				log::warn!(
-					target: TARGET,
+				tracing::warn!(
+					target: LOG_TARGET,
 					"received invalidity notice for a candidate we don't remember seconding"
 				);
 				return;
 			}
 		};
-		log::info!(
-			target: TARGET,
-			"received invalidity note for candidate {:?}",
-			candidate_receipt
+		tracing::info!(
+			target: LOG_TARGET,
+			candidate_receipt = ?candidate_receipt,
+			"received invalidity note for candidate",
 		);
 
 		let result =
 			if let Err(err) = forward_invalidity_note(received_from, &mut self.sender).await {
-				log::warn!(
-					target: TARGET,
-					"failed to forward invalidity note: {:?}",
-					err
+				tracing::warn!(
+					target: LOG_TARGET,
+					err = ?err,
+					"failed to forward invalidity note",
 				);
 				Err(())
 			} else {
@@ -271,6 +274,7 @@ impl CandidateSelectionJob {
 // get a collation from the Collator Protocol subsystem
 //
 // note that this gets an owned clone of the sender; that's becuase unlike `forward_invalidity_note`, it's expected to take a while longer
+#[tracing::instrument(level = "trace", skip(sender), fields(subsystem = LOG_TARGET))]
 async fn get_collation(
 	relay_parent: Hash,
 	para_id: ParaId,
@@ -305,7 +309,7 @@ async fn second_candidate(
 		.await
 	{
 		Err(err) => {
-			log::warn!(target: TARGET, "failed to send a seconding message");
+			tracing::warn!(target: LOG_TARGET, err = ?err, "failed to send a seconding message");
 			metrics.on_second(Err(()));
 			Err(err.into())
 		}

polkadot/node/core/candidate-validation/Cargo.toml

@@ -6,7 +6,8 @@ edition = "2018"
 
 [dependencies]
 futures = "0.3.8"
-log = "0.4.11"
+tracing = "0.1.21"
+tracing-futures = "0.2.4"
 
 sp-core = { package = "sp-core", git = "https://github.com/paritytech/substrate", branch = "master" }
 parity-scale-codec = { version = "1.3.5", default-features = false, features = ["bit-vec", "derive"] }

polkadot/node/core/candidate-validation/src/lib.rs

@@ -85,6 +85,7 @@ impl<S, C> Subsystem<C> for CandidateValidationSubsystem<S> where
 	}
 }
 
+#[tracing::instrument(skip(ctx, spawn, metrics), fields(subsystem = LOG_TARGET))]
 async fn run(
 	mut ctx: impl SubsystemContext<Message = CandidateValidationMessage>,
 	spawn: impl SpawnNamed + Clone + 'static,
@@ -139,7 +140,7 @@ async fn run(
 						Ok(x) => {
 							metrics.on_validation_event(&x);
 							if let Err(_e) = response_sender.send(x) {
-								log::warn!(
+								tracing::warn!(
 									target: LOG_TARGET,
 									"Requester of candidate validation dropped",
 								)
@@ -176,6 +177,7 @@ enum AssumptionCheckOutcome {
 	BadRequest,
 }
 
+#[tracing::instrument(level = "trace", skip(ctx), fields(subsystem = LOG_TARGET))]
 async fn check_assumption_validation_data(
 	ctx: &mut impl SubsystemContext<Message = CandidateValidationMessage>,
 	descriptor: &CandidateDescriptor,
@@ -226,6 +228,7 @@ async fn check_assumption_validation_data(
 	})
 }
 
+#[tracing::instrument(level = "trace", skip(ctx), fields(subsystem = LOG_TARGET))]
 async fn find_assumed_validation_data(
 	ctx: &mut impl SubsystemContext<Message = CandidateValidationMessage>,
 	descriptor: &CandidateDescriptor,
@@ -257,6 +260,7 @@ async fn find_assumed_validation_data(
 	Ok(AssumptionCheckOutcome::DoesNotMatch)
 }
 
+#[tracing::instrument(level = "trace", skip(ctx, pov, spawn), fields(subsystem = LOG_TARGET))]
 async fn spawn_validate_from_chain_state(
 	ctx: &mut impl SubsystemContext<Message = CandidateValidationMessage>,
 	isolation_strategy: IsolationStrategy,
@@ -316,6 +320,7 @@ async fn spawn_validate_from_chain_state(
 	validation_result
 }
 
+#[tracing::instrument(level = "trace", skip(ctx, validation_code, pov, spawn), fields(subsystem = LOG_TARGET))]
 async fn spawn_validate_exhaustive(
 	ctx: &mut impl SubsystemContext<Message = CandidateValidationMessage>,
 	isolation_strategy: IsolationStrategy,
@@ -345,6 +350,7 @@ async fn spawn_validate_exhaustive(
 
 /// Does basic checks of a candidate. Provide the encoded PoV-block. Returns `Ok` if basic checks
 /// are passed, `Err` otherwise.
+#[tracing::instrument(level = "trace", skip(pov), fields(subsystem = LOG_TARGET))]
 fn perform_basic_checks(
 	candidate: &CandidateDescriptor,
 	max_pov_size: u32,
@@ -402,6 +408,7 @@ impl ValidationBackend for RealValidationBackend {
 /// Validates the candidate from exhaustive parameters.
 ///
 /// Sends the result of validation on the channel once complete.
+#[tracing::instrument(level = "trace", skip(backend_arg, validation_code, pov, spawn), fields(subsystem = LOG_TARGET))]
 fn validate_candidate_exhaustive<B: ValidationBackend, S: SpawnNamed + 'static>(
 	backend_arg: B::Arg,
 	persisted_validation_data: PersistedValidationData,

polkadot/node/core/chain-api/Cargo.toml

@@ -6,6 +6,8 @@ edition = "2018"
 
 [dependencies]
 futures = "0.3.8"
+tracing = "0.1.21"
+tracing-futures = "0.2.4"
 sp-blockchain = { git = "https://github.com/paritytech/substrate", branch = "master" }
 polkadot-primitives = { path = "../../../primitives" }
 polkadot-subsystem = { package = "polkadot-node-subsystem", path = "../../subsystem" }

polkadot/node/core/chain-api/src/lib.rs

@@ -44,6 +44,8 @@ use std::sync::Arc;
 
 use futures::prelude::*;
 
+const LOG_TARGET: &str = "ChainApiSubsystem";
+
 /// The Chain API Subsystem implementation.
 pub struct ChainApiSubsystem<Client> {
 	client: Arc<Client>,
@@ -75,6 +77,7 @@ impl<Client, Context> Subsystem<Context> for ChainApiSubsystem<Client> where
 	}
 }
 
+#[tracing::instrument(skip(ctx, subsystem), fields(subsystem = LOG_TARGET))]
 async fn run<Client>(
 	mut ctx: impl SubsystemContext<Message = ChainApiMessage>,
 	subsystem: ChainApiSubsystem<Client>,
@@ -113,6 +116,7 @@ where
 					let _ = response_channel.send(Ok(result));
 				},
 				ChainApiMessage::Ancestors { hash, k, response_channel } => {
+					tracing::span!(tracing::Level::TRACE, "ChainApiMessage::Ancestors", subsystem=LOG_TARGET, hash=%hash, k=k);
 					let mut hash = hash;
 
 					let next_parent = core::iter::from_fn(|| {

polkadot/node/core/proposer/Cargo.toml

@@ -7,7 +7,7 @@ edition = "2018"
 [dependencies]
 futures = "0.3.8"
 futures-timer = "3.0.2"
-log = "0.4.11"
+tracing = "0.1.21"
 polkadot-node-subsystem = { path = "../../subsystem" }
 polkadot-overseer = { path = "../../overseer" }
 polkadot-primitives = { path = "../../../primitives" }

polkadot/node/core/proposer/src/lib.rs

@@ -193,7 +193,7 @@ where
 			let provisioner_data = match self.get_provisioner_data().await {
 				Ok(pd) => pd,
 				Err(err) => {
-					log::warn!("could not get provisioner inherent data; injecting default data: {}", err);
+					tracing::warn!(err = ?err, "could not get provisioner inherent data; injecting default data");
 					Default::default()
 				}
 			};

polkadot/node/core/provisioner/Cargo.toml

@@ -7,7 +7,8 @@ edition = "2018"
 [dependencies]
 bitvec = { version = "0.17.4", default-features = false, features = ["alloc"] }
 futures = "0.3.8"
-log = "0.4.11"
+tracing = "0.1.21"
+tracing-futures = "0.2.4"
 thiserror = "1.0.22"
 polkadot-primitives = { path = "../../../primitives" }
 polkadot-node-subsystem = { path = "../../subsystem" }

polkadot/node/core/provisioner/src/lib.rs

@@ -152,6 +152,7 @@ impl JobTrait for ProvisioningJob {
 	/// Run a job for the parent block indicated
 	//
 	// this function is in charge of creating and executing the job's main loop
+	#[tracing::instrument(skip(_run_args, metrics, receiver, sender), fields(subsystem = LOG_TARGET))]
 	fn run(
 		relay_parent: Hash,
 		_run_args: Self::RunArgs,
@@ -205,7 +206,7 @@ impl ProvisioningJob {
 					)
 					.await
 					{
-						log::warn!(target: LOG_TARGET, "failed to assemble or send inherent data: {:?}", err);
+						tracing::warn!(target: LOG_TARGET, err = ?err, "failed to assemble or send inherent data");
 						self.metrics.on_inherent_data_request(Err(()));
 					} else {
 						self.metrics.on_inherent_data_request(Ok(()));
@@ -254,6 +255,7 @@ impl ProvisioningJob {
 		Ok(())
 	}
 
+	#[tracing::instrument(level = "trace", skip(self), fields(subsystem = LOG_TARGET))]
 	fn note_provisionable_data(&mut self, provisionable_data: ProvisionableData) {
 		match provisionable_data {
 			ProvisionableData::Bitfield(_, signed_bitfield) => {
@@ -286,6 +288,7 @@ type CoreAvailability = BitVec<bitvec::order::Lsb0, u8>;
 /// When we're choosing bitfields to include, the rule should be simple:
 /// maximize availability. So basically, include all bitfields. And then
 /// choose a coherent set of candidates along with that.
+#[tracing::instrument(level = "trace", skip(return_sender, from_job), fields(subsystem = LOG_TARGET))]
 async fn send_inherent_data(
 	relay_parent: Hash,
 	bitfields: &[SignedAvailabilityBitfield],
@@ -323,6 +326,7 @@ async fn send_inherent_data(
 ///
 /// Note: This does not enforce any sorting precondition on the output; the ordering there will be unrelated
 /// to the sorting of the input.
+#[tracing::instrument(level = "trace", fields(subsystem = LOG_TARGET))]
 fn select_availability_bitfields(
 	cores: &[CoreState],
 	bitfields: &[SignedAvailabilityBitfield],
@@ -354,6 +358,7 @@ fn select_availability_bitfields(
 }
 
 /// Determine which cores are free, and then to the degree possible, pick a candidate appropriate to each free core.
+#[tracing::instrument(level = "trace", skip(sender), fields(subsystem = LOG_TARGET))]
 async fn select_candidates(
 	availability_cores: &[CoreState],
 	bitfields: &[SignedAvailabilityBitfield],
@@ -420,6 +425,7 @@ async fn select_candidates(
 
 /// Produces a block number 1 higher than that of the relay parent
 /// in the event of an invalid `relay_parent`, returns `Ok(0)`
+#[tracing::instrument(level = "trace", skip(sender), fields(subsystem = LOG_TARGET))]
 async fn get_block_number_under_construction(
 	relay_parent: Hash,
 	sender: &mut mpsc::Sender<FromJob>,
@@ -445,6 +451,7 @@ async fn get_block_number_under_construction(
 /// - construct a transverse slice along `core_idx`
 /// - bitwise-or it with the availability slice
 /// - count the 1 bits, compare to the total length; true on 2/3+
+#[tracing::instrument(level = "trace", fields(subsystem = LOG_TARGET))]
 fn bitfields_indicate_availability(
 	core_idx: usize,
 	bitfields: &[SignedAvailabilityBitfield],
@@ -460,8 +467,10 @@ fn bitfields_indicate_availability(
 				// in principle, this function might return a `Result<bool, Error>` so that we can more clearly express this error condition
 				// however, in practice, that would just push off an error-handling routine which would look a whole lot like this one.
 				// simpler to just handle the error internally here.
-				log::warn!(
+				tracing::warn!(
 					target: LOG_TARGET,
+					validator_idx = %validator_idx,
+					availability_len = %availability_len,
 					"attempted to set a transverse bit at idx {} which is greater than bitfield size {}",
 					validator_idx,
 					availability_len,

polkadot/node/core/runtime-api/Cargo.toml

@@ -6,6 +6,8 @@ edition = "2018"
 
 [dependencies]
 futures = "0.3.8"
+tracing = "0.1.21"
+tracing-futures = "0.2.4"
 sp-api = { git = "https://github.com/paritytech/substrate", branch = "master" }
 
 polkadot-primitives = { path = "../../../primitives" }

polkadot/node/core/runtime-api/src/lib.rs

@@ -40,6 +40,8 @@ use sp_api::{ProvideRuntimeApi};
 
 use futures::prelude::*;
 
+const LOG_TARGET: &str = "RuntimeApi";
+
 /// The `RuntimeApiSubsystem`. See module docs for more details.
 pub struct RuntimeApiSubsystem<Client> {
 	client: Arc<Client>,
@@ -66,6 +68,7 @@ impl<Client, Context> Subsystem<Context> for RuntimeApiSubsystem<Client> where
 	}
 }
 
+#[tracing::instrument(skip(ctx, subsystem), fields(subsystem = LOG_TARGET))]
 async fn run<Client>(
 	mut ctx: impl SubsystemContext<Message = RuntimeApiMessage>,
 	subsystem: RuntimeApiSubsystem<Client>,
@@ -90,6 +93,7 @@ async fn run<Client>(
 	}
 }
 
+#[tracing::instrument(level = "trace", skip(client, metrics), fields(subsystem = LOG_TARGET))]
 fn make_runtime_api_request<Client>(
 	client: &Client,
 	metrics: &Metrics,