Add tracing support to node (#1940)

* drop in tracing to replace log

* add structured logging to trace messages

* add structured logging to debug messages

* add structured logging to info messages

* add structured logging to warn messages

* add structured logging to error messages

* normalize spacing and Display vs Debug

* add instrumentation to the various 'fn run'

* use explicit tracing module throughout

* fix availability distribution test

* don't double-print errors

* remove further redundancy from logs

* fix test errors

* fix more test errors

* remove unused kv_log_macro

* fix unused variable

* add tracing spans to collation generation

* add tracing spans to av-store

* add tracing spans to backing

* add tracing spans to bitfield-signing

* add tracing spans to candidate-selection

* add tracing spans to candidate-validation

* add tracing spans to chain-api

* add tracing spans to provisioner

* add tracing spans to runtime-api

* add tracing spans to availability-distribution

* add tracing spans to bitfield-distribution

* add tracing spans to network-bridge

* add tracing spans to collator-protocol

* add tracing spans to pov-distribution

* add tracing spans to statement-distribution

* add tracing spans to overseer

* cleanup

polkadot/node/core/provisioner/src/lib.rs

@@ -152,6 +152,7 @@ impl JobTrait for ProvisioningJob {
 	/// Run a job for the parent block indicated
 	//
 	// this function is in charge of creating and executing the job's main loop
+	#[tracing::instrument(skip(_run_args, metrics, receiver, sender), fields(subsystem = LOG_TARGET))]
 	fn run(
 		relay_parent: Hash,
 		_run_args: Self::RunArgs,
@@ -205,7 +206,7 @@ impl ProvisioningJob {
 					)
 					.await
 					{
-						log::warn!(target: LOG_TARGET, "failed to assemble or send inherent data: {:?}", err);
+						tracing::warn!(target: LOG_TARGET, err = ?err, "failed to assemble or send inherent data");
 						self.metrics.on_inherent_data_request(Err(()));
 					} else {
 						self.metrics.on_inherent_data_request(Ok(()));
@@ -254,6 +255,7 @@ impl ProvisioningJob {
 		Ok(())
 	}
 
+	#[tracing::instrument(level = "trace", skip(self), fields(subsystem = LOG_TARGET))]
 	fn note_provisionable_data(&mut self, provisionable_data: ProvisionableData) {
 		match provisionable_data {
 			ProvisionableData::Bitfield(_, signed_bitfield) => {
@@ -286,6 +288,7 @@ type CoreAvailability = BitVec<bitvec::order::Lsb0, u8>;
 /// When we're choosing bitfields to include, the rule should be simple:
 /// maximize availability. So basically, include all bitfields. And then
 /// choose a coherent set of candidates along with that.
+#[tracing::instrument(level = "trace", skip(return_sender, from_job), fields(subsystem = LOG_TARGET))]
 async fn send_inherent_data(
 	relay_parent: Hash,
 	bitfields: &[SignedAvailabilityBitfield],
@@ -323,6 +326,7 @@ async fn send_inherent_data(
 ///
 /// Note: This does not enforce any sorting precondition on the output; the ordering there will be unrelated
 /// to the sorting of the input.
+#[tracing::instrument(level = "trace", fields(subsystem = LOG_TARGET))]
 fn select_availability_bitfields(
 	cores: &[CoreState],
 	bitfields: &[SignedAvailabilityBitfield],
@@ -354,6 +358,7 @@ fn select_availability_bitfields(
 }
 
 /// Determine which cores are free, and then to the degree possible, pick a candidate appropriate to each free core.
+#[tracing::instrument(level = "trace", skip(sender), fields(subsystem = LOG_TARGET))]
 async fn select_candidates(
 	availability_cores: &[CoreState],
 	bitfields: &[SignedAvailabilityBitfield],
@@ -420,6 +425,7 @@ async fn select_candidates(
 
 /// Produces a block number 1 higher than that of the relay parent
 /// in the event of an invalid `relay_parent`, returns `Ok(0)`
+#[tracing::instrument(level = "trace", skip(sender), fields(subsystem = LOG_TARGET))]
 async fn get_block_number_under_construction(
 	relay_parent: Hash,
 	sender: &mut mpsc::Sender<FromJob>,
@@ -445,6 +451,7 @@ async fn get_block_number_under_construction(
 /// - construct a transverse slice along `core_idx`
 /// - bitwise-or it with the availability slice
 /// - count the 1 bits, compare to the total length; true on 2/3+
+#[tracing::instrument(level = "trace", fields(subsystem = LOG_TARGET))]
 fn bitfields_indicate_availability(
 	core_idx: usize,
 	bitfields: &[SignedAvailabilityBitfield],
@@ -460,8 +467,10 @@ fn bitfields_indicate_availability(
 				// in principle, this function might return a `Result<bool, Error>` so that we can more clearly express this error condition
 				// however, in practice, that would just push off an error-handling routine which would look a whole lot like this one.
 				// simpler to just handle the error internally here.
-				log::warn!(
+				tracing::warn!(
 					target: LOG_TARGET,
+					validator_idx = %validator_idx,
+					availability_len = %availability_len,
 					"attempted to set a transverse bit at idx {} which is greater than bitfield size {}",
 					validator_idx,
 					availability_len,