Telemetry improvements (#1886)

* Fix typo

* Support multiple telemetry endpoints and verbosity levels

* Bump substrate-telemetry version

* Telemetrify Aura consensus

* Telemetrify Grandpa

* Fix CI version conflicts

* Implement style remarks

* Fix fixture

* Implement style remarks

* Clone only when necessary

* Get rid of Arc for URL

* Handle connection issues better
This commit is contained in:
Michael Müller
2019-02-28 12:22:05 +01:00
committed by Bastian Köcher
parent 8a72abffdd
commit 90e5c5ddfb
18 changed files with 285 additions and 66 deletions
+33 -1
View File
@@ -52,6 +52,7 @@ use srml_aura::{
InherentType as AuraInherent, AuraInherentData,
timestamp::{TimestampInherentData, InherentType as TimestampInherent, InherentError as TIError}
};
use substrate_telemetry::*;
use aura_slots::{CheckedHeader, SlotWorker, SlotInfo, SlotCompatible};
@@ -265,12 +266,18 @@ impl<B: Block, C, E, I, Error, SO> SlotWorker<B> for AuraWorker<C, E, I, SO> whe
chain_head.hash(),
e
);
telemetry!(CONSENSUS_WARN; "aura.unable_fetching_authorities";
"slot" => ?chain_head.hash(), "err" => ?e
);
return Box::new(future::ok(()));
}
};
if self.sync_oracle.is_offline() && authorities.len() > 1 {
debug!(target: "aura", "Skipping proposal slot. Waiting for the netork.");
debug!(target: "aura", "Skipping proposal slot. Waiting for the network.");
telemetry!(CONSENSUS_DEBUG; "aura.skipping_proposal_slot";
"authorities_len" => authorities.len()
);
return Box::new(future::ok(()));
}
@@ -282,12 +289,18 @@ impl<B: Block, C, E, I, Error, SO> SlotWorker<B> for AuraWorker<C, E, I, SO> whe
slot_num,
timestamp
);
telemetry!(CONSENSUS_DEBUG; "aura.starting_authorship";
"slot_num" => slot_num, "timestamp" => timestamp
);
// we are the slot author. make a block and sign it.
let proposer = match env.init(&chain_head, &authorities) {
Ok(p) => p,
Err(e) => {
warn!("Unable to author block in slot {:?}: {:?}", slot_num, e);
telemetry!(CONSENSUS_WARN; "aura.unable_authoring_block";
"slot" => slot_num, "err" => ?e
);
return Box::new(future::ok(()))
}
};
@@ -315,6 +328,9 @@ impl<B: Block, C, E, I, Error, SO> SlotWorker<B> for AuraWorker<C, E, I, SO> whe
"Discarding proposal for slot {}; block production took too long",
slot_num
);
telemetry!(CONSENSUS_INFO; "aura.discarding_proposal_took_too_long";
"slot" => slot_num
);
return
}
@@ -348,10 +364,18 @@ impl<B: Block, C, E, I, Error, SO> SlotWorker<B> for AuraWorker<C, E, I, SO> whe
import_block.post_header().hash(),
pre_hash
);
telemetry!(CONSENSUS_INFO; "aura.pre_sealed_block";
"header_num" => ?header_num,
"hash_now" => ?import_block.post_header().hash(),
"hash_previously" => ?pre_hash
);
if let Err(e) = block_import.import_block(import_block, None) {
warn!(target: "aura", "Error with block built on {:?}: {:?}",
parent_hash, e);
telemetry!(CONSENSUS_WARN; "aura.err_with_block_built_on";
"hash" => ?parent_hash, "err" => ?e
);
}
})
.map_err(|e| consensus_common::ErrorKind::ClientImport(format!("{:?}", e)).into())
@@ -456,6 +480,9 @@ impl<C, E> AuraVerifier<C, E>
"halting for block {} seconds in the future",
diff
);
telemetry!(CONSENSUS_INFO; "aura.halting_for_future_block";
"diff" => ?diff
);
thread::sleep(Duration::from_secs(diff));
Ok(())
},
@@ -504,6 +531,7 @@ impl<C, E> AuraVerifier<C, E>
"halting for block {} seconds in the future",
diff
);
telemetry!(CONSENSUS_INFO; "aura.halting_for_future_block"; "diff" => ?diff);
thread::sleep(Duration::from_secs(diff));
Ok(())
},
@@ -589,6 +617,7 @@ impl<B: Block, C, E> Verifier<B> for AuraVerifier<C, E> where
}
trace!(target: "aura", "Checked {:?}; importing.", pre_header);
telemetry!(CONSENSUS_TRACE; "aura.checked_and_importing"; "pre_header" => ?pre_header);
extra_verification.into_future().wait()?;
@@ -608,6 +637,9 @@ impl<B: Block, C, E> Verifier<B> for AuraVerifier<C, E> where
}
CheckedHeader::Deferred(a, b) => {
debug!(target: "aura", "Checking {:?} failed; {:?}, {:?}.", hash, a, b);
telemetry!(CONSENSUS_DEBUG; "aura.header_too_far_in_future";
"hash" => ?hash, "a" => ?a, "b" => ?b
);
Err(format!("Header {:?} rejected: too far in the future", hash))
}
}