mirror of
https://github.com/pezkuwichain/pezkuwi-subxt.git
synced 2026-05-31 21:31:02 +00:00
Rework telemetry to replace the use of tracing with an object we pass around (#8143)
polkadot companion: paritytech/polkadot#2535
This commit is contained in:
@@ -23,7 +23,7 @@ use parking_lot::RwLock;
|
||||
use finality_grandpa::voter_set::VoterSet;
|
||||
use parity_scale_codec::{Encode, Decode};
|
||||
use log::debug;
|
||||
use sc_telemetry::{telemetry, CONSENSUS_INFO};
|
||||
use sc_telemetry::{telemetry, TelemetryHandle, CONSENSUS_INFO};
|
||||
use sp_finality_grandpa::{AuthorityId, AuthorityList};
|
||||
|
||||
use std::cmp::Ord;
|
||||
@@ -43,8 +43,8 @@ pub enum Error<N, E> {
|
||||
#[display(fmt = "Multiple pending forced authority set changes are not allowed.")]
|
||||
MultiplePendingForcedAuthoritySetChanges,
|
||||
#[display(
|
||||
fmt = "A pending forced authority set change could not be applied since it must be applied after \
|
||||
the pending standard change at #{}",
|
||||
fmt = "A pending forced authority set change could not be applied since it must be applied \
|
||||
after the pending standard change at #{}",
|
||||
_0
|
||||
)]
|
||||
ForcedAuthoritySetChangeDependencyUnsatisfied(N),
|
||||
@@ -278,9 +278,13 @@ where
|
||||
let hash = pending.canon_hash.clone();
|
||||
let number = pending.canon_height.clone();
|
||||
|
||||
debug!(target: "afg", "Inserting potential standard set change signaled at block {:?} \
|
||||
(delayed by {:?} blocks).",
|
||||
(&number, &hash), pending.delay);
|
||||
debug!(
|
||||
target: "afg",
|
||||
"Inserting potential standard set change signaled at block {:?} (delayed by {:?}
|
||||
blocks).",
|
||||
(&number, &hash),
|
||||
pending.delay,
|
||||
);
|
||||
|
||||
self.pending_standard_changes.import(
|
||||
hash,
|
||||
@@ -289,8 +293,10 @@ where
|
||||
is_descendent_of,
|
||||
)?;
|
||||
|
||||
debug!(target: "afg", "There are now {} alternatives for the next pending standard change (roots), \
|
||||
and a total of {} pending standard changes (across all forks).",
|
||||
debug!(
|
||||
target: "afg",
|
||||
"There are now {} alternatives for the next pending standard change (roots), and a
|
||||
total of {} pending standard changes (across all forks).",
|
||||
self.pending_standard_changes.roots().count(),
|
||||
self.pending_standard_changes.iter().count(),
|
||||
);
|
||||
@@ -326,9 +332,12 @@ where
|
||||
))
|
||||
.unwrap_or_else(|i| i);
|
||||
|
||||
debug!(target: "afg", "Inserting potential forced set change at block {:?} \
|
||||
(delayed by {:?} blocks).",
|
||||
(&pending.canon_height, &pending.canon_hash), pending.delay);
|
||||
debug!(
|
||||
target: "afg",
|
||||
"Inserting potential forced set change at block {:?} (delayed by {:?} blocks).",
|
||||
(&pending.canon_height, &pending.canon_hash),
|
||||
pending.delay,
|
||||
);
|
||||
|
||||
self.pending_forced_changes.insert(idx, pending);
|
||||
|
||||
@@ -409,6 +418,7 @@ where
|
||||
best_number: N,
|
||||
is_descendent_of: &F,
|
||||
initial_sync: bool,
|
||||
telemetry: Option<TelemetryHandle>,
|
||||
) -> Result<Option<(N, Self)>, Error<N, E>>
|
||||
where
|
||||
F: Fn(&H, &H) -> Result<bool, E>,
|
||||
@@ -461,6 +471,7 @@ where
|
||||
);
|
||||
|
||||
telemetry!(
|
||||
telemetry;
|
||||
CONSENSUS_INFO;
|
||||
"afg.applying_forced_authority_set_change";
|
||||
"block" => ?change.canon_height
|
||||
@@ -505,6 +516,7 @@ where
|
||||
finalized_number: N,
|
||||
is_descendent_of: &F,
|
||||
initial_sync: bool,
|
||||
telemetry: Option<&TelemetryHandle>,
|
||||
) -> Result<Status<H, N>, Error<N, E>>
|
||||
where
|
||||
F: Fn(&H, &H) -> Result<bool, E>,
|
||||
@@ -544,7 +556,10 @@ where
|
||||
"👴 Applying authority set change scheduled at block #{:?}",
|
||||
change.canon_height,
|
||||
);
|
||||
telemetry!(CONSENSUS_INFO; "afg.applying_scheduled_authority_set_change";
|
||||
telemetry!(
|
||||
telemetry;
|
||||
CONSENSUS_INFO;
|
||||
"afg.applying_scheduled_authority_set_change";
|
||||
"block" => ?change.canon_height
|
||||
);
|
||||
|
||||
@@ -894,6 +909,7 @@ mod tests {
|
||||
_ => unreachable!(),
|
||||
}),
|
||||
false,
|
||||
None,
|
||||
).unwrap();
|
||||
|
||||
assert!(status.changed);
|
||||
@@ -913,6 +929,7 @@ mod tests {
|
||||
_ => unreachable!(),
|
||||
}),
|
||||
false,
|
||||
None,
|
||||
).unwrap();
|
||||
|
||||
assert!(status.changed);
|
||||
@@ -971,7 +988,7 @@ mod tests {
|
||||
|
||||
// trying to finalize past `change_c` without finalizing `change_a` first
|
||||
assert!(matches!(
|
||||
authorities.apply_standard_changes("hash_d", 40, &is_descendent_of, false),
|
||||
authorities.apply_standard_changes("hash_d", 40, &is_descendent_of, false, None),
|
||||
Err(Error::ForkTree(fork_tree::Error::UnfinalizedAncestor))
|
||||
));
|
||||
assert_eq!(authorities.authority_set_changes, AuthoritySetChanges::empty());
|
||||
@@ -981,6 +998,7 @@ mod tests {
|
||||
15,
|
||||
&is_descendent_of,
|
||||
false,
|
||||
None,
|
||||
).unwrap();
|
||||
|
||||
assert!(status.changed);
|
||||
@@ -996,6 +1014,7 @@ mod tests {
|
||||
40,
|
||||
&is_descendent_of,
|
||||
false,
|
||||
None,
|
||||
).unwrap();
|
||||
|
||||
assert!(status.changed);
|
||||
@@ -1138,7 +1157,7 @@ mod tests {
|
||||
// too early and there's no forced changes to apply.
|
||||
assert!(
|
||||
authorities
|
||||
.apply_forced_changes("hash_a10", 10, &static_is_descendent_of(true), false)
|
||||
.apply_forced_changes("hash_a10", 10, &static_is_descendent_of(true), false, None)
|
||||
.unwrap()
|
||||
.is_none()
|
||||
);
|
||||
@@ -1146,7 +1165,7 @@ mod tests {
|
||||
// too late.
|
||||
assert!(
|
||||
authorities
|
||||
.apply_forced_changes("hash_a16", 16, &is_descendent_of_a, false)
|
||||
.apply_forced_changes("hash_a16", 16, &is_descendent_of_a, false, None)
|
||||
.unwrap()
|
||||
.is_none()
|
||||
);
|
||||
@@ -1154,7 +1173,7 @@ mod tests {
|
||||
// on time -- chooses the right change for this fork.
|
||||
assert_eq!(
|
||||
authorities
|
||||
.apply_forced_changes("hash_a15", 15, &is_descendent_of_a, false)
|
||||
.apply_forced_changes("hash_a15", 15, &is_descendent_of_a, false, None)
|
||||
.unwrap()
|
||||
.unwrap(),
|
||||
(
|
||||
@@ -1202,7 +1221,7 @@ mod tests {
|
||||
// it should be enacted at the same block that signaled it
|
||||
assert!(
|
||||
authorities
|
||||
.apply_forced_changes("hash_a", 5, &static_is_descendent_of(false), false)
|
||||
.apply_forced_changes("hash_a", 5, &static_is_descendent_of(false), false, None)
|
||||
.unwrap()
|
||||
.is_some()
|
||||
);
|
||||
@@ -1269,27 +1288,27 @@ mod tests {
|
||||
// the forced change cannot be applied since the pending changes it depends on
|
||||
// have not been applied yet.
|
||||
assert!(matches!(
|
||||
authorities.apply_forced_changes("hash_d45", 45, &static_is_descendent_of(true), false),
|
||||
authorities.apply_forced_changes("hash_d45", 45, &static_is_descendent_of(true), false, None),
|
||||
Err(Error::ForcedAuthoritySetChangeDependencyUnsatisfied(15))
|
||||
));
|
||||
assert_eq!(authorities.authority_set_changes, AuthoritySetChanges::empty());
|
||||
|
||||
// we apply the first pending standard change at #15
|
||||
authorities
|
||||
.apply_standard_changes("hash_a15", 15, &static_is_descendent_of(true), false)
|
||||
.apply_standard_changes("hash_a15", 15, &static_is_descendent_of(true), false, None)
|
||||
.unwrap();
|
||||
assert_eq!(authorities.authority_set_changes, AuthoritySetChanges(vec![(0, 15)]));
|
||||
|
||||
// but the forced change still depends on the next standard change
|
||||
assert!(matches!(
|
||||
authorities.apply_forced_changes("hash_d", 45, &static_is_descendent_of(true), false),
|
||||
authorities.apply_forced_changes("hash_d", 45, &static_is_descendent_of(true), false, None),
|
||||
Err(Error::ForcedAuthoritySetChangeDependencyUnsatisfied(20))
|
||||
));
|
||||
assert_eq!(authorities.authority_set_changes, AuthoritySetChanges(vec![(0, 15)]));
|
||||
|
||||
// we apply the pending standard change at #20
|
||||
authorities
|
||||
.apply_standard_changes("hash_b", 20, &static_is_descendent_of(true), false)
|
||||
.apply_standard_changes("hash_b", 20, &static_is_descendent_of(true), false, None)
|
||||
.unwrap();
|
||||
assert_eq!(authorities.authority_set_changes, AuthoritySetChanges(vec![(0, 15), (1, 20)]));
|
||||
|
||||
@@ -1298,7 +1317,7 @@ mod tests {
|
||||
// at #35. subsequent forced changes on the same branch must be kept
|
||||
assert_eq!(
|
||||
authorities
|
||||
.apply_forced_changes("hash_d", 45, &static_is_descendent_of(true), false)
|
||||
.apply_forced_changes("hash_d", 45, &static_is_descendent_of(true), false, None)
|
||||
.unwrap()
|
||||
.unwrap(),
|
||||
(
|
||||
@@ -1395,7 +1414,7 @@ mod tests {
|
||||
|
||||
// we apply the change at A0 which should prune it and the fork at B
|
||||
authorities
|
||||
.apply_standard_changes("hash_a0", 5, &is_descendent_of, false)
|
||||
.apply_standard_changes("hash_a0", 5, &is_descendent_of, false, None)
|
||||
.unwrap();
|
||||
|
||||
// the next change is now at A1 (#10)
|
||||
@@ -1583,14 +1602,14 @@ mod tests {
|
||||
// applying the standard change at A should not prune anything
|
||||
// other then the change that was applied
|
||||
authorities
|
||||
.apply_standard_changes("A", 5, &is_descendent_of, false)
|
||||
.apply_standard_changes("A", 5, &is_descendent_of, false, None)
|
||||
.unwrap();
|
||||
|
||||
assert_eq!(authorities.pending_changes().count(), 6);
|
||||
|
||||
// same for B
|
||||
authorities
|
||||
.apply_standard_changes("B", 10, &is_descendent_of, false)
|
||||
.apply_standard_changes("B", 10, &is_descendent_of, false, None)
|
||||
.unwrap();
|
||||
|
||||
assert_eq!(authorities.pending_changes().count(), 5);
|
||||
@@ -1599,7 +1618,7 @@ mod tests {
|
||||
|
||||
// finalizing C2 should clear all forced changes
|
||||
authorities
|
||||
.apply_standard_changes("C2", 15, &is_descendent_of, false)
|
||||
.apply_standard_changes("C2", 15, &is_descendent_of, false, None)
|
||||
.unwrap();
|
||||
|
||||
assert_eq!(authorities.pending_forced_changes.len(), 0);
|
||||
@@ -1607,7 +1626,7 @@ mod tests {
|
||||
// finalizing C0 should clear all forced changes but D
|
||||
let mut authorities = authorities2;
|
||||
authorities
|
||||
.apply_standard_changes("C0", 15, &is_descendent_of, false)
|
||||
.apply_standard_changes("C0", 15, &is_descendent_of, false, None)
|
||||
.unwrap();
|
||||
|
||||
assert_eq!(authorities.pending_forced_changes.len(), 1);
|
||||
|
||||
@@ -137,7 +137,7 @@ struct V2AuthoritySet<H, N> {
|
||||
}
|
||||
|
||||
pub(crate) fn load_decode<B: AuxStore, T: Decode>(
|
||||
backend: &B,
|
||||
backend: &B,
|
||||
key: &[u8]
|
||||
) -> ClientResult<Option<T>> {
|
||||
match backend.get_aux(key)? {
|
||||
|
||||
@@ -90,7 +90,7 @@ use sc_network::{ObservedRole, PeerId, ReputationChange};
|
||||
use parity_scale_codec::{Encode, Decode};
|
||||
use sp_finality_grandpa::AuthorityId;
|
||||
|
||||
use sc_telemetry::{telemetry, CONSENSUS_DEBUG};
|
||||
use sc_telemetry::{telemetry, TelemetryHandle, CONSENSUS_DEBUG};
|
||||
use log::{trace, debug};
|
||||
use sp_utils::mpsc::{tracing_unbounded, TracingUnboundedReceiver, TracingUnboundedSender};
|
||||
use prometheus_endpoint::{CounterVec, Opts, PrometheusError, register, Registry, U64};
|
||||
@@ -744,7 +744,7 @@ impl<Block: BlockT> Inner<Block> {
|
||||
fn note_set(&mut self, set_id: SetId, authorities: Vec<AuthorityId>) -> MaybeMessage<Block> {
|
||||
{
|
||||
let local_view = match self.local_view {
|
||||
ref mut x @ None => x.get_or_insert(LocalView::new(
|
||||
ref mut x @ None => x.get_or_insert(LocalView::new(
|
||||
set_id,
|
||||
Round(1),
|
||||
)),
|
||||
@@ -828,7 +828,12 @@ impl<Block: BlockT> Inner<Block> {
|
||||
// ensure authority is part of the set.
|
||||
if !self.authorities.contains(&full.message.id) {
|
||||
debug!(target: "afg", "Message from unknown voter: {}", full.message.id);
|
||||
telemetry!(CONSENSUS_DEBUG; "afg.bad_msg_signature"; "signature" => ?full.message.id);
|
||||
telemetry!(
|
||||
self.config.telemetry;
|
||||
CONSENSUS_DEBUG;
|
||||
"afg.bad_msg_signature";
|
||||
"signature" => ?full.message.id,
|
||||
);
|
||||
return Action::Discard(cost::UNKNOWN_VOTER);
|
||||
}
|
||||
|
||||
@@ -840,7 +845,12 @@ impl<Block: BlockT> Inner<Block> {
|
||||
full.set_id.0,
|
||||
) {
|
||||
debug!(target: "afg", "Bad message signature {}", full.message.id);
|
||||
telemetry!(CONSENSUS_DEBUG; "afg.bad_msg_signature"; "signature" => ?full.message.id);
|
||||
telemetry!(
|
||||
self.config.telemetry;
|
||||
CONSENSUS_DEBUG;
|
||||
"afg.bad_msg_signature";
|
||||
"signature" => ?full.message.id,
|
||||
);
|
||||
return Action::Discard(cost::BAD_SIGNATURE);
|
||||
}
|
||||
|
||||
@@ -866,7 +876,10 @@ impl<Block: BlockT> Inner<Block> {
|
||||
|
||||
if full.message.precommits.len() != full.message.auth_data.len() || full.message.precommits.is_empty() {
|
||||
debug!(target: "afg", "Malformed compact commit");
|
||||
telemetry!(CONSENSUS_DEBUG; "afg.malformed_compact_commit";
|
||||
telemetry!(
|
||||
self.config.telemetry;
|
||||
CONSENSUS_DEBUG;
|
||||
"afg.malformed_compact_commit";
|
||||
"precommits_len" => ?full.message.precommits.len(),
|
||||
"auth_data_len" => ?full.message.auth_data.len(),
|
||||
"precommits_is_empty" => ?full.message.precommits.is_empty(),
|
||||
@@ -1277,6 +1290,7 @@ pub(super) struct GossipValidator<Block: BlockT> {
|
||||
set_state: environment::SharedVoterSetState<Block>,
|
||||
report_sender: TracingUnboundedSender<PeerReport>,
|
||||
metrics: Option<Metrics>,
|
||||
telemetry: Option<TelemetryHandle>,
|
||||
}
|
||||
|
||||
impl<Block: BlockT> GossipValidator<Block> {
|
||||
@@ -1287,6 +1301,7 @@ impl<Block: BlockT> GossipValidator<Block> {
|
||||
config: crate::Config,
|
||||
set_state: environment::SharedVoterSetState<Block>,
|
||||
prometheus_registry: Option<&Registry>,
|
||||
telemetry: Option<TelemetryHandle>,
|
||||
) -> (GossipValidator<Block>, TracingUnboundedReceiver<PeerReport>) {
|
||||
let metrics = match prometheus_registry.map(Metrics::register) {
|
||||
Some(Ok(metrics)) => Some(metrics),
|
||||
@@ -1303,6 +1318,7 @@ impl<Block: BlockT> GossipValidator<Block> {
|
||||
set_state,
|
||||
report_sender: tx,
|
||||
metrics,
|
||||
telemetry,
|
||||
};
|
||||
|
||||
(val, rx)
|
||||
@@ -1411,7 +1427,12 @@ impl<Block: BlockT> GossipValidator<Block> {
|
||||
Err(e) => {
|
||||
message_name = None;
|
||||
debug!(target: "afg", "Error decoding message: {}", e);
|
||||
telemetry!(CONSENSUS_DEBUG; "afg.err_decoding_msg"; "" => "");
|
||||
telemetry!(
|
||||
self.telemetry;
|
||||
CONSENSUS_DEBUG;
|
||||
"afg.err_decoding_msg";
|
||||
"" => "",
|
||||
);
|
||||
|
||||
let len = std::cmp::min(i32::max_value() as usize, data.len()) as i32;
|
||||
Action::Discard(Misbehavior::UndecodablePacket(len).cost())
|
||||
@@ -1630,6 +1651,7 @@ mod tests {
|
||||
name: None,
|
||||
is_authority: true,
|
||||
observer_enabled: true,
|
||||
telemetry: None,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1797,6 +1819,7 @@ mod tests {
|
||||
config(),
|
||||
voter_set_state(),
|
||||
None,
|
||||
None,
|
||||
);
|
||||
|
||||
let set_id = 1;
|
||||
@@ -1833,6 +1856,7 @@ mod tests {
|
||||
config(),
|
||||
voter_set_state(),
|
||||
None,
|
||||
None,
|
||||
);
|
||||
let set_id = 1;
|
||||
let auth = AuthorityId::from_slice(&[1u8; 32]);
|
||||
@@ -1878,6 +1902,7 @@ mod tests {
|
||||
config(),
|
||||
voter_set_state(),
|
||||
None,
|
||||
None,
|
||||
);
|
||||
|
||||
let set_id = 1;
|
||||
@@ -1947,6 +1972,7 @@ mod tests {
|
||||
config(),
|
||||
set_state.clone(),
|
||||
None,
|
||||
None,
|
||||
);
|
||||
|
||||
let set_id = 1;
|
||||
@@ -2002,6 +2028,7 @@ mod tests {
|
||||
config(),
|
||||
set_state.clone(),
|
||||
None,
|
||||
None,
|
||||
);
|
||||
|
||||
// the validator starts at set id 2
|
||||
@@ -2082,6 +2109,7 @@ mod tests {
|
||||
config(),
|
||||
voter_set_state(),
|
||||
None,
|
||||
None,
|
||||
);
|
||||
|
||||
// the validator starts at set id 1.
|
||||
@@ -2156,6 +2184,7 @@ mod tests {
|
||||
config,
|
||||
voter_set_state(),
|
||||
None,
|
||||
None,
|
||||
);
|
||||
|
||||
// the validator starts at set id 1.
|
||||
@@ -2190,6 +2219,7 @@ mod tests {
|
||||
config(),
|
||||
voter_set_state(),
|
||||
None,
|
||||
None,
|
||||
);
|
||||
|
||||
// the validator starts at set id 1.
|
||||
@@ -2250,6 +2280,7 @@ mod tests {
|
||||
config,
|
||||
voter_set_state(),
|
||||
None,
|
||||
None,
|
||||
);
|
||||
|
||||
// the validator starts at set id 1.
|
||||
@@ -2289,6 +2320,7 @@ mod tests {
|
||||
config(),
|
||||
voter_set_state(),
|
||||
None,
|
||||
None,
|
||||
);
|
||||
|
||||
// the validator starts at set id 1.
|
||||
@@ -2322,6 +2354,7 @@ mod tests {
|
||||
config,
|
||||
voter_set_state(),
|
||||
None,
|
||||
None,
|
||||
);
|
||||
|
||||
// the validator start at set id 0
|
||||
@@ -2401,6 +2434,7 @@ mod tests {
|
||||
config(),
|
||||
voter_set_state(),
|
||||
None,
|
||||
None,
|
||||
);
|
||||
|
||||
// the validator start at set id 0
|
||||
@@ -2441,6 +2475,7 @@ mod tests {
|
||||
config,
|
||||
voter_set_state(),
|
||||
None,
|
||||
None,
|
||||
);
|
||||
|
||||
// the validator start at set id 0
|
||||
@@ -2490,7 +2525,7 @@ mod tests {
|
||||
|
||||
#[test]
|
||||
fn only_gossip_commits_to_peers_on_same_set() {
|
||||
let (val, _) = GossipValidator::<Block>::new(config(), voter_set_state(), None);
|
||||
let (val, _) = GossipValidator::<Block>::new(config(), voter_set_state(), None, None);
|
||||
|
||||
// the validator start at set id 1
|
||||
val.note_set(SetId(1), Vec::new(), |_, _| {});
|
||||
@@ -2568,7 +2603,7 @@ mod tests {
|
||||
|
||||
#[test]
|
||||
fn expire_commits_from_older_rounds() {
|
||||
let (val, _) = GossipValidator::<Block>::new(config(), voter_set_state(), None);
|
||||
let (val, _) = GossipValidator::<Block>::new(config(), voter_set_state(), None, None);
|
||||
|
||||
let commit = |round, set_id, target_number| {
|
||||
let commit = finality_grandpa::CompactCommit {
|
||||
@@ -2619,7 +2654,7 @@ mod tests {
|
||||
|
||||
#[test]
|
||||
fn allow_noting_different_authorities_for_same_set() {
|
||||
let (val, _) = GossipValidator::<Block>::new(config(), voter_set_state(), None);
|
||||
let (val, _) = GossipValidator::<Block>::new(config(), voter_set_state(), None, None);
|
||||
|
||||
let a1 = vec![AuthorityId::from_slice(&[0; 32])];
|
||||
val.note_set(SetId(1), a1.clone(), |_, _| {});
|
||||
|
||||
@@ -42,7 +42,7 @@ use sc_network::{NetworkService, ReputationChange};
|
||||
use sc_network_gossip::{GossipEngine, Network as GossipNetwork};
|
||||
use parity_scale_codec::{Encode, Decode};
|
||||
use sp_runtime::traits::{Block as BlockT, Hash as HashT, Header as HeaderT, NumberFor};
|
||||
use sc_telemetry::{telemetry, CONSENSUS_DEBUG, CONSENSUS_INFO};
|
||||
use sc_telemetry::{telemetry, TelemetryHandle, CONSENSUS_DEBUG, CONSENSUS_INFO};
|
||||
|
||||
use crate::{
|
||||
CatchUp, Commit, CommunicationIn, CommunicationOutH,
|
||||
@@ -192,6 +192,8 @@ pub(crate) struct NetworkBridge<B: BlockT, N: Network<B>> {
|
||||
// just an `UnboundedReceiver`, one could also switch to a multi-producer-*multi*-consumer
|
||||
// channel implementation.
|
||||
gossip_validator_report_stream: Arc<Mutex<TracingUnboundedReceiver<PeerReport>>>,
|
||||
|
||||
telemetry: Option<TelemetryHandle>,
|
||||
}
|
||||
|
||||
impl<B: BlockT, N: Network<B>> Unpin for NetworkBridge<B, N> {}
|
||||
@@ -206,11 +208,13 @@ impl<B: BlockT, N: Network<B>> NetworkBridge<B, N> {
|
||||
config: crate::Config,
|
||||
set_state: crate::environment::SharedVoterSetState<B>,
|
||||
prometheus_registry: Option<&Registry>,
|
||||
telemetry: Option<TelemetryHandle>,
|
||||
) -> Self {
|
||||
let (validator, report_stream) = GossipValidator::new(
|
||||
config,
|
||||
set_state.clone(),
|
||||
prometheus_registry,
|
||||
telemetry.clone(),
|
||||
);
|
||||
|
||||
let validator = Arc::new(validator);
|
||||
@@ -268,6 +272,7 @@ impl<B: BlockT, N: Network<B>> NetworkBridge<B, N> {
|
||||
neighbor_sender: neighbor_packet_sender,
|
||||
neighbor_packet_worker: Arc::new(Mutex::new(neighbor_packet_worker)),
|
||||
gossip_validator_report_stream: Arc::new(Mutex::new(report_stream)),
|
||||
telemetry,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -320,6 +325,7 @@ impl<B: BlockT, N: Network<B>> NetworkBridge<B, N> {
|
||||
});
|
||||
|
||||
let topic = round_topic::<B>(round.0, set_id.0);
|
||||
let telemetry = self.telemetry.clone();
|
||||
let incoming = self.gossip_engine.lock().messages_for(topic)
|
||||
.filter_map(move |notification| {
|
||||
let decoded = GossipMessage::<B>::decode(&mut ¬ification.message[..]);
|
||||
@@ -339,21 +345,30 @@ impl<B: BlockT, N: Network<B>> NetworkBridge<B, N> {
|
||||
if voters.len().get() <= TELEMETRY_VOTERS_LIMIT {
|
||||
match &msg.message.message {
|
||||
PrimaryPropose(propose) => {
|
||||
telemetry!(CONSENSUS_INFO; "afg.received_propose";
|
||||
telemetry!(
|
||||
telemetry;
|
||||
CONSENSUS_INFO;
|
||||
"afg.received_propose";
|
||||
"voter" => ?format!("{}", msg.message.id),
|
||||
"target_number" => ?propose.target_number,
|
||||
"target_hash" => ?propose.target_hash,
|
||||
);
|
||||
},
|
||||
Prevote(prevote) => {
|
||||
telemetry!(CONSENSUS_INFO; "afg.received_prevote";
|
||||
telemetry!(
|
||||
telemetry;
|
||||
CONSENSUS_INFO;
|
||||
"afg.received_prevote";
|
||||
"voter" => ?format!("{}", msg.message.id),
|
||||
"target_number" => ?prevote.target_number,
|
||||
"target_hash" => ?prevote.target_hash,
|
||||
);
|
||||
},
|
||||
Precommit(precommit) => {
|
||||
telemetry!(CONSENSUS_INFO; "afg.received_precommit";
|
||||
telemetry!(
|
||||
telemetry;
|
||||
CONSENSUS_INFO;
|
||||
"afg.received_precommit";
|
||||
"voter" => ?format!("{}", msg.message.id),
|
||||
"target_number" => ?precommit.target_number,
|
||||
"target_hash" => ?precommit.target_hash,
|
||||
@@ -379,6 +394,7 @@ impl<B: BlockT, N: Network<B>> NetworkBridge<B, N> {
|
||||
network: self.gossip_engine.clone(),
|
||||
sender: tx,
|
||||
has_voted,
|
||||
telemetry: self.telemetry.clone(),
|
||||
};
|
||||
|
||||
// Combine incoming votes from external GRANDPA nodes with outgoing
|
||||
@@ -412,6 +428,7 @@ impl<B: BlockT, N: Network<B>> NetworkBridge<B, N> {
|
||||
voters,
|
||||
self.validator.clone(),
|
||||
self.neighbor_sender.clone(),
|
||||
self.telemetry.clone(),
|
||||
);
|
||||
|
||||
let outgoing = CommitsOut::<B>::new(
|
||||
@@ -420,6 +437,7 @@ impl<B: BlockT, N: Network<B>> NetworkBridge<B, N> {
|
||||
is_voter,
|
||||
self.validator.clone(),
|
||||
self.neighbor_sender.clone(),
|
||||
self.telemetry.clone(),
|
||||
);
|
||||
|
||||
let outgoing = outgoing.with(|out| {
|
||||
@@ -491,72 +509,80 @@ fn incoming_global<B: BlockT>(
|
||||
voters: Arc<VoterSet<AuthorityId>>,
|
||||
gossip_validator: Arc<GossipValidator<B>>,
|
||||
neighbor_sender: periodic::NeighborPacketSender<B>,
|
||||
telemetry: Option<TelemetryHandle>,
|
||||
) -> impl Stream<Item = CommunicationIn<B>> {
|
||||
let process_commit = move |
|
||||
msg: FullCommitMessage<B>,
|
||||
mut notification: sc_network_gossip::TopicNotification,
|
||||
gossip_engine: &Arc<Mutex<GossipEngine<B>>>,
|
||||
gossip_validator: &Arc<GossipValidator<B>>,
|
||||
voters: &VoterSet<AuthorityId>,
|
||||
| {
|
||||
if voters.len().get() <= TELEMETRY_VOTERS_LIMIT {
|
||||
let precommits_signed_by: Vec<String> =
|
||||
msg.message.auth_data.iter().map(move |(_, a)| {
|
||||
format!("{}", a)
|
||||
}).collect();
|
||||
let process_commit = {
|
||||
let telemetry = telemetry.clone();
|
||||
move |
|
||||
msg: FullCommitMessage<B>,
|
||||
mut notification: sc_network_gossip::TopicNotification,
|
||||
gossip_engine: &Arc<Mutex<GossipEngine<B>>>,
|
||||
gossip_validator: &Arc<GossipValidator<B>>,
|
||||
voters: &VoterSet<AuthorityId>,
|
||||
| {
|
||||
if voters.len().get() <= TELEMETRY_VOTERS_LIMIT {
|
||||
let precommits_signed_by: Vec<String> =
|
||||
msg.message.auth_data.iter().map(move |(_, a)| {
|
||||
format!("{}", a)
|
||||
}).collect();
|
||||
|
||||
telemetry!(CONSENSUS_INFO; "afg.received_commit";
|
||||
"contains_precommits_signed_by" => ?precommits_signed_by,
|
||||
"target_number" => ?msg.message.target_number.clone(),
|
||||
"target_hash" => ?msg.message.target_hash.clone(),
|
||||
);
|
||||
}
|
||||
|
||||
if let Err(cost) = check_compact_commit::<B>(
|
||||
&msg.message,
|
||||
voters,
|
||||
msg.round,
|
||||
msg.set_id,
|
||||
) {
|
||||
if let Some(who) = notification.sender {
|
||||
gossip_engine.lock().report(who, cost);
|
||||
}
|
||||
|
||||
return None;
|
||||
}
|
||||
|
||||
let round = msg.round;
|
||||
let set_id = msg.set_id;
|
||||
let commit = msg.message;
|
||||
let finalized_number = commit.target_number;
|
||||
let gossip_validator = gossip_validator.clone();
|
||||
let gossip_engine = gossip_engine.clone();
|
||||
let neighbor_sender = neighbor_sender.clone();
|
||||
let cb = move |outcome| match outcome {
|
||||
voter::CommitProcessingOutcome::Good(_) => {
|
||||
// if it checks out, gossip it. not accounting for
|
||||
// any discrepancy between the actual ghost and the claimed
|
||||
// finalized number.
|
||||
gossip_validator.note_commit_finalized(
|
||||
round,
|
||||
set_id,
|
||||
finalized_number,
|
||||
|to, neighbor| neighbor_sender.send(to, neighbor),
|
||||
telemetry!(
|
||||
telemetry;
|
||||
CONSENSUS_INFO;
|
||||
"afg.received_commit";
|
||||
"contains_precommits_signed_by" => ?precommits_signed_by,
|
||||
"target_number" => ?msg.message.target_number.clone(),
|
||||
"target_hash" => ?msg.message.target_hash.clone(),
|
||||
);
|
||||
|
||||
gossip_engine.lock().gossip_message(topic, notification.message.clone(), false);
|
||||
}
|
||||
voter::CommitProcessingOutcome::Bad(_) => {
|
||||
// report peer and do not gossip.
|
||||
if let Some(who) = notification.sender.take() {
|
||||
gossip_engine.lock().report(who, cost::INVALID_COMMIT);
|
||||
|
||||
if let Err(cost) = check_compact_commit::<B>(
|
||||
&msg.message,
|
||||
voters,
|
||||
msg.round,
|
||||
msg.set_id,
|
||||
telemetry.as_ref(),
|
||||
) {
|
||||
if let Some(who) = notification.sender {
|
||||
gossip_engine.lock().report(who, cost);
|
||||
}
|
||||
|
||||
return None;
|
||||
}
|
||||
};
|
||||
|
||||
let cb = voter::Callback::Work(Box::new(cb));
|
||||
let round = msg.round;
|
||||
let set_id = msg.set_id;
|
||||
let commit = msg.message;
|
||||
let finalized_number = commit.target_number;
|
||||
let gossip_validator = gossip_validator.clone();
|
||||
let gossip_engine = gossip_engine.clone();
|
||||
let neighbor_sender = neighbor_sender.clone();
|
||||
let cb = move |outcome| match outcome {
|
||||
voter::CommitProcessingOutcome::Good(_) => {
|
||||
// if it checks out, gossip it. not accounting for
|
||||
// any discrepancy between the actual ghost and the claimed
|
||||
// finalized number.
|
||||
gossip_validator.note_commit_finalized(
|
||||
round,
|
||||
set_id,
|
||||
finalized_number,
|
||||
|to, neighbor| neighbor_sender.send(to, neighbor),
|
||||
);
|
||||
|
||||
Some(voter::CommunicationIn::Commit(round.0, commit, cb))
|
||||
gossip_engine.lock().gossip_message(topic, notification.message.clone(), false);
|
||||
}
|
||||
voter::CommitProcessingOutcome::Bad(_) => {
|
||||
// report peer and do not gossip.
|
||||
if let Some(who) = notification.sender.take() {
|
||||
gossip_engine.lock().report(who, cost::INVALID_COMMIT);
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
let cb = voter::Callback::Work(Box::new(cb));
|
||||
|
||||
Some(voter::CommunicationIn::Commit(round.0, commit, cb))
|
||||
}
|
||||
};
|
||||
|
||||
let process_catch_up = move |
|
||||
@@ -573,6 +599,7 @@ fn incoming_global<B: BlockT>(
|
||||
&msg.message,
|
||||
voters,
|
||||
msg.set_id,
|
||||
telemetry.clone(),
|
||||
) {
|
||||
if let Some(who) = notification.sender {
|
||||
gossip_engine.lock().report(who, cost);
|
||||
@@ -629,6 +656,7 @@ impl<B: BlockT, N: Network<B>> Clone for NetworkBridge<B, N> {
|
||||
neighbor_sender: self.neighbor_sender.clone(),
|
||||
neighbor_packet_worker: self.neighbor_packet_worker.clone(),
|
||||
gossip_validator_report_stream: self.gossip_validator_report_stream.clone(),
|
||||
telemetry: self.telemetry.clone(),
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -655,6 +683,7 @@ pub(crate) struct OutgoingMessages<Block: BlockT> {
|
||||
sender: mpsc::Sender<SignedMessage<Block>>,
|
||||
network: Arc<Mutex<GossipEngine<Block>>>,
|
||||
has_voted: HasVoted<Block>,
|
||||
telemetry: Option<TelemetryHandle>,
|
||||
}
|
||||
|
||||
impl<B: BlockT> Unpin for OutgoingMessages<B> {}
|
||||
@@ -717,7 +746,9 @@ impl<Block: BlockT> Sink<Message<Block>> for OutgoingMessages<Block>
|
||||
);
|
||||
|
||||
telemetry!(
|
||||
CONSENSUS_DEBUG; "afg.announcing_blocks_to_voted_peers";
|
||||
self.telemetry;
|
||||
CONSENSUS_DEBUG;
|
||||
"afg.announcing_blocks_to_voted_peers";
|
||||
"block" => ?target_hash, "round" => ?self.round, "set_id" => ?self.set_id,
|
||||
);
|
||||
|
||||
@@ -756,6 +787,7 @@ fn check_compact_commit<Block: BlockT>(
|
||||
voters: &VoterSet<AuthorityId>,
|
||||
round: Round,
|
||||
set_id: SetId,
|
||||
telemetry: Option<&TelemetryHandle>,
|
||||
) -> Result<(), ReputationChange> {
|
||||
// 4f + 1 = equivocations from f voters.
|
||||
let f = voters.total_weight() - voters.threshold();
|
||||
@@ -797,7 +829,12 @@ fn check_compact_commit<Block: BlockT>(
|
||||
&mut buf,
|
||||
) {
|
||||
debug!(target: "afg", "Bad commit message signature {}", id);
|
||||
telemetry!(CONSENSUS_DEBUG; "afg.bad_commit_msg_signature"; "id" => ?id);
|
||||
telemetry!(
|
||||
telemetry;
|
||||
CONSENSUS_DEBUG;
|
||||
"afg.bad_commit_msg_signature";
|
||||
"id" => ?id,
|
||||
);
|
||||
let cost = Misbehavior::BadCommitMessage {
|
||||
signatures_checked: i as i32,
|
||||
blocks_loaded: 0,
|
||||
@@ -817,6 +854,7 @@ fn check_catch_up<Block: BlockT>(
|
||||
msg: &CatchUp<Block>,
|
||||
voters: &VoterSet<AuthorityId>,
|
||||
set_id: SetId,
|
||||
telemetry: Option<TelemetryHandle>,
|
||||
) -> Result<(), ReputationChange> {
|
||||
// 4f + 1 = equivocations from f voters.
|
||||
let f = voters.total_weight() - voters.threshold();
|
||||
@@ -867,6 +905,7 @@ fn check_catch_up<Block: BlockT>(
|
||||
set_id: SetIdNumber,
|
||||
mut signatures_checked: usize,
|
||||
buf: &mut Vec<u8>,
|
||||
telemetry: Option<TelemetryHandle>,
|
||||
) -> Result<usize, ReputationChange> where
|
||||
B: BlockT,
|
||||
I: Iterator<Item=(Message<B>, &'a AuthorityId, &'a AuthoritySignature)>,
|
||||
@@ -885,7 +924,12 @@ fn check_catch_up<Block: BlockT>(
|
||||
buf,
|
||||
) {
|
||||
debug!(target: "afg", "Bad catch up message signature {}", id);
|
||||
telemetry!(CONSENSUS_DEBUG; "afg.bad_catch_up_msg_signature"; "id" => ?id);
|
||||
telemetry!(
|
||||
telemetry;
|
||||
CONSENSUS_DEBUG;
|
||||
"afg.bad_catch_up_msg_signature";
|
||||
"id" => ?id,
|
||||
);
|
||||
|
||||
let cost = Misbehavior::BadCatchUpMessage {
|
||||
signatures_checked: signatures_checked as i32,
|
||||
@@ -909,6 +953,7 @@ fn check_catch_up<Block: BlockT>(
|
||||
set_id.0,
|
||||
0,
|
||||
&mut buf,
|
||||
telemetry.clone(),
|
||||
)?;
|
||||
|
||||
// check signatures on all contained precommits.
|
||||
@@ -920,6 +965,7 @@ fn check_catch_up<Block: BlockT>(
|
||||
set_id.0,
|
||||
signatures_checked,
|
||||
&mut buf,
|
||||
telemetry,
|
||||
)?;
|
||||
|
||||
Ok(())
|
||||
@@ -932,6 +978,7 @@ struct CommitsOut<Block: BlockT> {
|
||||
is_voter: bool,
|
||||
gossip_validator: Arc<GossipValidator<Block>>,
|
||||
neighbor_sender: periodic::NeighborPacketSender<Block>,
|
||||
telemetry: Option<TelemetryHandle>,
|
||||
}
|
||||
|
||||
impl<Block: BlockT> CommitsOut<Block> {
|
||||
@@ -942,6 +989,7 @@ impl<Block: BlockT> CommitsOut<Block> {
|
||||
is_voter: bool,
|
||||
gossip_validator: Arc<GossipValidator<Block>>,
|
||||
neighbor_sender: periodic::NeighborPacketSender<Block>,
|
||||
telemetry: Option<TelemetryHandle>,
|
||||
) -> Self {
|
||||
CommitsOut {
|
||||
network,
|
||||
@@ -949,6 +997,7 @@ impl<Block: BlockT> CommitsOut<Block> {
|
||||
is_voter,
|
||||
gossip_validator,
|
||||
neighbor_sender,
|
||||
telemetry,
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -968,8 +1017,12 @@ impl<Block: BlockT> Sink<(RoundNumber, Commit<Block>)> for CommitsOut<Block> {
|
||||
let (round, commit) = input;
|
||||
let round = Round(round);
|
||||
|
||||
telemetry!(CONSENSUS_DEBUG; "afg.commit_issued";
|
||||
"target_number" => ?commit.target_number, "target_hash" => ?commit.target_hash,
|
||||
telemetry!(
|
||||
self.telemetry;
|
||||
CONSENSUS_DEBUG;
|
||||
"afg.commit_issued";
|
||||
"target_number" => ?commit.target_number,
|
||||
"target_hash" => ?commit.target_hash,
|
||||
);
|
||||
let (precommits, auth_data) = commit.precommits.into_iter()
|
||||
.map(|signed| (signed.precommit, (signed.signature, signed.id)))
|
||||
|
||||
@@ -139,6 +139,7 @@ fn config() -> crate::Config {
|
||||
name: None,
|
||||
is_authority: true,
|
||||
observer_enabled: true,
|
||||
telemetry: None,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -189,6 +190,7 @@ pub(crate) fn make_test_network() -> (
|
||||
config(),
|
||||
voter_set_state(),
|
||||
None,
|
||||
None,
|
||||
);
|
||||
|
||||
(
|
||||
|
||||
@@ -39,7 +39,7 @@ use sp_runtime::generic::BlockId;
|
||||
use sp_runtime::traits::{
|
||||
Block as BlockT, Header as HeaderT, NumberFor, Zero,
|
||||
};
|
||||
use sc_telemetry::{telemetry, CONSENSUS_DEBUG, CONSENSUS_INFO};
|
||||
use sc_telemetry::{telemetry, TelemetryHandle, CONSENSUS_DEBUG, CONSENSUS_INFO};
|
||||
|
||||
use crate::{
|
||||
local_authority_id, CommandOrError, Commit, Config, Error, NewAuthoritySet, Precommit, Prevote,
|
||||
@@ -445,6 +445,7 @@ pub(crate) struct Environment<Backend, Block: BlockT, C, N: NetworkT<Block>, SC,
|
||||
pub(crate) voting_rule: VR,
|
||||
pub(crate) metrics: Option<Metrics>,
|
||||
pub(crate) justification_sender: Option<GrandpaJustificationSender<Block>>,
|
||||
pub(crate) telemetry: Option<TelemetryHandle>,
|
||||
pub(crate) _phantom: PhantomData<Backend>,
|
||||
}
|
||||
|
||||
@@ -891,7 +892,10 @@ where
|
||||
};
|
||||
|
||||
let report_prevote_metrics = |prevote: &Prevote<Block>| {
|
||||
telemetry!(CONSENSUS_DEBUG; "afg.prevote_issued";
|
||||
telemetry!(
|
||||
self.telemetry;
|
||||
CONSENSUS_DEBUG;
|
||||
"afg.prevote_issued";
|
||||
"round" => round,
|
||||
"target_number" => ?prevote.target_number,
|
||||
"target_hash" => ?prevote.target_hash,
|
||||
@@ -950,7 +954,10 @@ where
|
||||
};
|
||||
|
||||
let report_precommit_metrics = |precommit: &Precommit<Block>| {
|
||||
telemetry!(CONSENSUS_DEBUG; "afg.precommit_issued";
|
||||
telemetry!(
|
||||
self.telemetry;
|
||||
CONSENSUS_DEBUG;
|
||||
"afg.precommit_issued";
|
||||
"round" => round,
|
||||
"target_number" => ?precommit.target_number,
|
||||
"target_hash" => ?precommit.target_hash,
|
||||
@@ -1146,6 +1153,7 @@ where
|
||||
(round, commit).into(),
|
||||
false,
|
||||
self.justification_sender.as_ref(),
|
||||
self.telemetry.clone(),
|
||||
)
|
||||
}
|
||||
|
||||
@@ -1210,6 +1218,7 @@ pub(crate) fn finalize_block<BE, Block, Client>(
|
||||
justification_or_commit: JustificationOrCommit<Block>,
|
||||
initial_sync: bool,
|
||||
justification_sender: Option<&GrandpaJustificationSender<Block>>,
|
||||
telemetry: Option<TelemetryHandle>,
|
||||
) -> Result<(), CommandOrError<Block::Hash, NumberFor<Block>>>
|
||||
where
|
||||
Block: BlockT,
|
||||
@@ -1245,6 +1254,7 @@ where
|
||||
number,
|
||||
&is_descendent_of::<Block, _>(&*client, None),
|
||||
initial_sync,
|
||||
None,
|
||||
).map_err(|e| Error::Safety(e.to_string()))?;
|
||||
|
||||
// send a justification notification if a sender exists and in case of error log it.
|
||||
@@ -1320,7 +1330,10 @@ where
|
||||
warn!(target: "afg", "Error applying finality to block {:?}: {:?}", (hash, number), e);
|
||||
e
|
||||
})?;
|
||||
telemetry!(CONSENSUS_INFO; "afg.finalized_blocks_up_to";
|
||||
telemetry!(
|
||||
telemetry;
|
||||
CONSENSUS_INFO;
|
||||
"afg.finalized_blocks_up_to";
|
||||
"number" => ?number, "hash" => ?hash,
|
||||
);
|
||||
|
||||
@@ -1340,7 +1353,10 @@ where
|
||||
);
|
||||
}
|
||||
|
||||
telemetry!(CONSENSUS_INFO; "afg.generating_new_authority_set";
|
||||
telemetry!(
|
||||
telemetry;
|
||||
CONSENSUS_INFO;
|
||||
"afg.generating_new_authority_set";
|
||||
"number" => ?canon_number, "hash" => ?canon_hash,
|
||||
"authorities" => ?set_ref.to_vec(),
|
||||
"set_id" => ?new_id,
|
||||
|
||||
@@ -247,9 +247,6 @@ where
|
||||
.map_err(|_| ClientError::JustificationDecode)?;
|
||||
justification.verify(current_set_id, ¤t_authorities)?;
|
||||
|
||||
use sc_telemetry::{telemetry, CONSENSUS_INFO};
|
||||
telemetry!(CONSENSUS_INFO; "afg.finality_proof_ok";
|
||||
"finalized_header_hash" => ?proof.block);
|
||||
Ok(proof)
|
||||
}
|
||||
|
||||
|
||||
@@ -24,6 +24,7 @@ use parking_lot::RwLockWriteGuard;
|
||||
|
||||
use sp_blockchain::{BlockStatus, well_known_cache_keys};
|
||||
use sc_client_api::{backend::Backend, utils::is_descendent_of};
|
||||
use sc_telemetry::TelemetryHandle;
|
||||
use sp_utils::mpsc::TracingUnboundedSender;
|
||||
use sp_api::TransactionFor;
|
||||
|
||||
@@ -62,6 +63,7 @@ pub struct GrandpaBlockImport<Backend, Block: BlockT, Client, SC> {
|
||||
send_voter_commands: TracingUnboundedSender<VoterCommand<Block::Hash, NumberFor<Block>>>,
|
||||
authority_set_hard_forks: HashMap<Block::Hash, PendingChange<Block::Hash, NumberFor<Block>>>,
|
||||
justification_sender: GrandpaJustificationSender<Block>,
|
||||
telemetry: Option<TelemetryHandle>,
|
||||
_phantom: PhantomData<Backend>,
|
||||
}
|
||||
|
||||
@@ -76,6 +78,7 @@ impl<Backend, Block: BlockT, Client, SC: Clone> Clone for
|
||||
send_voter_commands: self.send_voter_commands.clone(),
|
||||
authority_set_hard_forks: self.authority_set_hard_forks.clone(),
|
||||
justification_sender: self.justification_sender.clone(),
|
||||
telemetry: self.telemetry.clone(),
|
||||
_phantom: PhantomData,
|
||||
}
|
||||
}
|
||||
@@ -338,7 +341,13 @@ where
|
||||
let applied_changes = {
|
||||
let forced_change_set = guard
|
||||
.as_mut()
|
||||
.apply_forced_changes(hash, number, &is_descendent_of, initial_sync)
|
||||
.apply_forced_changes(
|
||||
hash,
|
||||
number,
|
||||
&is_descendent_of,
|
||||
initial_sync,
|
||||
self.telemetry.clone(),
|
||||
)
|
||||
.map_err(|e| ConsensusError::ClientImport(e.to_string()))
|
||||
.map_err(ConsensusError::from)?;
|
||||
|
||||
@@ -355,8 +364,11 @@ where
|
||||
let canon_hash =
|
||||
self.inner.header(BlockId::Number(canon_number))
|
||||
.map_err(|e| ConsensusError::ClientImport(e.to_string()))?
|
||||
.expect("the given block number is less or equal than the current best finalized number; \
|
||||
current best finalized number must exist in chain; qed.")
|
||||
.expect(
|
||||
"the given block number is less or equal than the current best
|
||||
finalized number; current best finalized number must exist in
|
||||
chain; qed."
|
||||
)
|
||||
.hash();
|
||||
|
||||
NewAuthoritySet {
|
||||
@@ -557,6 +569,7 @@ impl<Backend, Block: BlockT, Client, SC> GrandpaBlockImport<Backend, Block, Clie
|
||||
send_voter_commands: TracingUnboundedSender<VoterCommand<Block::Hash, NumberFor<Block>>>,
|
||||
authority_set_hard_forks: Vec<(SetId, PendingChange<Block::Hash, NumberFor<Block>>)>,
|
||||
justification_sender: GrandpaJustificationSender<Block>,
|
||||
telemetry: Option<TelemetryHandle>,
|
||||
) -> GrandpaBlockImport<Backend, Block, Client, SC> {
|
||||
// check for and apply any forced authority set hard fork that applies
|
||||
// to the *current* authority set.
|
||||
@@ -600,6 +613,7 @@ impl<Backend, Block: BlockT, Client, SC> GrandpaBlockImport<Backend, Block, Clie
|
||||
send_voter_commands,
|
||||
authority_set_hard_forks,
|
||||
justification_sender,
|
||||
telemetry,
|
||||
_phantom: PhantomData,
|
||||
}
|
||||
}
|
||||
@@ -644,6 +658,7 @@ where
|
||||
justification.into(),
|
||||
initial_sync,
|
||||
Some(&self.justification_sender),
|
||||
self.telemetry.clone(),
|
||||
);
|
||||
|
||||
match result {
|
||||
|
||||
@@ -79,7 +79,7 @@ use sp_core::{
|
||||
use sp_keystore::{SyncCryptoStorePtr, SyncCryptoStore};
|
||||
use sp_application_crypto::AppKey;
|
||||
use sp_utils::mpsc::{tracing_unbounded, TracingUnboundedReceiver};
|
||||
use sc_telemetry::{telemetry, CONSENSUS_INFO, CONSENSUS_DEBUG};
|
||||
use sc_telemetry::{telemetry, TelemetryHandle, CONSENSUS_INFO, CONSENSUS_DEBUG};
|
||||
use parking_lot::RwLock;
|
||||
|
||||
use finality_grandpa::Error as GrandpaError;
|
||||
@@ -270,6 +270,8 @@ pub struct Config {
|
||||
pub name: Option<String>,
|
||||
/// The keystore that manages the keys of this node.
|
||||
pub keystore: Option<SyncCryptoStorePtr>,
|
||||
/// TelemetryHandle instance.
|
||||
pub telemetry: Option<TelemetryHandle>,
|
||||
}
|
||||
|
||||
impl Config {
|
||||
@@ -451,6 +453,7 @@ pub struct LinkHalf<Block: BlockT, C, SC> {
|
||||
voter_commands_rx: TracingUnboundedReceiver<VoterCommand<Block::Hash, NumberFor<Block>>>,
|
||||
justification_sender: GrandpaJustificationSender<Block>,
|
||||
justification_stream: GrandpaJustificationStream<Block>,
|
||||
telemetry: Option<TelemetryHandle>,
|
||||
}
|
||||
|
||||
impl<Block: BlockT, C, SC> LinkHalf<Block, C, SC> {
|
||||
@@ -501,6 +504,7 @@ pub fn block_import<BE, Block: BlockT, Client, SC>(
|
||||
client: Arc<Client>,
|
||||
genesis_authorities_provider: &dyn GenesisAuthoritySetProvider<Block>,
|
||||
select_chain: SC,
|
||||
telemetry: Option<TelemetryHandle>,
|
||||
) -> Result<
|
||||
(
|
||||
GrandpaBlockImport<BE, Block, Client, SC>,
|
||||
@@ -518,6 +522,7 @@ where
|
||||
genesis_authorities_provider,
|
||||
select_chain,
|
||||
Default::default(),
|
||||
telemetry,
|
||||
)
|
||||
}
|
||||
|
||||
@@ -531,6 +536,7 @@ pub fn block_import_with_authority_set_hard_forks<BE, Block: BlockT, Client, SC>
|
||||
genesis_authorities_provider: &dyn GenesisAuthoritySetProvider<Block>,
|
||||
select_chain: SC,
|
||||
authority_set_hard_forks: Vec<(SetId, (Block::Hash, NumberFor<Block>), AuthorityList)>,
|
||||
telemetry: Option<TelemetryHandle>,
|
||||
) -> Result<
|
||||
(
|
||||
GrandpaBlockImport<BE, Block, Client, SC>,
|
||||
@@ -550,13 +556,19 @@ where
|
||||
&*client,
|
||||
genesis_hash,
|
||||
<NumberFor<Block>>::zero(),
|
||||
|| {
|
||||
let authorities = genesis_authorities_provider.get()?;
|
||||
telemetry!(CONSENSUS_DEBUG; "afg.loading_authorities";
|
||||
"authorities_len" => ?authorities.len()
|
||||
);
|
||||
Ok(authorities)
|
||||
}
|
||||
{
|
||||
let telemetry = telemetry.clone();
|
||||
move || {
|
||||
let authorities = genesis_authorities_provider.get()?;
|
||||
telemetry!(
|
||||
telemetry;
|
||||
CONSENSUS_DEBUG;
|
||||
"afg.loading_authorities";
|
||||
"authorities_len" => ?authorities.len()
|
||||
);
|
||||
Ok(authorities)
|
||||
}
|
||||
},
|
||||
)?;
|
||||
|
||||
let (voter_commands_tx, voter_commands_rx) = tracing_unbounded("mpsc_grandpa_voter_command");
|
||||
@@ -590,6 +602,7 @@ where
|
||||
voter_commands_tx,
|
||||
authority_set_hard_forks,
|
||||
justification_sender.clone(),
|
||||
telemetry.clone(),
|
||||
),
|
||||
LinkHalf {
|
||||
client,
|
||||
@@ -598,6 +611,7 @@ where
|
||||
voter_commands_rx,
|
||||
justification_sender,
|
||||
justification_stream,
|
||||
telemetry,
|
||||
},
|
||||
))
|
||||
}
|
||||
@@ -660,14 +674,14 @@ pub struct GrandpaParams<Block: BlockT, C, N, SC, VR> {
|
||||
/// `sc_network` crate, it is assumed that the Grandpa notifications protocol has been passed
|
||||
/// to the configuration of the networking. See [`grandpa_peers_set_config`].
|
||||
pub network: N,
|
||||
/// If supplied, can be used to hook on telemetry connection established events.
|
||||
pub telemetry_on_connect: Option<TracingUnboundedReceiver<()>>,
|
||||
/// A voting rule used to potentially restrict target votes.
|
||||
pub voting_rule: VR,
|
||||
/// The prometheus metrics registry.
|
||||
pub prometheus_registry: Option<prometheus_endpoint::Registry>,
|
||||
/// The voter state is exposed at an RPC endpoint.
|
||||
pub shared_voter_state: SharedVoterState,
|
||||
/// TelemetryHandle instance.
|
||||
pub telemetry: Option<TelemetryHandle>,
|
||||
}
|
||||
|
||||
/// Returns the configuration value to put in
|
||||
@@ -706,10 +720,10 @@ where
|
||||
mut config,
|
||||
link,
|
||||
network,
|
||||
telemetry_on_connect,
|
||||
voting_rule,
|
||||
prometheus_registry,
|
||||
shared_voter_state,
|
||||
telemetry,
|
||||
} = grandpa_params;
|
||||
|
||||
// NOTE: we have recently removed `run_grandpa_observer` from the public
|
||||
@@ -725,6 +739,7 @@ where
|
||||
voter_commands_rx,
|
||||
justification_sender,
|
||||
justification_stream: _,
|
||||
telemetry: _,
|
||||
} = link;
|
||||
|
||||
let network = NetworkBridge::new(
|
||||
@@ -732,11 +747,16 @@ where
|
||||
config.clone(),
|
||||
persistent_data.set_state.clone(),
|
||||
prometheus_registry.as_ref(),
|
||||
telemetry.clone(),
|
||||
);
|
||||
|
||||
let conf = config.clone();
|
||||
let telemetry_task = if let Some(telemetry_on_connect) = telemetry_on_connect {
|
||||
let telemetry_task = if let Some(telemetry_on_connect) = telemetry
|
||||
.as_ref()
|
||||
.map(|x| x.on_connect_stream())
|
||||
{
|
||||
let authorities = persistent_data.authority_set.clone();
|
||||
let telemetry = telemetry.clone();
|
||||
let events = telemetry_on_connect
|
||||
.for_each(move |_| {
|
||||
let current_authorities = authorities.current_authorities();
|
||||
@@ -751,10 +771,13 @@ where
|
||||
|
||||
let authorities = serde_json::to_string(&authorities).expect(
|
||||
"authorities is always at least an empty vector; \
|
||||
elements are always of type string",
|
||||
elements are always of type string",
|
||||
);
|
||||
|
||||
telemetry!(CONSENSUS_INFO; "afg.authority_set";
|
||||
telemetry!(
|
||||
telemetry;
|
||||
CONSENSUS_INFO;
|
||||
"afg.authority_set";
|
||||
"authority_id" => authority_id.to_string(),
|
||||
"authority_set_id" => ?set_id,
|
||||
"authorities" => authorities,
|
||||
@@ -778,6 +801,7 @@ where
|
||||
prometheus_registry,
|
||||
shared_voter_state,
|
||||
justification_sender,
|
||||
telemetry,
|
||||
);
|
||||
|
||||
let voter_work = voter_work.map(|res| match res {
|
||||
@@ -816,7 +840,7 @@ struct VoterWork<B, Block: BlockT, C, N: NetworkT<Block>, SC, VR> {
|
||||
env: Arc<Environment<B, Block, C, N, SC, VR>>,
|
||||
voter_commands_rx: TracingUnboundedReceiver<VoterCommand<Block::Hash, NumberFor<Block>>>,
|
||||
network: NetworkBridge<Block, N>,
|
||||
|
||||
telemetry: Option<TelemetryHandle>,
|
||||
/// Prometheus metrics.
|
||||
metrics: Option<Metrics>,
|
||||
}
|
||||
@@ -843,6 +867,7 @@ where
|
||||
prometheus_registry: Option<prometheus_endpoint::Registry>,
|
||||
shared_voter_state: SharedVoterState,
|
||||
justification_sender: GrandpaJustificationSender<Block>,
|
||||
telemetry: Option<TelemetryHandle>,
|
||||
) -> Self {
|
||||
let metrics = match prometheus_registry.as_ref().map(Metrics::register) {
|
||||
Some(Ok(metrics)) => Some(metrics),
|
||||
@@ -866,6 +891,7 @@ where
|
||||
voter_set_state: persistent_data.set_state,
|
||||
metrics: metrics.as_ref().map(|m| m.environment.clone()),
|
||||
justification_sender: Some(justification_sender),
|
||||
telemetry: telemetry.clone(),
|
||||
_phantom: PhantomData,
|
||||
});
|
||||
|
||||
@@ -877,6 +903,7 @@ where
|
||||
env,
|
||||
voter_commands_rx,
|
||||
network,
|
||||
telemetry,
|
||||
metrics,
|
||||
};
|
||||
work.rebuild_voter();
|
||||
@@ -892,7 +919,10 @@ where
|
||||
let authority_id = local_authority_id(&self.env.voters, self.env.config.keystore.as_ref())
|
||||
.unwrap_or_default();
|
||||
|
||||
telemetry!(CONSENSUS_DEBUG; "afg.starting_new_voter";
|
||||
telemetry!(
|
||||
self.telemetry;
|
||||
CONSENSUS_DEBUG;
|
||||
"afg.starting_new_voter";
|
||||
"name" => ?self.env.config.name(),
|
||||
"set_id" => ?self.env.set_id,
|
||||
"authority_id" => authority_id.to_string(),
|
||||
@@ -911,7 +941,10 @@ where
|
||||
"authorities is always at least an empty vector; elements are always of type string",
|
||||
);
|
||||
|
||||
telemetry!(CONSENSUS_INFO; "afg.authority_set";
|
||||
telemetry!(
|
||||
self.telemetry;
|
||||
CONSENSUS_INFO;
|
||||
"afg.authority_set";
|
||||
"number" => ?chain_info.finalized_number,
|
||||
"hash" => ?chain_info.finalized_hash,
|
||||
"authority_id" => authority_id.to_string(),
|
||||
@@ -971,7 +1004,10 @@ where
|
||||
let voters: Vec<String> = new.authorities.iter().map(move |(a, _)| {
|
||||
format!("{}", a)
|
||||
}).collect();
|
||||
telemetry!(CONSENSUS_INFO; "afg.voter_command_change_authorities";
|
||||
telemetry!(
|
||||
self.telemetry;
|
||||
CONSENSUS_INFO;
|
||||
"afg.voter_command_change_authorities";
|
||||
"number" => ?new.canon_number,
|
||||
"hash" => ?new.canon_hash,
|
||||
"voters" => ?voters,
|
||||
@@ -992,10 +1028,11 @@ where
|
||||
})?;
|
||||
|
||||
let voters = Arc::new(VoterSet::new(new.authorities.into_iter())
|
||||
.expect("new authorities come from pending change; \
|
||||
pending change comes from `AuthoritySet`; \
|
||||
`AuthoritySet` validates authorities is non-empty and weights are non-zero; \
|
||||
qed."
|
||||
.expect(
|
||||
"new authorities come from pending change; \
|
||||
pending change comes from `AuthoritySet`; \
|
||||
`AuthoritySet` validates authorities is non-empty and weights are non-zero; \
|
||||
qed."
|
||||
)
|
||||
);
|
||||
|
||||
@@ -1011,6 +1048,7 @@ where
|
||||
voting_rule: self.env.voting_rule.clone(),
|
||||
metrics: self.env.metrics.clone(),
|
||||
justification_sender: self.env.justification_sender.clone(),
|
||||
telemetry: self.telemetry.clone(),
|
||||
_phantom: PhantomData,
|
||||
});
|
||||
|
||||
|
||||
@@ -29,6 +29,7 @@ use log::{debug, info, warn};
|
||||
use sp_keystore::SyncCryptoStorePtr;
|
||||
use sp_consensus::SelectChain;
|
||||
use sc_client_api::backend::Backend;
|
||||
use sc_telemetry::TelemetryHandle;
|
||||
use sp_utils::mpsc::TracingUnboundedReceiver;
|
||||
use sp_runtime::traits::{NumberFor, Block as BlockT};
|
||||
use sp_blockchain::HeaderMetadata;
|
||||
@@ -67,6 +68,7 @@ fn grandpa_observer<BE, Block: BlockT, Client, S, F>(
|
||||
last_finalized_number: NumberFor<Block>,
|
||||
commits: S,
|
||||
note_round: F,
|
||||
telemetry: Option<TelemetryHandle>,
|
||||
) -> impl Future<Output = Result<(), CommandOrError<Block::Hash, NumberFor<Block>>>>
|
||||
where
|
||||
NumberFor<Block>: BlockNumberOps,
|
||||
@@ -121,6 +123,7 @@ where
|
||||
(round, commit).into(),
|
||||
false,
|
||||
justification_sender.as_ref(),
|
||||
telemetry.clone(),
|
||||
) {
|
||||
Ok(_) => {},
|
||||
Err(e) => return future::err(e),
|
||||
@@ -172,7 +175,8 @@ where
|
||||
persistent_data,
|
||||
voter_commands_rx,
|
||||
justification_sender,
|
||||
..
|
||||
justification_stream: _,
|
||||
telemetry,
|
||||
} = link;
|
||||
|
||||
let network = NetworkBridge::new(
|
||||
@@ -180,15 +184,17 @@ where
|
||||
config.clone(),
|
||||
persistent_data.set_state.clone(),
|
||||
None,
|
||||
telemetry.clone(),
|
||||
);
|
||||
|
||||
let observer_work = ObserverWork::new(
|
||||
client,
|
||||
client.clone(),
|
||||
network,
|
||||
persistent_data,
|
||||
config.keystore,
|
||||
voter_commands_rx,
|
||||
Some(justification_sender),
|
||||
telemetry.clone(),
|
||||
);
|
||||
|
||||
let observer_work = observer_work
|
||||
@@ -210,6 +216,7 @@ struct ObserverWork<B: BlockT, BE, Client, N: NetworkT<B>> {
|
||||
keystore: Option<SyncCryptoStorePtr>,
|
||||
voter_commands_rx: TracingUnboundedReceiver<VoterCommand<B::Hash, NumberFor<B>>>,
|
||||
justification_sender: Option<GrandpaJustificationSender<B>>,
|
||||
telemetry: Option<TelemetryHandle>,
|
||||
_phantom: PhantomData<BE>,
|
||||
}
|
||||
|
||||
@@ -228,6 +235,7 @@ where
|
||||
keystore: Option<SyncCryptoStorePtr>,
|
||||
voter_commands_rx: TracingUnboundedReceiver<VoterCommand<B::Hash, NumberFor<B>>>,
|
||||
justification_sender: Option<GrandpaJustificationSender<B>>,
|
||||
telemetry: Option<TelemetryHandle>,
|
||||
) -> Self {
|
||||
|
||||
let mut work = ObserverWork {
|
||||
@@ -240,6 +248,7 @@ where
|
||||
keystore: keystore.clone(),
|
||||
voter_commands_rx,
|
||||
justification_sender,
|
||||
telemetry,
|
||||
_phantom: PhantomData,
|
||||
};
|
||||
work.rebuild_observer();
|
||||
@@ -289,6 +298,7 @@ where
|
||||
last_finalized_number,
|
||||
global_in,
|
||||
note_round,
|
||||
self.telemetry.clone(),
|
||||
);
|
||||
|
||||
self.observer = Box::pin(observer);
|
||||
@@ -429,6 +439,7 @@ mod tests {
|
||||
None,
|
||||
voter_command_rx,
|
||||
None,
|
||||
None,
|
||||
);
|
||||
|
||||
// Trigger a reputation change through the gossip validator.
|
||||
|
||||
@@ -120,6 +120,7 @@ impl TestNetFactory for GrandpaTestNet {
|
||||
client.clone(),
|
||||
&self.test_config,
|
||||
LongestChain::new(backend.clone()),
|
||||
None,
|
||||
).expect("Could not create block import for fresh peer.");
|
||||
let justification_import = Box::new(import.clone());
|
||||
(
|
||||
@@ -252,13 +253,14 @@ fn initialize_grandpa(
|
||||
name: Some(format!("peer#{}", peer_id)),
|
||||
is_authority: true,
|
||||
observer_enabled: true,
|
||||
telemetry: None,
|
||||
},
|
||||
link,
|
||||
network: net_service,
|
||||
telemetry_on_connect: None,
|
||||
voting_rule: (),
|
||||
prometheus_registry: None,
|
||||
shared_voter_state: SharedVoterState::empty(),
|
||||
telemetry: None,
|
||||
};
|
||||
let voter = run_grandpa_voter(grandpa_params).expect("all in order with client and network");
|
||||
|
||||
@@ -395,13 +397,14 @@ fn finalize_3_voters_1_full_observer() {
|
||||
name: Some(format!("peer#{}", peer_id)),
|
||||
is_authority: true,
|
||||
observer_enabled: true,
|
||||
telemetry: None,
|
||||
},
|
||||
link: link,
|
||||
network: net_service,
|
||||
telemetry_on_connect: None,
|
||||
voting_rule: (),
|
||||
prometheus_registry: None,
|
||||
shared_voter_state: SharedVoterState::empty(),
|
||||
telemetry: None,
|
||||
};
|
||||
|
||||
run_grandpa_voter(grandpa_params).expect("all in order with client and network")
|
||||
@@ -488,13 +491,14 @@ fn transition_3_voters_twice_1_full_observer() {
|
||||
name: Some(format!("peer#{}", peer_id)),
|
||||
is_authority: true,
|
||||
observer_enabled: true,
|
||||
telemetry: None,
|
||||
},
|
||||
link,
|
||||
network: net_service,
|
||||
telemetry_on_connect: None,
|
||||
voting_rule: (),
|
||||
prometheus_registry: None,
|
||||
shared_voter_state: SharedVoterState::empty(),
|
||||
telemetry: None,
|
||||
};
|
||||
|
||||
voters.push(run_grandpa_voter(grandpa_params).expect("all in order with client and network"));
|
||||
@@ -921,6 +925,7 @@ fn voter_persists_its_votes() {
|
||||
name: Some(format!("peer#{}", 1)),
|
||||
is_authority: true,
|
||||
observer_enabled: true,
|
||||
telemetry: None,
|
||||
};
|
||||
|
||||
let set_state = {
|
||||
@@ -939,6 +944,7 @@ fn voter_persists_its_votes() {
|
||||
config.clone(),
|
||||
set_state,
|
||||
None,
|
||||
None,
|
||||
)
|
||||
};
|
||||
|
||||
@@ -964,13 +970,14 @@ fn voter_persists_its_votes() {
|
||||
name: Some(format!("peer#{}", 0)),
|
||||
is_authority: true,
|
||||
observer_enabled: true,
|
||||
telemetry: None,
|
||||
},
|
||||
link,
|
||||
network: net_service,
|
||||
telemetry_on_connect: None,
|
||||
voting_rule: VotingRulesBuilder::default().build(),
|
||||
prometheus_registry: None,
|
||||
shared_voter_state: SharedVoterState::empty(),
|
||||
telemetry: None,
|
||||
};
|
||||
|
||||
run_grandpa_voter(grandpa_params).expect("all in order with client and network")
|
||||
@@ -1006,13 +1013,14 @@ fn voter_persists_its_votes() {
|
||||
name: Some(format!("peer#{}", 0)),
|
||||
is_authority: true,
|
||||
observer_enabled: true,
|
||||
telemetry: None,
|
||||
},
|
||||
link,
|
||||
network: net_service,
|
||||
telemetry_on_connect: None,
|
||||
voting_rule: VotingRulesBuilder::default().build(),
|
||||
prometheus_registry: None,
|
||||
shared_voter_state: SharedVoterState::empty(),
|
||||
telemetry: None,
|
||||
};
|
||||
|
||||
run_grandpa_voter(grandpa_params)
|
||||
@@ -1165,6 +1173,7 @@ fn finalize_3_voters_1_light_observer() {
|
||||
name: Some("observer".to_string()),
|
||||
is_authority: false,
|
||||
observer_enabled: true,
|
||||
telemetry: None,
|
||||
},
|
||||
net.peers[3].data.lock().take().expect("link initialized at startup; qed"),
|
||||
net.peers[3].network_service().clone(),
|
||||
@@ -1206,13 +1215,14 @@ fn voter_catches_up_to_latest_round_when_behind() {
|
||||
name: Some(format!("peer#{}", peer_id)),
|
||||
is_authority: true,
|
||||
observer_enabled: true,
|
||||
telemetry: None,
|
||||
},
|
||||
link,
|
||||
network: net.lock().peer(peer_id).network_service().clone(),
|
||||
telemetry_on_connect: None,
|
||||
voting_rule: (),
|
||||
prometheus_registry: None,
|
||||
shared_voter_state: SharedVoterState::empty(),
|
||||
telemetry: None,
|
||||
};
|
||||
|
||||
Box::pin(run_grandpa_voter(grandpa_params).expect("all in order with client and network"))
|
||||
@@ -1328,6 +1338,7 @@ where
|
||||
name: None,
|
||||
is_authority: true,
|
||||
observer_enabled: true,
|
||||
telemetry: None,
|
||||
};
|
||||
|
||||
let network = NetworkBridge::new(
|
||||
@@ -1335,6 +1346,7 @@ where
|
||||
config.clone(),
|
||||
set_state.clone(),
|
||||
None,
|
||||
None,
|
||||
);
|
||||
|
||||
Environment {
|
||||
@@ -1349,6 +1361,7 @@ where
|
||||
voting_rule,
|
||||
metrics: None,
|
||||
justification_sender: None,
|
||||
telemetry: None,
|
||||
_phantom: PhantomData,
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user