// This file is part of Bizinikiwi. // Copyright (C) Parity Technologies (UK) Ltd. // SPDX-License-Identifier: GPL-3.0-or-later WITH Classpath-exception-2.0 // This program is free software: you can redistribute it and/or modify // it under the terms of the GNU General Public License as published by // the Free Software Foundation, either version 3 of the License, or // (at your option) any later version. // This program is distributed in the hope that it will be useful, // but WITHOUT ANY WARRANTY; without even the implied warranty of // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the // GNU General Public License for more details. // You should have received a copy of the GNU General Public License // along with this program. If not, see . use std::{ collections::{BTreeMap, HashMap}, marker::PhantomData, pin::Pin, sync::Arc, time::Duration, }; use codec::{Decode, Encode}; use finality_grandpa::{ round::State as RoundState, voter, voter_set::VoterSet, BlockNumberOps, Error as GrandpaError, }; use futures::prelude::*; use futures_timer::Delay; use log::{debug, warn}; use parking_lot::RwLock; use prometheus_endpoint::{register, Counter, Gauge, PrometheusError, U64}; use pezsc_client_api::{ backend::{apply_aux, Backend as BackendT}, utils::is_descendent_of, }; use pezsc_telemetry::{telemetry, TelemetryHandle, CONSENSUS_DEBUG, CONSENSUS_INFO}; use pezsc_transaction_pool_api::OffchainTransactionPoolFactory; use pezsp_api::ApiExt; use pezsp_blockchain::HeaderMetadata; use pezsp_consensus::SelectChain as SelectChainT; use pezsp_consensus_grandpa::{ AuthorityId, AuthoritySignature, Equivocation, EquivocationProof, GrandpaApi, RoundNumber, SetId, GRANDPA_ENGINE_ID, }; use pezsp_runtime::traits::{Block as BlockT, Header as HeaderT, NumberFor, Zero}; use crate::{ authorities::{AuthoritySet, SharedAuthoritySet}, communication::{Network as NetworkT, Syncing as SyncingT}, justification::GrandpaJustification, local_authority_id, notification::GrandpaJustificationSender, until_imported::UntilVoteTargetImported, voting_rule::VotingRule as VotingRuleT, ClientForGrandpa, CommandOrError, Commit, Config, Error, NewAuthoritySet, Precommit, Prevote, PrimaryPropose, SignedMessage, VoterCommand, LOG_TARGET, }; type HistoricalVotes = finality_grandpa::HistoricalVotes< ::Hash, NumberFor, AuthoritySignature, AuthorityId, >; /// Data about a completed round. The set of votes that is stored must be /// minimal, i.e. at most one equivocation is stored per voter. #[derive(Debug, Clone, Decode, Encode, PartialEq)] pub struct CompletedRound { /// The round number. pub number: RoundNumber, /// The round state (prevote ghost, estimate, finalized, etc.) pub state: RoundState>, /// The target block base used for voting in the round. pub base: (Block::Hash, NumberFor), /// All the votes observed in the round. pub votes: Vec>, } // Data about last completed rounds within a single voter set. Stores // NUM_LAST_COMPLETED_ROUNDS and always contains data about at least one round // (genesis). #[derive(Debug, Clone, PartialEq)] pub struct CompletedRounds { rounds: Vec>, set_id: SetId, voters: Vec, } // NOTE: the current strategy for persisting completed rounds is very naive // (update everything) and we also rely on cloning to do atomic updates, // therefore this value should be kept small for now. const NUM_LAST_COMPLETED_ROUNDS: usize = 2; impl Encode for CompletedRounds { fn encode(&self) -> Vec { let v = Vec::from_iter(&self.rounds); (&v, &self.set_id, &self.voters).encode() } } impl codec::EncodeLike for CompletedRounds {} impl Decode for CompletedRounds { fn decode(value: &mut I) -> Result { <(Vec>, SetId, Vec)>::decode(value) .map(|(rounds, set_id, voters)| CompletedRounds { rounds, set_id, voters }) } } impl CompletedRounds { /// Create a new completed rounds tracker with NUM_LAST_COMPLETED_ROUNDS capacity. pub(crate) fn new( genesis: CompletedRound, set_id: SetId, voters: &AuthoritySet>, ) -> CompletedRounds { let mut rounds = Vec::with_capacity(NUM_LAST_COMPLETED_ROUNDS); rounds.push(genesis); let voters = voters.current_authorities.iter().map(|(a, _)| a.clone()).collect(); CompletedRounds { rounds, set_id, voters } } /// Get the set-id and voter set of the completed rounds. pub fn set_info(&self) -> (SetId, &[AuthorityId]) { (self.set_id, &self.voters[..]) } /// Iterate over all completed rounds. pub fn iter(&self) -> impl Iterator> { self.rounds.iter().rev() } /// Returns the last (latest) completed round. pub fn last(&self) -> &CompletedRound { self.rounds .first() .expect("inner is never empty; always contains at least genesis; qed") } /// Push a new completed round, oldest round is evicted if number of rounds /// is higher than `NUM_LAST_COMPLETED_ROUNDS`. pub fn push(&mut self, completed_round: CompletedRound) { use std::cmp::Reverse; match self .rounds .binary_search_by_key(&Reverse(completed_round.number), |completed_round| { Reverse(completed_round.number) }) { Ok(idx) => self.rounds[idx] = completed_round, Err(idx) => self.rounds.insert(idx, completed_round), }; if self.rounds.len() > NUM_LAST_COMPLETED_ROUNDS { self.rounds.pop(); } } } /// A map with voter status information for currently live rounds, /// which votes have we cast and what are they. pub type CurrentRounds = BTreeMap::Header>>; /// The state of the current voter set, whether it is currently active or not /// and information related to the previously completed rounds. Current round /// voting status is used when restarting the voter, i.e. it will re-use the /// previous votes for a given round if appropriate (same round and same local /// key). #[derive(Debug, Decode, Encode, PartialEq)] pub enum VoterSetState { /// The voter is live, i.e. participating in rounds. Live { /// The previously completed rounds. completed_rounds: CompletedRounds, /// Voter status for the currently live rounds. current_rounds: CurrentRounds, }, /// The voter is paused, i.e. not casting or importing any votes. Paused { /// The previously completed rounds. completed_rounds: CompletedRounds, }, } impl VoterSetState { /// Create a new live VoterSetState with round 0 as a completed round using /// the given genesis state and the given authorities. Round 1 is added as a /// current round (with state `HasVoted::No`). pub(crate) fn live( set_id: SetId, authority_set: &AuthoritySet>, genesis_state: (Block::Hash, NumberFor), ) -> VoterSetState { let state = RoundState::genesis((genesis_state.0, genesis_state.1)); let completed_rounds = CompletedRounds::new( CompletedRound { number: 0, state, base: (genesis_state.0, genesis_state.1), votes: Vec::new(), }, set_id, authority_set, ); let mut current_rounds = CurrentRounds::::new(); current_rounds.insert(1, HasVoted::No); VoterSetState::Live { completed_rounds, current_rounds } } /// Returns the last completed rounds. pub(crate) fn completed_rounds(&self) -> CompletedRounds { match self { VoterSetState::Live { completed_rounds, .. } => completed_rounds.clone(), VoterSetState::Paused { completed_rounds } => completed_rounds.clone(), } } /// Returns the last completed round. pub(crate) fn last_completed_round(&self) -> CompletedRound { match self { VoterSetState::Live { completed_rounds, .. } => completed_rounds.last().clone(), VoterSetState::Paused { completed_rounds } => completed_rounds.last().clone(), } } /// Returns the voter set state validating that it includes the given round /// in current rounds and that the voter isn't paused. pub fn with_current_round( &self, round: RoundNumber, ) -> Result<(&CompletedRounds, &CurrentRounds), Error> { if let VoterSetState::Live { completed_rounds, current_rounds } = self { if current_rounds.contains_key(&round) { Ok((completed_rounds, current_rounds)) } else { let msg = "Voter acting on a live round we are not tracking."; Err(Error::Safety(msg.to_string())) } } else { let msg = "Voter acting while in paused state."; Err(Error::Safety(msg.to_string())) } } } /// Whether we've voted already during a prior run of the program. #[derive(Clone, Debug, Decode, Encode, PartialEq)] pub enum HasVoted { /// Has not voted already in this round. No, /// Has voted in this round. Yes(AuthorityId, Vote
), } /// The votes cast by this voter already during a prior run of the program. #[derive(Debug, Clone, Decode, Encode, PartialEq)] pub enum Vote { /// Has cast a proposal. Propose(PrimaryPropose
), /// Has cast a prevote. Prevote(Option>, Prevote
), /// Has cast a precommit (implies prevote.) Precommit(Option>, Prevote
, Precommit
), } impl HasVoted
{ /// Returns the proposal we should vote with (if any.) pub fn propose(&self) -> Option<&PrimaryPropose
> { match self { HasVoted::Yes(_, Vote::Propose(propose)) => Some(propose), HasVoted::Yes(_, Vote::Prevote(propose, _)) | HasVoted::Yes(_, Vote::Precommit(propose, _, _)) => propose.as_ref(), _ => None, } } /// Returns the prevote we should vote with (if any.) pub fn prevote(&self) -> Option<&Prevote
> { match self { HasVoted::Yes(_, Vote::Prevote(_, prevote)) | HasVoted::Yes(_, Vote::Precommit(_, prevote, _)) => Some(prevote), _ => None, } } /// Returns the precommit we should vote with (if any.) pub fn precommit(&self) -> Option<&Precommit
> { match self { HasVoted::Yes(_, Vote::Precommit(_, _, precommit)) => Some(precommit), _ => None, } } /// Returns true if the voter can still propose, false otherwise. pub fn can_propose(&self) -> bool { self.propose().is_none() } /// Returns true if the voter can still prevote, false otherwise. pub fn can_prevote(&self) -> bool { self.prevote().is_none() } /// Returns true if the voter can still precommit, false otherwise. pub fn can_precommit(&self) -> bool { self.precommit().is_none() } } /// A voter set state meant to be shared safely across multiple owners. #[derive(Clone)] pub struct SharedVoterSetState { /// The inner shared `VoterSetState`. inner: Arc>>, /// A tracker for the rounds that we are actively participating on (i.e. voting) /// and the authority id under which we are doing it. voting: Arc>>, } impl From> for SharedVoterSetState { fn from(set_state: VoterSetState) -> Self { SharedVoterSetState::new(set_state) } } impl SharedVoterSetState { /// Create a new shared voter set tracker with the given state. pub(crate) fn new(state: VoterSetState) -> Self { SharedVoterSetState { inner: Arc::new(RwLock::new(state)), voting: Arc::new(RwLock::new(HashMap::new())), } } /// Read the inner voter set state. pub(crate) fn read(&self) -> parking_lot::RwLockReadGuard<'_, VoterSetState> { self.inner.read() } /// Get the authority id that we are using to vote on the given round, if any. pub(crate) fn voting_on(&self, round: RoundNumber) -> Option { self.voting.read().get(&round).cloned() } /// Note that we started voting on the give round with the given authority id. pub(crate) fn started_voting_on(&self, round: RoundNumber, local_id: AuthorityId) { self.voting.write().insert(round, local_id); } /// Note that we have finished voting on the given round. If we were voting on /// the given round, the authority id that we were using to do it will be /// cleared. pub(crate) fn finished_voting_on(&self, round: RoundNumber) { self.voting.write().remove(&round); } /// Return vote status information for the current round. pub(crate) fn has_voted(&self, round: RoundNumber) -> HasVoted { match &*self.inner.read() { VoterSetState::Live { current_rounds, .. } => current_rounds .get(&round) .and_then(|has_voted| match has_voted { HasVoted::Yes(id, vote) => Some(HasVoted::Yes(id.clone(), vote.clone())), _ => None, }) .unwrap_or(HasVoted::No), _ => HasVoted::No, } } // NOTE: not exposed outside of this module intentionally. fn with(&self, f: F) -> R where F: FnOnce(&mut VoterSetState) -> R, { f(&mut *self.inner.write()) } } /// Prometheus metrics for GRANDPA. #[derive(Clone)] pub(crate) struct Metrics { finality_grandpa_round: Gauge, finality_grandpa_prevotes: Counter, finality_grandpa_precommits: Counter, } impl Metrics { pub(crate) fn register( registry: &prometheus_endpoint::Registry, ) -> Result { Ok(Self { finality_grandpa_round: register( Gauge::new( "bizinikiwi_finality_grandpa_round", "Highest completed GRANDPA round.", )?, registry, )?, finality_grandpa_prevotes: register( Counter::new( "bizinikiwi_finality_grandpa_prevotes_total", "Total number of GRANDPA prevotes cast locally.", )?, registry, )?, finality_grandpa_precommits: register( Counter::new( "bizinikiwi_finality_grandpa_precommits_total", "Total number of GRANDPA precommits cast locally.", )?, registry, )?, }) } } /// The environment we run GRANDPA in. pub(crate) struct Environment< Backend, Block: BlockT, C, N: NetworkT, S: SyncingT, SC, VR, > { pub(crate) client: Arc, pub(crate) select_chain: SC, pub(crate) voters: Arc>, pub(crate) config: Config, pub(crate) authority_set: SharedAuthoritySet>, pub(crate) network: crate::communication::NetworkBridge, pub(crate) set_id: SetId, pub(crate) voter_set_state: SharedVoterSetState, pub(crate) voting_rule: VR, pub(crate) metrics: Option, pub(crate) justification_sender: Option>, pub(crate) telemetry: Option, pub(crate) offchain_tx_pool_factory: OffchainTransactionPoolFactory, pub(crate) _phantom: PhantomData, } impl, S: SyncingT, SC, VR> Environment { /// Updates the voter set state using the given closure. The write lock is /// held during evaluation of the closure and the environment's voter set /// state is set to its result if successful. pub(crate) fn update_voter_set_state(&self, f: F) -> Result<(), Error> where F: FnOnce(&VoterSetState) -> Result>, Error>, { self.voter_set_state.with(|voter_set_state| { if let Some(set_state) = f(voter_set_state)? { *voter_set_state = set_state; if let Some(metrics) = self.metrics.as_ref() { if let VoterSetState::Live { completed_rounds, .. } = voter_set_state { let highest = completed_rounds .rounds .iter() .map(|round| round.number) .max() .expect("There is always one completed round (genesis); qed"); metrics.finality_grandpa_round.set(highest); } } } Ok(()) }) } } impl Environment where Block: BlockT, BE: BackendT, C: ClientForGrandpa, C::Api: GrandpaApi, N: NetworkT, S: SyncingT, SC: SelectChainT, { /// Report the given equivocation to the GRANDPA runtime module. This method /// generates a session membership proof of the offender and then submits an /// extrinsic to report the equivocation. In particular, the session membership /// proof must be generated at the block at which the given set was active which /// isn't necessarily the best block if there are pending authority set changes. pub(crate) fn report_equivocation( &self, equivocation: Equivocation>, ) -> Result<(), Error> { if let Some(local_id) = self.voter_set_state.voting_on(equivocation.round_number()) { if *equivocation.offender() == local_id { return Err(Error::Safety( "Refraining from sending equivocation report for our own equivocation.".into(), )); } } let is_descendent_of = is_descendent_of(&*self.client, None); let (best_block_hash, best_block_number) = { // TODO [#9158]: Use SelectChain::best_chain() to get a potentially // more accurate best block let info = self.client.info(); (info.best_hash, info.best_number) }; let authority_set = self.authority_set.inner(); // block hash and number of the next pending authority set change in the // given best chain. let next_change = authority_set .next_change(&best_block_hash, &is_descendent_of) .map_err(|e| Error::Safety(e.to_string()))?; // find the hash of the latest block in the current set let current_set_latest_hash = match next_change { Some((_, n)) if n.is_zero() => { return Err(Error::Safety("Authority set change signalled at genesis.".to_string())) }, // the next set starts at `n` so the current one lasts until `n - 1`. if // `n` is later than the best block, then the current set is still live // at best block. Some((_, n)) if n > best_block_number => best_block_hash, Some((h, _)) => { // this is the header at which the new set will start let header = self.client.header(h)?.expect( "got block hash from registered pending change; \ pending changes are only registered on block import; qed.", ); // its parent block is the last block in the current set *header.parent_hash() }, // there is no pending change, the latest block for the current set is // the best block. None => best_block_hash, }; // generate key ownership proof at that block let key_owner_proof = match self .client .runtime_api() .generate_key_ownership_proof( current_set_latest_hash, authority_set.set_id, equivocation.offender().clone(), ) .map_err(Error::RuntimeApi)? { Some(proof) => proof, None => { debug!( target: LOG_TARGET, "Equivocation offender is not part of the authority set." ); return Ok(()); }, }; // submit equivocation report at **best** block let equivocation_proof = EquivocationProof::new(authority_set.set_id, equivocation); let mut runtime_api = self.client.runtime_api(); runtime_api.register_extension( self.offchain_tx_pool_factory.offchain_transaction_pool(best_block_hash), ); runtime_api .submit_report_equivocation_unsigned_extrinsic( best_block_hash, equivocation_proof, key_owner_proof, ) .map_err(Error::RuntimeApi)?; Ok(()) } } impl finality_grandpa::Chain> for Environment where Block: BlockT, BE: BackendT, C: ClientForGrandpa, N: NetworkT, S: SyncingT, SC: SelectChainT, VR: VotingRuleT, NumberFor: BlockNumberOps, { fn ancestry( &self, base: Block::Hash, block: Block::Hash, ) -> Result, GrandpaError> { ancestry(&self.client, base, block) } } pub(crate) fn ancestry( client: &Arc, base: Block::Hash, block: Block::Hash, ) -> Result, GrandpaError> where Client: HeaderMetadata, { if base == block { return Err(GrandpaError::NotDescendent); } let tree_route_res = pezsp_blockchain::tree_route(&**client, block, base); let tree_route = match tree_route_res { Ok(tree_route) => tree_route, Err(e) => { debug!( target: LOG_TARGET, "Encountered error computing ancestry between block {:?} and base {:?}: {}", block, base, e ); return Err(GrandpaError::NotDescendent); }, }; if tree_route.common_block().hash != base { return Err(GrandpaError::NotDescendent); } // skip one because our ancestry is meant to start from the parent of `block`, // and `tree_route` includes it. Ok(tree_route.retracted().iter().skip(1).map(|e| e.hash).collect()) } impl voter::Environment> for Environment where Block: BlockT, B: BackendT, C: ClientForGrandpa + 'static, C::Api: GrandpaApi, N: NetworkT, S: SyncingT, SC: SelectChainT + 'static, VR: VotingRuleT + Clone + 'static, NumberFor: BlockNumberOps, { type Timer = Pin> + Send>>; type BestChain = Pin< Box< dyn Future)>, Self::Error>> + Send, >, >; type Id = AuthorityId; type Signature = AuthoritySignature; // regular round message streams type In = Pin< Box< dyn Stream< Item = Result< ::finality_grandpa::SignedMessage< Block::Hash, NumberFor, Self::Signature, Self::Id, >, Self::Error, >, > + Send, >, >; type Out = Pin< Box< dyn Sink< ::finality_grandpa::Message>, Error = Self::Error, > + Send, >, >; type Error = CommandOrError>; fn best_chain_containing(&self, block: Block::Hash) -> Self::BestChain { let client = self.client.clone(); let authority_set = self.authority_set.clone(); let select_chain = self.select_chain.clone(); let voting_rule = self.voting_rule.clone(); let set_id = self.set_id; Box::pin(async move { // NOTE: when we finalize an authority set change through the sync protocol the voter is // signaled asynchronously. therefore the voter could still vote in the next round // before activating the new set. the `authority_set` is updated immediately thus // we restrict the voter based on that. if set_id != authority_set.set_id() { return Ok(None); } best_chain_containing(block, client, authority_set, select_chain, voting_rule) .await .map_err(|e| e.into()) }) } fn round_data( &self, round: RoundNumber, ) -> voter::RoundData { let prevote_timer = Delay::new(self.config.gossip_duration * 2); let precommit_timer = Delay::new(self.config.gossip_duration * 4); let local_id = local_authority_id(&self.voters, self.config.keystore.as_ref()); let has_voted = match self.voter_set_state.has_voted(round) { HasVoted::Yes(id, vote) => { if local_id.as_ref().map(|k| k == &id).unwrap_or(false) { HasVoted::Yes(id, vote) } else { HasVoted::No } }, HasVoted::No => HasVoted::No, }; // NOTE: we cache the local authority id that we'll be using to vote on the // given round. this is done to make sure we only check for available keys // from the keystore in this method when beginning the round, otherwise if // the keystore state changed during the round (e.g. a key was removed) it // could lead to internal state inconsistencies in the voter environment // (e.g. we wouldn't update the voter set state after prevoting since there's // no local authority id). if let Some(id) = local_id.as_ref() { self.voter_set_state.started_voting_on(round, id.clone()); } // we can only sign when we have a local key in the authority set // and we have a reference to the keystore. let keystore = match (local_id.as_ref(), self.config.keystore.as_ref()) { (Some(id), Some(keystore)) => Some((id.clone(), keystore.clone()).into()), _ => None, }; let (incoming, outgoing) = self.network.round_communication( keystore, crate::communication::Round(round), crate::communication::SetId(self.set_id), self.voters.clone(), has_voted, ); // schedule incoming messages from the network to be held until // corresponding blocks are imported. let incoming = Box::pin( UntilVoteTargetImported::new( self.client.import_notification_stream(), self.network.clone(), self.client.clone(), incoming, "round", None, ) .map_err(Into::into), ); // schedule network message cleanup when sink drops. let outgoing = Box::pin(outgoing.sink_err_into()); voter::RoundData { voter_id: local_id, prevote_timer: Box::pin(prevote_timer.map(Ok)), precommit_timer: Box::pin(precommit_timer.map(Ok)), incoming, outgoing, } } fn proposed( &self, round: RoundNumber, propose: PrimaryPropose, ) -> Result<(), Self::Error> { let local_id = match self.voter_set_state.voting_on(round) { Some(id) => id, None => return Ok(()), }; self.update_voter_set_state(|voter_set_state| { let (completed_rounds, current_rounds) = voter_set_state.with_current_round(round)?; let current_round = current_rounds .get(&round) .expect("checked in with_current_round that key exists; qed."); if !current_round.can_propose() { // we've already proposed in this round (in a previous run), // ignore the given vote and don't update the voter set // state return Ok(None); } let mut current_rounds = current_rounds.clone(); let current_round = current_rounds .get_mut(&round) .expect("checked previously that key exists; qed."); *current_round = HasVoted::Yes(local_id, Vote::Propose(propose)); let set_state = VoterSetState::::Live { completed_rounds: completed_rounds.clone(), current_rounds, }; crate::aux_schema::write_voter_set_state(&*self.client, &set_state)?; Ok(Some(set_state)) })?; Ok(()) } fn prevoted( &self, round: RoundNumber, prevote: Prevote, ) -> Result<(), Self::Error> { let local_id = match self.voter_set_state.voting_on(round) { Some(id) => id, None => return Ok(()), }; let report_prevote_metrics = |prevote: &Prevote| { telemetry!( self.telemetry; CONSENSUS_DEBUG; "afg.prevote_issued"; "round" => round, "target_number" => ?prevote.target_number, "target_hash" => ?prevote.target_hash, ); if let Some(metrics) = self.metrics.as_ref() { metrics.finality_grandpa_prevotes.inc(); } }; self.update_voter_set_state(|voter_set_state| { let (completed_rounds, current_rounds) = voter_set_state.with_current_round(round)?; let current_round = current_rounds .get(&round) .expect("checked in with_current_round that key exists; qed."); if !current_round.can_prevote() { // we've already prevoted in this round (in a previous run), // ignore the given vote and don't update the voter set // state return Ok(None); } // report to telemetry and prometheus report_prevote_metrics(&prevote); let propose = current_round.propose(); let mut current_rounds = current_rounds.clone(); let current_round = current_rounds .get_mut(&round) .expect("checked previously that key exists; qed."); *current_round = HasVoted::Yes(local_id, Vote::Prevote(propose.cloned(), prevote)); let set_state = VoterSetState::::Live { completed_rounds: completed_rounds.clone(), current_rounds, }; crate::aux_schema::write_voter_set_state(&*self.client, &set_state)?; Ok(Some(set_state)) })?; Ok(()) } fn precommitted( &self, round: RoundNumber, precommit: Precommit, ) -> Result<(), Self::Error> { let local_id = match self.voter_set_state.voting_on(round) { Some(id) => id, None => return Ok(()), }; let report_precommit_metrics = |precommit: &Precommit| { telemetry!( self.telemetry; CONSENSUS_DEBUG; "afg.precommit_issued"; "round" => round, "target_number" => ?precommit.target_number, "target_hash" => ?precommit.target_hash, ); if let Some(metrics) = self.metrics.as_ref() { metrics.finality_grandpa_precommits.inc(); } }; self.update_voter_set_state(|voter_set_state| { let (completed_rounds, current_rounds) = voter_set_state.with_current_round(round)?; let current_round = current_rounds .get(&round) .expect("checked in with_current_round that key exists; qed."); if !current_round.can_precommit() { // we've already precommitted in this round (in a previous run), // ignore the given vote and don't update the voter set // state return Ok(None); } // report to telemetry and prometheus report_precommit_metrics(&precommit); let propose = current_round.propose(); let prevote = match current_round { HasVoted::Yes(_, Vote::Prevote(_, prevote)) => prevote, _ => { let msg = "Voter precommitting before prevoting."; return Err(Error::Safety(msg.to_string())); }, }; let mut current_rounds = current_rounds.clone(); let current_round = current_rounds .get_mut(&round) .expect("checked previously that key exists; qed."); *current_round = HasVoted::Yes( local_id, Vote::Precommit(propose.cloned(), prevote.clone(), precommit), ); let set_state = VoterSetState::::Live { completed_rounds: completed_rounds.clone(), current_rounds, }; crate::aux_schema::write_voter_set_state(&*self.client, &set_state)?; Ok(Some(set_state)) })?; Ok(()) } fn completed( &self, round: RoundNumber, state: RoundState>, base: (Block::Hash, NumberFor), historical_votes: &HistoricalVotes, ) -> Result<(), Self::Error> { debug!( target: LOG_TARGET, "Voter {} completed round {} in set {}. Estimate = {:?}, Finalized in round = {:?}", self.config.name(), round, self.set_id, state.estimate.as_ref().map(|e| e.1), state.finalized.as_ref().map(|e| e.1), ); self.update_voter_set_state(|voter_set_state| { // NOTE: we don't use `with_current_round` here, it is possible that // we are not currently tracking this round if it is a round we // caught up to. let (completed_rounds, current_rounds) = if let VoterSetState::Live { completed_rounds, current_rounds } = voter_set_state { (completed_rounds, current_rounds) } else { let msg = "Voter acting while in paused state."; return Err(Error::Safety(msg.to_string())); }; let mut completed_rounds = completed_rounds.clone(); // TODO: Future integration will store the prevote and precommit index. See #2611. let votes = historical_votes.seen().to_vec(); completed_rounds.push(CompletedRound { number: round, state: state.clone(), base, votes, }); // remove the round from live rounds and start tracking the next round let mut current_rounds = current_rounds.clone(); current_rounds.remove(&round); // NOTE: this entry should always exist as GRANDPA rounds are always // started in increasing order, still it's better to play it safe. current_rounds.entry(round + 1).or_insert(HasVoted::No); let set_state = VoterSetState::::Live { completed_rounds, current_rounds }; crate::aux_schema::write_voter_set_state(&*self.client, &set_state)?; Ok(Some(set_state)) })?; // clear any cached local authority id associated with this round self.voter_set_state.finished_voting_on(round); Ok(()) } fn concluded( &self, round: RoundNumber, state: RoundState>, _base: (Block::Hash, NumberFor), historical_votes: &HistoricalVotes, ) -> Result<(), Self::Error> { debug!( target: LOG_TARGET, "Voter {} concluded round {} in set {}. Estimate = {:?}, Finalized in round = {:?}", self.config.name(), round, self.set_id, state.estimate.as_ref().map(|e| e.1), state.finalized.as_ref().map(|e| e.1), ); self.update_voter_set_state(|voter_set_state| { // NOTE: we don't use `with_current_round` here, because a concluded // round is completed and cannot be current. let (completed_rounds, current_rounds) = if let VoterSetState::Live { completed_rounds, current_rounds } = voter_set_state { (completed_rounds, current_rounds) } else { let msg = "Voter acting while in paused state."; return Err(Error::Safety(msg.to_string())); }; let mut completed_rounds = completed_rounds.clone(); if let Some(already_completed) = completed_rounds.rounds.iter_mut().find(|r| r.number == round) { let n_existing_votes = already_completed.votes.len(); // the interface of Environment guarantees that the previous `historical_votes` // from `completable` is a prefix of what is passed to `concluded`. already_completed .votes .extend(historical_votes.seen().iter().skip(n_existing_votes).cloned()); already_completed.state = state; crate::aux_schema::write_concluded_round(&*self.client, already_completed)?; } let set_state = VoterSetState::::Live { completed_rounds, current_rounds: current_rounds.clone(), }; crate::aux_schema::write_voter_set_state(&*self.client, &set_state)?; Ok(Some(set_state)) })?; Ok(()) } fn finalize_block( &self, hash: Block::Hash, number: NumberFor, round: RoundNumber, commit: Commit, ) -> Result<(), Self::Error> { finalize_block( self.client.clone(), &self.authority_set, Some(self.config.justification_generation_period), hash, number, (round, commit).into(), false, self.justification_sender.as_ref(), self.telemetry.clone(), ) } fn round_commit_timer(&self) -> Self::Timer { use rand::{thread_rng, Rng}; // random between `[0, 2 * gossip_duration]` seconds. let delay: u64 = thread_rng().gen_range(0..2 * self.config.gossip_duration.as_millis() as u64); Box::pin(Delay::new(Duration::from_millis(delay)).map(Ok)) } fn prevote_equivocation( &self, _round: RoundNumber, equivocation: finality_grandpa::Equivocation< Self::Id, Prevote, Self::Signature, >, ) { warn!( target: LOG_TARGET, "Detected prevote equivocation in the finality worker: {:?}", equivocation ); if let Err(err) = self.report_equivocation(equivocation.into()) { warn!(target: LOG_TARGET, "Error reporting prevote equivocation: {}", err); } } fn precommit_equivocation( &self, _round: RoundNumber, equivocation: finality_grandpa::Equivocation< Self::Id, Precommit, Self::Signature, >, ) { warn!( target: LOG_TARGET, "Detected precommit equivocation in the finality worker: {:?}", equivocation ); if let Err(err) = self.report_equivocation(equivocation.into()) { warn!(target: LOG_TARGET, "Error reporting precommit equivocation: {}", err); } } } pub(crate) enum JustificationOrCommit { Justification(GrandpaJustification), Commit((RoundNumber, Commit)), } impl From<(RoundNumber, Commit)> for JustificationOrCommit { fn from(commit: (RoundNumber, Commit)) -> JustificationOrCommit { JustificationOrCommit::Commit(commit) } } impl From> for JustificationOrCommit { fn from(justification: GrandpaJustification) -> JustificationOrCommit { JustificationOrCommit::Justification(justification) } } async fn best_chain_containing( block: Block::Hash, client: Arc, authority_set: SharedAuthoritySet>, select_chain: SelectChain, voting_rule: VotingRule, ) -> Result)>, Error> where Backend: BackendT, Block: BlockT, Client: ClientForGrandpa, SelectChain: SelectChainT + 'static, VotingRule: VotingRuleT, { let base_header = match client.header(block)? { Some(h) => h, None => { warn!( target: LOG_TARGET, "Encountered error finding best chain containing {:?}: couldn't find base block", block, ); return Ok(None); }, }; // we refuse to vote beyond the current limit number where transitions are scheduled to occur. // once blocks are finalized that make that transition irrelevant or activate it, we will // proceed onwards. most of the time there will be no pending transition. the limit, if any, is // guaranteed to be higher than or equal to the given base number. let limit = authority_set.current_limit(*base_header.number()); debug!( target: LOG_TARGET, "Finding best chain containing block {:?} with number limit {:?}", block, limit ); let mut target_header = match select_chain.finality_target(block, None).await { Ok(target_hash) => client .header(target_hash)? .expect("Header known to exist after `finality_target` call; qed"), Err(err) => { debug!( target: LOG_TARGET, "Encountered error finding best chain containing {:?}: couldn't find target block: {}", block, err, ); // NOTE: in case the given `SelectChain` doesn't provide any block we fallback to using // the given base block provided by the GRANDPA voter. // // For example, `LongestChain` will error if the given block to use as base isn't part // of the best chain (as defined by `LongestChain`), which could happen if there was a // re-org. base_header.clone() }, }; // NOTE: this is purposefully done after `finality_target` to prevent a case // where in-between these two requests there is a block import and // `finality_target` returns something higher than `best_chain`. let mut best_header = match select_chain.best_chain().await { Ok(best_header) => best_header, Err(err) => { warn!( target: LOG_TARGET, "Encountered error finding best chain containing {:?}: couldn't find best block: {}", block, err, ); return Ok(None); }, }; let is_descendent_of = is_descendent_of(&*client, None); if target_header.number() > best_header.number() || target_header.number() == best_header.number() && target_header.hash() != best_header.hash() || !is_descendent_of(&target_header.hash(), &best_header.hash())? { debug!( target: LOG_TARGET, "SelectChain returned a finality target inconsistent with its best block. Restricting best block to target block" ); best_header = target_header.clone(); } debug!( target: LOG_TARGET, "SelectChain: finality target: #{} ({}), best block: #{} ({})", target_header.number(), target_header.hash(), best_header.number(), best_header.hash(), ); // check if our vote is currently being limited due to a pending change, // in which case we will restrict our target header to the given limit if let Some(target_number) = limit.filter(|limit| limit < target_header.number()) { // walk backwards until we find the target block loop { if *target_header.number() < target_number { unreachable!( "we are traversing backwards from a known block; \ blocks are stored contiguously; \ qed" ); } if *target_header.number() == target_number { break; } target_header = client .header(*target_header.parent_hash())? .expect("Header known to exist after `finality_target` call; qed"); } debug!( target: LOG_TARGET, "Finality target restricted to #{} ({}) due to pending authority set change", target_header.number(), target_header.hash() ) } // restrict vote according to the given voting rule, if the voting rule // doesn't restrict the vote then we keep the previous target. // // we also make sure that the restricted vote is higher than the round base // (i.e. last finalized), otherwise the value returned by the given voting // rule is ignored and the original target is used instead. Ok(voting_rule .restrict_vote(client.clone(), &base_header, &best_header, &target_header) .await .filter(|(_, restricted_number)| { // we can only restrict votes within the interval [base, target] restricted_number >= base_header.number() && restricted_number < target_header.number() }) .or_else(|| Some((target_header.hash(), *target_header.number())))) } /// Whether we should process a justification for the given block. /// /// This can be used to decide whether to import a justification (when /// importing a block), or whether to generate a justification from a /// commit (when validating). Justifications for blocks that change the /// authority set will always be processed, otherwise we'll only process /// justifications if the last one was `justification_period` blocks ago. pub(crate) fn should_process_justification( client: &Client, justification_period: u32, number: NumberFor, enacts_change: bool, ) -> bool where Block: BlockT, BE: BackendT, Client: ClientForGrandpa, { if enacts_change { return true; } let last_finalized_number = client.info().finalized_number; // keep the first justification before reaching the justification period if last_finalized_number.is_zero() { return true; } last_finalized_number / justification_period.into() != number / justification_period.into() } /// Finalize the given block and apply any authority set changes. If an /// authority set change is enacted then a justification is created (if not /// given) and stored with the block when finalizing it. /// This method assumes that the block being finalized has already been imported. pub(crate) fn finalize_block( client: Arc, authority_set: &SharedAuthoritySet>, justification_generation_period: Option, hash: Block::Hash, number: NumberFor, justification_or_commit: JustificationOrCommit, initial_sync: bool, justification_sender: Option<&GrandpaJustificationSender>, telemetry: Option, ) -> Result<(), CommandOrError>> where Block: BlockT, BE: BackendT, Client: ClientForGrandpa, { // NOTE: lock must be held through writing to DB to avoid race. this lock // also implicitly synchronizes the check for last finalized number // below. let mut authority_set = authority_set.inner(); let status = client.info(); if number <= status.finalized_number && client.hash(number)? == Some(hash) { // This can happen after a forced change (triggered manually from the runtime when // finality is stalled), since the voter will be restarted at the median last finalized // block, which can be lower than the local best finalized block. warn!(target: LOG_TARGET, "Re-finalized block #{:?} ({:?}) in the canonical chain, current best finalized is #{:?}", hash, number, status.finalized_number, ); return Ok(()); } // FIXME #1483: clone only when changed let old_authority_set = authority_set.clone(); let update_res: Result<_, Error> = client.lock_import_and_run(|import_op| { let status = authority_set .apply_standard_changes( hash, number, &is_descendent_of::(&*client, None), initial_sync, None, ) .map_err(|e| Error::Safety(e.to_string()))?; // send a justification notification if a sender exists and in case of error log it. fn notify_justification( justification_sender: Option<&GrandpaJustificationSender>, justification: impl FnOnce() -> Result, Error>, ) { if let Some(sender) = justification_sender { if let Err(err) = sender.notify(justification) { warn!( target: LOG_TARGET, "Error creating justification for subscriber: {}", err ); } } } // NOTE: this code assumes that honest voters will never vote past a // transition block, thus we don't have to worry about the case where // we have a transition with `effective_block = N`, but we finalize // `N+1`. this assumption is required to make sure we store // justifications for transition blocks which will be requested by // syncing clients. let (justification_required, justification) = match justification_or_commit { JustificationOrCommit::Justification(justification) => (true, justification), JustificationOrCommit::Commit((round_number, commit)) => { let enacts_change = status.new_set_block.is_some(); let justification_required = justification_generation_period .map(|period| { should_process_justification(&*client, period, number, enacts_change) }) .unwrap_or(enacts_change); let justification = GrandpaJustification::from_commit(&client, round_number, commit)?; (justification_required, justification) }, }; notify_justification(justification_sender, || Ok(justification.clone())); let persisted_justification = if justification_required { Some((GRANDPA_ENGINE_ID, justification.encode())) } else { None }; // ideally some handle to a synchronization oracle would be used // to avoid unconditionally notifying. client .apply_finality(import_op, hash, persisted_justification, true) .map_err(|e| { warn!( target: LOG_TARGET, "Error applying finality to block {:?}: {}", (hash, number), e ); e })?; debug!(target: LOG_TARGET, "Finalizing blocks up to ({:?}, {})", number, hash); telemetry!( telemetry; CONSENSUS_INFO; "afg.finalized_blocks_up_to"; "number" => ?number, "hash" => ?hash, ); crate::aux_schema::update_best_justification(&justification, |insert| { apply_aux(import_op, insert, &[]) })?; let new_authorities = if let Some((canon_hash, canon_number)) = status.new_set_block { // the authority set has changed. let (new_id, set_ref) = authority_set.current(); if set_ref.len() > 16 { grandpa_log!( initial_sync, "👴 Applying GRANDPA set change to new set with {} authorities", set_ref.len(), ); } else { grandpa_log!( initial_sync, "👴 Applying GRANDPA set change to new set {:?}", set_ref ); } telemetry!( telemetry; CONSENSUS_INFO; "afg.generating_new_authority_set"; "number" => ?canon_number, "hash" => ?canon_hash, "authorities" => ?set_ref.to_vec(), "set_id" => ?new_id, ); Some(NewAuthoritySet { canon_hash, canon_number, set_id: new_id, authorities: set_ref.to_vec(), }) } else { None }; if status.changed { let write_result = crate::aux_schema::update_authority_set::( &authority_set, new_authorities.as_ref(), |insert| apply_aux(import_op, insert, &[]), ); if let Err(e) = write_result { warn!( target: LOG_TARGET, "Failed to write updated authority set to disk. Bailing." ); warn!(target: LOG_TARGET, "Node is in a potentially inconsistent state."); return Err(e.into()); } } Ok(new_authorities.map(VoterCommand::ChangeAuthorities)) }); match update_res { Ok(Some(command)) => Err(CommandOrError::VoterCommand(command)), Ok(None) => Ok(()), Err(e) => { *authority_set = old_authority_set; Err(CommandOrError::Error(e)) }, } }