From 08890f2bc13a0bbfac881ca7407a1c5c49eda3f8 Mon Sep 17 00:00:00 2001 From: Robert Habermeier Date: Thu, 25 Oct 2018 16:48:51 +0200 Subject: [PATCH] note future changes when importing block --- substrate/Cargo.lock | 10 +- substrate/core/consensus/common/Cargo.toml | 5 +- substrate/core/consensus/common/src/lib.rs | 11 ++- .../consensus/common/src/offline_tracker.rs | 14 +-- substrate/core/finality-grandpa/Cargo.toml | 5 +- .../core/finality-grandpa/src/authorities.rs | 42 +++++--- substrate/core/finality-grandpa/src/lib.rs | 95 +++++++++++++++++-- 7 files changed, 138 insertions(+), 44 deletions(-) diff --git a/substrate/Cargo.lock b/substrate/Cargo.lock index e3e4b66e00..0bbfc82ad0 100644 --- a/substrate/Cargo.lock +++ b/substrate/Cargo.lock @@ -580,8 +580,8 @@ dependencies = [ [[package]] name = "finality-grandpa" -version = "0.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" +version = "0.3.0" +source = "git+https://github.com/paritytech/finality-grandpa#fffce2dc450e72ff8b3f43818f8aceb5cfacc2d7" dependencies = [ "futures 0.1.25 (registry+https://github.com/rust-lang/crates.io-index)", "log 0.4.5 (registry+https://github.com/rust-lang/crates.io-index)", @@ -2965,6 +2965,7 @@ dependencies = [ name = "substrate-consensus-common" version = "0.1.0" dependencies = [ + "sr-primitives 0.1.0", "substrate-primitives 0.1.0", ] @@ -3019,7 +3020,7 @@ dependencies = [ name = "substrate-finality-grandpa" version = "0.1.0" dependencies = [ - "finality-grandpa 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)", + "finality-grandpa 0.3.0 (git+https://github.com/paritytech/finality-grandpa)", "futures 0.1.25 (registry+https://github.com/rust-lang/crates.io-index)", "log 0.4.5 (registry+https://github.com/rust-lang/crates.io-index)", "parity-codec 2.1.5 (registry+https://github.com/rust-lang/crates.io-index)", @@ -3027,6 +3028,7 @@ dependencies = [ "parking_lot 0.4.8 (registry+https://github.com/rust-lang/crates.io-index)", "sr-primitives 0.1.0", "substrate-client 0.1.0", + "substrate-consensus-common 0.1.0", "substrate-keyring 0.1.0", "substrate-network 0.1.0", "substrate-primitives 0.1.0", @@ -4179,7 +4181,7 @@ dependencies = [ "checksum failure_derive 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)" = "946d0e98a50d9831f5d589038d2ca7f8f455b1c21028c0db0e84116a12696426" "checksum fake-simd 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)" = "e88a8acf291dafb59c2d96e8f59828f3838bb1a70398823ade51a84de6a6deed" "checksum fdlimit 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)" = "b1ee15a7050e5580b3712877157068ea713b245b080ff302ae2ca973cfcd9baa" -"checksum finality-grandpa 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)" = "20d8cf871510f0d57630e75f9e65f87cba29581ccab1f78666d8b2e422d0baa6" +"checksum finality-grandpa 0.3.0 (git+https://github.com/paritytech/finality-grandpa)" = "" "checksum fixed-hash 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)" = "0d5ec8112f00ea8a483e04748a85522184418fd1cf02890b626d8fc28683f7de" "checksum fnv 1.0.6 (registry+https://github.com/rust-lang/crates.io-index)" = "2fad85553e09a6f881f739c29f0b00b0f01357c743266d478b68951ce23285f3" "checksum foreign-types 0.3.2 (registry+https://github.com/rust-lang/crates.io-index)" = "f6f339eb8adc052cd2ca78910fda869aefa38d22d5cb648e6485e4d3fc06f3b1" diff --git a/substrate/core/consensus/common/Cargo.toml b/substrate/core/consensus/common/Cargo.toml index ff5ebae0f8..ed39cf8fd4 100644 --- a/substrate/core/consensus/common/Cargo.toml +++ b/substrate/core/consensus/common/Cargo.toml @@ -4,5 +4,6 @@ version = "0.1.0" authors = ["Parity Technologies "] description = "Common utilities for substrate consensus" -[dev-dependencies] -substrate-primitives = { path= "../../primitives"} \ No newline at end of file +[dependencies] +substrate-primitives = { path = "../../primitives"} +sr-primitives = { path = "../../sr-primitives" } diff --git a/substrate/core/consensus/common/src/lib.rs b/substrate/core/consensus/common/src/lib.rs index a5bf6d218a..390e7beade 100644 --- a/substrate/core/consensus/common/src/lib.rs +++ b/substrate/core/consensus/common/src/lib.rs @@ -17,12 +17,13 @@ //! Tracks offline validators. #![allow(dead_code)] -#![cfg(feature="rhd")] - extern crate substrate_primitives as primitives; +extern crate sr_primitives; -use primitives::{generic::BlockId, Justification}; -use primitives::traits::{Block, Header}; +use sr_primitives::{generic::BlockId}; +use sr_primitives::traits::{Block, Header}; +use sr_primitives::Justification; +use primitives::AuthorityId; /// Block import trait. pub trait BlockImport { @@ -30,4 +31,4 @@ pub trait BlockImport { fn import_block(&self, block: B, justification: Justification, authorities: &[AuthorityId]) -> bool; } -pub mod offline_tracker; \ No newline at end of file +pub mod offline_tracker; diff --git a/substrate/core/consensus/common/src/offline_tracker.rs b/substrate/core/consensus/common/src/offline_tracker.rs index 18845dd68b..bd8eab8b1b 100644 --- a/substrate/core/consensus/common/src/offline_tracker.rs +++ b/substrate/core/consensus/common/src/offline_tracker.rs @@ -16,7 +16,7 @@ //! Tracks offline validators. -use node_primitives::AccountId; +use primitives::AuthorityId; use std::collections::HashMap; use std::time::{Instant, Duration}; @@ -56,7 +56,7 @@ impl Observed { /// Tracks offline validators and can issue a report for those offline. pub struct OfflineTracker { - observed: HashMap, + observed: HashMap, } impl OfflineTracker { @@ -66,7 +66,7 @@ impl OfflineTracker { } /// Note new consensus is starting with the given set of validators. - pub fn note_new_block(&mut self, validators: &[AccountId]) { + pub fn note_new_block(&mut self, validators: &[AuthorityId]) { use std::collections::HashSet; let set: HashSet<_> = validators.iter().cloned().collect(); @@ -74,14 +74,14 @@ impl OfflineTracker { } /// Note that a round has ended. - pub fn note_round_end(&mut self, validator: AccountId, was_online: bool) { + pub fn note_round_end(&mut self, validator: AuthorityId, was_online: bool) { self.observed.entry(validator) .or_insert_with(Observed::new) .note_round_end(was_online); } /// Generate a vector of indices for offline account IDs. - pub fn reports(&self, validators: &[AccountId]) -> Vec { + pub fn reports(&self, validators: &[AuthorityId]) -> Vec { validators.iter() .enumerate() .filter_map(|(i, v)| if self.is_online(v) { @@ -93,7 +93,7 @@ impl OfflineTracker { } /// Whether reports on a validator set are consistent with our view of things. - pub fn check_consistency(&self, validators: &[AccountId], reports: &[u32]) -> bool { + pub fn check_consistency(&self, validators: &[AuthorityId], reports: &[u32]) -> bool { reports.iter().cloned().all(|r| { let v = match validators.get(r as usize) { Some(v) => v, @@ -106,7 +106,7 @@ impl OfflineTracker { }) } - fn is_online(&self, v: &AccountId) -> bool { + fn is_online(&self, v: &AuthorityId) -> bool { self.observed.get(v).map(Observed::is_active).unwrap_or(true) } } diff --git a/substrate/core/finality-grandpa/Cargo.toml b/substrate/core/finality-grandpa/Cargo.toml index 88b11d7c75..34c0e99b63 100644 --- a/substrate/core/finality-grandpa/Cargo.toml +++ b/substrate/core/finality-grandpa/Cargo.toml @@ -8,14 +8,17 @@ futures = "0.1.17" parity-codec = "2.1" parity-codec-derive = "2.0" sr-primitives = { path = "../sr-primitives" } +substrate-consensus-common = { path = "../consensus/common" } substrate-primitives = { path = "../primitives" } substrate-client = { path = "../client" } substrate-network = { path = "../network" } log = "0.4" +parking_lot = "0.4" tokio = "0.1.7" [dependencies.finality-grandpa] -version = "0.2.0" +#version = "0.3.0" +git = "https://github.com/paritytech/finality-grandpa" features = ["derive-codec"] [dev-dependencies] diff --git a/substrate/core/finality-grandpa/src/authorities.rs b/substrate/core/finality-grandpa/src/authorities.rs index 31d61ef5e1..0ed2bcb36c 100644 --- a/substrate/core/finality-grandpa/src/authorities.rs +++ b/substrate/core/finality-grandpa/src/authorities.rs @@ -16,23 +16,33 @@ //! Utilities for dealing with authorities, authority sets, and handoffs. +use parking_lot::RwLock; use substrate_primitives::AuthorityId; + +use std::cmp::Ord; use std::ops::Add; +use std::sync::Arc; /// A shared authority set. pub(crate) struct SharedAuthoritySet { - inner: RwLock>, + inner: Arc>>, +} + +impl Clone for SharedAuthoritySet { + fn clone(&self) -> Self { + SharedAuthoritySet { inner: self.inner.clone() } + } } impl SharedAuthoritySet { /// The genesis authority set. - pub(crate) fn genesis(initial: Vec<(AuthorityId, usize)>) -> Self { + pub(crate) fn genesis(initial: Vec<(AuthorityId, u64)>) -> Self { SharedAuthoritySet { - inner: RwLock::new(AuthoritySet { + inner: Arc::new(RwLock::new(AuthoritySet { current_authorities: initial, set_id: 0, pending_changes: Vec::new(), - }) + })) } } @@ -42,33 +52,35 @@ impl SharedAuthoritySet { { f(&*self.inner.read()) } +} -impl SharedAuthoritySet { +impl + Ord + Clone> SharedAuthoritySet { /// Note an upcoming pending transition. pub(crate) fn add_pending_change(&self, pending: PendingChange) { - let idx = self.pending_changes - .binary_search_by_key(|change| change.effective_number()) + let mut inner = self.inner.write(); + let idx = inner.pending_changes + .binary_search_by_key(&pending.effective_number(), |change| change.effective_number()) .unwrap_or_else(|i| i); - self.pending_changes.insert(idx); + inner.pending_changes.insert(idx, pending); } /// Get the earliest limit-block number, if any. - pub(crate) fn current_limit(&self) -> Option<&N> { - self.pending_changes.get(0).map(|change| &change.effective_number()); + pub(crate) fn current_limit(&self) -> Option { + self.inner.read().pending_changes.get(0).map(|change| change.effective_number().clone()) } } impl From> for SharedAuthoritySet { fn from(set: AuthoritySet) -> Self { - SharedAuthoritySet { inner: RwLock::new(set) } + SharedAuthoritySet { inner: Arc::new(RwLock::new(set)) } } } /// A set of authorities. #[derive(Encode, Decode)] pub(crate) struct AuthoritySet { - current_authorities: Vec<(AuthorityId, usize)>, + current_authorities: Vec<(AuthorityId, u64)>, set_id: u64, pending_changes: Vec>, } @@ -80,7 +92,7 @@ pub(crate) struct AuthoritySet { #[derive(Encode, Decode)] pub(crate) struct PendingChange { /// The new authorities and weights to apply. - pub(crate) next_authorities: Vec<(AuthorityId, usize)>, + pub(crate) next_authorities: Vec<(AuthorityId, u64)>, /// How deep in the finalized chain the announcing block must be /// before the change is applied. pub(crate) finalization_depth: N, @@ -90,9 +102,9 @@ pub(crate) struct PendingChange { pub(crate) canon_hash: H, } -impl PendingChange { +impl + Clone> PendingChange { /// Returns the effective number. fn effective_number(&self) -> N { - self.canon_height + self.finalization_depth + self.canon_height.clone() + self.finalization_depth.clone() } } diff --git a/substrate/core/finality-grandpa/src/lib.rs b/substrate/core/finality-grandpa/src/lib.rs index 2a6e7814e4..6411fee6ad 100644 --- a/substrate/core/finality-grandpa/src/lib.rs +++ b/substrate/core/finality-grandpa/src/lib.rs @@ -22,9 +22,11 @@ extern crate finality_grandpa as grandpa; extern crate futures; extern crate substrate_client as client; extern crate sr_primitives as runtime_primitives; +extern crate substrate_consensus_common as consensus_common; extern crate substrate_primitives; extern crate substrate_network as network; extern crate tokio; +extern crate parking_lot; extern crate parity_codec as codec; #[macro_use] @@ -47,10 +49,11 @@ use futures::stream::Fuse; use futures::sync::mpsc; use client::{Client, ImportNotifications, backend::Backend, CallExecutor}; use codec::{Encode, Decode}; +use consensus_common::BlockImport; use runtime_primitives::traits::{ As, NumberFor, Block as BlockT, Header as HeaderT, DigestItemFor, }; -use runtime_primitives::generic::BlockId; +use runtime_primitives::{generic::BlockId, Justification}; use substrate_primitives::{ed25519, AuthorityId, Blake2Hasher}; use tokio::timer::Interval; @@ -61,6 +64,8 @@ use std::collections::{VecDeque, HashMap}; use std::sync::Arc; use std::time::{Instant, Duration}; +use authorities::SharedAuthoritySet; + mod authorities; const LAST_COMPLETED_KEY: &[u8] = b"grandpa_completed_round"; @@ -404,8 +409,9 @@ fn outgoing_messages( /// The environment we run GRANDPA in. pub struct Environment { inner: Arc>, - voters: HashMap, + voters: HashMap, config: Config, + authority_set: SharedAuthoritySet>, network: N, } @@ -461,14 +467,23 @@ impl grandpa::Chain for Environment { + /// The new authorities after the change, along with their respective weights. + pub next_authorities: Vec<(AuthorityId, u64)>, + /// The number of blocks to delay. + pub delay: N, +} + /// A GRANDPA-compatible DigestItem. This can describe when GRANDPA set changes /// are scheduled. // TODO: with specialization, do a blanket implementation so this trait // doesn't have to be implemented by users. pub trait CompatibleDigestItem { - /// If this digest item notes a GRANDPA set change, return the number of - /// blocks the change should occur after. - fn scheduled_change_in(&self) -> Option { None } + /// If this digest item notes a GRANDPA set change, return information about + /// the scheduled change. + fn scheduled_change(&self) -> Option> { None } } impl voter::Environment for Environment where @@ -487,6 +502,7 @@ impl voter::Environment for Environment, SinkError = Self::Error>>; type Error = Error; + #[allow(unreachable_code)] fn round_data( &self, round: u64 @@ -498,7 +514,9 @@ impl voter::Environment for Environment( round, @@ -511,7 +529,7 @@ impl voter::Environment for Environment voter::Environment for Environment { + inner: Arc>, + authority_set: SharedAuthoritySet>, +} + +impl BlockImport for GrandpaBlockImport where + B: Backend + 'static, + E: CallExecutor + 'static, + DigestItemFor: CompatibleDigestItem>, +{ + fn import_block(&self, block: Block, _justification: Justification, _authorities: &[AuthorityId]) -> bool { + use runtime_primitives::traits::Digest; + use authorities::PendingChange; + + let maybe_event = block.header().digest().logs().iter() + .filter_map(|log| log.scheduled_change()) + .next() + .map(|change| (block.header().hash(), *block.header().number(), change)); + + // TODO [now]: use import-block trait for client when implemented + let result = self.inner.import_block(unimplemented!(), unimplemented!()).is_ok(); + if let (true, Some((hash, number, change))) = (result, maybe_event) { + self.authority_set.add_pending_change(PendingChange { + next_authorities: change.next_authorities, + finalization_depth: number + change.delay, + canon_height: number, + canon_hash: hash, + }); + + // TODO [now]: write to DB, and what to do on failure? + } + result + } +} + +/// Run a GRANDPA voter as a task. This returns two pieces of data: a task to run, +/// and a `BlockImport` implementation. pub fn run_grandpa( config: Config, client: Arc>, - voters: HashMap, + voters: HashMap, network: N, -) -> Result,client::error::Error> where +) -> ::client::error::Result<( + impl Future, + GrandpaBlockImport, +)> where Block::Hash: Ord, B: Backend + 'static, E: CallExecutor + 'static, @@ -614,11 +676,22 @@ pub fn run_grandpa( ))? }; + // TODO: attempt to load from disk. + let authority_set = SharedAuthoritySet::genesis( + voters.iter().map(|(&id, &weight)| (id, weight)).collect(), + ); + + let block_import = GrandpaBlockImport { + inner: client.clone(), + authority_set: authority_set.clone(), + }; + let environment = Arc::new(Environment { inner: client, config, voters, network, + authority_set, }); let voter = voter::Voter::new( @@ -628,7 +701,9 @@ pub fn run_grandpa( last_finalized, ); - Ok(voter.map_err(|e| warn!("GRANDPA Voter failed: {:?}", e))) + let work = voter.map_err(|e| warn!("GRANDPA Voter failed: {:?}", e)); + + Ok((work, block_import)) } #[cfg(test)]