client/beefy: persist voter state (#12712)

* client/beefy: prepare worker for persisting state

* client/beefy: persist voter state

* client/beefy: initialize persistent state

* client/beefy: try to vote from the very beginning

Now that voter is initialized from persistent state, it makes
sense that it can attempt voting right away. This also helps
the genesis case when we consider block `One` as mandatory.

* client/beefy: add tests for voter state db
* client/beefy: persist voter state as soon as initialized
* client/beefy: make sure min-block-delta is at least 1
* client/beefy: persist state after voting

Persist state after handling self vote to avoid double voting in case
of voter restarts.

* client/beefy: persist state after handling mandatory block vote

For mandatory blocks we want to make sure we're not losing votes
in case of crashes or restarts, since voter will not make further
progress without finalizing them.

* frame/beefy: use GENESIS_AUTHORITY_SET_ID on pallet genesis

* client/beefy: initialize voter at either genesis or last finalized
To guarantee unbroken chain of mandatory blocks justifications, voter
will always resume from either last BEEFY-justified block or
`pallet-beefy` genesis, whichever is more recent.

Initialization walks back the chain from latest GRANDPA finalized
block looking for one of the above. Along the way, it also records
and enqueues for processing any BEEFY mandatory blocks that have
been already GRANDPA finalized but not BEEFY finalized.

* client/beefy: decouple voter init from aux db state load
* client/beefy: fix voter init tests
* remove debug prints
* gadget future must be type ()
* fix init from last justification

Signed-off-by: Adrian Catangiu <adrian@parity.io>

substrate/client/beefy/src/aux_schema.rs

@@ -0,0 +1,105 @@
+// This file is part of Substrate.
+
+// Copyright (C) 2019-2022 Parity Technologies (UK) Ltd.
+// SPDX-License-Identifier: GPL-3.0-or-later WITH Classpath-exception-2.0
+
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with this program. If not, see <https://www.gnu.org/licenses/>.
+
+//! Schema for BEEFY state persisted in the aux-db.
+
+use crate::worker::PersistedState;
+use codec::{Decode, Encode};
+use log::{info, trace};
+use sc_client_api::{backend::AuxStore, Backend};
+use sp_blockchain::{Error as ClientError, Result as ClientResult};
+use sp_runtime::traits::Block as BlockT;
+
+const VERSION_KEY: &[u8] = b"beefy_auxschema_version";
+const WORKER_STATE: &[u8] = b"beefy_voter_state";
+
+const CURRENT_VERSION: u32 = 1;
+
+pub(crate) fn write_current_version<B: AuxStore>(backend: &B) -> ClientResult<()> {
+	info!(target: "beefy", "🥩 write aux schema version {:?}", CURRENT_VERSION);
+	AuxStore::insert_aux(backend, &[(VERSION_KEY, CURRENT_VERSION.encode().as_slice())], &[])
+}
+
+/// Write voter state.
+pub(crate) fn write_voter_state<Block: BlockT, B: AuxStore>(
+	backend: &B,
+	state: &PersistedState<Block>,
+) -> ClientResult<()> {
+	trace!(target: "beefy", "🥩 persisting {:?}", state);
+	backend.insert_aux(&[(WORKER_STATE, state.encode().as_slice())], &[])
+}
+
+fn load_decode<B: AuxStore, T: Decode>(backend: &B, key: &[u8]) -> ClientResult<Option<T>> {
+	match backend.get_aux(key)? {
+		None => Ok(None),
+		Some(t) => T::decode(&mut &t[..])
+			.map_err(|e| ClientError::Backend(format!("BEEFY DB is corrupted: {}", e)))
+			.map(Some),
+	}
+}
+
+/// Load or initialize persistent data from backend.
+pub(crate) fn load_persistent<B, BE>(backend: &BE) -> ClientResult<Option<PersistedState<B>>>
+where
+	B: BlockT,
+	BE: Backend<B>,
+{
+	let version: Option<u32> = load_decode(backend, VERSION_KEY)?;
+
+	match version {
+		None => (),
+		Some(1) => return load_decode::<_, PersistedState<B>>(backend, WORKER_STATE),
+		other =>
+			return Err(ClientError::Backend(format!("Unsupported BEEFY DB version: {:?}", other))),
+	}
+
+	// No persistent state found in DB.
+	Ok(None)
+}
+
+#[cfg(test)]
+pub(crate) mod tests {
+	use super::*;
+	use crate::tests::BeefyTestNet;
+	use sc_network_test::TestNetFactory;
+
+	// also used in tests.rs
+	pub fn verify_persisted_version<B: BlockT, BE: Backend<B>>(backend: &BE) -> bool {
+		let version: u32 = load_decode(backend, VERSION_KEY).unwrap().unwrap();
+		version == CURRENT_VERSION
+	}
+
+	#[test]
+	fn should_load_persistent_sanity_checks() {
+		let mut net = BeefyTestNet::new(1);
+		let backend = net.peer(0).client().as_backend();
+
+		// version not available in db -> None
+		assert_eq!(load_persistent(&*backend).unwrap(), None);
+
+		// populate version in db
+		write_current_version(&*backend).unwrap();
+		// verify correct version is retrieved
+		assert_eq!(load_decode(&*backend, VERSION_KEY).unwrap(), Some(CURRENT_VERSION));
+
+		// version is available in db but state isn't -> None
+		assert_eq!(load_persistent(&*backend).unwrap(), None);
+
+		// full `PersistedState` load is tested in `tests.rs`.
+	}
+}

substrate/client/beefy/src/lib.rs

@@ -16,35 +16,6 @@
 // You should have received a copy of the GNU General Public License
 // along with this program. If not, see <https://www.gnu.org/licenses/>.
 
-use beefy_primitives::{BeefyApi, MmrRootHash, PayloadProvider};
-use parking_lot::Mutex;
-use prometheus::Registry;
-use sc_client_api::{Backend, BlockBackend, BlockchainEvents, Finalizer};
-use sc_consensus::BlockImport;
-use sc_network::ProtocolName;
-use sc_network_common::service::NetworkRequest;
-use sc_network_gossip::Network as GossipNetwork;
-use sp_api::{NumberFor, ProvideRuntimeApi};
-use sp_blockchain::HeaderBackend;
-use sp_consensus::{Error as ConsensusError, SyncOracle};
-use sp_keystore::SyncCryptoStorePtr;
-use sp_mmr_primitives::MmrApi;
-use sp_runtime::traits::Block;
-use std::{marker::PhantomData, sync::Arc};
-
-mod error;
-mod keystore;
-mod metrics;
-mod round;
-mod worker;
-
-pub mod communication;
-pub mod import;
-pub mod justification;
-
-#[cfg(test)]
-mod tests;
-
 use crate::{
 	communication::{
 		notification::{
@@ -57,12 +28,53 @@ use crate::{
 		},
 	},
 	import::BeefyBlockImport,
+	round::Rounds,
+	worker::PersistedState,
 };
+use beefy_primitives::{
+	crypto::AuthorityId, BeefyApi, MmrRootHash, PayloadProvider, ValidatorSet, BEEFY_ENGINE_ID,
+	GENESIS_AUTHORITY_SET_ID,
+};
+use futures::{stream::Fuse, StreamExt};
+use log::{debug, error, info};
+use parking_lot::Mutex;
+use prometheus::Registry;
+use sc_client_api::{Backend, BlockBackend, BlockchainEvents, FinalityNotifications, Finalizer};
+use sc_consensus::BlockImport;
+use sc_network::ProtocolName;
+use sc_network_common::service::NetworkRequest;
+use sc_network_gossip::{GossipEngine, Network as GossipNetwork};
+use sp_api::{HeaderT, NumberFor, ProvideRuntimeApi};
+use sp_blockchain::{
+	Backend as BlockchainBackend, Error as ClientError, HeaderBackend, Result as ClientResult,
+};
+use sp_consensus::{Error as ConsensusError, SyncOracle};
+use sp_keystore::SyncCryptoStorePtr;
+use sp_mmr_primitives::MmrApi;
+use sp_runtime::{
+	generic::BlockId,
+	traits::{Block, One, Zero},
+};
+use std::{collections::VecDeque, marker::PhantomData, sync::Arc};
+
+mod aux_schema;
+mod error;
+mod keystore;
+mod metrics;
+mod round;
+mod worker;
+
+pub mod communication;
+pub mod import;
+pub mod justification;
 
 pub use communication::beefy_protocol_name::{
 	gossip_protocol_name, justifications_protocol_name as justifs_protocol_name,
 };
 
+#[cfg(test)]
+mod tests;
+
 /// A convenience BEEFY client trait that defines all the type bounds a BEEFY client
 /// has to satisfy. Ideally that should actually be a trait alias. Unfortunately as
 /// of today, Rust does not allow a type alias to be used as a trait bound. Tracking
@@ -222,7 +234,7 @@ where
 	let known_peers = Arc::new(Mutex::new(KnownPeers::new()));
 	let gossip_validator =
 		Arc::new(communication::gossip::GossipValidator::new(known_peers.clone()));
-	let gossip_engine = sc_network_gossip::GossipEngine::new(
+	let mut gossip_engine = sc_network_gossip::GossipEngine::new(
 		network.clone(),
 		gossip_protocol_name,
 		gossip_validator.clone(),
@@ -240,21 +252,38 @@ where
 		prometheus_registry.as_ref().map(metrics::Metrics::register).and_then(
 			|result| match result {
 				Ok(metrics) => {
-					log::debug!(target: "beefy", "🥩 Registered metrics");
+					debug!(target: "beefy", "🥩 Registered metrics");
 					Some(metrics)
 				},
 				Err(err) => {
-					log::debug!(target: "beefy", "🥩 Failed to register metrics: {:?}", err);
+					debug!(target: "beefy", "🥩 Failed to register metrics: {:?}", err);
 					None
 				},
 			},
 		);
 
+	// Subscribe to finality notifications and justifications before waiting for runtime pallet and
+	// reuse the streams, so we don't miss notifications while waiting for pallet to be available.
+	let mut finality_notifications = client.finality_notification_stream().fuse();
+	let block_import_justif = links.from_block_import_justif_stream.subscribe().fuse();
+
+	// Wait for BEEFY pallet to be active before starting voter.
+	let persisted_state =
+		match wait_for_runtime_pallet(&*runtime, &mut gossip_engine, &mut finality_notifications)
+			.await
+			.and_then(|best_grandpa| {
+				load_or_init_voter_state(&*backend, &*runtime, best_grandpa, min_block_delta)
+			}) {
+			Ok(state) => state,
+			Err(e) => {
+				error!(target: "beefy", "Error: {:?}. Terminating.", e);
+				return
+			},
+		};
+
 	let worker_params = worker::WorkerParams {
-		client,
 		backend,
 		payload_provider,
-		runtime,
 		network,
 		key_store: key_store.into(),
 		known_peers,
@@ -263,10 +292,195 @@ where
 		on_demand_justifications,
 		links,
 		metrics,
-		min_block_delta,
+		persisted_state,
 	};
 
-	let worker = worker::BeefyWorker::<_, _, _, _, _, _>::new(worker_params);
+	let worker = worker::BeefyWorker::<_, _, _, _, _>::new(worker_params);
 
-	futures::future::join(worker.run(), on_demand_justifications_handler.run()).await;
+	futures::future::join(
+		worker.run(block_import_justif, finality_notifications),
+		on_demand_justifications_handler.run(),
+	)
+	.await;
+}
+
+fn load_or_init_voter_state<B, BE, R>(
+	backend: &BE,
+	runtime: &R,
+	best_grandpa: <B as Block>::Header,
+	min_block_delta: u32,
+) -> ClientResult<PersistedState<B>>
+where
+	B: Block,
+	BE: Backend<B>,
+	R: ProvideRuntimeApi<B>,
+	R::Api: BeefyApi<B>,
+{
+	// Initialize voter state from AUX DB or from pallet genesis.
+	if let Some(mut state) = crate::aux_schema::load_persistent(backend)? {
+		// Overwrite persisted state with current best GRANDPA block.
+		state.set_best_grandpa(best_grandpa);
+		// Overwrite persisted data with newly provided `min_block_delta`.
+		state.set_min_block_delta(min_block_delta);
+		info!(target: "beefy", "🥩 Loading BEEFY voter state from db: {:?}.", state);
+		Ok(state)
+	} else {
+		initialize_voter_state(backend, runtime, best_grandpa, min_block_delta)
+	}
+}
+
+// If no persisted state present, walk back the chain from first GRANDPA notification to either:
+//  - latest BEEFY finalized block, or if none found on the way,
+//  - BEEFY pallet genesis;
+// Enqueue any BEEFY mandatory blocks (session boundaries) found on the way, for voter to finalize.
+fn initialize_voter_state<B, BE, R>(
+	backend: &BE,
+	runtime: &R,
+	best_grandpa: <B as Block>::Header,
+	min_block_delta: u32,
+) -> ClientResult<PersistedState<B>>
+where
+	B: Block,
+	BE: Backend<B>,
+	R: ProvideRuntimeApi<B>,
+	R::Api: BeefyApi<B>,
+{
+	// Walk back the imported blocks and initialize voter either, at the last block with
+	// a BEEFY justification, or at pallet genesis block; voter will resume from there.
+	let blockchain = backend.blockchain();
+	let mut sessions = VecDeque::new();
+	let mut header = best_grandpa.clone();
+	let state = loop {
+		if let Some(true) = blockchain
+			.justifications(header.hash())
+			.ok()
+			.flatten()
+			.map(|justifs| justifs.get(BEEFY_ENGINE_ID).is_some())
+		{
+			info!(
+				target: "beefy",
+				"🥩 Initialize BEEFY voter at last BEEFY finalized block: {:?}.",
+				*header.number()
+			);
+			let best_beefy = *header.number();
+			// If no session boundaries detected so far, just initialize new rounds here.
+			if sessions.is_empty() {
+				let active_set = expect_validator_set(runtime, BlockId::hash(header.hash()))?;
+				let mut rounds = Rounds::new(best_beefy, active_set);
+				// Mark the round as already finalized.
+				rounds.conclude(best_beefy);
+				sessions.push_front(rounds);
+			}
+			let state =
+				PersistedState::checked_new(best_grandpa, best_beefy, sessions, min_block_delta)
+					.ok_or_else(|| ClientError::Backend("Invalid BEEFY chain".into()))?;
+			break state
+		}
+
+		// Check if we should move up the chain.
+		let parent_hash = *header.parent_hash();
+		if *header.number() == One::one() ||
+			runtime
+				.runtime_api()
+				.validator_set(&BlockId::hash(parent_hash))
+				.ok()
+				.flatten()
+				.is_none()
+		{
+			// We've reached pallet genesis, initialize voter here.
+			let genesis_num = *header.number();
+			let genesis_set = expect_validator_set(runtime, BlockId::hash(header.hash()))
+				.and_then(genesis_set_sanity_check)?;
+			info!(
+				target: "beefy",
+				"🥩 Loading BEEFY voter state from genesis on what appears to be first startup. \
+				Starting voting rounds at block {:?}, genesis validator set {:?}.",
+				genesis_num, genesis_set,
+			);
+
+			sessions.push_front(Rounds::new(genesis_num, genesis_set));
+			break PersistedState::checked_new(best_grandpa, Zero::zero(), sessions, min_block_delta)
+				.ok_or_else(|| ClientError::Backend("Invalid BEEFY chain".into()))?
+		}
+
+		if let Some(active) = worker::find_authorities_change::<B>(&header) {
+			info!(target: "beefy", "🥩 Marking block {:?} as BEEFY Mandatory.", *header.number());
+			sessions.push_front(Rounds::new(*header.number(), active));
+		}
+
+		// Move up the chain.
+		header = blockchain.expect_header(BlockId::Hash(parent_hash))?;
+	};
+
+	aux_schema::write_current_version(backend)?;
+	aux_schema::write_voter_state(backend, &state)?;
+	Ok(state)
+}
+
+/// Wait for BEEFY runtime pallet to be available, return active validator set.
+/// Should be called only once during worker initialization.
+async fn wait_for_runtime_pallet<B, R>(
+	runtime: &R,
+	mut gossip_engine: &mut GossipEngine<B>,
+	finality: &mut Fuse<FinalityNotifications<B>>,
+) -> ClientResult<<B as Block>::Header>
+where
+	B: Block,
+	R: ProvideRuntimeApi<B>,
+	R::Api: BeefyApi<B>,
+{
+	info!(target: "beefy", "🥩 BEEFY gadget waiting for BEEFY pallet to become available...");
+	loop {
+		futures::select! {
+			notif = finality.next() => {
+				let notif = match notif {
+					Some(notif) => notif,
+					None => break
+				};
+				let at = BlockId::hash(notif.header.hash());
+				if let Some(active) = runtime.runtime_api().validator_set(&at).ok().flatten() {
+					// Beefy pallet available, return best grandpa at the time.
+					info!(
+						target: "beefy", "🥩 BEEFY pallet available: block {:?} validator set {:?}",
+						notif.header.number(), active
+					);
+					return Ok(notif.header)
+				}
+			},
+			_ = gossip_engine => {
+				break
+			}
+		}
+	}
+	let err_msg = "🥩 Gossip engine has unexpectedly terminated.".into();
+	error!(target: "beefy", "{}", err_msg);
+	Err(ClientError::Backend(err_msg))
+}
+
+fn genesis_set_sanity_check(
+	active: ValidatorSet<AuthorityId>,
+) -> ClientResult<ValidatorSet<AuthorityId>> {
+	if active.id() == GENESIS_AUTHORITY_SET_ID {
+		Ok(active)
+	} else {
+		error!(target: "beefy", "🥩 Unexpected ID for genesis validator set {:?}.", active);
+		Err(ClientError::Backend("BEEFY Genesis sanity check failed.".into()))
+	}
+}
+
+fn expect_validator_set<B, R>(
+	runtime: &R,
+	at: BlockId<B>,
+) -> ClientResult<ValidatorSet<AuthorityId>>
+where
+	B: Block,
+	R: ProvideRuntimeApi<B>,
+	R::Api: BeefyApi<B>,
+{
+	runtime
+		.runtime_api()
+		.validator_set(&at)
+		.ok()
+		.flatten()
+		.ok_or_else(|| ClientError::Backend("BEEFY pallet expected to be active.".into()))
 }

substrate/client/beefy/src/round.rs

@@ -16,27 +16,23 @@
 // You should have received a copy of the GNU General Public License
 // along with this program. If not, see <https://www.gnu.org/licenses/>.
 
-use std::{
-	collections::{BTreeMap, HashMap},
-	hash::Hash,
-};
-
-use log::{debug, trace};
-
 use beefy_primitives::{
 	crypto::{Public, Signature},
 	ValidatorSet, ValidatorSetId,
 };
+use codec::{Decode, Encode};
+use log::{debug, trace};
 use sp_runtime::traits::{Block, NumberFor};
+use std::{collections::BTreeMap, hash::Hash};
 
 /// Tracks for each round which validators have voted/signed and
 /// whether the local `self` validator has voted/signed.
 ///
 /// Does not do any validation on votes or signatures, layers above need to handle that (gossip).
-#[derive(Debug, Default)]
+#[derive(Debug, Decode, Default, Encode, PartialEq)]
 struct RoundTracker {
 	self_vote: bool,
-	votes: HashMap<Public, Signature>,
+	votes: BTreeMap<Public, Signature>,
 }
 
 impl RoundTracker {
@@ -69,7 +65,7 @@ pub fn threshold(authorities: usize) -> usize {
 /// Only round numbers > `best_done` are of interest, all others are considered stale.
 ///
 /// Does not do any validation on votes or signatures, layers above need to handle that (gossip).
-#[derive(Debug)]
+#[derive(Debug, Decode, Encode, PartialEq)]
 pub(crate) struct Rounds<Payload, B: Block> {
 	rounds: BTreeMap<(Payload, NumberFor<B>), RoundTracker>,
 	session_start: NumberFor<B>,

substrate/client/beefy/src/tests.rs

@@ -18,47 +18,8 @@
 
 //! Tests and test helpers for BEEFY.
 
-use futures::{future, stream::FuturesUnordered, Future, StreamExt};
-use parking_lot::Mutex;
-use serde::{Deserialize, Serialize};
-use std::{collections::HashMap, marker::PhantomData, sync::Arc, task::Poll};
-use tokio::{runtime::Runtime, time::Duration};
-
-use sc_client_api::HeaderBackend;
-use sc_consensus::{
-	BlockImport, BlockImportParams, BoxJustificationImport, ForkChoiceStrategy, ImportResult,
-	ImportedAux,
-};
-use sc_network_test::{
-	Block, BlockImportAdapter, FullPeerConfig, PassThroughVerifier, Peer, PeersClient,
-	PeersFullClient, TestNetFactory,
-};
-use sc_utils::notification::NotificationReceiver;
-use sp_keystore::testing::KeyStore as TestKeystore;
-
-use beefy_primitives::{
-	crypto::{AuthorityId, Signature},
-	mmr::MmrRootProvider,
-	BeefyApi, ConsensusLog, MmrRootHash, ValidatorSet, VersionedFinalityProof, BEEFY_ENGINE_ID,
-	KEY_TYPE as BeefyKeyType,
-};
-use sc_network::{config::RequestResponseConfig, ProtocolName};
-use sp_mmr_primitives::{EncodableOpaqueLeaf, Error as MmrError, MmrApi, Proof};
-
-use sp_api::{ApiRef, ProvideRuntimeApi};
-use sp_consensus::BlockOrigin;
-use sp_core::H256;
-use sp_keystore::{SyncCryptoStore, SyncCryptoStorePtr};
-use sp_runtime::{
-	codec::Encode,
-	generic::BlockId,
-	traits::{Header as HeaderT, NumberFor},
-	BuildStorage, DigestItem, Justifications, Storage,
-};
-
-use substrate_test_runtime_client::{runtime::Header, ClientExt};
-
 use crate::{
+	aux_schema::{load_persistent, tests::verify_persisted_version},
 	beefy_block_import_and_links,
 	communication::request_response::{
 		on_demand_justifications_protocol_config, BeefyJustifsRequestHandler,
@@ -66,8 +27,44 @@ use crate::{
 	gossip_protocol_name,
 	justification::*,
 	keystore::tests::Keyring as BeefyKeyring,
-	BeefyRPCLinks, BeefyVoterLinks,
+	load_or_init_voter_state, wait_for_runtime_pallet, BeefyRPCLinks, BeefyVoterLinks, KnownPeers,
+	PersistedState,
 };
+use beefy_primitives::{
+	crypto::{AuthorityId, Signature},
+	known_payloads,
+	mmr::MmrRootProvider,
+	BeefyApi, Commitment, ConsensusLog, MmrRootHash, Payload, SignedCommitment, ValidatorSet,
+	VersionedFinalityProof, BEEFY_ENGINE_ID, KEY_TYPE as BeefyKeyType,
+};
+use futures::{future, stream::FuturesUnordered, Future, StreamExt};
+use parking_lot::Mutex;
+use sc_client_api::{Backend as BackendT, BlockchainEvents, FinalityNotifications, HeaderBackend};
+use sc_consensus::{
+	BlockImport, BlockImportParams, BoxJustificationImport, ForkChoiceStrategy, ImportResult,
+	ImportedAux,
+};
+use sc_network::{config::RequestResponseConfig, ProtocolName};
+use sc_network_test::{
+	Block, BlockImportAdapter, FullPeerConfig, PassThroughVerifier, Peer, PeersClient,
+	PeersFullClient, TestNetFactory,
+};
+use sc_utils::notification::NotificationReceiver;
+use serde::{Deserialize, Serialize};
+use sp_api::{ApiRef, ProvideRuntimeApi};
+use sp_consensus::BlockOrigin;
+use sp_core::H256;
+use sp_keystore::{testing::KeyStore as TestKeystore, SyncCryptoStore, SyncCryptoStorePtr};
+use sp_mmr_primitives::{EncodableOpaqueLeaf, Error as MmrError, MmrApi, Proof};
+use sp_runtime::{
+	codec::Encode,
+	generic::BlockId,
+	traits::{Header as HeaderT, NumberFor},
+	BuildStorage, DigestItem, Justifications, Storage,
+};
+use std::{collections::HashMap, marker::PhantomData, sync::Arc, task::Poll};
+use substrate_test_runtime_client::{runtime::Header, ClientExt};
+use tokio::{runtime::Runtime, time::Duration};
 
 const GENESIS_HASH: H256 = H256::zero();
 fn beefy_gossip_proto_name() -> ProtocolName {
@@ -531,7 +528,7 @@ fn beefy_finalizing_blocks() {
 
 	let peers = peers.into_iter().enumerate();
 	// finalize block #5 -> BEEFY should finalize #1 (mandatory) and #5 from diff-power-of-two rule.
-	finalize_block_and_wait_for_beefy(&net, peers.clone(), &mut runtime, &[5], &[1, 5]);
+	finalize_block_and_wait_for_beefy(&net, peers.clone(), &mut runtime, &[1, 5], &[1, 5]);
 
 	// GRANDPA finalize #10 -> BEEFY finalize #10 (mandatory)
 	finalize_block_and_wait_for_beefy(&net, peers.clone(), &mut runtime, &[10], &[10]);
@@ -573,7 +570,7 @@ fn lagging_validators() {
 		&net,
 		peers.clone(),
 		&mut runtime,
-		&[15],
+		&[1, 15],
 		&[1, 9, 13, 14, 15],
 	);
 
@@ -661,7 +658,7 @@ fn correct_beefy_payload() {
 	let net = Arc::new(Mutex::new(net));
 	let peers = peers.into_iter().enumerate();
 	// with 3 good voters and 1 bad one, consensus should happen and best blocks produced.
-	finalize_block_and_wait_for_beefy(&net, peers, &mut runtime, &[10], &[1, 9]);
+	finalize_block_and_wait_for_beefy(&net, peers, &mut runtime, &[1, 10], &[1, 9]);
 
 	let (best_blocks, versioned_finality_proof) =
 		get_beefy_streams(&mut net.lock(), [(0, BeefyKeyring::Alice)].into_iter());
@@ -945,3 +942,187 @@ fn on_demand_beefy_justification_sync() {
 	// Now that Dave has caught up, sanity check voting works for all of them.
 	finalize_block_and_wait_for_beefy(&net, all_peers, &mut runtime, &[30], &[30]);
 }
+
+fn test_voter_init_setup(
+	net: &mut BeefyTestNet,
+	finality: &mut futures::stream::Fuse<FinalityNotifications<Block>>,
+) -> sp_blockchain::Result<PersistedState<Block>> {
+	let backend = net.peer(0).client().as_backend();
+	let api = Arc::new(crate::tests::two_validators::TestApi {});
+	let known_peers = Arc::new(Mutex::new(KnownPeers::new()));
+	let gossip_validator =
+		Arc::new(crate::communication::gossip::GossipValidator::new(known_peers));
+	let mut gossip_engine = sc_network_gossip::GossipEngine::new(
+		net.peer(0).network_service().clone(),
+		"/beefy/whatever",
+		gossip_validator,
+		None,
+	);
+	let best_grandpa =
+		futures::executor::block_on(wait_for_runtime_pallet(&*api, &mut gossip_engine, finality))
+			.unwrap();
+	load_or_init_voter_state(&*backend, &*api, best_grandpa, 1)
+}
+
+#[test]
+fn should_initialize_voter_at_genesis() {
+	let keys = &[BeefyKeyring::Alice];
+	let validator_set = ValidatorSet::new(make_beefy_ids(keys), 0).unwrap();
+	let mut net = BeefyTestNet::new(1);
+	let backend = net.peer(0).client().as_backend();
+
+	// push 15 blocks with `AuthorityChange` digests every 10 blocks
+	net.generate_blocks_and_sync(15, 10, &validator_set, false);
+
+	let mut finality = net.peer(0).client().as_client().finality_notification_stream().fuse();
+
+	// finalize 13 without justifications
+	let hashof13 = backend.blockchain().expect_block_hash_from_id(&BlockId::Number(13)).unwrap();
+	net.peer(0).client().as_client().finalize_block(hashof13, None).unwrap();
+
+	// load persistent state - nothing in DB, should init at session boundary
+	let persisted_state = test_voter_init_setup(&mut net, &mut finality).unwrap();
+
+	// Test initialization at session boundary.
+	// verify voter initialized with two sessions starting at blocks 1 and 10
+	let sessions = persisted_state.voting_oracle().sessions();
+	assert_eq!(sessions.len(), 2);
+	assert_eq!(sessions[0].session_start(), 1);
+	assert_eq!(sessions[1].session_start(), 10);
+	let rounds = persisted_state.active_round().unwrap();
+	assert_eq!(rounds.session_start(), 1);
+	assert_eq!(rounds.validator_set_id(), validator_set.id());
+
+	// verify next vote target is mandatory block 1
+	assert_eq!(persisted_state.best_beefy_block(), 0);
+	assert_eq!(persisted_state.best_grandpa_block(), 13);
+	assert_eq!(
+		persisted_state
+			.voting_oracle()
+			.voting_target(persisted_state.best_beefy_block(), 13),
+		Some(1)
+	);
+
+	// verify state also saved to db
+	assert!(verify_persisted_version(&*backend));
+	let state = load_persistent(&*backend).unwrap().unwrap();
+	assert_eq!(state, persisted_state);
+}
+
+#[test]
+fn should_initialize_voter_when_last_final_is_session_boundary() {
+	let keys = &[BeefyKeyring::Alice];
+	let validator_set = ValidatorSet::new(make_beefy_ids(keys), 0).unwrap();
+	let mut net = BeefyTestNet::new(1);
+	let backend = net.peer(0).client().as_backend();
+
+	// push 15 blocks with `AuthorityChange` digests every 10 blocks
+	net.generate_blocks_and_sync(15, 10, &validator_set, false);
+
+	let mut finality = net.peer(0).client().as_client().finality_notification_stream().fuse();
+
+	// finalize 13 without justifications
+	let hashof13 = backend.blockchain().expect_block_hash_from_id(&BlockId::Number(13)).unwrap();
+	net.peer(0).client().as_client().finalize_block(hashof13, None).unwrap();
+
+	// import/append BEEFY justification for session boundary block 10
+	let commitment = Commitment {
+		payload: Payload::from_single_entry(known_payloads::MMR_ROOT_ID, vec![]),
+		block_number: 10,
+		validator_set_id: validator_set.id(),
+	};
+	let justif = VersionedFinalityProof::<_, Signature>::V1(SignedCommitment {
+		commitment,
+		signatures: vec![None],
+	});
+	let hashof10 = backend.blockchain().expect_block_hash_from_id(&BlockId::Number(10)).unwrap();
+	backend
+		.append_justification(hashof10, (BEEFY_ENGINE_ID, justif.encode()))
+		.unwrap();
+
+	// Test corner-case where session boundary == last beefy finalized,
+	// expect rounds initialized at last beefy finalized 10.
+
+	// load persistent state - nothing in DB, should init at session boundary
+	let persisted_state = test_voter_init_setup(&mut net, &mut finality).unwrap();
+
+	// verify voter initialized with single session starting at block 10
+	assert_eq!(persisted_state.voting_oracle().sessions().len(), 1);
+	let rounds = persisted_state.active_round().unwrap();
+	assert_eq!(rounds.session_start(), 10);
+	assert_eq!(rounds.validator_set_id(), validator_set.id());
+
+	// verify block 10 is correctly marked as finalized
+	assert_eq!(persisted_state.best_beefy_block(), 10);
+	assert_eq!(persisted_state.best_grandpa_block(), 13);
+	// verify next vote target is diff-power-of-two block 12
+	assert_eq!(
+		persisted_state
+			.voting_oracle()
+			.voting_target(persisted_state.best_beefy_block(), 13),
+		Some(12)
+	);
+
+	// verify state also saved to db
+	assert!(verify_persisted_version(&*backend));
+	let state = load_persistent(&*backend).unwrap().unwrap();
+	assert_eq!(state, persisted_state);
+}
+
+#[test]
+fn should_initialize_voter_at_latest_finalized() {
+	let keys = &[BeefyKeyring::Alice];
+	let validator_set = ValidatorSet::new(make_beefy_ids(keys), 0).unwrap();
+	let mut net = BeefyTestNet::new(1);
+	let backend = net.peer(0).client().as_backend();
+
+	// push 15 blocks with `AuthorityChange` digests every 10 blocks
+	net.generate_blocks_and_sync(15, 10, &validator_set, false);
+
+	let mut finality = net.peer(0).client().as_client().finality_notification_stream().fuse();
+
+	// finalize 13 without justifications
+	let hashof13 = backend.blockchain().expect_block_hash_from_id(&BlockId::Number(13)).unwrap();
+	net.peer(0).client().as_client().finalize_block(hashof13, None).unwrap();
+
+	// import/append BEEFY justification for block 12
+	let commitment = Commitment {
+		payload: Payload::from_single_entry(known_payloads::MMR_ROOT_ID, vec![]),
+		block_number: 12,
+		validator_set_id: validator_set.id(),
+	};
+	let justif = VersionedFinalityProof::<_, Signature>::V1(SignedCommitment {
+		commitment,
+		signatures: vec![None],
+	});
+	let hashof12 = backend.blockchain().expect_block_hash_from_id(&BlockId::Number(12)).unwrap();
+	backend
+		.append_justification(hashof12, (BEEFY_ENGINE_ID, justif.encode()))
+		.unwrap();
+
+	// Test initialization at last BEEFY finalized.
+
+	// load persistent state - nothing in DB, should init at last BEEFY finalized
+	let persisted_state = test_voter_init_setup(&mut net, &mut finality).unwrap();
+
+	// verify voter initialized with single session starting at block 12
+	assert_eq!(persisted_state.voting_oracle().sessions().len(), 1);
+	let rounds = persisted_state.active_round().unwrap();
+	assert_eq!(rounds.session_start(), 12);
+	assert_eq!(rounds.validator_set_id(), validator_set.id());
+
+	// verify next vote target is 13
+	assert_eq!(persisted_state.best_beefy_block(), 12);
+	assert_eq!(persisted_state.best_grandpa_block(), 13);
+	assert_eq!(
+		persisted_state
+			.voting_oracle()
+			.voting_target(persisted_state.best_beefy_block(), 13),
+		Some(13)
+	);
+
+	// verify state also saved to db
+	assert!(verify_persisted_version(&*backend));
+	let state = load_persistent(&*backend).unwrap().unwrap();
+	assert_eq!(state, persisted_state);
+}

substrate/frame/beefy/src/lib.rs

@@ -34,6 +34,7 @@ use sp_std::prelude::*;
 
 use beefy_primitives::{
 	AuthorityIndex, ConsensusLog, OnNewValidatorSet, ValidatorSet, BEEFY_ENGINE_ID,
+	GENESIS_AUTHORITY_SET_ID,
 };
 
 #[cfg(test)]
@@ -162,7 +163,7 @@ impl<T: Config> Pallet<T> {
 			BoundedSlice::<T::BeefyId, T::MaxAuthorities>::try_from(authorities.as_slice())
 				.map_err(|_| ())?;
 
-		let id = 0;
+		let id = GENESIS_AUTHORITY_SET_ID;
 		<Authorities<T>>::put(bounded_authorities);
 		<ValidatorSetId<T>>::put(id);
 		// Like `pallet_session`, initialize the next validator set as well.

substrate/primitives/beefy/src/lib.rs

@@ -113,7 +113,7 @@ pub mod crypto {
 /// The `ConsensusEngineId` of BEEFY.
 pub const BEEFY_ENGINE_ID: sp_runtime::ConsensusEngineId = *b"BEEF";
 
-/// Authority set id starts with zero at genesis
+/// Authority set id starts with zero at BEEFY pallet genesis.
 pub const GENESIS_AUTHORITY_SET_ID: u64 = 0;
 
 /// A typedef for validator set id.