Beefy on-demand justifications as a custom RequestResponse protocol (#12124)

* client/beefy: create communication module and move gossip there

* client/beefy: move beefy_protocol_name module to communication

* client/beefy: move notification module under communication

* client/beefy: add incoming request_response protocol handler

* client/beefy: keep track of connected peers and their progress

* client/beefy: add logic for generating Justif requests

* client/beefy: cancel outdated on-demand justification requests

* try Andre's suggestion for JustificationEngine

* justif engine add justifs validation

* client/beefy: impl OnDemandJustificationsEngine async next()

* move beefy proto name test

* client/beefy: initialize OnDemandJustificationsEngine

* client/tests: allow for custom req-resp protocols

* client/beefy: on-demand-justif: implement simple peer selection strategy

* client/beefy: fix voter initialization

Fix corner case where voter gets a single burst of finality
notifications just when it starts.

The notification stream was consumed by "wait_for_pallet" logic,
then main loop would subscribe to finality notifications, but by that
time some notifications might've been lost.

Fix this by subscribing the main loop to notifications before waiting
for pallet to become available. Share the same stream with the main loop
so that notifications for blocks before pallet available are ignored,
while _all_ notifications after pallet available are processed.

Add regression test for this.

Signed-off-by: acatangiu <adrian@parity.io>

* client/beefy: make sure justif requests are always out for mandatory blocks

* client/beefy: add test for on-demand justifications sync

* client/beefy: tweak main loop event processing order

* client/beefy: run on-demand-justif-handler under same async task as voter

* client/beefy: add test for known-peers

* client/beefy: reorg request-response module

* client/beefy: add issue references for future work todos

* client/beefy: consolidate on-demand-justifications engine state machine

Signed-off-by: acatangiu <adrian@parity.io>

* client/beefy: fix for polkadot companion

* client/beefy: implement review suggestions

* cargo fmt and clippy

* fix merge damage

* fix rust-doc

* fix merge damage

* fix merge damage

* client/beefy: add test for justif proto name

Signed-off-by: acatangiu <adrian@parity.io>

substrate/client/beefy/src/tests.rs

@@ -21,10 +21,9 @@
 use futures::{future, stream::FuturesUnordered, Future, StreamExt};
 use parking_lot::Mutex;
 use serde::{Deserialize, Serialize};
-use std::{collections::HashMap, sync::Arc, task::Poll};
+use std::{collections::HashMap, marker::PhantomData, sync::Arc, task::Poll};
 use tokio::{runtime::Runtime, time::Duration};
 
-use sc_chain_spec::{ChainSpec, GenericChainSpec};
 use sc_client_api::HeaderBackend;
 use sc_consensus::{
 	BlockImport, BlockImportParams, BoxJustificationImport, ForkChoiceStrategy, ImportResult,
@@ -33,7 +32,7 @@ use sc_consensus::{
 use sc_keystore::LocalKeystore;
 use sc_network_test::{
 	Block, BlockImportAdapter, FullPeerConfig, PassThroughVerifier, Peer, PeersClient,
-	TestNetFactory,
+	PeersFullClient, TestNetFactory,
 };
 use sc_utils::notification::NotificationReceiver;
 
@@ -42,6 +41,7 @@ use beefy_primitives::{
 	BeefyApi, ConsensusLog, MmrRootHash, ValidatorSet, VersionedFinalityProof, BEEFY_ENGINE_ID,
 	KEY_TYPE as BeefyKeyType,
 };
+use sc_network::{config::RequestResponseConfig, ProtocolName};
 use sp_mmr_primitives::{
 	BatchProof, EncodableOpaqueLeaf, Error as MmrError, LeafIndex, MmrApi, Proof,
 };
@@ -60,11 +60,21 @@ use sp_runtime::{
 use substrate_test_runtime_client::{runtime::Header, ClientExt};
 
 use crate::{
-	beefy_block_import_and_links, beefy_protocol_name, justification::*,
-	keystore::tests::Keyring as BeefyKeyring, BeefyRPCLinks, BeefyVoterLinks,
+	beefy_block_import_and_links,
+	communication::request_response::{
+		on_demand_justifications_protocol_config, BeefyJustifsRequestHandler,
+	},
+	gossip_protocol_name,
+	justification::*,
+	keystore::tests::Keyring as BeefyKeyring,
+	BeefyRPCLinks, BeefyVoterLinks,
 };
 
-pub(crate) const BEEFY_PROTOCOL_NAME: &'static str = "/beefy/1";
+const GENESIS_HASH: H256 = H256::zero();
+fn beefy_gossip_proto_name() -> ProtocolName {
+	gossip_protocol_name(GENESIS_HASH, None)
+}
+
 const GOOD_MMR_ROOT: MmrRootHash = MmrRootHash::repeat_byte(0xbf);
 const BAD_MMR_ROOT: MmrRootHash = MmrRootHash::repeat_byte(0x42);
 
@@ -89,35 +99,12 @@ impl BuildStorage for Genesis {
 	}
 }
 
-#[test]
-fn beefy_protocol_name() {
-	let chain_spec = GenericChainSpec::<Genesis>::from_json_bytes(
-		&include_bytes!("../../chain-spec/res/chain_spec.json")[..],
-	)
-	.unwrap()
-	.cloned_box();
-
-	// Create protocol name using random genesis hash.
-	let genesis_hash = H256::random();
-	let expected = format!("/{}/beefy/1", array_bytes::bytes2hex("", genesis_hash.as_ref()));
-	let proto_name = beefy_protocol_name::standard_name(&genesis_hash, &chain_spec);
-	assert_eq!(proto_name.to_string(), expected);
-
-	// Create protocol name using hardcoded genesis hash. Verify exact representation.
-	let genesis_hash = [
-		50, 4, 60, 123, 58, 106, 216, 246, 194, 188, 139, 193, 33, 212, 202, 171, 9, 55, 123, 94,
-		8, 43, 12, 251, 187, 57, 173, 19, 188, 74, 205, 147,
-	];
-	let expected =
-		"/32043c7b3a6ad8f6c2bc8bc121d4caab09377b5e082b0cfbbb39ad13bc4acd93/beefy/1".to_string();
-	let proto_name = beefy_protocol_name::standard_name(&genesis_hash, &chain_spec);
-	assert_eq!(proto_name.to_string(), expected);
-}
-
 #[derive(Default)]
 pub(crate) struct PeerData {
 	pub(crate) beefy_rpc_links: Mutex<Option<BeefyRPCLinks<Block>>>,
 	pub(crate) beefy_voter_links: Mutex<Option<BeefyVoterLinks<Block>>>,
+	pub(crate) beefy_justif_req_handler:
+		Mutex<Option<BeefyJustifsRequestHandler<Block, PeersFullClient>>>,
 }
 
 #[derive(Default)]
@@ -126,23 +113,34 @@ pub(crate) struct BeefyTestNet {
 }
 
 impl BeefyTestNet {
-	pub(crate) fn new(n_authority: usize, n_full: usize) -> Self {
-		let mut net = BeefyTestNet { peers: Vec::with_capacity(n_authority + n_full) };
-		for _ in 0..n_authority {
-			net.add_authority_peer();
-		}
-		for _ in 0..n_full {
-			net.add_full_peer();
+	pub(crate) fn new(n_authority: usize) -> Self {
+		let mut net = BeefyTestNet { peers: Vec::with_capacity(n_authority) };
+
+		for i in 0..n_authority {
+			let (rx, cfg) = on_demand_justifications_protocol_config(GENESIS_HASH, None);
+			let justif_protocol_name = cfg.name.clone();
+
+			net.add_authority_peer(vec![cfg]);
+
+			let client = net.peers[i].client().as_client();
+			let justif_handler = BeefyJustifsRequestHandler {
+				request_receiver: rx,
+				justif_protocol_name,
+				client,
+				_block: PhantomData,
+			};
+			*net.peers[i].data.beefy_justif_req_handler.lock() = Some(justif_handler);
 		}
 		net
 	}
 
-	pub(crate) fn add_authority_peer(&mut self) {
+	pub(crate) fn add_authority_peer(&mut self, req_resp_cfgs: Vec<RequestResponseConfig>) {
 		self.add_full_peer_with_config(FullPeerConfig {
-			notifications_protocols: vec![BEEFY_PROTOCOL_NAME.into()],
+			notifications_protocols: vec![beefy_gossip_proto_name()],
+			request_response_protocols: req_resp_cfgs,
 			is_authority: true,
 			..Default::default()
-		})
+		});
 	}
 
 	pub(crate) fn generate_blocks_and_sync(
@@ -198,6 +196,7 @@ impl TestNetFactory for BeefyTestNet {
 		let peer_data = PeerData {
 			beefy_rpc_links: Mutex::new(Some(rpc_links)),
 			beefy_voter_links: Mutex::new(Some(voter_links)),
+			..Default::default()
 		};
 		(BlockImportAdapter::new(block_import), None, peer_data)
 	}
@@ -215,11 +214,8 @@ impl TestNetFactory for BeefyTestNet {
 	}
 
 	fn add_full_peer(&mut self) {
-		self.add_full_peer_with_config(FullPeerConfig {
-			notifications_protocols: vec![BEEFY_PROTOCOL_NAME.into()],
-			is_authority: false,
-			..Default::default()
-		})
+		// `add_authority_peer()` used instead.
+		unimplemented!()
 	}
 }
 
@@ -354,7 +350,7 @@ where
 	API: ProvideRuntimeApi<Block> + Default + Sync + Send,
 	API::Api: BeefyApi<Block> + MmrApi<Block, MmrRootHash>,
 {
-	let voters = FuturesUnordered::new();
+	let tasks = FuturesUnordered::new();
 
 	for (peer_id, key, api) in peers.into_iter() {
 		let peer = &net.peers[peer_id];
@@ -362,31 +358,40 @@ where
 		let keystore = create_beefy_keystore(*key);
 
 		let (_, _, peer_data) = net.make_block_import(peer.client().clone());
-		let PeerData { beefy_rpc_links, beefy_voter_links } = peer_data;
+		let PeerData { beefy_rpc_links, beefy_voter_links, .. } = peer_data;
 
 		let beefy_voter_links = beefy_voter_links.lock().take();
 		*peer.data.beefy_rpc_links.lock() = beefy_rpc_links.lock().take();
 		*peer.data.beefy_voter_links.lock() = beefy_voter_links.clone();
 
+		let on_demand_justif_handler = peer.data.beefy_justif_req_handler.lock().take().unwrap();
+
+		let network_params = crate::BeefyNetworkParams {
+			network: peer.network_service().clone(),
+			gossip_protocol_name: beefy_gossip_proto_name(),
+			justifications_protocol_name: on_demand_justif_handler.protocol_name(),
+			_phantom: PhantomData,
+		};
+
 		let beefy_params = crate::BeefyParams {
 			client: peer.client().as_client(),
 			backend: peer.client().as_backend(),
 			runtime: api.clone(),
 			key_store: Some(keystore),
-			network: peer.network_service().clone(),
+			network_params,
 			links: beefy_voter_links.unwrap(),
 			min_block_delta,
 			prometheus_registry: None,
-			protocol_name: BEEFY_PROTOCOL_NAME.into(),
+			on_demand_justifications_handler: on_demand_justif_handler,
 		};
-		let gadget = crate::start_beefy_gadget::<_, _, _, _, _>(beefy_params);
+		let task = crate::start_beefy_gadget::<_, _, _, _, _>(beefy_params);
 
 		fn assert_send<T: Send>(_: &T) {}
-		assert_send(&gadget);
-		voters.push(gadget);
+		assert_send(&task);
+		tasks.push(task);
 	}
 
-	voters.for_each(|_| async move {})
+	tasks.for_each(|_| async move {})
 }
 
 fn block_until(future: impl Future + Unpin, net: &Arc<Mutex<BeefyTestNet>>, runtime: &mut Runtime) {
@@ -404,18 +409,19 @@ fn run_for(duration: Duration, net: &Arc<Mutex<BeefyTestNet>>, runtime: &mut Run
 
 pub(crate) fn get_beefy_streams(
 	net: &mut BeefyTestNet,
-	peers: &[BeefyKeyring],
+	// peer index and key
+	peers: impl Iterator<Item = (usize, BeefyKeyring)>,
 ) -> (Vec<NotificationReceiver<H256>>, Vec<NotificationReceiver<BeefyVersionedFinalityProof<Block>>>)
 {
 	let mut best_block_streams = Vec::new();
 	let mut versioned_finality_proof_streams = Vec::new();
-	for peer_id in 0..peers.len() {
-		let beefy_rpc_links = net.peer(peer_id).data.beefy_rpc_links.lock().clone().unwrap();
+	peers.for_each(|(index, _)| {
+		let beefy_rpc_links = net.peer(index).data.beefy_rpc_links.lock().clone().unwrap();
 		let BeefyRPCLinks { from_voter_justif_stream, from_voter_best_beefy_stream } =
 			beefy_rpc_links;
 		best_block_streams.push(from_voter_best_beefy_stream.subscribe());
 		versioned_finality_proof_streams.push(from_voter_justif_stream.subscribe());
-	}
+	});
 	(best_block_streams, versioned_finality_proof_streams)
 }
 
@@ -493,18 +499,24 @@ fn streams_empty_after_timeout<T>(
 
 fn finalize_block_and_wait_for_beefy(
 	net: &Arc<Mutex<BeefyTestNet>>,
-	peers: &[BeefyKeyring],
+	// peer index and key
+	peers: impl Iterator<Item = (usize, BeefyKeyring)> + Clone,
 	runtime: &mut Runtime,
 	finalize_targets: &[u64],
 	expected_beefy: &[u64],
 ) {
-	let (best_blocks, versioned_finality_proof) = get_beefy_streams(&mut net.lock(), peers);
+	let (best_blocks, versioned_finality_proof) = get_beefy_streams(&mut net.lock(), peers.clone());
 
 	for block in finalize_targets {
 		let finalize = BlockId::number(*block);
-		for i in 0..peers.len() {
-			net.lock().peer(i).client().as_client().finalize_block(finalize, None).unwrap();
-		}
+		peers.clone().for_each(|(index, _)| {
+			net.lock()
+				.peer(index)
+				.client()
+				.as_client()
+				.finalize_block(finalize, None)
+				.unwrap();
+		})
 	}
 
 	if expected_beefy.is_empty() {
@@ -524,12 +536,12 @@ fn beefy_finalizing_blocks() {
 	sp_tracing::try_init_simple();
 
 	let mut runtime = Runtime::new().unwrap();
-	let peers = &[BeefyKeyring::Alice, BeefyKeyring::Bob];
-	let validator_set = ValidatorSet::new(make_beefy_ids(peers), 0).unwrap();
+	let peers = [BeefyKeyring::Alice, BeefyKeyring::Bob];
+	let validator_set = ValidatorSet::new(make_beefy_ids(&peers), 0).unwrap();
 	let session_len = 10;
 	let min_block_delta = 4;
 
-	let mut net = BeefyTestNet::new(2, 0);
+	let mut net = BeefyTestNet::new(2);
 
 	let api = Arc::new(two_validators::TestApi {});
 	let beefy_peers = peers.iter().enumerate().map(|(id, key)| (id, key, api.clone())).collect();
@@ -542,17 +554,18 @@ fn beefy_finalizing_blocks() {
 
 	// Minimum BEEFY block delta is 4.
 
+	let peers = peers.into_iter().enumerate();
 	// finalize block #5 -> BEEFY should finalize #1 (mandatory) and #5 from diff-power-of-two rule.
-	finalize_block_and_wait_for_beefy(&net, peers, &mut runtime, &[5], &[1, 5]);
+	finalize_block_and_wait_for_beefy(&net, peers.clone(), &mut runtime, &[5], &[1, 5]);
 
 	// GRANDPA finalize #10 -> BEEFY finalize #10 (mandatory)
-	finalize_block_and_wait_for_beefy(&net, peers, &mut runtime, &[10], &[10]);
+	finalize_block_and_wait_for_beefy(&net, peers.clone(), &mut runtime, &[10], &[10]);
 
 	// GRANDPA finalize #18 -> BEEFY finalize #14, then #18 (diff-power-of-two rule)
-	finalize_block_and_wait_for_beefy(&net, peers, &mut runtime, &[18], &[14, 18]);
+	finalize_block_and_wait_for_beefy(&net, peers.clone(), &mut runtime, &[18], &[14, 18]);
 
 	// GRANDPA finalize #20 -> BEEFY finalize #20 (mandatory)
-	finalize_block_and_wait_for_beefy(&net, peers, &mut runtime, &[20], &[20]);
+	finalize_block_and_wait_for_beefy(&net, peers.clone(), &mut runtime, &[20], &[20]);
 
 	// GRANDPA finalize #21 -> BEEFY finalize nothing (yet) because min delta is 4
 	finalize_block_and_wait_for_beefy(&net, peers, &mut runtime, &[21], &[]);
@@ -563,12 +576,12 @@ fn lagging_validators() {
 	sp_tracing::try_init_simple();
 
 	let mut runtime = Runtime::new().unwrap();
-	let peers = &[BeefyKeyring::Alice, BeefyKeyring::Bob];
-	let validator_set = ValidatorSet::new(make_beefy_ids(peers), 0).unwrap();
+	let peers = [BeefyKeyring::Alice, BeefyKeyring::Bob];
+	let validator_set = ValidatorSet::new(make_beefy_ids(&peers), 0).unwrap();
 	let session_len = 30;
 	let min_block_delta = 1;
 
-	let mut net = BeefyTestNet::new(2, 0);
+	let mut net = BeefyTestNet::new(2);
 	let api = Arc::new(two_validators::TestApi {});
 	let beefy_peers = peers.iter().enumerate().map(|(id, key)| (id, key, api.clone())).collect();
 	runtime.spawn(initialize_beefy(&mut net, beefy_peers, min_block_delta));
@@ -578,13 +591,20 @@ fn lagging_validators() {
 
 	let net = Arc::new(Mutex::new(net));
 
+	let peers = peers.into_iter().enumerate();
 	// finalize block #15 -> BEEFY should finalize #1 (mandatory) and #9, #13, #14, #15 from
 	// diff-power-of-two rule.
-	finalize_block_and_wait_for_beefy(&net, peers, &mut runtime, &[15], &[1, 9, 13, 14, 15]);
+	finalize_block_and_wait_for_beefy(
+		&net,
+		peers.clone(),
+		&mut runtime,
+		&[15],
+		&[1, 9, 13, 14, 15],
+	);
 
 	// Alice finalizes #25, Bob lags behind
 	let finalize = BlockId::number(25);
-	let (best_blocks, versioned_finality_proof) = get_beefy_streams(&mut net.lock(), peers);
+	let (best_blocks, versioned_finality_proof) = get_beefy_streams(&mut net.lock(), peers.clone());
 	net.lock().peer(0).client().as_client().finalize_block(finalize, None).unwrap();
 	// verify nothing gets finalized by BEEFY
 	let timeout = Some(Duration::from_millis(250));
@@ -592,21 +612,21 @@ fn lagging_validators() {
 	streams_empty_after_timeout(versioned_finality_proof, &net, &mut runtime, None);
 
 	// Bob catches up and also finalizes #25
-	let (best_blocks, versioned_finality_proof) = get_beefy_streams(&mut net.lock(), peers);
+	let (best_blocks, versioned_finality_proof) = get_beefy_streams(&mut net.lock(), peers.clone());
 	net.lock().peer(1).client().as_client().finalize_block(finalize, None).unwrap();
 	// expected beefy finalizes block #17 from diff-power-of-two
 	wait_for_best_beefy_blocks(best_blocks, &net, &mut runtime, &[23, 24, 25]);
 	wait_for_beefy_signed_commitments(versioned_finality_proof, &net, &mut runtime, &[23, 24, 25]);
 
 	// Both finalize #30 (mandatory session) and #32 -> BEEFY finalize #30 (mandatory), #31, #32
-	finalize_block_and_wait_for_beefy(&net, peers, &mut runtime, &[30, 32], &[30, 31, 32]);
+	finalize_block_and_wait_for_beefy(&net, peers.clone(), &mut runtime, &[30, 32], &[30, 31, 32]);
 
 	// Verify that session-boundary votes get buffered by client and only processed once
 	// session-boundary block is GRANDPA-finalized (this guarantees authenticity for the new session
 	// validator set).
 
 	// Alice finalizes session-boundary mandatory block #60, Bob lags behind
-	let (best_blocks, versioned_finality_proof) = get_beefy_streams(&mut net.lock(), peers);
+	let (best_blocks, versioned_finality_proof) = get_beefy_streams(&mut net.lock(), peers.clone());
 	let finalize = BlockId::number(60);
 	net.lock().peer(0).client().as_client().finalize_block(finalize, None).unwrap();
 	// verify nothing gets finalized by BEEFY
@@ -617,7 +637,7 @@ fn lagging_validators() {
 	// Bob catches up and also finalizes #60 (and should have buffered Alice's vote on #60)
 	let (best_blocks, versioned_finality_proof) = get_beefy_streams(&mut net.lock(), peers);
 	net.lock().peer(1).client().as_client().finalize_block(finalize, None).unwrap();
-	// verify beefy skips intermediary votes, and successfully finalizes mandatory block #40
+	// verify beefy skips intermediary votes, and successfully finalizes mandatory block #60
 	wait_for_best_beefy_blocks(best_blocks, &net, &mut runtime, &[60]);
 	wait_for_beefy_signed_commitments(versioned_finality_proof, &net, &mut runtime, &[60]);
 }
@@ -627,13 +647,12 @@ fn correct_beefy_payload() {
 	sp_tracing::try_init_simple();
 
 	let mut runtime = Runtime::new().unwrap();
-	let peers =
-		&[BeefyKeyring::Alice, BeefyKeyring::Bob, BeefyKeyring::Charlie, BeefyKeyring::Dave];
-	let validator_set = ValidatorSet::new(make_beefy_ids(peers), 0).unwrap();
+	let peers = [BeefyKeyring::Alice, BeefyKeyring::Bob, BeefyKeyring::Charlie, BeefyKeyring::Dave];
+	let validator_set = ValidatorSet::new(make_beefy_ids(&peers), 0).unwrap();
 	let session_len = 20;
 	let min_block_delta = 2;
 
-	let mut net = BeefyTestNet::new(4, 0);
+	let mut net = BeefyTestNet::new(4);
 
 	// Alice, Bob, Charlie will vote on good payloads
 	let good_api = Arc::new(four_validators::TestApi {});
@@ -649,15 +668,16 @@ fn correct_beefy_payload() {
 	let bad_peers = vec![(3, &BeefyKeyring::Dave, bad_api)];
 	runtime.spawn(initialize_beefy(&mut net, bad_peers, min_block_delta));
 
-	// push 10 blocks
+	// push 12 blocks
 	net.generate_blocks_and_sync(12, session_len, &validator_set, false);
 
 	let net = Arc::new(Mutex::new(net));
+	let peers = peers.into_iter().enumerate();
 	// with 3 good voters and 1 bad one, consensus should happen and best blocks produced.
 	finalize_block_and_wait_for_beefy(&net, peers, &mut runtime, &[10], &[1, 9]);
 
 	let (best_blocks, versioned_finality_proof) =
-		get_beefy_streams(&mut net.lock(), &[BeefyKeyring::Alice]);
+		get_beefy_streams(&mut net.lock(), [(0, BeefyKeyring::Alice)].into_iter());
 
 	// now 2 good validators and 1 bad one are voting
 	net.lock()
@@ -686,7 +706,7 @@ fn correct_beefy_payload() {
 
 	// 3rd good validator catches up and votes as well
 	let (best_blocks, versioned_finality_proof) =
-		get_beefy_streams(&mut net.lock(), &[BeefyKeyring::Alice]);
+		get_beefy_streams(&mut net.lock(), [(0, BeefyKeyring::Alice)].into_iter());
 	net.lock()
 		.peer(2)
 		.client()
@@ -707,11 +727,11 @@ fn beefy_importing_blocks() {
 
 	sp_tracing::try_init_simple();
 
-	let mut net = BeefyTestNet::new(2, 0);
+	let mut net = BeefyTestNet::new(2);
 
 	let client = net.peer(0).client().clone();
 	let (mut block_import, _, peer_data) = net.make_block_import(client.clone());
-	let PeerData { beefy_rpc_links: _, beefy_voter_links } = peer_data;
+	let PeerData { beefy_voter_links, .. } = peer_data;
 	let justif_stream = beefy_voter_links.lock().take().unwrap().from_block_import_justif_stream;
 
 	let params = |block: Block, justifications: Option<Justifications>| {
@@ -826,18 +846,18 @@ fn voter_initialization() {
 	// after waiting for BEEFY pallet availability.
 
 	let mut runtime = Runtime::new().unwrap();
-	let peers = &[BeefyKeyring::Alice, BeefyKeyring::Bob];
-	let validator_set = ValidatorSet::new(make_beefy_ids(peers), 0).unwrap();
+	let peers = [BeefyKeyring::Alice, BeefyKeyring::Bob];
+	let validator_set = ValidatorSet::new(make_beefy_ids(&peers), 0).unwrap();
 	let session_len = 5;
 	// Should vote on all mandatory blocks no matter the `min_block_delta`.
 	let min_block_delta = 10;
 
-	let mut net = BeefyTestNet::new(2, 0);
+	let mut net = BeefyTestNet::new(2);
 	let api = Arc::new(two_validators::TestApi {});
 	let beefy_peers = peers.iter().enumerate().map(|(id, key)| (id, key, api.clone())).collect();
 	runtime.spawn(initialize_beefy(&mut net, beefy_peers, min_block_delta));
 
-	// push 30 blocks
+	// push 26 blocks
 	net.generate_blocks_and_sync(26, session_len, &validator_set, false);
 	let net = Arc::new(Mutex::new(net));
 
@@ -846,9 +866,90 @@ fn voter_initialization() {
 	// Expect voters to pick up all of them and BEEFY-finalize the mandatory blocks of each session.
 	finalize_block_and_wait_for_beefy(
 		&net,
-		peers,
+		peers.into_iter().enumerate(),
 		&mut runtime,
 		&[1, 6, 10, 17, 24, 26],
 		&[1, 5, 10, 15, 20, 25],
 	);
 }
+
+#[test]
+fn on_demand_beefy_justification_sync() {
+	sp_tracing::try_init_simple();
+
+	let mut runtime = Runtime::new().unwrap();
+	let all_peers =
+		[BeefyKeyring::Alice, BeefyKeyring::Bob, BeefyKeyring::Charlie, BeefyKeyring::Dave];
+	let validator_set = ValidatorSet::new(make_beefy_ids(&all_peers), 0).unwrap();
+	let session_len = 5;
+	let min_block_delta = 5;
+
+	let mut net = BeefyTestNet::new(4);
+
+	// Alice, Bob, Charlie start first and make progress through voting.
+	let api = Arc::new(four_validators::TestApi {});
+	let fast_peers = [BeefyKeyring::Alice, BeefyKeyring::Bob, BeefyKeyring::Charlie];
+	let voting_peers =
+		fast_peers.iter().enumerate().map(|(id, key)| (id, key, api.clone())).collect();
+	runtime.spawn(initialize_beefy(&mut net, voting_peers, min_block_delta));
+
+	// Dave will start late and have to catch up using on-demand justification requests (since
+	// in this test there is no block import queue to automatically import justifications).
+	let dave = vec![(3, &BeefyKeyring::Dave, api)];
+	// Instantiate but don't run Dave, yet.
+	let dave_task = initialize_beefy(&mut net, dave, min_block_delta);
+	let dave_index = 3;
+
+	// push 30 blocks
+	net.generate_blocks_and_sync(30, session_len, &validator_set, false);
+
+	let fast_peers = fast_peers.into_iter().enumerate();
+	let net = Arc::new(Mutex::new(net));
+	// With 3 active voters and one inactive, consensus should happen and blocks BEEFY-finalized.
+	// Need to finalize at least one block in each session, choose randomly.
+	finalize_block_and_wait_for_beefy(
+		&net,
+		fast_peers.clone(),
+		&mut runtime,
+		&[1, 6, 10, 17, 24],
+		&[1, 5, 10, 15, 20],
+	);
+
+	// Spawn Dave, he's now way behind voting and can only catch up through on-demand justif sync.
+	runtime.spawn(dave_task);
+	// give Dave a chance to spawn and init.
+	run_for(Duration::from_millis(400), &net, &mut runtime);
+
+	let (dave_best_blocks, _) =
+		get_beefy_streams(&mut net.lock(), [(dave_index, BeefyKeyring::Dave)].into_iter());
+	net.lock()
+		.peer(dave_index)
+		.client()
+		.as_client()
+		.finalize_block(BlockId::number(1), None)
+		.unwrap();
+	// Give Dave task some cpu cycles to process the finality notification,
+	run_for(Duration::from_millis(100), &net, &mut runtime);
+	// freshly spun up Dave now needs to listen for gossip to figure out the state of his peers.
+
+	// Have the other peers do some gossip so Dave finds out about their progress.
+	finalize_block_and_wait_for_beefy(&net, fast_peers, &mut runtime, &[25], &[25]);
+
+	// Now verify Dave successfully finalized #1 (through on-demand justification request).
+	wait_for_best_beefy_blocks(dave_best_blocks, &net, &mut runtime, &[1]);
+
+	// Give Dave all tasks some cpu cycles to burn through their events queues,
+	run_for(Duration::from_millis(100), &net, &mut runtime);
+	// then verify Dave catches up through on-demand justification requests.
+	finalize_block_and_wait_for_beefy(
+		&net,
+		[(dave_index, BeefyKeyring::Dave)].into_iter(),
+		&mut runtime,
+		&[6, 10, 17, 24, 26],
+		&[5, 10, 15, 20, 25],
+	);
+
+	let all_peers = all_peers.into_iter().enumerate();
+	// Now that Dave has caught up, sanity check voting works for all of them.
+	finalize_block_and_wait_for_beefy(&net, all_peers, &mut runtime, &[30], &[30]);
+}