Sync block justifications (#1410)

* core: sync protocol for justifications

* core: basic test for justification sync

* core: pass block number with justification

* grandpa: request justifications when importing change blocks

* core: pass finality notifications to chain sync

* core: require justifications for pending change blocks on start

* core: avoid requesting justifications from previous failed peers

* core: timeout block justification requests

* core: add some docs

* core: fix unused variables warning

* core: tick pending justifications fetch periodically

* grandpa: add test for syncing justifications

* core: early exit dispatch of pending justifications

* core: style fix

* core: grandpa: change logging level

* core: sync: add missing docs

* core: network: report peer on bad justification

* core: replace mem::replace with Option::take

* core: revert authority set changes on failed block finalization

* core: grandpa: add docs to import_justification

* core: warn on re-finalization of last finalized block

* core: only notify sync with last finality notification

* core: style fix

* core: add docs for PendingJustifications

* core: network: use BlockRequest messages for justification requests

* core: reference issues in todo comments

* core: grandpa: revert authority set changes on db

* core: grandpa: remove inconsistent state warning

substrate/core/network/src/message.rs

@@ -48,7 +48,6 @@ pub type BlockRequest<B> = generic::BlockRequest<
 	<<B as BlockT>::Header as HeaderT>::Number,
 >;
 
-
 /// Type alias for using the BlockData type using block type parameters.
 pub type BlockData<B> = generic::BlockData<
 	<B as BlockT>::Header,

substrate/core/network/src/protocol.rs

@@ -15,7 +15,7 @@
 // along with Substrate.  If not, see <http://www.gnu.org/licenses/>.
 
 use std::collections::{HashMap, HashSet, BTreeMap};
-use std::{mem, cmp};
+use std::cmp;
 use std::sync::Arc;
 use std::time;
 use parking_lot::RwLock;
@@ -273,7 +273,7 @@ impl<B: BlockT, S: NetworkSpecialization<B>, H: ExHashT> Protocol<B, S, H> {
 					let mut peers = self.context_data.peers.write();
 					if let Some(ref mut peer) = peers.get_mut(&who) {
 						peer.request_timestamp = None;
-						match mem::replace(&mut peer.block_request, None) {
+						match peer.block_request.take() {
 							Some(r) => r,
 							None => {
 								io.report_peer(who, Severity::Bad("Unexpected response packet received from peer"));
@@ -285,10 +285,12 @@ impl<B: BlockT, S: NetworkSpecialization<B>, H: ExHashT> Protocol<B, S, H> {
 						return;
 					}
 				};
+
 				if request.id != r.id {
 					trace!(target: "sync", "Ignoring mismatched response packet from {} (expected {} got {})", who, request.id, r.id);
 					return;
 				}
+
 				self.on_block_response(io, who, request, r);
 			},
 			GenericMessage::BlockAnnounce(announce) => self.on_block_announce(io, who, announce),
@@ -330,7 +332,6 @@ impl<B: BlockT, S: NetworkSpecialization<B>, H: ExHashT> Protocol<B, S, H> {
 	pub fn on_peer_disconnected(&self, io: &mut SyncIo, peer: NodeIndex) {
 		trace!(target: "sync", "Disconnecting {}: {}", peer, io.peer_debug_info(peer));
 
-
 		// lock all the the peer lists so that add/remove peer events are in order
 		let mut sync = self.sync.write();
 		let mut spec = self.specialization.write();
@@ -351,7 +352,15 @@ impl<B: BlockT, S: NetworkSpecialization<B>, H: ExHashT> Protocol<B, S, H> {
 	}
 
 	fn on_block_request(&self, io: &mut SyncIo, peer: NodeIndex, request: message::BlockRequest<B>) {
-		trace!(target: "sync", "BlockRequest {} from {}: from {:?} to {:?} max {:?}", request.id, peer, request.from, request.to, request.max);
+		trace!(target: "sync", "BlockRequest {} from {} with fields {:?}: from {:?} to {:?} max {:?}",
+			request.id,
+			peer,
+			request.fields,
+			request.from,
+			request.to,
+			request.max,
+		);
+
 		let mut blocks = Vec::new();
 		let mut id = match request.from {
 			message::FromBlock::Hash(h) => BlockId::Hash(h),
@@ -409,25 +418,36 @@ impl<B: BlockT, S: NetworkSpecialization<B>, H: ExHashT> Protocol<B, S, H> {
 		trace!(target: "sync", "BlockResponse {} from {} with {} blocks{}",
 			response.id, peer, response.blocks.len(), blocks_range);
 
-		// import_queue.import_blocks also acquires sync.write();
-		// Break the cycle by doing these separately from the outside;
-		let new_blocks = {
+		// TODO [andre]: move this logic to the import queue so that
+		// justifications are imported asynchronously (#1482)
+		if request.fields == message::BlockAttributes::JUSTIFICATION {
 			let mut sync = self.sync.write();
-			sync.on_block_data(&mut ProtocolContext::new(&self.context_data, io), peer, request, response)
-		};
+			sync.on_block_justification_data(
+				&mut ProtocolContext::new(&self.context_data, io),
+				peer,
+				request,
+				response,
+			);
+		} else {
+			// import_queue.import_blocks also acquires sync.write();
+			// Break the cycle by doing these separately from the outside;
+			let new_blocks = {
+				let mut sync = self.sync.write();
+				sync.on_block_data(&mut ProtocolContext::new(&self.context_data, io), peer, request, response)
+			};
 
-		if let Some((origin, new_blocks)) = new_blocks {
-			let import_queue = self.sync.read().import_queue();
-			import_queue.import_blocks(origin, new_blocks);
+			if let Some((origin, new_blocks)) = new_blocks {
+				let import_queue = self.sync.read().import_queue();
+				import_queue.import_blocks(origin, new_blocks);
+			}
 		}
-
-
 	}
 
 	/// Perform time based maintenance.
 	pub fn tick(&self, io: &mut SyncIo) {
 		self.consensus_gossip.write().collect_garbage(|_| true);
 		self.maintain_peers(io);
+		self.sync.write().tick(&mut ProtocolContext::new(&self.context_data, io));
 		self.on_demand.as_ref().map(|s| s.maintain_peers(io));
 	}
 
@@ -439,7 +459,8 @@ impl<B: BlockT, S: NetworkSpecialization<B>, H: ExHashT> Protocol<B, S, H> {
 			let handshaking_peers = self.handshaking_peers.read();
 			for (who, timestamp) in peers.iter()
 				.filter_map(|(id, peer)| peer.request_timestamp.as_ref().map(|r| (id, r)))
-				.chain(handshaking_peers.iter()) {
+				.chain(handshaking_peers.iter())
+			{
 				if (tick - *timestamp).as_secs() > REQUEST_TIMEOUT_SEC {
 					trace!(target: "sync", "Timeout {}", who);
 					aborting.push(*who);
@@ -648,6 +669,10 @@ impl<B: BlockT, S: NetworkSpecialization<B>, H: ExHashT> Protocol<B, S, H> {
 		}
 	}
 
+	pub fn on_block_finalized(&self, _io: &mut SyncIo, hash: B::Hash, header: &B::Header) {
+		self.sync.write().block_finalized(&hash, *header.number());
+	}
+
 	fn on_remote_call_request(&self, io: &mut SyncIo, who: NodeIndex, request: message::RemoteCallRequest<B::Hash>) {
 		trace!(target: "sync", "Remote call request {} from {} ({} at {})", request.id, who, request.method, request.block);
 		let proof = match self.context_data.chain.execution_proof(&request.block, &request.method, &request.data) {
@@ -752,8 +777,8 @@ impl<B: BlockT, S: NetworkSpecialization<B>, H: ExHashT> Protocol<B, S, H> {
 }
 
 fn send_message<B: BlockT, H: ExHashT>(peers: &RwLock<HashMap<NodeIndex, Peer<B, H>>>, io: &mut SyncIo, who: NodeIndex, mut message: Message<B>) {
-	match &mut message {
-		&mut GenericMessage::BlockRequest(ref mut r) => {
+	match message {
+		GenericMessage::BlockRequest(ref mut r) => {
 			let mut peers = peers.write();
 			if let Some(ref mut peer) = peers.get_mut(&who) {
 				r.id = peer.next_request_id;

substrate/core/network/src/service.rs

@@ -94,6 +94,10 @@ impl<B: BlockT, E: ExecuteInContext<B>> Link<B> for NetworkLink<B, E> {
 		self.with_sync(|sync, _| sync.block_imported(&hash, number))
 	}
 
+	fn request_justification(&self, hash: &B::Hash, number: NumberFor<B>) {
+		self.with_sync(|sync, protocol| sync.request_justification(hash, number, protocol))
+	}
+
 	fn maintain_sync(&self) {
 		self.with_sync(|sync, protocol| sync.maintain_sync(protocol))
 	}
@@ -174,6 +178,11 @@ impl<B: BlockT + 'static, S: NetworkSpecialization<B>, H: ExHashT> Service<B, S,
 		self.handler.on_block_imported(&mut NetSyncIo::new(&self.network, self.protocol_id), hash, header)
 	}
 
+	/// Called when a new block is finalized by the client.
+	pub fn on_block_finalized(&self, hash: B::Hash, header: &B::Header) {
+		self.handler.on_block_finalized(&mut NetSyncIo::new(&self.network, self.protocol_id), hash, header)
+	}
+
 	/// Called when new transactons are imported by the client.
 	pub fn trigger_repropagate(&self) {
 		self.handler.propagate_extrinsics(&mut NetSyncIo::new(&self.network, self.protocol_id));

substrate/core/network/src/sync.rs

@@ -14,8 +14,9 @@
 // You should have received a copy of the GNU General Public License
 // along with Substrate.  If not, see <http://www.gnu.org/licenses/>.
 
-use std::collections::HashMap;
+use std::collections::{HashMap, HashSet, VecDeque};
 use std::sync::Arc;
+use std::time::{Duration, Instant};
 use protocol::Context;
 use network_libp2p::{Severity, NodeIndex};
 use client::{BlockStatus, ClientInfo};
@@ -23,6 +24,7 @@ use consensus::BlockOrigin;
 use consensus::import_queue::{ImportQueue, IncomingBlock};
 use client::error::Error as ClientError;
 use blocks::BlockCollection;
+use runtime_primitives::Justification;
 use runtime_primitives::traits::{Block as BlockT, Header as HeaderT, As, NumberFor, Zero};
 use runtime_primitives::generic::BlockId;
 use message::{self, generic::Message as GenericMessage};
@@ -34,6 +36,8 @@ const MAX_BLOCKS_TO_REQUEST: usize = 128;
 const MAX_IMPORTING_BLOCKS: usize = 2048;
 // Number of blocks in the queue that prevents ancestry search.
 const MAJOR_SYNC_BLOCKS: usize = 5;
+// Time to wait before trying to get a justification from the same peer.
+const JUSTIFICATION_RETRY_WAIT: Duration = Duration::from_secs(10);
 
 struct PeerSync<B: BlockT> {
 	pub common_number: NumberFor<B>,
@@ -48,6 +52,180 @@ enum PeerSyncState<B: BlockT> {
 	Available,
 	DownloadingNew(NumberFor<B>),
 	DownloadingStale(B::Hash),
+	DownloadingJustification(B::Hash),
+}
+
+/// Pending justification request for the given block (hash and number).
+type PendingJustification<B> = (<B as BlockT>::Hash, NumberFor<B>);
+
+/// Manages pending block justification requests.
+struct PendingJustifications<B: BlockT> {
+	justifications: HashSet<PendingJustification<B>>,
+	pending_requests: VecDeque<PendingJustification<B>>,
+	peer_requests: HashMap<NodeIndex, PendingJustification<B>>,
+	previous_requests: HashMap<PendingJustification<B>, Vec<(NodeIndex, Instant)>>,
+}
+
+impl<B: BlockT> PendingJustifications<B> {
+	fn new() -> PendingJustifications<B> {
+		PendingJustifications {
+			justifications: HashSet::new(),
+			pending_requests: VecDeque::new(),
+			peer_requests: HashMap::new(),
+			previous_requests: HashMap::new(),
+		}
+	}
+
+	/// Dispatches all possible pending requests to the given peers. Peers are
+	/// filtered according to the current known best block (i.e. we won't send a
+	/// justification request for block #10 to a peer at block #2), and we also
+	/// throttle requests to the same peer if a previous justification request
+	/// yielded no results.
+	fn dispatch(&mut self, peers: &mut HashMap<NodeIndex, PeerSync<B>>, protocol: &mut Context<B>) {
+		if self.pending_requests.is_empty() {
+			return;
+		}
+
+		// clean up previous failed requests so we can retry again
+		for (_, requests) in self.previous_requests.iter_mut() {
+			requests.retain(|(_, instant)| instant.elapsed() < JUSTIFICATION_RETRY_WAIT);
+		}
+
+		let mut available_peers = peers.iter().filter_map(|(peer, sync)| {
+			// don't request to any peers that already have pending requests
+			if let PeerSyncState::Available = sync.state {
+				Some((*peer, sync.best_number))
+			} else {
+				None
+			}
+		}).collect::<VecDeque<_>>();
+
+		let mut last_peer = available_peers.back().map(|p| p.0);
+		let mut unhandled_requests = VecDeque::new();
+
+		loop {
+			let (peer, peer_best_number) = match available_peers.pop_front() {
+				Some(p) => p,
+				_ => break,
+			};
+
+			// only ask peers that have synced past the block number that we're
+			// asking the justification for and to whom we haven't already made
+			// the same request recently
+			let peer_eligible = {
+				let request = match self.pending_requests.front() {
+					Some(r) => r.clone(),
+					_ => break,
+				};
+
+				peer_best_number >= request.1 &&
+					!self.previous_requests
+						 .get(&request)
+						 .map(|requests| requests.iter().any(|i| i.0 == peer))
+						 .unwrap_or(false)
+			};
+
+			if !peer_eligible {
+				available_peers.push_back((peer, peer_best_number));
+
+				// we tried all peers and none can answer this request
+				if Some(peer) == last_peer {
+					last_peer = available_peers.back().map(|p| p.0);
+
+					let request = self.pending_requests.pop_front()
+						.expect("verified to be Some in the beginning of the loop; qed");
+
+					unhandled_requests.push_back(request);
+				}
+
+				continue;
+			}
+
+			last_peer = available_peers.back().map(|p| p.0);
+
+			let request = self.pending_requests.pop_front()
+				.expect("verified to be Some in the beginning of the loop; qed");
+
+			self.peer_requests.insert(peer, request);
+
+			peers.get_mut(&peer)
+				.expect("peer was is taken from available_peers; available_peers is a subset of peers; qed")
+				.state = PeerSyncState::DownloadingJustification(request.0);
+
+			let request = message::generic::BlockRequest {
+				id: 0,
+				fields: message::BlockAttributes::JUSTIFICATION,
+				from: message::FromBlock::Hash(request.0),
+				to: None,
+				direction: message::Direction::Ascending,
+				max: Some(1),
+			};
+
+			protocol.send_message(peer, GenericMessage::BlockRequest(request));
+		}
+
+		self.pending_requests.append(&mut unhandled_requests);
+	}
+
+	/// Queue a justification request (without dispatching it).
+	fn queue_request(&mut self, justification: &PendingJustification<B>) {
+		if !self.justifications.insert(*justification) {
+			return;
+		}
+		self.pending_requests.push_back(*justification);
+	}
+
+	/// Retry any pending request if a peer disconnected.
+	fn peer_disconnected(&mut self, who: NodeIndex) {
+		if let Some(request) = self.peer_requests.remove(&who) {
+			self.pending_requests.push_front(request);
+		}
+	}
+
+	/// Processes the response for the request previously sent to the given
+	/// peer. Queues a retry in case the import fails or the given justification
+	/// was `None`.
+	fn on_response(
+		&mut self,
+		who: NodeIndex,
+		justification: Option<Justification>,
+		protocol: &mut Context<B>,
+		import_queue: &ImportQueue<B>,
+	) {
+		// we assume that the request maps to the given response, this is
+		// currently enforced by the outer network protocol before passing on
+		// messages to chain sync.
+		if let Some(request) = self.peer_requests.remove(&who) {
+			if let Some(justification) = justification {
+				if import_queue.import_justification(request.0, request.1, justification) {
+					self.justifications.remove(&request);
+					self.previous_requests.remove(&request);
+					return;
+				} else {
+					protocol.report_peer(
+						who,
+						Severity::Bad(&format!("Invalid justification provided for #{}", request.0)),
+					);
+				}
+			} else {
+				self.previous_requests
+					.entry(request)
+					.or_insert(Vec::new())
+					.push((who, Instant::now()));
+			}
+
+			self.pending_requests.push_front(request);
+		}
+	}
+
+	/// Removes any pending justification requests for blocks lower than the
+	/// given best finalized.
+	fn collect_garbage(&mut self, best_finalized: NumberFor<B>) {
+		self.justifications.retain(|(_, n)| *n > best_finalized);
+		self.pending_requests.retain(|(_, n)| *n > best_finalized);
+		self.peer_requests.retain(|_, (_, n)| *n > best_finalized);
+		self.previous_requests.retain(|(_, n), _| *n > best_finalized);
+	}
 }
 
 /// Relay chain sync strategy.
@@ -59,6 +237,7 @@ pub struct ChainSync<B: BlockT> {
 	best_queued_hash: B::Hash,
 	required_block_attributes: message::BlockAttributes,
 	import_queue: Arc<ImportQueue<B>>,
+	justifications: PendingJustifications<B>,
 }
 
 /// Reported sync state.
@@ -104,6 +283,7 @@ impl<B: BlockT> ChainSync<B> {
 			blocks: BlockCollection::new(),
 			best_queued_hash: info.best_queued_hash.unwrap_or(info.chain.best_hash),
 			best_queued_number: info.best_queued_number.unwrap_or(info.chain.best_number),
+			justifications: PendingJustifications::new(),
 			required_block_attributes,
 			import_queue,
 		}
@@ -192,6 +372,7 @@ impl<B: BlockT> ChainSync<B> {
 		}
 	}
 
+	/// Handle new block data.
 	pub(crate) fn on_block_data(
 		&mut self,
 		protocol: &mut Context<B>,
@@ -269,10 +450,10 @@ impl<B: BlockT> ChainSync<B> {
 						}
 					}
 				},
-				PeerSyncState::Available => Vec::new(),
+				PeerSyncState::Available | PeerSyncState::DownloadingJustification(..) => Vec::new(),
 			}
 		} else {
-			vec![]
+			Vec::new()
 		};
 
 		let best_seen = self.best_seen_block();
@@ -288,17 +469,87 @@ impl<B: BlockT> ChainSync<B> {
 		Some((origin, new_blocks))
 	}
 
+	/// Handle new justification data.
+	pub(crate) fn on_block_justification_data(
+		&mut self,
+		protocol: &mut Context<B>,
+		who: NodeIndex,
+		_request: message::BlockRequest<B>,
+		response: message::BlockResponse<B>,
+	) {
+		if let Some(ref mut peer) = self.peers.get_mut(&who) {
+			if let PeerSyncState::DownloadingJustification(hash) = peer.state {
+				peer.state = PeerSyncState::Available;
+
+				// we only request one justification at a time
+				match response.blocks.into_iter().next() {
+					Some(response) => {
+						if hash != response.hash {
+							let msg = format!(
+								"Invalid block justification provided: requested: {:?} got: {:?}",
+								hash,
+								response.hash,
+							);
+
+							protocol.report_peer(who, Severity::Bad(&msg));
+							return;
+						}
+
+						self.justifications.on_response(
+							who,
+							response.justification,
+							protocol,
+							&*self.import_queue,
+						);
+					},
+					None => {
+						let msg = format!(
+							"Provided empty response for justification request {:?}",
+							hash,
+						);
+
+						protocol.report_peer(who, Severity::Useless(&msg));
+						return;
+					},
+				}
+			}
+		}
+
+		self.maintain_sync(protocol);
+	}
+
+	/// Maintain the sync process (download new blocks, fetch justifications).
 	pub fn maintain_sync(&mut self, protocol: &mut Context<B>) {
 		let peers: Vec<NodeIndex> = self.peers.keys().map(|p| *p).collect();
 		for peer in peers {
 			self.download_new(protocol, peer);
 		}
+		self.justifications.dispatch(&mut self.peers, protocol);
 	}
 
+	/// Called periodically to perform any time-based actions.
+	pub fn tick(&mut self, protocol: &mut Context<B>) {
+		self.justifications.dispatch(&mut self.peers, protocol);
+	}
+
+	/// Request a justification for the given block.
+	///
+	/// Queues a new justification request and tries to dispatch all pending requests.
+	pub fn request_justification(&mut self, hash: &B::Hash, number: NumberFor<B>, protocol: &mut Context<B>) {
+		self.justifications.queue_request(&(*hash, number));
+		self.justifications.dispatch(&mut self.peers, protocol);
+	}
+
+	/// Notify about successful import of the given block.
 	pub fn block_imported(&mut self, hash: &B::Hash, number: NumberFor<B>) {
 		trace!(target: "sync", "Block imported successfully {} ({})", number, hash);
 	}
 
+	/// Notify about finalization of the given block.
+	pub fn block_finalized(&mut self, _hash: &B::Hash, number: NumberFor<B>) {
+		self.justifications.collect_garbage(number);
+	}
+
 	fn block_queued(&mut self, hash: &B::Hash, number: NumberFor<B>) {
 		if number > self.best_queued_number {
 			self.best_queued_number = number;
@@ -324,6 +575,7 @@ impl<B: BlockT> ChainSync<B> {
 		self.block_queued(&hash, best_header.number().clone())
 	}
 
+	/// Handle new block announcement.
 	pub(crate) fn on_block_announce(&mut self, protocol: &mut Context<B>, who: NodeIndex, hash: B::Hash, header: &B::Header) {
 		let number = *header.number();
 		if number <= As::sa(0) {
@@ -376,12 +628,15 @@ impl<B: BlockT> ChainSync<B> {
 		block_status(&*protocol.client(), &*self.import_queue, *hash).ok().map_or(false, |s| s != BlockStatus::Unknown)
 	}
 
+	/// Handle disconnected peer.
 	pub(crate) fn peer_disconnected(&mut self, protocol: &mut Context<B>, who: NodeIndex) {
 		self.blocks.clear_peer_download(who);
 		self.peers.remove(&who);
+		self.justifications.peer_disconnected(who);
 		self.maintain_sync(protocol);
 	}
 
+	/// Restart the sync process.
 	pub(crate) fn restart(&mut self, protocol: &mut Context<B>) {
 		self.import_queue.clear();
 		self.blocks.clear();
@@ -403,6 +658,7 @@ impl<B: BlockT> ChainSync<B> {
 		}
 	}
 
+	/// Clear all sync data.
 	pub(crate) fn clear(&mut self) {
 		self.blocks.clear();
 		self.peers.clear();

substrate/core/network/src/test/mod.rs

@@ -30,7 +30,7 @@ use client::block_builder::BlockBuilder;
 use primitives::Ed25519AuthorityId;
 use runtime_primitives::Justification;
 use runtime_primitives::generic::BlockId;
-use runtime_primitives::traits::{Block as BlockT, Zero, Header, Digest, DigestItem, AuthorityIdFor};
+use runtime_primitives::traits::{AuthorityIdFor, Block as BlockT, Digest, DigestItem, Header, NumberFor, Zero};
 use io::SyncIo;
 use protocol::{Context, Protocol, ProtocolContext};
 use config::ProtocolConfig;
@@ -190,6 +190,15 @@ impl<B: 'static + BlockT, V: 'static + Verifier<B>> ImportQueue<B> for SyncImpor
 	fn import_blocks(&self, origin: BlockOrigin, blocks: Vec<IncomingBlock<B>>) {
 		self.link.call(origin, blocks);
 	}
+
+	fn import_justification(
+		&self,
+		hash: B::Hash,
+		number: NumberFor<B>,
+		justification: Justification,
+	) -> bool {
+		self.block_import.import_justification(hash, number, justification).is_ok()
+	}
 }
 
 struct DummyContextExecutor(Arc<Protocol<Block, DummySpecialization, Hash>>, Arc<RwLock<VecDeque<TestPacket>>>);
@@ -366,6 +375,13 @@ impl<V: 'static + Verifier<Block>, D> Peer<V, D> {
 		self.sync.on_block_imported(&mut TestIo::new(&self.queue, None), info.chain.best_hash, &header);
 	}
 
+	/// Send block finalization notifications.
+	pub fn send_finality_notifications(&self) {
+		let info = self.client.info().expect("In-mem client does not fail");
+		let header = self.client.header(&BlockId::Hash(info.chain.finalized_hash)).unwrap().unwrap();
+		self.sync.on_block_finalized(&mut TestIo::new(&self.queue, None), info.chain.finalized_hash, &header);
+	}
+
 	/// Restart sync for a peer.
 	fn restart_sync(&self) {
 		self.sync.abort();
@@ -380,6 +396,14 @@ impl<V: 'static + Verifier<Block>, D> Peer<V, D> {
 		self.sync.gossip_consensus_message(&mut TestIo::new(&self.queue, None), topic, data, broadcast);
 	}
 
+	/// Request a justification for the given block.
+	#[cfg(test)]
+	fn request_justification(&self, hash: &::primitives::H256, number: NumberFor<Block>) {
+		self.executor.execute_in_context(|context| {
+			self.sync.sync().write().request_justification(hash, number, context);
+		})
+	}
+
 	/// Add blocks to the peer -- edit the block before adding
 	pub fn generate_blocks<F>(&self, count: usize, origin: BlockOrigin, mut edit_block: F)
 		where F: FnMut(BlockBuilder<Block, (), PeersClient>) -> Block
@@ -601,6 +625,15 @@ pub trait TestNetFactory: Sized {
 		})
 	}
 
+	/// Send block finalization notifications for all peers.
+	fn send_finality_notifications(&mut self) {
+		self.mut_peers(|peers| {
+			for peer in peers {
+				peer.send_finality_notifications();
+			}
+		})
+	}
+
 	/// Restart sync for a peer.
 	fn restart_peer(&mut self, i: usize) {
 		self.peers()[i].restart_sync();

substrate/core/network/src/test/sync.rs

@@ -65,6 +65,27 @@ fn sync_no_common_longer_chain_fails() {
 	assert!(!net.peer(0).client.backend().blockchain().canon_equals_to(net.peer(1).client.backend().blockchain()));
 }
 
+#[test]
+fn sync_justifications() {
+	::env_logger::init().ok();
+	let mut net = TestNet::new(3);
+	net.peer(0).push_blocks(20, false);
+	net.sync();
+
+	// there's currently no justification for block #10
+	assert_eq!(net.peer(0).client().justification(&BlockId::Number(10)).unwrap(), None);
+	// we finalize block #10 for peer 0 with a justification
+	net.peer(0).client().finalize_block(BlockId::Number(10), Some(Vec::new()), true).unwrap();
+
+	let header = net.peer(1).client().header(&BlockId::Number(10)).unwrap().unwrap();
+	net.peer(1).request_justification(&header.hash().into(), 10);
+
+	net.sync();
+
+	assert_eq!(net.peer(0).client().justification(&BlockId::Number(10)).unwrap(), Some(Vec::new()));
+	assert_eq!(net.peer(1).client().justification(&BlockId::Number(10)).unwrap(), Some(Vec::new()));
+}
+
 #[test]
 fn sync_after_fork_works() {
 	::env_logger::init().ok();