Sync block justifications (#1410)

* core: sync protocol for justifications * core: basic test for justification sync * core: pass block number with justification * grandpa: request justifications when importing change blocks * core: pass finality notifications to chain sync * core: require justifications for pending change blocks on start * core: avoid requesting justifications from previous failed peers * core: timeout block justification requests * core: add some docs * core: fix unused variables warning * core: tick pending justifications fetch periodically * grandpa: add test for syncing justifications * core: early exit dispatch of pending justifications * core: style fix * core: grandpa: change logging level * core: sync: add missing docs * core: network: report peer on bad justification * core: replace mem::replace with Option::take * core: revert authority set changes on failed block finalization * core: grandpa: add docs to import_justification * core: warn on re-finalization of last finalized block * core: only notify sync with last finality notification * core: style fix * core: add docs for PendingJustifications * core: network: use BlockRequest messages for justification requests * core: reference issues in todo comments * core: grandpa: revert authority set changes on db * core: grandpa: remove inconsistent state warning
2026-05-08 00:48:01 +00:00 · 2019-01-21 06:04:01 +00:00
parent 3ea681998a
commit 399cee310a
13 changed files with 747 additions and 140 deletions
@@ -14,8 +14,9 @@
 // You should have received a copy of the GNU General Public License
 // along with Substrate.  If not, see <http://www.gnu.org/licenses/>.

-use std::collections::HashMap;
+use std::collections::{HashMap, HashSet, VecDeque};
 use std::sync::Arc;
+use std::time::{Duration, Instant};
 use protocol::Context;
 use network_libp2p::{Severity, NodeIndex};
 use client::{BlockStatus, ClientInfo};
@@ -23,6 +24,7 @@ use consensus::BlockOrigin;
 use consensus::import_queue::{ImportQueue, IncomingBlock};
 use client::error::Error as ClientError;
 use blocks::BlockCollection;
+use runtime_primitives::Justification;
 use runtime_primitives::traits::{Block as BlockT, Header as HeaderT, As, NumberFor, Zero};
 use runtime_primitives::generic::BlockId;
 use message::{self, generic::Message as GenericMessage};
@@ -34,6 +36,8 @@ const MAX_BLOCKS_TO_REQUEST: usize = 128;
 const MAX_IMPORTING_BLOCKS: usize = 2048;
 // Number of blocks in the queue that prevents ancestry search.
 const MAJOR_SYNC_BLOCKS: usize = 5;
+// Time to wait before trying to get a justification from the same peer.
+const JUSTIFICATION_RETRY_WAIT: Duration = Duration::from_secs(10);

 struct PeerSync<B: BlockT> {
 	pub common_number: NumberFor<B>,
@@ -48,6 +52,180 @@ enum PeerSyncState<B: BlockT> {
 	Available,
 	DownloadingNew(NumberFor<B>),
 	DownloadingStale(B::Hash),
+	DownloadingJustification(B::Hash),
+}
+
+/// Pending justification request for the given block (hash and number).
+type PendingJustification<B> = (<B as BlockT>::Hash, NumberFor<B>);
+
+/// Manages pending block justification requests.
+struct PendingJustifications<B: BlockT> {
+	justifications: HashSet<PendingJustification<B>>,
+	pending_requests: VecDeque<PendingJustification<B>>,
+	peer_requests: HashMap<NodeIndex, PendingJustification<B>>,
+	previous_requests: HashMap<PendingJustification<B>, Vec<(NodeIndex, Instant)>>,
+}
+
+impl<B: BlockT> PendingJustifications<B> {
+	fn new() -> PendingJustifications<B> {
+		PendingJustifications {
+			justifications: HashSet::new(),
+			pending_requests: VecDeque::new(),
+			peer_requests: HashMap::new(),
+			previous_requests: HashMap::new(),
+		}
+	}
+
+	/// Dispatches all possible pending requests to the given peers. Peers are
+	/// filtered according to the current known best block (i.e. we won't send a
+	/// justification request for block #10 to a peer at block #2), and we also
+	/// throttle requests to the same peer if a previous justification request
+	/// yielded no results.
+	fn dispatch(&mut self, peers: &mut HashMap<NodeIndex, PeerSync<B>>, protocol: &mut Context<B>) {
+		if self.pending_requests.is_empty() {
+			return;
+		}
+
+		// clean up previous failed requests so we can retry again
+		for (_, requests) in self.previous_requests.iter_mut() {
+			requests.retain(|(_, instant)| instant.elapsed() < JUSTIFICATION_RETRY_WAIT);
+		}
+
+		let mut available_peers = peers.iter().filter_map(|(peer, sync)| {
+			// don't request to any peers that already have pending requests
+			if let PeerSyncState::Available = sync.state {
+				Some((*peer, sync.best_number))
+			} else {
+				None
+			}
+		}).collect::<VecDeque<_>>();
+
+		let mut last_peer = available_peers.back().map(|p| p.0);
+		let mut unhandled_requests = VecDeque::new();
+
+		loop {
+			let (peer, peer_best_number) = match available_peers.pop_front() {
+				Some(p) => p,
+				_ => break,
+			};
+
+			// only ask peers that have synced past the block number that we're
+			// asking the justification for and to whom we haven't already made
+			// the same request recently
+			let peer_eligible = {
+				let request = match self.pending_requests.front() {
+					Some(r) => r.clone(),
+					_ => break,
+				};
+
+				peer_best_number >= request.1 &&
+					!self.previous_requests
+						 .get(&request)
+						 .map(|requests| requests.iter().any(|i| i.0 == peer))
+						 .unwrap_or(false)
+			};
+
+			if !peer_eligible {
+				available_peers.push_back((peer, peer_best_number));
+
+				// we tried all peers and none can answer this request
+				if Some(peer) == last_peer {
+					last_peer = available_peers.back().map(|p| p.0);
+
+					let request = self.pending_requests.pop_front()
+						.expect("verified to be Some in the beginning of the loop; qed");
+
+					unhandled_requests.push_back(request);
+				}
+
+				continue;
+			}
+
+			last_peer = available_peers.back().map(|p| p.0);
+
+			let request = self.pending_requests.pop_front()
+				.expect("verified to be Some in the beginning of the loop; qed");
+
+			self.peer_requests.insert(peer, request);
+
+			peers.get_mut(&peer)
+				.expect("peer was is taken from available_peers; available_peers is a subset of peers; qed")
+				.state = PeerSyncState::DownloadingJustification(request.0);
+
+			let request = message::generic::BlockRequest {
+				id: 0,
+				fields: message::BlockAttributes::JUSTIFICATION,
+				from: message::FromBlock::Hash(request.0),
+				to: None,
+				direction: message::Direction::Ascending,
+				max: Some(1),
+			};
+
+			protocol.send_message(peer, GenericMessage::BlockRequest(request));
+		}
+
+		self.pending_requests.append(&mut unhandled_requests);
+	}
+
+	/// Queue a justification request (without dispatching it).
+	fn queue_request(&mut self, justification: &PendingJustification<B>) {
+		if !self.justifications.insert(*justification) {
+			return;
+		}
+		self.pending_requests.push_back(*justification);
+	}
+
+	/// Retry any pending request if a peer disconnected.
+	fn peer_disconnected(&mut self, who: NodeIndex) {
+		if let Some(request) = self.peer_requests.remove(&who) {
+			self.pending_requests.push_front(request);
+		}
+	}
+
+	/// Processes the response for the request previously sent to the given
+	/// peer. Queues a retry in case the import fails or the given justification
+	/// was `None`.
+	fn on_response(
+		&mut self,
+		who: NodeIndex,
+		justification: Option<Justification>,
+		protocol: &mut Context<B>,
+		import_queue: &ImportQueue<B>,
+	) {
+		// we assume that the request maps to the given response, this is
+		// currently enforced by the outer network protocol before passing on
+		// messages to chain sync.
+		if let Some(request) = self.peer_requests.remove(&who) {
+			if let Some(justification) = justification {
+				if import_queue.import_justification(request.0, request.1, justification) {
+					self.justifications.remove(&request);
+					self.previous_requests.remove(&request);
+					return;
+				} else {
+					protocol.report_peer(
+						who,
+						Severity::Bad(&format!("Invalid justification provided for #{}", request.0)),
+					);
+				}
+			} else {
+				self.previous_requests
+					.entry(request)
+					.or_insert(Vec::new())
+					.push((who, Instant::now()));
+			}
+
+			self.pending_requests.push_front(request);
+		}
+	}
+
+	/// Removes any pending justification requests for blocks lower than the
+	/// given best finalized.
+	fn collect_garbage(&mut self, best_finalized: NumberFor<B>) {
+		self.justifications.retain(|(_, n)| *n > best_finalized);
+		self.pending_requests.retain(|(_, n)| *n > best_finalized);
+		self.peer_requests.retain(|_, (_, n)| *n > best_finalized);
+		self.previous_requests.retain(|(_, n), _| *n > best_finalized);
+	}
 }

 /// Relay chain sync strategy.
@@ -59,6 +237,7 @@ pub struct ChainSync<B: BlockT> {
 	best_queued_hash: B::Hash,
 	required_block_attributes: message::BlockAttributes,
 	import_queue: Arc<ImportQueue<B>>,
+	justifications: PendingJustifications<B>,
 }

 /// Reported sync state.
@@ -104,6 +283,7 @@ impl<B: BlockT> ChainSync<B> {
 			blocks: BlockCollection::new(),
 			best_queued_hash: info.best_queued_hash.unwrap_or(info.chain.best_hash),
 			best_queued_number: info.best_queued_number.unwrap_or(info.chain.best_number),
+			justifications: PendingJustifications::new(),
 			required_block_attributes,
 			import_queue,
 		}
@@ -192,6 +372,7 @@ impl<B: BlockT> ChainSync<B> {
 		}
 	}

+	/// Handle new block data.
 	pub(crate) fn on_block_data(
 		&mut self,
 		protocol: &mut Context<B>,
@@ -269,10 +450,10 @@ impl<B: BlockT> ChainSync<B> {
 						}
 					}
 				},
-				PeerSyncState::Available => Vec::new(),
+				PeerSyncState::Available | PeerSyncState::DownloadingJustification(..) => Vec::new(),
 			}
 		} else {
-			vec![]
+			Vec::new()
 		};

 		let best_seen = self.best_seen_block();
@@ -288,17 +469,87 @@ impl<B: BlockT> ChainSync<B> {
 		Some((origin, new_blocks))
 	}

+	/// Handle new justification data.
+	pub(crate) fn on_block_justification_data(
+		&mut self,
+		protocol: &mut Context<B>,
+		who: NodeIndex,
+		_request: message::BlockRequest<B>,
+		response: message::BlockResponse<B>,
+	) {
+		if let Some(ref mut peer) = self.peers.get_mut(&who) {
+			if let PeerSyncState::DownloadingJustification(hash) = peer.state {
+				peer.state = PeerSyncState::Available;
+
+				// we only request one justification at a time
+				match response.blocks.into_iter().next() {
+					Some(response) => {
+						if hash != response.hash {
+							let msg = format!(
+								"Invalid block justification provided: requested: {:?} got: {:?}",
+								hash,
+								response.hash,
+							);
+
+							protocol.report_peer(who, Severity::Bad(&msg));
+							return;
+						}
+
+						self.justifications.on_response(
+							who,
+							response.justification,
+							protocol,
+							&*self.import_queue,
+						);
+					},
+					None => {
+						let msg = format!(
+							"Provided empty response for justification request {:?}",
+							hash,
+						);
+
+						protocol.report_peer(who, Severity::Useless(&msg));
+						return;
+					},
+				}
+			}
+		}
+
+		self.maintain_sync(protocol);
+	}
+
+	/// Maintain the sync process (download new blocks, fetch justifications).
 	pub fn maintain_sync(&mut self, protocol: &mut Context<B>) {
 		let peers: Vec<NodeIndex> = self.peers.keys().map(|p| *p).collect();
 		for peer in peers {
 			self.download_new(protocol, peer);
 		}
+		self.justifications.dispatch(&mut self.peers, protocol);
 	}

+	/// Called periodically to perform any time-based actions.
+	pub fn tick(&mut self, protocol: &mut Context<B>) {
+		self.justifications.dispatch(&mut self.peers, protocol);
+	}
+
+	/// Request a justification for the given block.
+	///
+	/// Queues a new justification request and tries to dispatch all pending requests.
+	pub fn request_justification(&mut self, hash: &B::Hash, number: NumberFor<B>, protocol: &mut Context<B>) {
+		self.justifications.queue_request(&(*hash, number));
+		self.justifications.dispatch(&mut self.peers, protocol);
+	}
+
+	/// Notify about successful import of the given block.
 	pub fn block_imported(&mut self, hash: &B::Hash, number: NumberFor<B>) {
 		trace!(target: "sync", "Block imported successfully {} ({})", number, hash);
 	}

+	/// Notify about finalization of the given block.
+	pub fn block_finalized(&mut self, _hash: &B::Hash, number: NumberFor<B>) {
+		self.justifications.collect_garbage(number);
+	}
+
 	fn block_queued(&mut self, hash: &B::Hash, number: NumberFor<B>) {
 		if number > self.best_queued_number {
 			self.best_queued_number = number;
@@ -324,6 +575,7 @@ impl<B: BlockT> ChainSync<B> {
 		self.block_queued(&hash, best_header.number().clone())
 	}

+	/// Handle new block announcement.
 	pub(crate) fn on_block_announce(&mut self, protocol: &mut Context<B>, who: NodeIndex, hash: B::Hash, header: &B::Header) {
 		let number = *header.number();
 		if number <= As::sa(0) {
@@ -376,12 +628,15 @@ impl<B: BlockT> ChainSync<B> {
 		block_status(&*protocol.client(), &*self.import_queue, *hash).ok().map_or(false, |s| s != BlockStatus::Unknown)
 	}

+	/// Handle disconnected peer.
 	pub(crate) fn peer_disconnected(&mut self, protocol: &mut Context<B>, who: NodeIndex) {
 		self.blocks.clear_peer_download(who);
 		self.peers.remove(&who);
+		self.justifications.peer_disconnected(who);
 		self.maintain_sync(protocol);
 	}

+	/// Restart the sync process.
 	pub(crate) fn restart(&mut self, protocol: &mut Context<B>) {
 		self.import_queue.clear();
 		self.blocks.clear();
@@ -403,6 +658,7 @@ impl<B: BlockT> ChainSync<B> {
 		}
 	}

+	/// Clear all sync data.
 	pub(crate) fn clear(&mut self) {
 		self.blocks.clear();
 		self.peers.clear();