Beefy on-demand justifications as a custom RequestResponse protocol (#12124)

* client/beefy: create communication module and move gossip there * client/beefy: move beefy_protocol_name module to communication * client/beefy: move notification module under communication * client/beefy: add incoming request_response protocol handler * client/beefy: keep track of connected peers and their progress * client/beefy: add logic for generating Justif requests * client/beefy: cancel outdated on-demand justification requests * try Andre's suggestion for JustificationEngine * justif engine add justifs validation * client/beefy: impl OnDemandJustificationsEngine async next() * move beefy proto name test * client/beefy: initialize OnDemandJustificationsEngine * client/tests: allow for custom req-resp protocols * client/beefy: on-demand-justif: implement simple peer selection strategy * client/beefy: fix voter initialization Fix corner case where voter gets a single burst of finality notifications just when it starts. The notification stream was consumed by "wait_for_pallet" logic, then main loop would subscribe to finality notifications, but by that time some notifications might've been lost. Fix this by subscribing the main loop to notifications before waiting for pallet to become available. Share the same stream with the main loop so that notifications for blocks before pallet available are ignored, while _all_ notifications after pallet available are processed. Add regression test for this. Signed-off-by: acatangiu <adrian@parity.io> * client/beefy: make sure justif requests are always out for mandatory blocks * client/beefy: add test for on-demand justifications sync * client/beefy: tweak main loop event processing order * client/beefy: run on-demand-justif-handler under same async task as voter * client/beefy: add test for known-peers * client/beefy: reorg request-response module * client/beefy: add issue references for future work todos * client/beefy: consolidate on-demand-justifications engine state machine Signed-off-by: acatangiu <adrian@parity.io> * client/beefy: fix for polkadot companion * client/beefy: implement review suggestions * cargo fmt and clippy * fix merge damage * fix rust-doc * fix merge damage * fix merge damage * client/beefy: add test for justif proto name Signed-off-by: acatangiu <adrian@parity.io>
2026-07-02 23:27:24 +00:00 · 2022-10-03 16:00:57 +03:00
parent bb9d2fa75a
commit 2a27545afe
14 changed files with 1208 additions and 219 deletions
@@ -0,0 +1,193 @@
+// Copyright 2022 Parity Technologies (UK) Ltd.
+// This file is part of Substrate.
+
+// Substrate is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// Substrate is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with Substrate.  If not, see <http://www.gnu.org/licenses/>.
+
+//! Helper for handling (i.e. answering) BEEFY justifications requests from a remote peer.
+
+use beefy_primitives::BEEFY_ENGINE_ID;
+use codec::Decode;
+use futures::{
+	channel::{mpsc, oneshot},
+	StreamExt,
+};
+use log::{debug, trace};
+use sc_client_api::BlockBackend;
+use sc_network::{config as netconfig, config::RequestResponseConfig, PeerId, ReputationChange};
+use sc_network_common::protocol::ProtocolName;
+use sp_runtime::{generic::BlockId, traits::Block};
+use std::{marker::PhantomData, sync::Arc};
+
+use crate::communication::request_response::{
+	on_demand_justifications_protocol_config, Error, JustificationRequest,
+};
+
+/// A request coming in, including a sender for sending responses.
+#[derive(Debug)]
+pub(crate) struct IncomingRequest<B: Block> {
+	/// `PeerId` of sending peer.
+	pub peer: PeerId,
+	/// The sent request.
+	pub payload: JustificationRequest<B>,
+	/// Sender for sending response back.
+	pub pending_response: oneshot::Sender<netconfig::OutgoingResponse>,
+}
+
+impl<B: Block> IncomingRequest<B> {
+	/// Create new `IncomingRequest`.
+	pub fn new(
+		peer: PeerId,
+		payload: JustificationRequest<B>,
+		pending_response: oneshot::Sender<netconfig::OutgoingResponse>,
+	) -> Self {
+		Self { peer, payload, pending_response }
+	}
+
+	/// Try building from raw network request.
+	///
+	/// This function will fail if the request cannot be decoded and will apply passed in
+	/// reputation changes in that case.
+	///
+	/// Params:
+	/// 	- The raw request to decode
+	/// 	- Reputation changes to apply for the peer in case decoding fails.
+	pub fn try_from_raw(
+		raw: netconfig::IncomingRequest,
+		reputation_changes: Vec<ReputationChange>,
+	) -> Result<Self, Error> {
+		let netconfig::IncomingRequest { payload, peer, pending_response } = raw;
+		let payload = match JustificationRequest::decode(&mut payload.as_ref()) {
+			Ok(payload) => payload,
+			Err(err) => {
+				let response = netconfig::OutgoingResponse {
+					result: Err(()),
+					reputation_changes,
+					sent_feedback: None,
+				};
+				if let Err(_) = pending_response.send(response) {
+					return Err(Error::DecodingErrorNoReputationChange(peer, err))
+				}
+				return Err(Error::DecodingError(peer, err))
+			},
+		};
+		Ok(Self::new(peer, payload, pending_response))
+	}
+}
+
+/// Receiver for incoming BEEFY justifications requests.
+///
+/// Takes care of decoding and handling of invalid encoded requests.
+pub(crate) struct IncomingRequestReceiver {
+	raw: mpsc::Receiver<netconfig::IncomingRequest>,
+}
+
+impl IncomingRequestReceiver {
+	pub fn new(inner: mpsc::Receiver<netconfig::IncomingRequest>) -> Self {
+		Self { raw: inner }
+	}
+
+	/// Try to receive the next incoming request.
+	///
+	/// Any received request will be decoded, on decoding errors the provided reputation changes
+	/// will be applied and an error will be reported.
+	pub async fn recv<B, F>(&mut self, reputation_changes: F) -> Result<IncomingRequest<B>, Error>
+	where
+		B: Block,
+		F: FnOnce() -> Vec<ReputationChange>,
+	{
+		let req = match self.raw.next().await {
+			None => return Err(Error::RequestChannelExhausted),
+			Some(raw) => IncomingRequest::<B>::try_from_raw(raw, reputation_changes())?,
+		};
+		Ok(req)
+	}
+}
+
+/// Handler for incoming BEEFY justifications requests from a remote peer.
+pub struct BeefyJustifsRequestHandler<B, Client> {
+	pub(crate) request_receiver: IncomingRequestReceiver,
+	pub(crate) justif_protocol_name: ProtocolName,
+	pub(crate) client: Arc<Client>,
+	pub(crate) _block: PhantomData<B>,
+}
+
+impl<B, Client> BeefyJustifsRequestHandler<B, Client>
+where
+	B: Block,
+	Client: BlockBackend<B> + Send + Sync,
+{
+	/// Create a new [`BeefyJustifsRequestHandler`].
+	pub fn new<Hash: AsRef<[u8]>>(
+		genesis_hash: Hash,
+		fork_id: Option<&str>,
+		client: Arc<Client>,
+	) -> (Self, RequestResponseConfig) {
+		let (request_receiver, config) =
+			on_demand_justifications_protocol_config(genesis_hash, fork_id);
+		let justif_protocol_name = config.name.clone();
+
+		(Self { request_receiver, justif_protocol_name, client, _block: PhantomData }, config)
+	}
+
+	/// Network request-response protocol name used by this handler.
+	pub fn protocol_name(&self) -> ProtocolName {
+		self.justif_protocol_name.clone()
+	}
+
+	// Sends back justification response if justification found in client backend.
+	fn handle_request(&self, request: IncomingRequest<B>) -> Result<(), Error> {
+		// TODO (issue #12293): validate `request` and change peer reputation for invalid requests.
+
+		let maybe_encoded_proof = self
+			.client
+			.justifications(&BlockId::Number(request.payload.begin))
+			.map_err(Error::Client)?
+			.and_then(|justifs| justifs.get(BEEFY_ENGINE_ID).cloned())
+			// No BEEFY justification present.
+			.ok_or(());
+
+		request
+			.pending_response
+			.send(netconfig::OutgoingResponse {
+				result: maybe_encoded_proof,
+				reputation_changes: Vec::new(),
+				sent_feedback: None,
+			})
+			.map_err(|_| Error::SendResponse)
+	}
+
+	/// Run [`BeefyJustifsRequestHandler`].
+	pub async fn run(mut self) {
+		trace!(target: "beefy::sync", "🥩 Running BeefyJustifsRequestHandler");
+
+		while let Ok(request) = self.request_receiver.recv(|| vec![]).await {
+			let peer = request.peer;
+			match self.handle_request(request) {
+				Ok(()) => {
+					debug!(
+						target: "beefy::sync",
+						"🥩 Handled BEEFY justification request from {:?}.", peer
+					)
+				},
+				Err(e) => {
+					// TODO (issue #12293): apply reputation changes here based on error type.
+					debug!(
+						target: "beefy::sync",
+						"🥩 Failed to handle BEEFY justification request from {:?}: {}", peer, e,
+					)
+				},
+			}
+		}
+	}
+}
@@ -0,0 +1,101 @@
+// This file is part of Substrate.
+
+// Copyright (C) 2022 Parity Technologies (UK) Ltd.
+// SPDX-License-Identifier: GPL-3.0-or-later WITH Classpath-exception-2.0
+
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with this program. If not, see <https://www.gnu.org/licenses/>.
+
+//! Request/response protocol for syncing BEEFY justifications.
+
+mod incoming_requests_handler;
+pub(crate) mod outgoing_requests_engine;
+
+pub use incoming_requests_handler::BeefyJustifsRequestHandler;
+
+use futures::channel::mpsc;
+use std::time::Duration;
+
+use codec::{Decode, Encode, Error as CodecError};
+use sc_network::{config::RequestResponseConfig, PeerId};
+use sp_runtime::traits::{Block, NumberFor};
+
+use crate::communication::beefy_protocol_name::justifications_protocol_name;
+use incoming_requests_handler::IncomingRequestReceiver;
+
+// 10 seems reasonable, considering justifs are explicitly requested only
+// for mandatory blocks, by nodes that are syncing/catching-up.
+const JUSTIF_CHANNEL_SIZE: usize = 10;
+
+const MAX_RESPONSE_SIZE: u64 = 1024 * 1024;
+const JUSTIF_REQUEST_TIMEOUT: Duration = Duration::from_secs(3);
+
+/// Get the configuration for the BEEFY justifications Request/response protocol.
+///
+/// Returns a receiver for messages received on this protocol and the requested
+/// `ProtocolConfig`.
+///
+/// Consider using [`BeefyJustifsRequestHandler`] instead of this low-level function.
+pub(crate) fn on_demand_justifications_protocol_config<Hash: AsRef<[u8]>>(
+	genesis_hash: Hash,
+	fork_id: Option<&str>,
+) -> (IncomingRequestReceiver, RequestResponseConfig) {
+	let name = justifications_protocol_name(genesis_hash, fork_id);
+	let fallback_names = vec![];
+	let (tx, rx) = mpsc::channel(JUSTIF_CHANNEL_SIZE);
+	let rx = IncomingRequestReceiver::new(rx);
+	let cfg = RequestResponseConfig {
+		name,
+		fallback_names,
+		max_request_size: 32,
+		max_response_size: MAX_RESPONSE_SIZE,
+		// We are connected to all validators:
+		request_timeout: JUSTIF_REQUEST_TIMEOUT,
+		inbound_queue: Some(tx),
+	};
+	(rx, cfg)
+}
+
+/// BEEFY justification request.
+#[derive(Debug, Clone, Encode, Decode)]
+pub struct JustificationRequest<B: Block> {
+	/// Start collecting proofs from this block.
+	pub begin: NumberFor<B>,
+}
+
+#[derive(Debug, thiserror::Error)]
+pub enum Error {
+	#[error(transparent)]
+	Client(#[from] sp_blockchain::Error),
+
+	#[error(transparent)]
+	RuntimeApi(#[from] sp_api::ApiError),
+
+	/// Decoding failed, we were able to change the peer's reputation accordingly.
+	#[error("Decoding request failed for peer {0}.")]
+	DecodingError(PeerId, #[source] CodecError),
+
+	/// Decoding failed, but sending reputation change failed.
+	#[error("Decoding request failed for peer {0}, and changing reputation failed.")]
+	DecodingErrorNoReputationChange(PeerId, #[source] CodecError),
+
+	/// Incoming request stream exhausted. Should only happen on shutdown.
+	#[error("Incoming request channel got closed.")]
+	RequestChannelExhausted,
+
+	#[error("Failed to send response.")]
+	SendResponse,
+
+	#[error("Received invalid response.")]
+	InvalidResponse,
+}
@@ -0,0 +1,245 @@
+// This file is part of Substrate.
+
+// Copyright (C) 2022 Parity Technologies (UK) Ltd.
+// SPDX-License-Identifier: GPL-3.0-or-later WITH Classpath-exception-2.0
+
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with this program. If not, see <https://www.gnu.org/licenses/>.
+
+//! Generating request logic for request/response protocol for syncing BEEFY justifications.
+
+use beefy_primitives::{crypto::AuthorityId, BeefyApi, ValidatorSet};
+use codec::Encode;
+use futures::{
+	channel::{oneshot, oneshot::Canceled},
+	stream::{self, StreamExt},
+};
+use log::{debug, error, warn};
+use parking_lot::Mutex;
+use sc_network::{PeerId, ProtocolName};
+use sc_network_common::{
+	request_responses::{IfDisconnected, RequestFailure},
+	service::NetworkRequest,
+};
+use sp_api::ProvideRuntimeApi;
+use sp_runtime::{
+	generic::BlockId,
+	traits::{Block, NumberFor},
+};
+use std::{collections::VecDeque, result::Result, sync::Arc};
+
+use crate::{
+	communication::request_response::{Error, JustificationRequest},
+	justification::{decode_and_verify_finality_proof, BeefyVersionedFinalityProof},
+	KnownPeers,
+};
+
+/// Response type received from network.
+type Response = Result<Vec<u8>, RequestFailure>;
+/// Used to receive a response from the network.
+type ResponseReceiver = oneshot::Receiver<Response>;
+
+enum State<B: Block> {
+	Idle(stream::Pending<Result<Response, Canceled>>),
+	AwaitingResponse(PeerId, NumberFor<B>, stream::Once<ResponseReceiver>),
+}
+
+pub struct OnDemandJustificationsEngine<B: Block, R> {
+	network: Arc<dyn NetworkRequest + Send + Sync>,
+	runtime: Arc<R>,
+	protocol_name: ProtocolName,
+
+	live_peers: Arc<Mutex<KnownPeers<B>>>,
+	peers_cache: VecDeque<PeerId>,
+
+	state: State<B>,
+}
+
+impl<B, R> OnDemandJustificationsEngine<B, R>
+where
+	B: Block,
+	R: ProvideRuntimeApi<B>,
+	R::Api: BeefyApi<B>,
+{
+	pub fn new(
+		network: Arc<dyn NetworkRequest + Send + Sync>,
+		runtime: Arc<R>,
+		protocol_name: ProtocolName,
+		live_peers: Arc<Mutex<KnownPeers<B>>>,
+	) -> Self {
+		Self {
+			network,
+			runtime,
+			protocol_name,
+			live_peers,
+			peers_cache: VecDeque::new(),
+			state: State::Idle(stream::pending()),
+		}
+	}
+
+	fn reset_peers_cache_for_block(&mut self, block: NumberFor<B>) {
+		// TODO (issue #12296): replace peer selection with generic one that involves all protocols.
+		self.peers_cache = self.live_peers.lock().at_least_at_block(block);
+	}
+
+	fn try_next_peer(&mut self) -> Option<PeerId> {
+		// TODO (issue #12296): replace peer selection with generic one that involves all protocols.
+		let live = self.live_peers.lock();
+		while let Some(peer) = self.peers_cache.pop_front() {
+			if live.contains(&peer) {
+				return Some(peer)
+			}
+		}
+		None
+	}
+
+	fn request_from_peer(&mut self, peer: PeerId, block: NumberFor<B>) {
+		debug!(target: "beefy::sync", "🥩 requesting justif #{:?} from peer {:?}", block, peer);
+
+		let payload = JustificationRequest::<B> { begin: block }.encode();
+
+		let (tx, rx) = oneshot::channel();
+
+		self.network.start_request(
+			peer,
+			self.protocol_name.clone(),
+			payload,
+			tx,
+			IfDisconnected::ImmediateError,
+		);
+
+		self.state = State::AwaitingResponse(peer, block, stream::once(rx));
+	}
+
+	/// If no other request is in progress, start new justification request for `block`.
+	pub fn request(&mut self, block: NumberFor<B>) {
+		// ignore new requests while there's already one pending
+		match &self.state {
+			State::AwaitingResponse(_, _, _) => return,
+			State::Idle(_) => (),
+		}
+		self.reset_peers_cache_for_block(block);
+
+		// Start the requests engine - each unsuccessful received response will automatically
+		// trigger a new request to the next peer in the `peers_cache` until there are none left.
+		if let Some(peer) = self.try_next_peer() {
+			self.request_from_peer(peer, block);
+		} else {
+			debug!(target: "beefy::sync", "🥩 no good peers to request justif #{:?} from", block);
+		}
+	}
+
+	/// Cancel any pending request for block numbers smaller or equal to `block`.
+	pub fn cancel_requests_older_than(&mut self, block: NumberFor<B>) {
+		match &self.state {
+			State::AwaitingResponse(_, number, _) if *number <= block => {
+				debug!(
+					target: "beefy::sync",
+					"🥩 cancel pending request for justification #{:?}",
+					number
+				);
+				self.state = State::Idle(stream::pending());
+			},
+			_ => (),
+		}
+	}
+
+	fn process_response(
+		&mut self,
+		peer: PeerId,
+		block: NumberFor<B>,
+		validator_set: &ValidatorSet<AuthorityId>,
+		response: Result<Response, Canceled>,
+	) -> Result<BeefyVersionedFinalityProof<B>, Error> {
+		response
+			.map_err(|e| {
+				debug!(
+					target: "beefy::sync",
+					"🥩 for on demand justification #{:?}, peer {:?} hung up: {:?}",
+					block, peer, e
+				);
+				Error::InvalidResponse
+			})?
+			.map_err(|e| {
+				debug!(
+					target: "beefy::sync",
+					"🥩 for on demand justification #{:?}, peer {:?} error: {:?}",
+					block, peer, e
+				);
+				Error::InvalidResponse
+			})
+			.and_then(|encoded| {
+				decode_and_verify_finality_proof::<B>(&encoded[..], block, &validator_set).map_err(
+					|e| {
+						debug!(
+							target: "beefy::sync",
+							"🥩 for on demand justification #{:?}, peer {:?} responded with invalid proof: {:?}",
+							block, peer, e
+						);
+						Error::InvalidResponse
+					},
+				)
+			})
+	}
+
+	pub async fn next(&mut self) -> Option<BeefyVersionedFinalityProof<B>> {
+		let (peer, block, resp) = match &mut self.state {
+			State::Idle(pending) => {
+				let _ = pending.next().await;
+				// This never happens since 'stream::pending' never generates any items.
+				return None
+			},
+			State::AwaitingResponse(peer, block, receiver) => {
+				let resp = receiver.next().await?;
+				(*peer, *block, resp)
+			},
+		};
+		// We received the awaited response. Our 'stream::once()' receiver will never generate any
+		// other response, meaning we're done with current state. Move the engine to `State::Idle`.
+		self.state = State::Idle(stream::pending());
+
+		let block_id = BlockId::number(block);
+		let validator_set = self
+			.runtime
+			.runtime_api()
+			.validator_set(&block_id)
+			.map_err(|e| {
+				error!(target: "beefy::sync", "🥩 Runtime API error {:?} in on-demand justif engine.", e);
+				e
+			})
+			.ok()?
+			.or_else(|| {
+				error!(target: "beefy::sync", "🥩 BEEFY pallet not available for block {:?}.", block);
+				None
+			})?;
+
+		self.process_response(peer, block, &validator_set, resp)
+			.map_err(|_| {
+				// No valid justification received, try next peer in our set.
+				if let Some(peer) = self.try_next_peer() {
+					self.request_from_peer(peer, block);
+				} else {
+					warn!(target: "beefy::sync", "🥩 ran out of peers to request justif #{:?} from", block);
+				}
+			})
+			.map(|proof| {
+				debug!(
+					target: "beefy::sync",
+					"🥩 received valid on-demand justif #{:?} from {:?}",
+					block, peer
+				);
+				proof
+			})
+			.ok()
+	}
+}