Storing multiple Justifications per block (#7640)

* primitives/runtime: initial changes on supporting multiple Justifications

* primitives/runtime: make Justifications strongly typed

* Encode/decode Justifications

* primitives/runtime: add Justification type

* backend: apply_finality and finalize_block takes a single Justification

* manual-seal: create engine id and let rpc take encoded justification

* backend: skeleton functions for appending justifications

* backend: initial implementation append_justification

Initial implementation of append_justification on the Backend trait, and also remove unused skeleton
functions for append_justificaton on Finaziler trait.
k

* backend: guard against duplicate consensus engine id

* client/db: add check for block finality

* client/api: add append_justification to in_mem db

* client/light: add no-op append_justification

* network: fix decode call for Justification

* network: only send a single Justification in BlockData

* network: minor comment update

* protocol: update field names to distinguish single justification

* client: further field renames to plural

* client: update function names to plural justifications

* client/db: upgrade existing database for new format

* network: remove dependency on grandpa crate

* db: fix check for finalized block

* grandpa: check for multiple grandpa justifications hwne importing

* backend: update Finalizer trait to take multiple Justifications

* db: remove debugging statements in migration code

* manual-seal: update note about engine id

* db: fix check for finalized block

* client: update variable name to reflect it is now plural

* grandpa: fix incorrect empty Justications in test

* primitives: make Justifications opaque to avoid being empty

* network: fix detecting empty Justification

* runtime: doc strings for Justifications functions

* runtime: add into_justifications

* primitives: check for duplicates in when adding to Justifications

* network/test: use real grandpa engine id in test

* client: fix reviewer comments

* primitives: rename Justifications::push to append

* backend: revert changes to Finalizer trait

* backend: revert mark_finalized

* backend: revert changes to finalize_block

* backend: revert finalized_blocks

* db: add a quick early return for performance

* client: minor reviewer comments

* service/test: use local ConsensusEngineId

* network: add link to issue for sending multiple Justifications

* Apply suggestions from code review

Co-authored-by: Pierre Krieger <pierre.krieger1708@gmail.com>

* Apply suggestions from code review

Co-authored-by: Pierre Krieger <pierre.krieger1708@gmail.com>

* network: tweaks to review suggestions

* network: revert change to BlockData for backwards compatibility

* Apply suggestion from code review

Co-authored-by: Pierre Krieger <pierre.krieger1708@gmail.com>

* Apply suggestions from code review

Co-authored-by: André Silva <123550+andresilva@users.noreply.github.com>

* primitives: update doc comment for Justifications

* client/db/upgrade: avoid grandpa crate dependency

* consensus: revert to single Justification for import_justification

* primitives: improve justifications docs

* style cleanups

* use and_then

* client: rename JUSTIFICATIONS db column

* network: revert to using FRNK in network-test

Co-authored-by: Pierre Krieger <pierre.krieger1708@gmail.com>
Co-authored-by: André Silva <123550+andresilva@users.noreply.github.com>
Co-authored-by: André Silva <andrerfosilva@gmail.com>

substrate/client/network/src/behaviour.rs

@@ -37,7 +37,7 @@ use libp2p::swarm::{
 use log::debug;
 use prost::Message;
 use sp_consensus::{BlockOrigin, import_queue::{IncomingBlock, Origin}};
-use sp_runtime::{traits::{Block as BlockT, NumberFor}, Justification};
+use sp_runtime::{traits::{Block as BlockT, NumberFor}, Justifications};
 use std::{
 	borrow::Cow,
 	collections::{HashSet, VecDeque},
@@ -84,7 +84,7 @@ pub struct Behaviour<B: BlockT> {
 /// Event generated by `Behaviour`.
 pub enum BehaviourOut<B: BlockT> {
 	BlockImport(BlockOrigin, Vec<IncomingBlock<B>>),
-	JustificationImport(Origin, B::Hash, NumberFor<B>, Justification),
+	JustificationImport(Origin, B::Hash, NumberFor<B>, Justifications),
 
 	/// Started a random iterative Kademlia discovery query.
 	RandomKademliaStarted(ProtocolId),

substrate/client/network/src/block_request_handler.rs

@@ -275,12 +275,28 @@ impl<B: BlockT> BlockRequestHandler<B> {
 			let number = *header.number();
 			let hash = header.hash();
 			let parent_hash = *header.parent_hash();
-			let justification = if get_justification {
-				self.client.justification(&BlockId::Hash(hash))?
+			let justifications = if get_justification {
+				self.client.justifications(&BlockId::Hash(hash))?
 			} else {
 				None
 			};
-			let is_empty_justification = justification.as_ref().map(|j| j.is_empty()).unwrap_or(false);
+
+			// TODO: In a follow up PR tracked by https://github.com/paritytech/substrate/issues/8172
+			// we want to send/receive all justifications.
+			// For now we keep compatibility by selecting precisely the GRANDPA one, and not just
+			// the first one. When sending we could have just taken the first one, since we don't
+			// expect there to be any other kind currently, but when receiving we need to add the
+			// engine ID tag.
+			// The ID tag is hardcoded here to avoid depending on the GRANDPA crate, and will be
+			// removed when resolving the above issue.
+			let justification = justifications.and_then(|just| just.into_justification(*b"FRNK"));
+
+			let is_empty_justification = justification
+				.as_ref()
+				.map(|j| j.is_empty())
+				.unwrap_or(false);
+
+			let justification = justification.unwrap_or_default();
 
 			let body = if get_body {
 				match self.client.block_body(&BlockId::Hash(hash))? {
@@ -306,7 +322,7 @@ impl<B: BlockT> BlockRequestHandler<B> {
 				body,
 				receipt: Vec::new(),
 				message_queue: Vec::new(),
-				justification: justification.unwrap_or_default(),
+				justification,
 				is_empty_justification,
 			};
 

substrate/client/network/src/gossip/tests.rs

@@ -52,7 +52,7 @@ fn build_test_full_node(network_config: config::NetworkConfiguration)
 			&mut self,
 			origin: sp_consensus::BlockOrigin,
 			header: B::Header,
-			justification: Option<sp_runtime::Justification>,
+			justifications: Option<sp_runtime::Justifications>,
 			body: Option<Vec<B::Extrinsic>>,
 		) -> Result<
 			(
@@ -79,7 +79,7 @@ fn build_test_full_node(network_config: config::NetworkConfiguration)
 			let mut import = sp_consensus::BlockImportParams::new(origin, header);
 			import.body = body;
 			import.finalized = self.0;
-			import.justification = justification;
+			import.justifications = justifications;
 			import.fork_choice = Some(sp_consensus::ForkChoiceStrategy::LongestChain);
 			Ok((import, maybe_keys))
 		}

substrate/client/network/src/protocol.rs

@@ -43,9 +43,10 @@ use sp_consensus::{
 	block_validation::BlockAnnounceValidator,
 	import_queue::{BlockImportResult, BlockImportError, IncomingBlock, Origin}
 };
-use sp_runtime::{generic::BlockId, Justification};
-use sp_runtime::traits::{
-	Block as BlockT, Header as HeaderT, NumberFor, Zero, CheckedSub
+use sp_runtime::{
+	Justifications,
+	generic::BlockId,
+	traits::{Block as BlockT, Header as HeaderT, NumberFor, Zero, CheckedSub},
 };
 use sp_arithmetic::traits::SaturatedConversion;
 use sync::{ChainSync, SyncState};
@@ -612,8 +613,8 @@ impl<B: BlockT> Protocol<B> {
 		if request.fields == message::BlockAttributes::JUSTIFICATION {
 			match self.sync.on_block_justification(peer_id, block_response) {
 				Ok(sync::OnBlockJustification::Nothing) => CustomMessageOutcome::None,
-				Ok(sync::OnBlockJustification::Import { peer, hash, number, justification }) =>
-					CustomMessageOutcome::JustificationImport(peer, hash, number, justification),
+				Ok(sync::OnBlockJustification::Import { peer, hash, number, justifications }) =>
+					CustomMessageOutcome::JustificationImport(peer, hash, number, justifications),
 				Err(sync::BadPeer(id, repu)) => {
 					self.behaviour.disconnect_peer(&id, HARDCODED_PEERSETS_SYNC);
 					self.peerset_handle.report_peer(id, repu);
@@ -1134,7 +1135,7 @@ fn prepare_block_request<B: BlockT>(
 #[must_use]
 pub enum CustomMessageOutcome<B: BlockT> {
 	BlockImport(BlockOrigin, Vec<IncomingBlock<B>>),
-	JustificationImport(Origin, B::Hash, NumberFor<B>, Justification),
+	JustificationImport(Origin, B::Hash, NumberFor<B>, Justifications),
 	/// Notification protocols have been opened with a remote.
 	NotificationStreamOpened {
 		remote: PeerId,

substrate/client/network/src/protocol/message.rs

@@ -148,7 +148,7 @@ pub struct RemoteReadResponse {
 pub mod generic {
 	use bitflags::bitflags;
 	use codec::{Encode, Decode, Input, Output};
-	use sp_runtime::Justification;
+	use sp_runtime::EncodedJustification;
 	use super::{
 		RemoteReadResponse, Transactions, Direction,
 		RequestId, BlockAttributes, RemoteCallResponse, ConsensusEngineId,
@@ -233,7 +233,7 @@ pub mod generic {
 		/// Block message queue if requested.
 		pub message_queue: Option<Vec<u8>>,
 		/// Justification if requested.
-		pub justification: Option<Justification>,
+		pub justification: Option<EncodedJustification>,
 	}
 
 	/// Identifies starting point of a block sequence.

substrate/client/network/src/protocol/sync.rs

@@ -44,7 +44,7 @@ use extra_requests::ExtraRequests;
 use libp2p::PeerId;
 use log::{debug, trace, warn, info, error};
 use sp_runtime::{
-	Justification,
+	EncodedJustification, Justifications,
 	generic::BlockId,
 	traits::{
 		Block as BlockT, Header as HeaderT, NumberFor, Zero, One, CheckedSub, SaturatedConversion,
@@ -425,7 +425,7 @@ pub enum OnBlockJustification<B: BlockT> {
 		peer: PeerId,
 		hash: B::Hash,
 		number: NumberFor<B>,
-		justification: Justification
+		justifications: Justifications
 	}
 }
 
@@ -823,11 +823,13 @@ impl<B: BlockT> ChainSync<B> {
 								.drain(self.best_queued_number + One::one())
 								.into_iter()
 								.map(|block_data| {
+									let justifications =
+										legacy_justification_mapping(block_data.block.justification);
 									IncomingBlock {
 										hash: block_data.block.hash,
 										header: block_data.block.header,
 										body: block_data.block.body,
-										justification: block_data.block.justification,
+										justifications,
 										origin: block_data.origin,
 										allow_missing_state: true,
 										import_existing: false,
@@ -846,7 +848,7 @@ impl<B: BlockT> ChainSync<B> {
 									hash: b.hash,
 									header: b.header,
 									body: b.body,
-									justification: b.justification,
+									justifications: legacy_justification_mapping(b.justification),
 									origin: Some(who.clone()),
 									allow_missing_state: true,
 									import_existing: false,
@@ -955,7 +957,7 @@ impl<B: BlockT> ChainSync<B> {
 							hash: b.hash,
 							header: b.header,
 							body: b.body,
-							justification: b.justification,
+							justifications: legacy_justification_mapping(b.justification),
 							origin: Some(who.clone()),
 							allow_missing_state: true,
 							import_existing: false,
@@ -1039,8 +1041,11 @@ impl<B: BlockT> ChainSync<B> {
 				None
 			};
 
-			if let Some((peer, hash, number, j)) = self.extra_justifications.on_response(who, justification) {
-				return Ok(OnBlockJustification::Import { peer, hash, number, justification: j })
+			if let Some((peer, hash, number, j)) = self
+				.extra_justifications
+				.on_response(who, legacy_justification_mapping(justification))
+			{
+				return Ok(OnBlockJustification::Import { peer, hash, number, justifications: j })
 			}
 		}
 
@@ -1597,6 +1602,14 @@ impl<B: BlockT> ChainSync<B> {
 	}
 }
 
+// This is purely during a backwards compatible transitionary period and should be removed
+// once we can assume all nodes can send and receive multiple Justifications
+// The ID tag is hardcoded here to avoid depending on the GRANDPA crate.
+// TODO: https://github.com/paritytech/substrate/issues/8172
+fn legacy_justification_mapping(justification: Option<EncodedJustification>) -> Option<Justifications> {
+	justification.map(|just| (*b"FRNK", just).into())
+}
+
 #[derive(Debug)]
 pub(crate) struct Metrics {
 	pub(crate) queued_blocks: u32,
@@ -2396,7 +2409,8 @@ mod test {
 		);
 
 		let finalized_block = blocks[MAX_BLOCKS_TO_LOOK_BACKWARDS as usize * 2 - 1].clone();
-		client.finalize_block(BlockId::Hash(finalized_block.hash()), Some(Vec::new())).unwrap();
+		let just = (*b"TEST", Vec::new());
+		client.finalize_block(BlockId::Hash(finalized_block.hash()), Some(just)).unwrap();
 		sync.update_chain_info(&info.best_hash, info.best_number);
 
 		let peer_id1 = PeerId::random();

substrate/client/network/src/service.rs

@@ -1452,11 +1452,11 @@ impl<B: BlockT + 'static, H: ExHashT> Future for NetworkWorker<B, H> {
 					}
 					this.import_queue.import_blocks(origin, blocks);
 				},
-				Poll::Ready(SwarmEvent::Behaviour(BehaviourOut::JustificationImport(origin, hash, nb, justification))) => {
+				Poll::Ready(SwarmEvent::Behaviour(BehaviourOut::JustificationImport(origin, hash, nb, justifications))) => {
 					if let Some(metrics) = this.metrics.as_ref() {
 						metrics.import_queue_justifications_submitted.inc();
 					}
-					this.import_queue.import_justification(origin, hash, nb, justification);
+					this.import_queue.import_justifications(origin, hash, nb, justifications);
 				},
 				Poll::Ready(SwarmEvent::Behaviour(BehaviourOut::InboundRequest { protocol, result, .. })) => {
 					if let Some(metrics) = this.metrics.as_ref() {

substrate/client/network/src/service/tests.rs

@@ -52,7 +52,7 @@ fn build_test_full_node(config: config::NetworkConfiguration)
 			&mut self,
 			origin: sp_consensus::BlockOrigin,
 			header: B::Header,
-			justification: Option<sp_runtime::Justification>,
+			justifications: Option<sp_runtime::Justifications>,
 			body: Option<Vec<B::Extrinsic>>,
 		) -> Result<
 			(
@@ -79,7 +79,7 @@ fn build_test_full_node(config: config::NetworkConfiguration)
 			let mut import = sp_consensus::BlockImportParams::new(origin, header);
 			import.body = body;
 			import.finalized = self.0;
-			import.justification = justification;
+			import.justifications = justifications;
 			import.fork_choice = Some(sp_consensus::ForkChoiceStrategy::LongestChain);
 			Ok((import, maybe_keys))
 		}