Storing multiple Justifications per block (#7640)

* primitives/runtime: initial changes on supporting multiple Justifications

* primitives/runtime: make Justifications strongly typed

* Encode/decode Justifications

* primitives/runtime: add Justification type

* backend: apply_finality and finalize_block takes a single Justification

* manual-seal: create engine id and let rpc take encoded justification

* backend: skeleton functions for appending justifications

* backend: initial implementation append_justification

Initial implementation of append_justification on the Backend trait, and also remove unused skeleton
functions for append_justificaton on Finaziler trait.
k

* backend: guard against duplicate consensus engine id

* client/db: add check for block finality

* client/api: add append_justification to in_mem db

* client/light: add no-op append_justification

* network: fix decode call for Justification

* network: only send a single Justification in BlockData

* network: minor comment update

* protocol: update field names to distinguish single justification

* client: further field renames to plural

* client: update function names to plural justifications

* client/db: upgrade existing database for new format

* network: remove dependency on grandpa crate

* db: fix check for finalized block

* grandpa: check for multiple grandpa justifications hwne importing

* backend: update Finalizer trait to take multiple Justifications

* db: remove debugging statements in migration code

* manual-seal: update note about engine id

* db: fix check for finalized block

* client: update variable name to reflect it is now plural

* grandpa: fix incorrect empty Justications in test

* primitives: make Justifications opaque to avoid being empty

* network: fix detecting empty Justification

* runtime: doc strings for Justifications functions

* runtime: add into_justifications

* primitives: check for duplicates in when adding to Justifications

* network/test: use real grandpa engine id in test

* client: fix reviewer comments

* primitives: rename Justifications::push to append

* backend: revert changes to Finalizer trait

* backend: revert mark_finalized

* backend: revert changes to finalize_block

* backend: revert finalized_blocks

* db: add a quick early return for performance

* client: minor reviewer comments

* service/test: use local ConsensusEngineId

* network: add link to issue for sending multiple Justifications

* Apply suggestions from code review

Co-authored-by: Pierre Krieger <pierre.krieger1708@gmail.com>

* Apply suggestions from code review

Co-authored-by: Pierre Krieger <pierre.krieger1708@gmail.com>

* network: tweaks to review suggestions

* network: revert change to BlockData for backwards compatibility

* Apply suggestion from code review

Co-authored-by: Pierre Krieger <pierre.krieger1708@gmail.com>

* Apply suggestions from code review

Co-authored-by: André Silva <123550+andresilva@users.noreply.github.com>

* primitives: update doc comment for Justifications

* client/db/upgrade: avoid grandpa crate dependency

* consensus: revert to single Justification for import_justification

* primitives: improve justifications docs

* style cleanups

* use and_then

* client: rename JUSTIFICATIONS db column

* network: revert to using FRNK in network-test

Co-authored-by: Pierre Krieger <pierre.krieger1708@gmail.com>
Co-authored-by: André Silva <123550+andresilva@users.noreply.github.com>
Co-authored-by: André Silva <andrerfosilva@gmail.com>

substrate/client/consensus/aura/src/import_queue.rs

@@ -39,7 +39,7 @@ use sp_consensus::{
 use sc_client_api::{backend::AuxStore, BlockOf};
 use sp_blockchain::{well_known_cache_keys::{self, Id as CacheKeyId}, ProvideCache, HeaderBackend};
 use sp_block_builder::BlockBuilder as BlockBuilderApi;
-use sp_runtime::{generic::{BlockId, OpaqueDigestItemId}, Justification};
+use sp_runtime::{generic::{BlockId, OpaqueDigestItemId}, Justifications};
 use sp_runtime::traits::{Block as BlockT, Header, DigestItemFor, Zero};
 use sp_api::ProvideRuntimeApi;
 use sp_core::crypto::Pair;
@@ -238,7 +238,7 @@ impl<B: BlockT, C, P, CAW> Verifier<B> for AuraVerifier<C, P, CAW> where
 		&mut self,
 		origin: BlockOrigin,
 		header: B::Header,
-		justification: Option<Justification>,
+		justifications: Option<Justifications>,
 		mut body: Option<Vec<B::Extrinsic>>,
 	) -> Result<(BlockImportParams<B, ()>, Option<Vec<(CacheKeyId, Vec<u8>)>>), String> {
 		let mut inherent_data = self.inherent_data_providers
@@ -317,7 +317,7 @@ impl<B: BlockT, C, P, CAW> Verifier<B> for AuraVerifier<C, P, CAW> where
 				let mut import_block = BlockImportParams::new(origin, pre_header);
 				import_block.post_digests.push(seal);
 				import_block.body = body;
-				import_block.justification = justification;
+				import_block.justifications = justifications;
 				import_block.fork_choice = Some(ForkChoiceStrategy::LongestChain);
 				import_block.post_hash = Some(hash);
 

substrate/client/consensus/babe/src/lib.rs

@@ -84,7 +84,7 @@ use sp_core::crypto::Public;
 use sp_application_crypto::AppKey;
 use sp_keystore::{SyncCryptoStorePtr, SyncCryptoStore};
 use sp_runtime::{
-	generic::{BlockId, OpaqueDigestItemId}, Justification,
+	generic::{BlockId, OpaqueDigestItemId}, Justifications,
 	traits::{Block as BlockT, Header, DigestItemFor, Zero},
 };
 use sp_api::{ProvideRuntimeApi, NumberFor};
@@ -1097,15 +1097,15 @@ where
 		&mut self,
 		origin: BlockOrigin,
 		header: Block::Header,
-		justification: Option<Justification>,
+		justifications: Option<Justifications>,
 		mut body: Option<Vec<Block::Extrinsic>>,
 	) -> Result<(BlockImportParams<Block, ()>, Option<Vec<(CacheKeyId, Vec<u8>)>>), String> {
 		trace!(
 			target: "babe",
-			"Verifying origin: {:?} header: {:?} justification: {:?} body: {:?}",
+			"Verifying origin: {:?} header: {:?} justification(s): {:?} body: {:?}",
 			origin,
 			header,
-			justification,
+			justifications,
 			body,
 		);
 
@@ -1194,7 +1194,7 @@ where
 				let mut import_block = BlockImportParams::new(origin, pre_header);
 				import_block.post_digests.push(verified_info.seal);
 				import_block.body = body;
-				import_block.justification = justification;
+				import_block.justifications = justifications;
 				import_block.intermediates.insert(
 					Cow::from(INTERMEDIATE_KEY),
 					Box::new(BabeIntermediate::<Block> { epoch_descriptor }) as Box<dyn Any>,

substrate/client/consensus/babe/src/tests.rs

@@ -235,12 +235,12 @@ impl Verifier<TestBlock> for TestVerifier {
 		&mut self,
 		origin: BlockOrigin,
 		mut header: TestHeader,
-		justification: Option<Justification>,
+		justifications: Option<Justifications>,
 		body: Option<Vec<TestExtrinsic>>,
 	) -> Result<(BlockImportParams<TestBlock, ()>, Option<Vec<(CacheKeyId, Vec<u8>)>>), String> {
 		// apply post-sealing mutations (i.e. stripping seal, if desired).
 		(self.mutator)(&mut header, Stage::PostSeal);
-		self.inner.verify(origin, header, justification, body)
+		self.inner.verify(origin, header, justifications, body)
 	}
 }
 

substrate/client/consensus/manual-seal/src/lib.rs

@@ -27,7 +27,7 @@ use sp_consensus::{
 };
 use sp_blockchain::HeaderBackend;
 use sp_inherents::InherentDataProviders;
-use sp_runtime::{traits::Block as BlockT, Justification};
+use sp_runtime::{traits::Block as BlockT, Justifications, ConsensusEngineId};
 use sc_client_api::backend::{Backend as ClientBackend, Finalizer};
 use sc_transaction_pool::txpool;
 use std::{sync::Arc, marker::PhantomData};
@@ -49,6 +49,9 @@ pub use self::{
 };
 use sp_api::{ProvideRuntimeApi, TransactionFor};
 
+/// The `ConsensusEngineId` of Manual Seal.
+pub const MANUAL_SEAL_ENGINE_ID: ConsensusEngineId = [b'm', b'a', b'n', b'l'];
+
 /// The verifier for the manual seal engine; instantly finalizes.
 struct ManualSealVerifier;
 
@@ -57,11 +60,11 @@ impl<B: BlockT> Verifier<B> for ManualSealVerifier {
 		&mut self,
 		origin: BlockOrigin,
 		header: B::Header,
-		justification: Option<Justification>,
+		justifications: Option<Justifications>,
 		body: Option<Vec<B::Extrinsic>>,
 	) -> Result<(BlockImportParams<B, ()>, Option<Vec<(CacheKeyId, Vec<u8>)>>), String> {
 		let mut import_params = BlockImportParams::new(origin, header);
-		import_params.justification = justification;
+		import_params.justifications = justifications;
 		import_params.body = body;
 		import_params.finalized = false;
 		import_params.fork_choice = Some(ForkChoiceStrategy::LongestChain);
@@ -193,6 +196,7 @@ pub async fn run_manual_seal<B, BI, CB, E, C, A, SC, CS>(
 				).await;
 			}
 			EngineCommand::FinalizeBlock { hash, sender, justification } => {
+				let justification = justification.map(|j| (MANUAL_SEAL_ENGINE_ID, j));
 				finalize_block(
 					FinalizeBlockParams {
 						hash,

substrate/client/consensus/manual-seal/src/rpc.rs

@@ -28,7 +28,7 @@ use futures::{
 	SinkExt
 };
 use serde::{Deserialize, Serialize};
-use sp_runtime::Justification;
+use sp_runtime::EncodedJustification;
 pub use self::gen_client::Client as ManualSealClient;
 
 /// Future's type for jsonrpc
@@ -62,7 +62,7 @@ pub enum EngineCommand<Hash> {
 		/// sender to report errors/success to the rpc.
 		sender: Sender<()>,
 		/// finalization justification
-		justification: Option<Justification>,
+		justification: Option<EncodedJustification>,
 	}
 }
 
@@ -83,7 +83,7 @@ pub trait ManualSealApi<Hash> {
 	fn finalize_block(
 		&self,
 		hash: Hash,
-		justification: Option<Justification>
+		justification: Option<EncodedJustification>
 	) -> FutureResult<bool>;
 }
 
@@ -131,7 +131,7 @@ impl<Hash: Send + 'static> ManualSealApi<Hash> for ManualSeal<Hash> {
 		Box::new(future.map_err(Error::from).compat())
 	}
 
-	fn finalize_block(&self, hash: Hash, justification: Option<Justification>) -> FutureResult<bool> {
+	fn finalize_block(&self, hash: Hash, justification: Option<EncodedJustification>) -> FutureResult<bool> {
 		let mut sink = self.import_block_channel.clone();
 		let future = async move {
 			let (sender, receiver) = oneshot::channel();

substrate/client/consensus/pow/src/lib.rs

@@ -44,7 +44,7 @@ use parking_lot::Mutex;
 use sc_client_api::{BlockOf, backend::AuxStore, BlockchainEvents};
 use sp_blockchain::{HeaderBackend, ProvideCache, well_known_cache_keys::Id as CacheKeyId};
 use sp_block_builder::BlockBuilder as BlockBuilderApi;
-use sp_runtime::{Justification, RuntimeString};
+use sp_runtime::{Justifications, RuntimeString};
 use sp_runtime::generic::{BlockId, Digest, DigestItem};
 use sp_runtime::traits::{Block as BlockT, Header as HeaderT};
 use sp_api::ProvideRuntimeApi;
@@ -457,7 +457,7 @@ impl<B: BlockT, Algorithm> Verifier<B> for PowVerifier<B, Algorithm> where
 		&mut self,
 		origin: BlockOrigin,
 		header: B::Header,
-		justification: Option<Justification>,
+		justifications: Option<Justifications>,
 		body: Option<Vec<B::Extrinsic>>,
 	) -> Result<(BlockImportParams<B, ()>, Option<Vec<(CacheKeyId, Vec<u8>)>>), String> {
 		let hash = header.hash();
@@ -470,7 +470,7 @@ impl<B: BlockT, Algorithm> Verifier<B> for PowVerifier<B, Algorithm> where
 		let mut import_block = BlockImportParams::new(origin, checked_header);
 		import_block.post_digests.push(seal);
 		import_block.body = body;
-		import_block.justification = justification;
+		import_block.justifications = justifications;
 		import_block.intermediates.insert(
 			Cow::from(INTERMEDIATE_KEY),
 			Box::new(intermediate) as Box<dyn Any>