A more comprehensive model for PoV-Blocks and Candidate receipts (#843)

* encode the candidate statement as only the hash

* refactor CandidateReceipt and CollationInfo

* introduce an abridged candidate receipt type

* erasure coding stores candidate receipt

* store omitted data instead and introduce AvailableData type

* refactor availability-store schema

* tweak schema and APIs a bit more

* get availability-store tests passing

* accept AbridgedCandidateReceipt in `set_heads`

* change statement type in primitives to be hash-only

* fix parachains runtime tests

* fix bad merge

* rewrite validation pipeline

* remove evaluation module

* use abridged candidate hash as canonical

* statement table uses abridged candidate receipts

* kill availability_store::Data struct

* port shared table to new validation pipelines

* extract full validation pipeline to helper

* remove old validation pipeline from collation module

* polkadot-validation compiles

* polkadot-validation tests compile

* make local collation available in validation service

* port legacy network code

* polkadot-network fully ported

* network: ensure fresh statement is propagated

* remove pov_block_hash from LocalValidationData

* remove candidate_hash field from AttestedCandidate and update runtime

* port runtimes to new ParachainHost definition

* port over polkadot-collator

* fix test compilation

* better fix

* remove unrelated validation work dispatch fix

* address grumbles

* fix equality check

polkadot/network/src/legacy/collator_pool.rs

@@ -236,7 +236,7 @@ impl CollatorPool {
 mod tests {
 	use super::*;
 	use sp_core::crypto::UncheckedInto;
-	use polkadot_primitives::parachain::{CandidateReceipt, BlockData, PoVBlock, HeadData};
+	use polkadot_primitives::parachain::{CollationInfo, BlockData, PoVBlock};
 	use futures::executor::block_on;
 
 	fn make_pov(block_data: Vec<u8>) -> PoVBlock {
@@ -284,18 +284,11 @@ mod tests {
 		let (tx2, rx2) = oneshot::channel();
 		pool.await_collation(relay_parent, para_id, tx1);
 		pool.await_collation(relay_parent, para_id, tx2);
+		let mut collation_info = CollationInfo::default();
+		collation_info.parachain_index = para_id;
+		collation_info.collator = primary.clone().into();
 		pool.on_collation(primary.clone(), relay_parent, Collation {
-			info: CandidateReceipt {
-				parachain_index: para_id,
-				collator: primary.clone().into(),
-				signature: Default::default(),
-				head_data: HeadData(vec![1, 2, 3]),
-				parent_head: HeadData(vec![]),
-				fees: 0,
-				block_data_hash: [3; 32].into(),
-				upward_messages: Vec::new(),
-				erasure_root: [1u8; 32].into(),
-			}.into(),
+			info: collation_info,
 			pov: make_pov(vec![4, 5, 6]),
 		});
 
@@ -313,18 +306,11 @@ mod tests {
 
 		assert_eq!(pool.on_new_collator(primary.clone(), para_id.clone(), PeerId::random()), Role::Primary);
 
+		let mut collation_info = CollationInfo::default();
+		collation_info.parachain_index = para_id;
+		collation_info.collator = primary.clone();
 		pool.on_collation(primary.clone(), relay_parent, Collation {
-			info: CandidateReceipt {
-				parachain_index: para_id,
-				collator: primary,
-				signature: Default::default(),
-				head_data: HeadData(vec![1, 2, 3]),
-				parent_head: HeadData(vec![]),
-				fees: 0,
-				block_data_hash: [3; 32].into(),
-				upward_messages: Vec::new(),
-				erasure_root: [1u8; 32].into(),
-			}.into(),
+			info: collation_info,
 			pov: make_pov(vec![4, 5, 6]),
 		});
 

polkadot/network/src/legacy/gossip/mod.rs

@@ -540,7 +540,7 @@ impl<C: ?Sized + ChainContext> Inner<C> {
 		if let Some(store) = &self.availability_store {
 			if let Some(receipt) = store.get_candidate(&msg.candidate_hash) {
 				let chunk_hash = erasure::branch_hash(
-					&receipt.erasure_root,
+					&receipt.commitments.erasure_root,
 					&msg.chunk.proof,
 					msg.chunk.index as usize
 				);
@@ -552,15 +552,15 @@ impl<C: ?Sized + ChainContext> Inner<C> {
 					)
 				} else {
 					if let Some(awaited_chunks) = store.awaited_chunks() {
-						if awaited_chunks.contains(&(
-								msg.relay_parent,
-								receipt.erasure_root,
-								receipt.hash(),
-								msg.chunk.index,
-							)) {
+						let frontier_entry = av_store::AwaitedFrontierEntry {
+							relay_parent: msg.relay_parent,
+							erasure_root: receipt.commitments.erasure_root,
+							validator_index: msg.chunk.index,
+						};
+						if awaited_chunks.contains(&frontier_entry) {
 							let topic = av_store::erasure_coding_topic(
 								msg.relay_parent,
-								receipt.erasure_root,
+								receipt.commitments.erasure_root,
 								msg.chunk.index,
 							);
 
@@ -722,8 +722,7 @@ mod tests {
 	use sc_network_gossip::Validator as ValidatorT;
 	use std::sync::mpsc;
 	use parking_lot::Mutex;
-	use polkadot_primitives::parachain::{CandidateReceipt, HeadData};
-	use sp_core::crypto::UncheckedInto;
+	use polkadot_primitives::parachain::AbridgedCandidateReceipt;
 	use sp_core::sr25519::Signature as Sr25519Signature;
 	use polkadot_validation::GenericStatement;
 
@@ -807,18 +806,7 @@ mod tests {
 
 		validator_context.clear();
 
-		let candidate_receipt = CandidateReceipt {
-			parachain_index: 5.into(),
-			collator: [255; 32].unchecked_into(),
-			head_data: HeadData(vec![9, 9, 9]),
-			parent_head: HeadData(vec![]),
-			signature: Default::default(),
-			fees: 1_000_000,
-			block_data_hash: [20u8; 32].into(),
-			upward_messages: Vec::new(),
-			erasure_root: [1u8; 32].into(),
-		};
-
+		let candidate_receipt = AbridgedCandidateReceipt::default();
 		let statement = GossipMessage::Statement(GossipStatement {
 			relay_chain_leaf: hash_a,
 			signed_statement: SignedStatement {

polkadot/network/src/legacy/mod.rs

@@ -30,7 +30,7 @@ use futures::channel::oneshot;
 use futures::prelude::*;
 use polkadot_primitives::{Block, Hash, Header};
 use polkadot_primitives::parachain::{
-	Id as ParaId, CollatorId, CandidateReceipt, Collation, PoVBlock,
+	Id as ParaId, CollatorId, AbridgedCandidateReceipt, Collation, PoVBlock,
 	ValidatorId, ErasureChunk,
 };
 use sc_network::{
@@ -168,7 +168,7 @@ struct PoVBlockRequest {
 	attempted_peers: HashSet<ValidatorId>,
 	validation_leaf: Hash,
 	candidate_hash: Hash,
-	block_data_hash: Hash,
+	pov_block_hash: Hash,
 	sender: oneshot::Sender<PoVBlock>,
 }
 
@@ -178,7 +178,7 @@ impl PoVBlockRequest {
 	//
 	// If `Ok(())` is returned, that indicates that the request has been processed.
 	fn process_response(self, pov_block: PoVBlock) -> Result<(), Self> {
-		if pov_block.block_data.hash() != self.block_data_hash {
+		if pov_block.hash() != self.pov_block_hash {
 			return Err(self);
 		}
 
@@ -292,7 +292,7 @@ impl PolkadotProtocol {
 	fn fetch_pov_block(
 		&mut self,
 		ctx: &mut dyn Context<Block>,
-		candidate: &CandidateReceipt,
+		candidate: &AbridgedCandidateReceipt,
 		relay_parent: Hash,
 	) -> oneshot::Receiver<PoVBlock> {
 		let (tx, rx) = oneshot::channel();
@@ -301,7 +301,7 @@ impl PolkadotProtocol {
 			attempted_peers: Default::default(),
 			validation_leaf: relay_parent,
 			candidate_hash: candidate.hash(),
-			block_data_hash: candidate.block_data_hash,
+			pov_block_hash: candidate.pov_block_hash,
 			sender: tx,
 		});
 
@@ -608,7 +608,7 @@ impl Specialization<Block> for PolkadotProtocol {
 							attempted_peers: Default::default(),
 							validation_leaf: Default::default(),
 							candidate_hash: Default::default(),
-							block_data_hash: Default::default(),
+							pov_block_hash: Default::default(),
 							sender,
 						}));
 					}
@@ -738,7 +738,7 @@ impl PolkadotProtocol {
 		relay_parent: Hash,
 		targets: HashSet<ValidatorId>,
 		collation: Collation,
-	) -> impl Future<Output = ()> {
+	) {
 		debug!(target: "p_net", "Importing local collation on relay parent {:?} and parachain {:?}",
 			relay_parent, collation.info.parachain_index);
 
@@ -756,19 +756,6 @@ impl PolkadotProtocol {
 					warn!(target: "polkadot_network", "Encountered tracked but disconnected validator {:?}", primary),
 			}
 		}
-
-		let availability_store = self.availability_store.clone();
-		let collation_cloned = collation.clone();
-
-		async move {
-			if let Some(availability_store) = availability_store {
-				let _ = availability_store.make_available(av_store::Data {
-					relay_parent,
-					parachain_id: collation_cloned.info.parachain_index,
-					block_data: collation_cloned.pov.block_data.clone(),
-				}).await;
-			}
-		}
 	}
 
 	/// Give the network protocol a handle to an availability store, used for

polkadot/network/src/legacy/router.rs

@@ -29,7 +29,7 @@ use polkadot_validation::{
 };
 use polkadot_primitives::{Block, Hash};
 use polkadot_primitives::parachain::{
-	CandidateReceipt, ParachainHost, ValidatorIndex, Collation, PoVBlock, ErasureChunk,
+	AbridgedCandidateReceipt, ParachainHost, ValidatorIndex, PoVBlock, ErasureChunk,
 };
 use sp_api::ProvideRuntimeApi;
 
@@ -204,17 +204,17 @@ impl<P: ProvideRuntimeApi<Block> + Send + Sync + 'static, T> Router<P, T> where
 					// store the data before broadcasting statements, so other peers can fetch.
 					knowledge.lock().note_candidate(
 						candidate_hash,
-						Some(validated.0.pov_block().clone()),
+						Some(validated.pov_block().clone()),
 					);
 
 					// propagate the statement.
 					// consider something more targeted than gossip in the future.
 					let statement = GossipStatement::new(
-					parent_hash,
-					match table.import_validated(validated.0) {
-					None => return,
-					Some(s) => s,
-					}
+						parent_hash,
+						match table.import_validated(validated) {
+							None => return,
+							Some(s) => s,
+						}
 					);
 
 					network.gossip_message(attestation_topic, statement.into());
@@ -238,16 +238,16 @@ impl<P: ProvideRuntimeApi<Block> + Send, T> TableRouter for Router<P, T> where
 	// We have fetched from a collator and here the receipt should have been already formed.
 	fn local_collation(
 		&self,
-		collation: Collation,
-		receipt: CandidateReceipt,
+		receipt: AbridgedCandidateReceipt,
+		pov_block: PoVBlock,
 		chunks: (ValidatorIndex, &[ErasureChunk])
 	) -> Self::SendLocalCollation {
 		// produce a signed statement
 		let hash = receipt.hash();
-		let erasure_root = receipt.erasure_root;
+		let erasure_root = receipt.commitments.erasure_root;
 		let validated = Validated::collated_local(
 			receipt,
-			collation.pov.clone(),
+			pov_block.clone(),
 		);
 
 		let statement = GossipStatement::new(
@@ -259,7 +259,7 @@ impl<P: ProvideRuntimeApi<Block> + Send, T> TableRouter for Router<P, T> where
 		);
 
 		// give to network to make available.
-		self.fetcher.knowledge().lock().note_candidate(hash, Some(collation.pov));
+		self.fetcher.knowledge().lock().note_candidate(hash, Some(pov_block));
 		self.network().gossip_message(self.attestation_topic, statement.into());
 
 		for chunk in chunks.1 {
@@ -279,7 +279,7 @@ impl<P: ProvideRuntimeApi<Block> + Send, T> TableRouter for Router<P, T> where
 		future::ready(Ok(()))
 	}
 
-	fn fetch_pov_block(&self, candidate: &CandidateReceipt) -> Self::FetchValidationProof {
+	fn fetch_pov_block(&self, candidate: &AbridgedCandidateReceipt) -> Self::FetchValidationProof {
 		self.fetcher.fetch_pov_block(candidate)
 	}
 }

polkadot/network/src/legacy/tests/validation.rs

@@ -28,8 +28,9 @@ use crate::legacy::{PolkadotProtocol, NetworkService, GossipService, GossipMessa
 use polkadot_validation::{SharedTable, Network};
 use polkadot_primitives::{Block, BlockNumber, Hash, Header, BlockId};
 use polkadot_primitives::parachain::{
-	Id as ParaId, Chain, DutyRoster, ParachainHost, ValidatorId, Status,
-	FeeSchedule, HeadData, Retriable, CollatorId, ErasureChunk, CandidateReceipt,
+	Id as ParaId, Chain, DutyRoster, ParachainHost, ValidatorId,
+	FeeSchedule, HeadData, Retriable, CollatorId, ErasureChunk, AbridgedCandidateReceipt,
+	GlobalValidationSchedule, LocalValidationData,
 };
 use parking_lot::Mutex;
 use sp_blockchain::Result as ClientResult;
@@ -270,23 +271,6 @@ impl ParachainHost<Block> for RuntimeApi {
 		Ok(NativeOrEncoded::Native(self.data.lock().active_parachains.clone()))
 	}
 
-	fn ParachainHost_parachain_status_runtime_api_impl(
-		&self,
-		_at: &BlockId,
-		_: ExecutionContext,
-		_: Option<ParaId>,
-		_: Vec<u8>,
-	) -> ClientResult<NativeOrEncoded<Option<Status>>> {
-		Ok(NativeOrEncoded::Native(Some(Status {
-			head_data: HeadData(Vec::new()),
-			balance: 0,
-			fee_schedule: FeeSchedule {
-				base: 0,
-				per_byte: 0,
-			}
-		})))
-	}
-
 	fn ParachainHost_parachain_code_runtime_api_impl(
 		&self,
 		_at: &BlockId,
@@ -297,13 +281,33 @@ impl ParachainHost<Block> for RuntimeApi {
 		Ok(NativeOrEncoded::Native(Some(Vec::new())))
 	}
 
+	fn ParachainHost_global_validation_schedule_runtime_api_impl(
+		&self,
+		_at: &BlockId,
+		_: ExecutionContext,
+		_: Option<()>,
+		_: Vec<u8>,
+	) -> ClientResult<NativeOrEncoded<GlobalValidationSchedule>> {
+		Ok(NativeOrEncoded::Native(Default::default()))
+	}
+
+	fn ParachainHost_local_validation_data_runtime_api_impl(
+		&self,
+		_at: &BlockId,
+		_: ExecutionContext,
+		_: Option<ParaId>,
+		_: Vec<u8>,
+	) -> ClientResult<NativeOrEncoded<Option<LocalValidationData>>> {
+		Ok(NativeOrEncoded::Native(Some(Default::default())))
+	}
+
 	fn ParachainHost_get_heads_runtime_api_impl(
 		&self,
 		_at: &BlockId,
 		_: ExecutionContext,
 		_extrinsics: Option<Vec<<Block as BlockT>::Extrinsic>>,
 		_: Vec<u8>,
-	) -> ClientResult<NativeOrEncoded<Option<Vec<CandidateReceipt>>>> {
+	) -> ClientResult<NativeOrEncoded<Option<Vec<AbridgedCandidateReceipt>>>> {
 		Ok(NativeOrEncoded::Native(Some(Vec::new())))
 	}
 }

polkadot/network/src/legacy/validation.rs

@@ -25,7 +25,7 @@ use polkadot_validation::{
 };
 use polkadot_primitives::{Block, Hash};
 use polkadot_primitives::parachain::{
-	Id as ParaId, Collation, ParachainHost, CandidateReceipt, CollatorId,
+	Id as ParaId, Collation, ParachainHost, AbridgedCandidateReceipt, CollatorId,
 	ValidatorId, PoVBlock,
 };
 use sp_api::ProvideRuntimeApi;
@@ -557,7 +557,7 @@ impl<P: ProvideRuntimeApi<Block> + Send, T> LeafWorkDataFetcher<P, T> where
 	T: Clone + Executor + Send + 'static,
 {
 	/// Fetch PoV block for the given candidate receipt.
-	pub fn fetch_pov_block(&self, candidate: &CandidateReceipt)
+	pub fn fetch_pov_block(&self, candidate: &AbridgedCandidateReceipt)
 		-> Pin<Box<dyn Future<Output = Result<PoVBlock, io::Error>> + Send>> {
 
 		let parent_hash = self.parent_hash;

polkadot/network/src/protocol.rs

@@ -31,7 +31,7 @@ use log::{debug, trace};
 use polkadot_primitives::{
 	Hash, Block,
 	parachain::{
-		PoVBlock, ValidatorId, ValidatorIndex, Collation, CandidateReceipt,
+		PoVBlock, ValidatorId, ValidatorIndex, Collation, AbridgedCandidateReceipt,
 		ErasureChunk, ParachainHost, Id as ParaId, CollatorId,
 	},
 };
@@ -72,13 +72,13 @@ enum ServiceToWorkerMsg {
 	DropConsensusNetworking(Hash),
 	LocalCollation(
 		Hash, // relay-parent
-		Collation,
-		CandidateReceipt,
+		AbridgedCandidateReceipt,
+		PoVBlock,
 		(ValidatorIndex, Vec<ErasureChunk>),
 	),
 	FetchPoVBlock(
 		Hash, // relay-parent
-		CandidateReceipt,
+		AbridgedCandidateReceipt,
 		oneshot::Sender<PoVBlock>,
 	),
 	AwaitCollation(
@@ -650,7 +650,7 @@ async fn worker_loop<Api, Sp>(
 			ServiceToWorkerMsg::DropConsensusNetworking(relay_parent) => {
 				consensus_instances.remove(&relay_parent);
 			}
-			ServiceToWorkerMsg::LocalCollation(relay_parent, collation, receipt, chunks) => {
+			ServiceToWorkerMsg::LocalCollation(relay_parent, receipt, pov_block, chunks) => {
 				let instance = match consensus_instances.get(&relay_parent) {
 					None => continue,
 					Some(instance) => instance,
@@ -658,8 +658,8 @@ async fn worker_loop<Api, Sp>(
 
 				distribute_local_collation(
 					instance,
-					collation,
 					receipt,
+					pov_block,
 					chunks,
 					&gossip_handle,
 				);
@@ -686,7 +686,7 @@ async fn statement_import_loop<Api>(
 	table: Arc<SharedTable>,
 	api: Arc<Api>,
 	weak_router: Weak<RouterInner>,
-	validator: RegisteredMessageValidator,
+	gossip_handle: RegisteredMessageValidator,
 	mut exit: exit_future::Exit,
 	executor: impl Spawn,
 ) where
@@ -694,7 +694,7 @@ async fn statement_import_loop<Api>(
 	Api::Api: ParachainHost<Block, Error = sp_blockchain::Error>,
 {
 	let topic = crate::legacy::router::attestation_topic(relay_parent);
-	let mut checked_messages = validator.gossip_messages_for(topic)
+	let mut checked_messages = gossip_handle.gossip_messages_for(topic)
 		.filter_map(|msg| match msg.0 {
 			crate::legacy::gossip::GossipMessage::Statement(s) => future::ready(Some(s.signed_statement)),
 			_ => future::ready(None),
@@ -758,8 +758,10 @@ async fn statement_import_loop<Api>(
 				if let Some(producer) = producer {
 					trace!(target: "validation", "driving statement work to completion");
 
-					let work = producer.prime(api.clone()).validate();
+					let table = table.clone();
+					let gossip_handle = gossip_handle.clone();
 
+					let work = producer.prime(api.clone()).validate();
 					let work = future::select(work.boxed(), exit.clone()).map(drop);
 					let _ = executor.spawn(work);
 				}
@@ -773,17 +775,17 @@ async fn statement_import_loop<Api>(
 // group.
 fn distribute_local_collation(
 	instance: &ConsensusNetworkingInstance,
-	collation: Collation,
-	receipt: CandidateReceipt,
+	receipt: AbridgedCandidateReceipt,
+	pov_block: PoVBlock,
 	chunks: (ValidatorIndex, Vec<ErasureChunk>),
 	gossip_handle: &RegisteredMessageValidator,
 ) {
 	// produce a signed statement.
 	let hash = receipt.hash();
-	let erasure_root = receipt.erasure_root;
+	let erasure_root = receipt.commitments.erasure_root;
 	let validated = Validated::collated_local(
 		receipt,
-		collation.pov.clone(),
+		pov_block,
 	);
 
 	let statement = crate::legacy::gossip::GossipStatement::new(
@@ -906,14 +908,14 @@ impl TableRouter for Router {
 
 	fn local_collation(
 		&self,
-		collation: Collation,
-		receipt: CandidateReceipt,
+		receipt: AbridgedCandidateReceipt,
+		pov_block: PoVBlock,
 		chunks: (ValidatorIndex, &[ErasureChunk]),
 	) -> Self::SendLocalCollation {
 		let message = ServiceToWorkerMsg::LocalCollation(
 			self.inner.relay_parent.clone(),
-			collation,
 			receipt,
+			pov_block,
 			(chunks.0, chunks.1.to_vec()),
 		);
 		let mut sender = self.inner.sender.clone();
@@ -922,7 +924,7 @@ impl TableRouter for Router {
 		})
 	}
 
-	fn fetch_pov_block(&self, candidate: &CandidateReceipt) -> Self::FetchValidationProof {
+	fn fetch_pov_block(&self, candidate: &AbridgedCandidateReceipt) -> Self::FetchValidationProof {
 		let (tx, rx) = oneshot::channel();
 		let message = ServiceToWorkerMsg::FetchPoVBlock(
 			self.inner.relay_parent.clone(),