A more comprehensive model for PoV-Blocks and Candidate receipts (#843)

* encode the candidate statement as only the hash

* refactor CandidateReceipt and CollationInfo

* introduce an abridged candidate receipt type

* erasure coding stores candidate receipt

* store omitted data instead and introduce AvailableData type

* refactor availability-store schema

* tweak schema and APIs a bit more

* get availability-store tests passing

* accept AbridgedCandidateReceipt in `set_heads`

* change statement type in primitives to be hash-only

* fix parachains runtime tests

* fix bad merge

* rewrite validation pipeline

* remove evaluation module

* use abridged candidate hash as canonical

* statement table uses abridged candidate receipts

* kill availability_store::Data struct

* port shared table to new validation pipelines

* extract full validation pipeline to helper

* remove old validation pipeline from collation module

* polkadot-validation compiles

* polkadot-validation tests compile

* make local collation available in validation service

* port legacy network code

* polkadot-network fully ported

* network: ensure fresh statement is propagated

* remove pov_block_hash from LocalValidationData

* remove candidate_hash field from AttestedCandidate and update runtime

* port runtimes to new ParachainHost definition

* port over polkadot-collator

* fix test compilation

* better fix

* remove unrelated validation work dispatch fix

* address grumbles

* fix equality check

polkadot/validation/src/shared_table/mod.rs

@@ -22,10 +22,10 @@ use std::sync::Arc;
 
 use availability_store::{Store as AvailabilityStore};
 use table::{self, Table, Context as TableContextTrait};
-use polkadot_primitives::{Block, BlockId, Hash};
+use polkadot_primitives::{Block, Hash};
 use polkadot_primitives::parachain::{
-	Id as ParaId, CandidateReceipt, ValidatorPair, ValidatorId,
-	AttestedCandidate, ParachainHost, PoVBlock, ValidatorIndex, ErasureChunk,
+	Id as ParaId, AbridgedCandidateReceipt, ValidatorPair, ValidatorId,
+	AttestedCandidate, ParachainHost, PoVBlock, ValidatorIndex,
 };
 
 use parking_lot::Mutex;
@@ -39,6 +39,9 @@ use self::includable::IncludabilitySender;
 use primitives::Pair;
 use sp_api::ProvideRuntimeApi;
 
+use crate::pipeline::FullOutput;
+use crate::Error;
+
 mod includable;
 
 pub use table::{SignedStatement, Statement};
@@ -188,8 +191,8 @@ impl SharedTableInner {
 			availability_store: self.availability_store.clone(),
 			relay_parent: context.parent_hash.clone(),
 			work,
-			local_index: local_index as usize,
 			max_block_data_size,
+			n_validators: context.validators.len(),
 		})
 	}
 
@@ -232,7 +235,7 @@ impl Validated {
 	/// Note that we've collated a candidate.
 	/// outgoing message required.
 	pub fn collated_local(
-		receipt: CandidateReceipt,
+		receipt: AbridgedCandidateReceipt,
 		collation: PoVBlock,
 	) -> Self {
 		Validated {
@@ -253,9 +256,9 @@ impl Validated {
 pub struct ParachainWork<Fetch> {
 	work: Work<Fetch>,
 	relay_parent: Hash,
-	local_index: usize,
 	availability_store: AvailabilityStore,
 	max_block_data_size: Option<u64>,
+	n_validators: usize,
 }
 
 impl<Fetch: Future + Unpin> ParachainWork<Fetch> {
@@ -264,48 +267,47 @@ impl<Fetch: Future + Unpin> ParachainWork<Fetch> {
 	pub fn prime<P: ProvideRuntimeApi<Block>>(self, api: Arc<P>)
 		-> PrimedParachainWork<
 			Fetch,
-			impl Send + FnMut(&BlockId, &PoVBlock, &CandidateReceipt) -> Result<ErasureChunk, ()> + Unpin,
+			impl Send + FnMut(&PoVBlock, &AbridgedCandidateReceipt)
+				-> Result<FullOutput, Error> + Unpin,
 		>
 		where
 			P: Send + Sync + 'static,
 			P::Api: ParachainHost<Block, Error = sp_blockchain::Error>,
 	{
 		let max_block_data_size = self.max_block_data_size;
-		let local_index = self.local_index;
+		let n_validators = self.n_validators;
+		let expected_relay_parent = self.relay_parent;
 
-		let validate = move |id: &_, pov_block: &_, receipt: &_| {
-			let res = crate::collation::validate_receipt(
+		let validate = move |pov_block: &PoVBlock, candidate: &AbridgedCandidateReceipt| {
+			let collation_info = candidate.to_collation_info();
+			let full_output = crate::pipeline::full_output_validation_with_api(
 				&*api,
-				id,
+				&collation_info,
 				pov_block,
-				receipt,
+				&expected_relay_parent,
 				max_block_data_size,
-			);
+				n_validators,
+			)?;
 
-			match res {
-				Ok(mut chunks) => {
-					Ok(chunks.swap_remove(local_index))
-				}
-				Err(e) => {
-					debug!(target: "validation", "Encountered bad collation: {}", e);
-					Err(())
-				}
-			}
+			full_output.check_consistency(candidate)?;
+			Ok(full_output)
 		};
 
 		PrimedParachainWork { inner: self, validate }
 	}
 
 	/// Prime the parachain work with a custom validation function.
+	#[cfg(test)]
 	pub fn prime_with<F>(self, validate: F) -> PrimedParachainWork<Fetch, F>
-		where F: FnMut(&BlockId, &PoVBlock, &CandidateReceipt) -> Result<ErasureChunk, ()>
+		where F: FnMut(&PoVBlock, &AbridgedCandidateReceipt)
+			-> Result<FullOutput, Error>
 	{
 		PrimedParachainWork { inner: self, validate }
 	}
 }
 
 struct Work<Fetch> {
-	candidate_receipt: CandidateReceipt,
+	candidate_receipt: AbridgedCandidateReceipt,
 	fetch: Fetch
 }
 
@@ -318,46 +320,62 @@ pub struct PrimedParachainWork<Fetch, F> {
 impl<Fetch, F, Err> PrimedParachainWork<Fetch, F>
 	where
 		Fetch: Future<Output=Result<PoVBlock,Err>> + Unpin,
-		F: FnMut(&BlockId, &PoVBlock, &CandidateReceipt) -> Result<ErasureChunk, ()> + Unpin,
-		Err: From<::std::io::Error>,
+		F: FnMut(&PoVBlock, &AbridgedCandidateReceipt) -> Result<FullOutput, Error> + Unpin,
+		Err: From<std::io::Error>,
 {
-	pub async fn validate(mut self) -> Result<(Validated, Option<ErasureChunk>), Err> {
-		let candidate = &self.inner.work.candidate_receipt;
+	pub async fn validate(self) -> Result<Validated, Err> {
+		let candidate = self.inner.work.candidate_receipt;
 		let pov_block = self.inner.work.fetch.await?;
 
-		let validation_res = (self.validate)(
-			&BlockId::hash(self.inner.relay_parent),
-			&pov_block,
-			&candidate,
-		);
+		let mut validate = self.validate;
+		let relay_parent = self.inner.relay_parent;
 
+		// create a wrapper around the custom validation function that does
+		// some more general pre and post checks.
+		let mut validate = move |pov_block: &_, candidate: &AbridgedCandidateReceipt| {
+			if candidate.relay_parent != relay_parent {
+				return Err(Error::DisallowedRelayParent(candidate.relay_parent));
+			}
+
+			let full_output = validate(pov_block, candidate)?;
+
+			if full_output.commitments != candidate.commitments {
+				return Err(Error::CommitmentsMismatch);
+			}
+
+			Ok(full_output)
+		};
+
+		let validation_res = (validate)(&pov_block, &candidate);
 		let candidate_hash = candidate.hash();
 
 		debug!(target: "validation", "Making validity statement about candidate {}: is_good? {:?}",
 			candidate_hash, validation_res.is_ok());
 
 		match validation_res {
-			Err(()) => Ok((
-				Validated {
+			Err(err) => {
+				debug!(target: "validation", "candidate is invalid: {}", err);
+				Ok(Validated {
 					statement: GenericStatement::Invalid(candidate_hash),
 					result: Validation::Invalid(pov_block),
-				},
-				None,
-			)),
-			Ok(our_chunk) => {
-				self.inner.availability_store.add_erasure_chunk(
-					self.inner.relay_parent,
-					candidate.clone(),
-					our_chunk.clone(),
+				})
+			}
+			Ok(full_output) => {
+				// make data and all erasure chunks available. The chunk
+				// must be fully available before we add the chunks.
+				self.inner.availability_store.make_available(
+					candidate_hash,
+					full_output.available_data,
+				).await?;
+				self.inner.availability_store.add_erasure_chunks(
+					candidate,
+					full_output.erasure_chunks,
 				).await?;
 
-				Ok((
-					Validated {
-						statement: GenericStatement::Valid(candidate_hash),
-						result: Validation::Valid(pov_block),
-					},
-					Some(our_chunk),
-				))
+				Ok(Validated {
+					statement: GenericStatement::Valid(candidate_hash),
+					result: Validation::Valid(pov_block),
+				})
 			}
 		}
 	}
@@ -461,7 +479,8 @@ impl SharedTable {
 	}
 
 	/// Sign and import the result of candidate validation. Returns `None` if the table
-	/// was instantiated without a local key.
+	/// was instantiated without a local key. Otherwise, returns a copy of the signed
+	/// statement.
 	pub fn import_validated(&self, validated: Validated)
 		-> Option<SignedStatement>
 	{
@@ -485,7 +504,7 @@ impl SharedTable {
 	///
 	/// Deadlocks if called recursively.
 	pub fn with_candidate<F, U>(&self, digest: &Hash, f: F) -> U
-		where F: FnOnce(Option<&CandidateReceipt>) -> U
+		where F: FnOnce(Option<&AbridgedCandidateReceipt>) -> U
 	{
 		let inner = self.inner.lock();
 		f(inner.table.get_candidate(digest))
@@ -570,8 +589,9 @@ impl SharedTable {
 mod tests {
 	use super::*;
 	use sp_keyring::Sr25519Keyring;
-	use primitives::crypto::UncheckedInto;
-	use polkadot_primitives::parachain::{BlockData, Collation, HeadData};
+	use polkadot_primitives::parachain::{
+		BlockData, ErasureChunk, AvailableData,
+	};
 	use polkadot_erasure_coding::{self as erasure};
 	use availability_store::ProvideGossipMessages;
 	use futures::future;
@@ -613,12 +633,12 @@ mod tests {
 
 		fn local_collation(
 			&self,
-			_collation: Collation,
-			_candidate: CandidateReceipt,
+			_candidate: AbridgedCandidateReceipt,
+			_pov_block: PoVBlock,
 			_chunks: (ValidatorIndex, &[ErasureChunk])
 		) -> Self::SendLocalCollation { future::ready(Ok(())) }
 
-		fn fetch_pov_block(&self, _candidate: &CandidateReceipt) -> Self::FetchValidationProof {
+		fn fetch_pov_block(&self, _candidate: &AbridgedCandidateReceipt) -> Self::FetchValidationProof {
 			future::ok(pov_block_with_data(vec![1, 2, 3, 4, 5]))
 		}
 	}
@@ -652,17 +672,9 @@ mod tests {
 			None,
 		);
 
-		let candidate = CandidateReceipt {
-			parachain_index: para_id,
-			collator: [1; 32].unchecked_into(),
-			signature: Default::default(),
-			head_data: HeadData(vec![1, 2, 3, 4]),
-			parent_head: HeadData(vec![]),
-			fees: 1_000_000,
-			block_data_hash: [2; 32].into(),
-			upward_messages: Vec::new(),
-			erasure_root: [1u8; 32].into(),
-		};
+		let mut candidate = AbridgedCandidateReceipt::default();
+		candidate.parachain_index = para_id;
+		candidate.relay_parent = parent_hash;
 
 		let candidate_statement = GenericStatement::Candidate(candidate);
 
@@ -708,17 +720,9 @@ mod tests {
 			None,
 		);
 
-		let candidate = CandidateReceipt {
-			parachain_index: para_id,
-			collator: [1; 32].unchecked_into(),
-			signature: Default::default(),
-			head_data: HeadData(vec![1, 2, 3, 4]),
-			parent_head: HeadData(vec![]),
-			fees: 1_000_000,
-			block_data_hash: [2; 32].into(),
-			upward_messages: Vec::new(),
-			erasure_root: [1u8; 32].into(),
-		};
+		let mut candidate = AbridgedCandidateReceipt::default();
+		candidate.parachain_index = para_id;
+		candidate.relay_parent = parent_hash;
 
 		let candidate_statement = GenericStatement::Candidate(candidate);
 
@@ -741,25 +745,18 @@ mod tests {
 		let relay_parent = [0; 32].into();
 		let para_id = 5.into();
 		let pov_block = pov_block_with_data(vec![1, 2, 3]);
-		let block_data_hash = [2; 32].into();
+		let pov_block_hash = pov_block.hash();
 		let local_index = 0;
 		let n_validators = 2;
 
-		let candidate = CandidateReceipt {
-			parachain_index: para_id,
-			collator: [1; 32].unchecked_into(),
-			signature: Default::default(),
-			head_data: HeadData(vec![1, 2, 3, 4]),
-			parent_head: HeadData(vec![]),
-			fees: 1_000_000,
-			block_data_hash,
-			upward_messages: Vec::new(),
-			erasure_root: [1u8; 32].into(),
-		};
+		let mut candidate = AbridgedCandidateReceipt::default();
+		candidate.parachain_index = para_id;
+		candidate.relay_parent = relay_parent;
+		candidate.pov_block_hash = pov_block_hash;
 
-		let hash = candidate.hash();
+		let candidate_hash = candidate.hash();
 
-		store.add_validator_index_and_n_validators(
+		store.note_validator_index_and_n_validators(
 			&relay_parent,
 			local_index as u32,
 			n_validators as u32,
@@ -770,25 +767,37 @@ mod tests {
 				candidate_receipt: candidate,
 				fetch: future::ok(pov_block.clone()),
 			},
-			local_index,
 			relay_parent,
 			availability_store: store.clone(),
 			max_block_data_size: None,
+			n_validators,
 		};
 
-		let validated = block_on(producer.prime_with(|_, _, _| Ok(
-			ErasureChunk {
-				chunk: vec![1, 2, 3],
-				index: local_index as u32,
-				proof: vec![],
+		for i in 0..n_validators {
+			assert!(store.get_erasure_chunk(&candidate_hash, i).is_none());
+		}
+
+		let validated = block_on(producer.prime_with(|_, _| Ok(
+			FullOutput {
+				available_data: AvailableData {
+					pov_block: pov_block.clone(),
+					omitted_validation: Default::default(),
+				},
+				erasure_chunks: (0..n_validators).map(|i| ErasureChunk {
+					chunk: vec![1, 2, 3],
+					index: i as u32,
+					proof: vec![],
+				}).collect(),
+				commitments: Default::default(),
 			}
 		)).validate()).unwrap();
 
-		assert_eq!(validated.0.pov_block(), &pov_block);
-		assert_eq!(validated.0.statement, GenericStatement::Valid(hash));
+		assert_eq!(validated.pov_block(), &pov_block);
+		assert_eq!(validated.statement, GenericStatement::Valid(candidate_hash));
 
-		assert!(store.get_erasure_chunk(&relay_parent, block_data_hash, local_index).is_some());
-		assert!(store.get_erasure_chunk(&relay_parent, block_data_hash, local_index + 1).is_none());
+		for i in 0..n_validators {
+			assert!(store.get_erasure_chunk(&candidate_hash, i).is_some());
+		}
 	}
 
 	#[test]
@@ -797,25 +806,25 @@ mod tests {
 		let relay_parent = [0; 32].into();
 		let para_id = 5.into();
 		let pov_block = pov_block_with_data(vec![1, 2, 3]);
-		let block_data_hash = pov_block.block_data.hash();
+		let pov_block_hash = pov_block.hash();
 		let local_index = 0;
 		let n_validators = 2;
 
-		let candidate = CandidateReceipt {
-			parachain_index: para_id,
-			collator: [1; 32].unchecked_into(),
-			signature: Default::default(),
-			head_data: HeadData(vec![1, 2, 3, 4]),
-			parent_head: HeadData(vec![]),
-			fees: 1_000_000,
-			block_data_hash: [2; 32].into(),
-			upward_messages: Vec::new(),
-			erasure_root: [1u8; 32].into(),
+		let mut candidate = AbridgedCandidateReceipt::default();
+		candidate.parachain_index = para_id;
+		candidate.pov_block_hash = pov_block_hash;
+		candidate.relay_parent = relay_parent;
+
+		let candidate_hash = candidate.hash();
+
+		let available_data = AvailableData {
+			pov_block: pov_block.clone(),
+			omitted_validation: Default::default(),
 		};
 
-		let chunks = erasure::obtain_chunks(n_validators, &pov_block.block_data).unwrap();
+		let chunks = erasure::obtain_chunks(n_validators, &available_data).unwrap();
 
-		store.add_validator_index_and_n_validators(
+		store.note_validator_index_and_n_validators(
 			&relay_parent,
 			local_index as u32,
 			n_validators as u32,
@@ -826,26 +835,30 @@ mod tests {
 				candidate_receipt: candidate,
 				fetch: future::ok::<_, ::std::io::Error>(pov_block.clone()),
 			},
-			local_index,
 			relay_parent,
 			availability_store: store.clone(),
 			max_block_data_size: None,
+			n_validators,
 		};
 
-		let validated = block_on(producer.prime_with(|_, _, _| Ok(
-				ErasureChunk {
-					chunk: chunks[local_index].clone(),
-					index: local_index as u32,
-					proof: vec![],
+		let validated = block_on(producer.prime_with(|_, _| Ok(
+			FullOutput {
+				available_data: AvailableData {
+					pov_block: pov_block.clone(),
+					omitted_validation: Default::default(),
 				},
-			)).validate()).unwrap();
+				erasure_chunks: (0..n_validators).map(|i| ErasureChunk {
+					chunk: chunks[i].clone(),
+					index: i as u32,
+					proof: vec![],
+				}).collect(),
+				commitments: Default::default(),
+			}
+		)).validate()).unwrap();
 
-		assert_eq!(validated.0.pov_block(), &pov_block);
+		assert_eq!(validated.pov_block(), &pov_block);
 
-		// This works since there are only two validators and one erasure chunk should be
-		// enough to reconstruct the block data.
-		assert_eq!(store.block_data(relay_parent, block_data_hash).unwrap(), pov_block.block_data);
-		// TODO: check that a message queue is included by root.
+		assert_eq!(store.execution_data(&candidate_hash).unwrap().pov_block, pov_block);
 	}
 
 	#[test]
@@ -877,19 +890,11 @@ mod tests {
 			None,
 		);
 
-		let candidate = CandidateReceipt {
-			parachain_index: para_id,
-			collator: [1; 32].unchecked_into(),
-			signature: Default::default(),
-			head_data: HeadData(vec![1, 2, 3, 4]),
-			parent_head: HeadData(vec![]),
-			fees: 1_000_000,
-			block_data_hash: [2; 32].into(),
-			upward_messages: Vec::new(),
-			erasure_root: [1u8; 32].into(),
-		};
+		let mut candidate = AbridgedCandidateReceipt::default();
+		candidate.parachain_index = para_id;
+		candidate.relay_parent = parent_hash;
 
-		let hash = candidate.hash();
+		let candidate_hash = candidate.hash();
 		let candidate_statement = GenericStatement::Candidate(candidate);
 
 		let signature = crate::sign_table_statement(&candidate_statement, &validity_other_key.into(), &parent_hash);
@@ -904,7 +909,8 @@ mod tests {
 			signed_statement.clone(),
 		).expect("should produce work");
 
-		assert!(shared_table.inner.lock().validated.get(&hash).expect("validation has started").is_in_progress());
+		assert!(shared_table.inner.lock().validated.get(&candidate_hash)
+			.expect("validation has started").is_in_progress());
 
 		let b = shared_table.import_remote_statement(
 			&DummyRouter,
@@ -943,25 +949,18 @@ mod tests {
 			None,
 		);
 
-		let candidate = CandidateReceipt {
-			parachain_index: para_id,
-			collator: [1; 32].unchecked_into(),
-			signature: Default::default(),
-			head_data: HeadData(vec![1, 2, 3, 4]),
-			parent_head: HeadData(vec![]),
-			fees: 1_000_000,
-			block_data_hash: [2; 32].into(),
-			upward_messages: Vec::new(),
-			erasure_root: [1u8; 32].into(),
-		};
+		let mut candidate = AbridgedCandidateReceipt::default();
+		candidate.parachain_index = para_id;
+		candidate.relay_parent = parent_hash;
 
-		let hash = candidate.hash();
+		let candidate_hash = candidate.hash();
 		let signed_statement = shared_table.import_validated(Validated::collated_local(
 			candidate,
 			pov_block,
 		)).unwrap();
 
-		assert!(shared_table.inner.lock().validated.get(&hash).expect("validation has started").is_done());
+		assert!(shared_table.inner.lock().validated.get(&candidate_hash)
+			.expect("validation has started").is_done());
 
 		let a = shared_table.import_remote_statement(
 			&DummyRouter,