A more comprehensive model for PoV-Blocks and Candidate receipts (#843)

* encode the candidate statement as only the hash

* refactor CandidateReceipt and CollationInfo

* introduce an abridged candidate receipt type

* erasure coding stores candidate receipt

* store omitted data instead and introduce AvailableData type

* refactor availability-store schema

* tweak schema and APIs a bit more

* get availability-store tests passing

* accept AbridgedCandidateReceipt in `set_heads`

* change statement type in primitives to be hash-only

* fix parachains runtime tests

* fix bad merge

* rewrite validation pipeline

* remove evaluation module

* use abridged candidate hash as canonical

* statement table uses abridged candidate receipts

* kill availability_store::Data struct

* port shared table to new validation pipelines

* extract full validation pipeline to helper

* remove old validation pipeline from collation module

* polkadot-validation compiles

* polkadot-validation tests compile

* make local collation available in validation service

* port legacy network code

* polkadot-network fully ported

* network: ensure fresh statement is propagated

* remove pov_block_hash from LocalValidationData

* remove candidate_hash field from AttestedCandidate and update runtime

* port runtimes to new ParachainHost definition

* port over polkadot-collator

* fix test compilation

* better fix

* remove unrelated validation work dispatch fix

* address grumbles

* fix equality check

polkadot/validation/src/block_production.rs

@@ -29,7 +29,7 @@ use sp_blockchain::HeaderBackend;
 use block_builder::BlockBuilderApi;
 use codec::Encode;
 use consensus::{Proposal, RecordProof};
-use polkadot_primitives::{Hash, Block, BlockId, BlockNumber, Header};
+use polkadot_primitives::{Hash, Block, BlockId, Header};
 use polkadot_primitives::parachain::{
 	ParachainHost, AttestedCandidate, NEW_HEADS_IDENTIFIER,
 };
@@ -47,6 +47,9 @@ use crate::validation_service::ServiceHandle;
 use crate::dynamic_inclusion::DynamicInclusion;
 use crate::Error;
 
+// block size limit.
+pub(crate) const MAX_TRANSACTIONS_SIZE: usize = 4 * 1024 * 1024;
+
 // Polkadot proposer factory.
 pub struct ProposerFactory<Client, TxPool, Backend> {
 	client: Arc<Client>,
@@ -100,7 +103,6 @@ where
 		parent_header: &Header,
 	) -> Self::CreateProposer {
 		let parent_hash = parent_header.hash();
-		let parent_number = parent_header.number;
 		let parent_id = BlockId::hash(parent_hash);
 
 		let client = self.client.clone();
@@ -114,9 +116,7 @@ where
 			.and_then(move |tracker| future::ready(Ok(Proposer {
 				client,
 				tracker,
-				parent_hash,
 				parent_id,
-				parent_number,
 				transaction_pool,
 				slot_duration,
 				backend,
@@ -129,9 +129,7 @@ where
 /// The Polkadot proposer logic.
 pub struct Proposer<Client, TxPool, Backend> {
 	client: Arc<Client>,
-	parent_hash: Hash,
 	parent_id: BlockId,
-	parent_number: BlockNumber,
 	tracker: crate::validation_service::ValidationInstanceHandle,
 	transaction_pool: Arc<TxPool>,
 	slot_duration: u64,
@@ -172,8 +170,6 @@ impl<Client, TxPool, Backend> consensus::Proposer<Block> for Proposer<Client, Tx
 			Duration::from_millis(self.slot_duration / SLOT_DURATION_DENOMINATOR),
 		);
 
-		let parent_hash = self.parent_hash.clone();
-		let parent_number = self.parent_number.clone();
 		let parent_id = self.parent_id.clone();
 		let client = self.client.clone();
 		let transaction_pool = self.transaction_pool.clone();
@@ -198,13 +194,10 @@ impl<Client, TxPool, Backend> consensus::Proposer<Block> for Proposer<Client, Tx
 			};
 
 			let data = CreateProposalData {
-				parent_hash,
-				parent_number,
 				parent_id,
 				client,
 				transaction_pool,
 				table,
-				believed_minimum_timestamp: believed_timestamp,
 				inherent_data: Some(inherent_data),
 				inherent_digests,
 				// leave some time for the proposal finalisation
@@ -239,13 +232,10 @@ fn current_timestamp() -> u64 {
 
 /// Inner data of the create proposal.
 struct CreateProposalData<Client, TxPool, Backend> {
-	parent_hash: Hash,
-	parent_number: BlockNumber,
 	parent_id: BlockId,
 	client: Arc<Client>,
 	transaction_pool: Arc<TxPool>,
 	table: Arc<crate::SharedTable>,
-	believed_minimum_timestamp: u64,
 	inherent_data: Option<InherentData>,
 	inherent_digests: DigestFor<Block>,
 	deadline: Instant,
@@ -298,7 +288,7 @@ impl<Client, TxPool, Backend> CreateProposalData<Client, TxPool, Backend> where
 			let ready_iter = self.transaction_pool.ready();
 			for ready in ready_iter.take(MAX_TRANSACTIONS) {
 				let encoded_size = ready.data().encode().len();
-				if pending_size + encoded_size >= crate::evaluation::MAX_TRANSACTIONS_SIZE {
+				if pending_size + encoded_size >= MAX_TRANSACTIONS_SIZE {
 					break;
 				}
 				if Instant::now() > self.deadline {
@@ -339,16 +329,6 @@ impl<Client, TxPool, Backend> CreateProposalData<Client, TxPool, Backend> where
 				.join(", ")
 		);
 
-		// TODO: full re-evaluation (https://github.com/paritytech/polkadot/issues/216)
-		let active_parachains = runtime_api.active_parachains(&self.parent_id)?;
-		assert!(crate::evaluation::evaluate_initial(
-			&new_block,
-			self.believed_minimum_timestamp,
-			&self.parent_hash,
-			self.parent_number,
-			&active_parachains[..],
-		).is_ok());
-
 		Ok(Proposal { block: new_block, storage_changes, proof })
 	}
 }

polkadot/validation/src/collation.rs

@@ -22,22 +22,15 @@
 use std::sync::Arc;
 
 use polkadot_primitives::{
-	BlakeTwo256, Block, Hash, HashT, BlockId, Balance,
+	BlakeTwo256, Block, Hash, HashT,
 	parachain::{
-		CollatorId, CandidateReceipt, CollationInfo,
-		ParachainHost, Id as ParaId, Collation, FeeSchedule, ErasureChunk,
-		HeadData, PoVBlock,
+		CollatorId, ParachainHost, Id as ParaId, Collation, ErasureChunk, CollationInfo,
 	},
 };
 use polkadot_erasure_coding as erasure;
 use sp_api::ProvideRuntimeApi;
-use parachain::{
-	wasm_executor::{self, ExecutionMode}, UpwardMessage,
-};
-use trie::TrieConfiguration;
 use futures::prelude::*;
 use log::debug;
-use parking_lot::Mutex;
 
 /// Encapsulates connections to collators and allows collation on any parachain.
 ///
@@ -67,530 +60,55 @@ pub trait Collators: Clone {
 /// A future which resolves when a collation is available.
 pub async fn collation_fetch<C: Collators, P>(
 	parachain: ParaId,
-	relay_parent_hash: Hash,
+	relay_parent: Hash,
 	collators: C,
 	client: Arc<P>,
 	max_block_data_size: Option<u64>,
-) -> Result<(Collation, HeadData, Balance),C::Error>
+	n_validators: usize,
+) -> Result<(CollationInfo, crate::pipeline::FullOutput), C::Error>
 	where
 		P::Api: ParachainHost<Block, Error = sp_blockchain::Error>,
 		C: Collators + Unpin,
 		P: ProvideRuntimeApi<Block>,
 		<C as Collators>::Collation: Unpin,
 {
-	let relay_parent = BlockId::hash(relay_parent_hash);
-
 	loop {
-		let collation = collators.collate(parachain, relay_parent_hash)
-			.await?;
-
-		let res = validate_collation(
+		let collation = collators.collate(parachain, relay_parent).await?;
+		let Collation { info, pov } = collation;
+		let res = crate::pipeline::full_output_validation_with_api(
 			&*client,
+			&info,
+			&pov,
 			&relay_parent,
-			&collation,
 			max_block_data_size,
+			n_validators,
 		);
 
 		match res {
-			Ok((parent_head, fees)) => {
-				return Ok((collation, parent_head, fees))
+			Ok(full_output) => {
+				return Ok((info, full_output))
 			}
 			Err(e) => {
 				debug!("Failed to validate parachain due to API error: {}", e);
 
 				// just continue if we got a bad collation or failed to validate
-				collators.note_bad_collator(collation.info.collator)
+				collators.note_bad_collator(info.collator)
 			}
 		}
 	}
 }
 
-// Errors that can occur when validating a parachain.
-#[derive(Debug, derive_more::Display, derive_more::From)]
-pub enum Error {
-	/// Client error
-	Client(sp_blockchain::Error),
-	/// Wasm validation error
-	WasmValidation(wasm_executor::Error),
-	/// Erasure-encoding error.
-	Erasure(erasure::Error),
-	/// Collated for inactive parachain
-	#[display(fmt = "Collated for inactive parachain: {:?}", _0)]
-	InactiveParachain(ParaId),
-	/// Unexpected egress root
-	#[display(fmt = "Got unexpected egress root to {:?}. (expected: {:?}, got {:?})", id, expected, got)]
-	EgressRootMismatch { id: ParaId, expected: Hash, got: Hash },
-	/// Unexpected ingress root
-	#[display(fmt = "Got unexpected ingress root to {:?}. (expected: {:?}, got {:?})", id, expected, got)]
-	IngressRootMismatch { id: ParaId, expected: Hash, got: Hash },
-	/// Ingress from wrong chain
-	#[display(fmt = "Got ingress from wrong chain. (expected: {:?}, got {:?})", expected, got)]
-	IngressChainMismatch { expected: ParaId, got: ParaId },
-	/// Ingress canonicality mismatch
-	#[display(fmt = "Got data for {} roots, expected {}", expected, got)]
-	IngressCanonicalityMismatch { expected: usize, got: usize },
-	/// Missing or extra egress root
-	#[display(fmt = "Missing or extra egress root. (expected: {:?}, got {:?})", expected, got)]
-	MissingEgressRoot { expected: Option<ParaId>, got: Option<ParaId>, },
-	/// Parachain validation produced wrong head data
-	#[display(fmt = "Parachain validation produced wrong head data (expected: {:?}, got {:?})", expected, got)]
-	WrongHeadData { expected: Vec<u8>, got: Vec<u8> },
-	/// Block data is too big
-	#[display(fmt = "Block data is too big (maximum allowed size: {}, actual size: {})", size, max_size)]
-	BlockDataTooBig { size: u64, max_size: u64 },
-	/// Parachain validation produced wrong relay-chain messages
-	#[display(fmt = "Parachain validation produced wrong relay-chain messages (expected: {:?}, got {:?})", expected, got)]
-	UpwardMessagesInvalid { expected: Vec<UpwardMessage>, got: Vec<UpwardMessage> },
-	/// Parachain validation produced wrong fees to charge to parachain.
-	#[display(fmt = "Parachain validation produced wrong relay-chain fees (expected: {:?}, got {:?})", expected, got)]
-	FeesChargedInvalid { expected: Balance, got: Balance },
-	/// Candidate block has an erasure-encoded root that mismatches the actual
-	/// erasure-encoded root of block data and extrinsics.
-	#[display(fmt = "Got unexpected erasure root (expected: {:?}, got {:?})", expected, got)]
-	ErasureRootMismatch { expected: Hash, got: Hash },
-	/// Candidate block collation info doesn't match candidate receipt.
-	#[display(fmt = "Got receipt mismatch for candidate {:?}", candidate)]
-	CandidateReceiptMismatch { candidate: Hash },
-	/// The parent header given in the candidate did not match current relay-chain
-	/// state.
-	#[display(fmt = "Got unexpected parachain parent.")]
-	ParentMismatch { expected: HeadData, got: HeadData },
-}
-
-impl std::error::Error for Error {
-	fn source(&self) -> Option<&(dyn std::error::Error + 'static)> {
-		match self {
-			Error::Client(ref err) => Some(err),
-			Error::WasmValidation(ref err) => Some(err),
-			_ => None,
-		}
-	}
-}
-
-/// Compute a trie root for a set of messages, given the raw message data.
-pub fn message_queue_root<A, I: IntoIterator<Item=A>>(messages: I) -> Hash
-	where A: AsRef<[u8]>
-{
-	trie::trie_types::Layout::<primitives::Blake2Hasher>::ordered_trie_root(messages)
-}
-
-struct ExternalitiesInner {
-	upward: Vec<UpwardMessage>,
-	fees_charged: Balance,
-	free_balance: Balance,
-	fee_schedule: FeeSchedule,
-}
-
-impl wasm_executor::Externalities for ExternalitiesInner {
-	fn post_upward_message(&mut self, message: UpwardMessage) -> Result<(), String> {
-		self.apply_message_fee(message.data.len())?;
-
-		self.upward.push(message);
-
-		Ok(())
-	}
-}
-
-impl ExternalitiesInner {
-	fn new(free_balance: Balance, fee_schedule: FeeSchedule) -> Self {
-		Self {
-			free_balance,
-			fee_schedule,
-			fees_charged: 0,
-			upward: Vec::new(),
-		}
-	}
-
-	fn apply_message_fee(&mut self, message_len: usize) -> Result<(), String> {
-		let fee = self.fee_schedule.compute_fee(message_len);
-		let new_fees_charged = self.fees_charged.saturating_add(fee);
-		if new_fees_charged > self.free_balance {
-			Err("could not cover fee.".into())
-		} else {
-			self.fees_charged = new_fees_charged;
-			Ok(())
-		}
-	}
-
-	// Performs final checks of validity, producing the outgoing message data.
-	fn final_checks(
-		&mut self,
-		upward_messages: &[UpwardMessage],
-		fees_charged: Option<Balance>,
-	) -> Result<Balance, Error> {
-		if self.upward != upward_messages {
-			return Err(Error::UpwardMessagesInvalid {
-				expected: upward_messages.to_vec(),
-				got: self.upward.clone(),
-			});
-		}
-
-		if let Some(fees_charged) = fees_charged {
-			if self.fees_charged != fees_charged {
-				return Err(Error::FeesChargedInvalid {
-					expected: fees_charged.clone(),
-					got: self.fees_charged.clone(),
-				});
-			}
-		}
-
-
-		Ok(self.fees_charged)
-	}
-}
-
-#[derive(Clone)]
-struct Externalities(Arc<Mutex<ExternalitiesInner>>);
-
-impl Externalities {
-	fn new(free_balance: Balance, fee_schedule: FeeSchedule) -> Self {
-		Self(Arc::new(Mutex::new(
-			ExternalitiesInner::new(free_balance, fee_schedule)
-		)))
-	}
-}
-
-impl wasm_executor::Externalities for Externalities {
-	fn post_upward_message(&mut self, message: UpwardMessage) -> Result<(), String> {
-		self.0.lock().post_upward_message(message)
-	}
-}
-
 /// Validate an erasure chunk against an expected root.
 pub fn validate_chunk(
 	root: &Hash,
 	chunk: &ErasureChunk,
-) -> Result<(), Error> {
-	let expected = erasure::branch_hash(root, &chunk.proof, chunk.index as usize)?;
+) -> Result<(), ()> {
+	let expected = erasure::branch_hash(root, &chunk.proof, chunk.index as usize).map_err(|_| ())?;
 	let got = BlakeTwo256::hash(&chunk.chunk);
 
 	if expected != got {
-		return Err(Error::ErasureRootMismatch {
-			expected,
-			got,
-		})
+		return Err(())
 	}
 
 	Ok(())
 }
-
-// A utility function that implements most of the collation validation logic.
-//
-// Reused by `validate_collation` and `validate_receipt`.
-// Returns outgoing messages, parent nead data, and fees charged for later reuse.
-fn do_validation<P>(
-	client: &P,
-	relay_parent: &BlockId,
-	pov_block: &PoVBlock,
-	para_id: ParaId,
-	max_block_data_size: Option<u64>,
-	fees_charged: Option<Balance>,
-	head_data: &HeadData,
-	upward_messages: &Vec<UpwardMessage>,
-) -> Result<(HeadData, Balance), Error> where
-	P: ProvideRuntimeApi<Block>,
-	P::Api: ParachainHost<Block, Error = sp_blockchain::Error>,
-{
-	use parachain::ValidationParams;
-
-	if let Some(max_size) = max_block_data_size {
-		let block_data_size = pov_block.block_data.0.len() as u64;
-		if block_data_size > max_size {
-			return Err(Error::BlockDataTooBig { size: block_data_size, max_size });
-		}
-	}
-
-	let api = client.runtime_api();
-	let validation_code = api.parachain_code(relay_parent, para_id)?
-		.ok_or_else(|| Error::InactiveParachain(para_id))?;
-
-	let chain_status = api.parachain_status(relay_parent, para_id)?
-		.ok_or_else(|| Error::InactiveParachain(para_id))?;
-
-
-	let params = ValidationParams {
-		parent_head: chain_status.head_data.0.clone(),
-		block_data: pov_block.block_data.0.clone(),
-	};
-
-	let ext = Externalities::new(chain_status.balance, chain_status.fee_schedule);
-
-	match wasm_executor::validate_candidate(
-		&validation_code,
-		params,
-		ext.clone(),
-		ExecutionMode::Remote,
-	) {
-		Ok(result) => {
-			if result.head_data == head_data.0 {
-				let fees = ext.0.lock().final_checks(
-					upward_messages,
-					fees_charged
-				)?;
-
-				Ok((chain_status.head_data, fees))
-			} else {
-				Err(Error::WrongHeadData {
-					expected: head_data.0.clone(),
-					got: result.head_data
-				})
-			}
-		}
-		Err(e) => Err(e.into())
-	}
-}
-
-/// Produce a `CandidateReceipt` and erasure encoding chunks with a given collation.
-///
-/// To produce a `CandidateReceipt` among other things the root of erasure encoding of
-/// the block data and messages needs to be known. To avoid redundant re-computations
-/// of erasure encoding this method creates an encoding and produces a candidate with
-/// encoding's root returning both for re-use.
-pub fn produce_receipt_and_chunks(
-	n_validators: usize,
-	parent_head: HeadData,
-	pov: &PoVBlock,
-	fees: Balance,
-	info: &CollationInfo,
-) -> Result<(CandidateReceipt, Vec<ErasureChunk>), Error>
-{
-	let erasure_chunks = erasure::obtain_chunks(
-		n_validators,
-		&pov.block_data,
-	)?;
-
-	let branches = erasure::branches(erasure_chunks.as_ref());
-	let erasure_root = branches.root();
-
-	let chunks: Vec<_> = erasure_chunks
-			.iter()
-			.zip(branches.map(|(proof, _)| proof))
-			.enumerate()
-			.map(|(index, (chunk, proof))| ErasureChunk {
-				// branches borrows the original chunks, but this clone could probably be dodged.
-				chunk: chunk.clone(),
-				index: index as u32,
-				proof,
-			})
-			.collect();
-
-	let receipt = CandidateReceipt {
-		parachain_index: info.parachain_index,
-		collator: info.collator.clone(),
-		signature: info.signature.clone(),
-		head_data: info.head_data.clone(),
-		parent_head,
-		fees,
-		block_data_hash: info.block_data_hash.clone(),
-		upward_messages: info.upward_messages.clone(),
-		erasure_root,
-	};
-
-	Ok((receipt, chunks))
-}
-
-/// Check if a given candidate receipt is valid with a given collation.
-///
-/// This assumes that basic validity checks have been done:
-///   - Block data hash is the same as linked in collation info and a receipt.
-pub fn validate_receipt<P>(
-	client: &P,
-	relay_parent: &BlockId,
-	pov_block: &PoVBlock,
-	receipt: &CandidateReceipt,
-	max_block_data_size: Option<u64>,
-) -> Result<Vec<ErasureChunk>, Error> where
-	P: ProvideRuntimeApi<Block>,
-	P::Api: ParachainHost<Block, Error = sp_blockchain::Error>,
-{
-	let (parent_head, _fees) = do_validation(
-		client,
-		relay_parent,
-		pov_block,
-		receipt.parachain_index,
-		max_block_data_size,
-		Some(receipt.fees),
-		&receipt.head_data,
-		&receipt.upward_messages,
-	)?;
-
-	if parent_head != receipt.parent_head {
-		return Err(Error::ParentMismatch {
-			expected: receipt.parent_head.clone(),
-			got: parent_head,
-		});
-	}
-
-	let api = client.runtime_api();
-	let validators = api.validators(&relay_parent)?;
-	let n_validators = validators.len();
-
-	let (validated_receipt, chunks) = produce_receipt_and_chunks(
-		n_validators,
-		parent_head,
-		pov_block,
-		receipt.fees,
-		&receipt.clone().into(),
-	)?;
-
-	if validated_receipt.erasure_root != receipt.erasure_root {
-		return Err(Error::ErasureRootMismatch {
-			expected: validated_receipt.erasure_root,
-			got: receipt.erasure_root,
-		});
-	}
-
-	Ok(chunks)
-}
-
-/// Check whether a given collation is valid. Returns `Ok` on success, error otherwise.
-/// Returns outgoing messages, parent head-data, and fees.
-///
-/// This assumes that basic validity checks have been done:
-///   - Block data hash is the same as linked in collation info.
-pub fn validate_collation<P>(
-	client: &P,
-	relay_parent: &BlockId,
-	collation: &Collation,
-	max_block_data_size: Option<u64>,
-) -> Result<(HeadData, Balance), Error> where
-	P: ProvideRuntimeApi<Block>,
-	P::Api: ParachainHost<Block, Error = sp_blockchain::Error>,
-{
-	let para_id = collation.info.parachain_index;
-
-	debug!("Validating collation for parachain {} at relay parent: {}", para_id, relay_parent);
-
-	do_validation(
-		client,
-		relay_parent,
-		&collation.pov,
-		para_id,
-		max_block_data_size,
-		None,
-		&collation.info.head_data,
-		&collation.info.upward_messages,
-	)
-}
-
-#[cfg(test)]
-mod tests {
-	use super::*;
-	use parachain::wasm_executor::Externalities as ExternalitiesTrait;
-	use parachain::ParachainDispatchOrigin;
-	use polkadot_primitives::parachain::{CandidateReceipt, HeadData};
-
-	#[test]
-	fn ext_checks_upward_messages() {
-		let ext = || ExternalitiesInner {
-			upward: vec![
-				UpwardMessage{ data: vec![42], origin: ParachainDispatchOrigin::Parachain },
-			],
-			fees_charged: 0,
-			free_balance: 1_000_000,
-			fee_schedule: FeeSchedule {
-				base: 1000,
-				per_byte: 10,
-			},
-		};
-		let receipt = CandidateReceipt {
-			parachain_index: 5.into(),
-			collator: Default::default(),
-			signature: Default::default(),
-			head_data: HeadData(Vec::new()),
-			parent_head: HeadData(Vec::new()),
-			fees: 0,
-			block_data_hash: Default::default(),
-			upward_messages: vec![
-				UpwardMessage{ data: vec![42], origin: ParachainDispatchOrigin::Signed },
-				UpwardMessage{ data: vec![69], origin: ParachainDispatchOrigin::Parachain },
-			],
-			erasure_root: [1u8; 32].into(),
-		};
-		assert!(ext().final_checks(
-			&receipt.upward_messages,
-			Some(receipt.fees),
-		).is_err());
-		let receipt = CandidateReceipt {
-			parachain_index: 5.into(),
-			collator: Default::default(),
-			signature: Default::default(),
-			head_data: HeadData(Vec::new()),
-			parent_head: HeadData(Vec::new()),
-			fees: 0,
-			block_data_hash: Default::default(),
-			upward_messages: vec![
-				UpwardMessage{ data: vec![42], origin: ParachainDispatchOrigin::Signed },
-			],
-			erasure_root: [1u8; 32].into(),
-		};
-		assert!(ext().final_checks(
-			&receipt.upward_messages,
-			Some(receipt.fees),
-		).is_err());
-		let receipt = CandidateReceipt {
-			parachain_index: 5.into(),
-			collator: Default::default(),
-			signature: Default::default(),
-			head_data: HeadData(Vec::new()),
-			parent_head: HeadData(Vec::new()),
-			fees: 0,
-			block_data_hash: Default::default(),
-			upward_messages: vec![
-				UpwardMessage{ data: vec![69], origin: ParachainDispatchOrigin::Parachain },
-			],
-			erasure_root: [1u8; 32].into(),
-		};
-		assert!(ext().final_checks(
-			&receipt.upward_messages,
-			Some(receipt.fees),
-		).is_err());
-		let receipt = CandidateReceipt {
-			parachain_index: 5.into(),
-			collator: Default::default(),
-			signature: Default::default(),
-			head_data: HeadData(Vec::new()),
-			parent_head: HeadData(Vec::new()),
-			fees: 0,
-			block_data_hash: Default::default(),
-			upward_messages: vec![
-				UpwardMessage{ data: vec![42], origin: ParachainDispatchOrigin::Parachain },
-			],
-			erasure_root: [1u8; 32].into(),
-		};
-		assert!(ext().final_checks(
-			&receipt.upward_messages,
-			Some(receipt.fees),
-		).is_ok());
-	}
-
-	#[test]
-	fn ext_checks_fees_and_updates_correctly() {
-		let mut ext = ExternalitiesInner {
-			upward: vec![
-				UpwardMessage{ data: vec![42], origin: ParachainDispatchOrigin::Parachain },
-			],
-			fees_charged: 0,
-			free_balance: 1_000_000,
-			fee_schedule: FeeSchedule {
-				base: 1000,
-				per_byte: 10,
-			},
-		};
-
-		ext.apply_message_fee(100).unwrap();
-		assert_eq!(ext.fees_charged, 2000);
-
-		ext.post_upward_message(UpwardMessage {
-			origin: ParachainDispatchOrigin::Signed,
-			data: vec![0u8; 100],
-		}).unwrap();
-		assert_eq!(ext.fees_charged, 4000);
-
-		ext.apply_message_fee((1_000_000 - 4000 - 1000) / 10).unwrap();
-		assert_eq!(ext.fees_charged, 1_000_000);
-
-		// cannot pay fee.
-		assert!(ext.apply_message_fee(1).is_err());
-	}
-}

polkadot/validation/src/error.rs

@@ -16,7 +16,7 @@
 
 //! Errors that can occur during the validation process.
 
-use polkadot_primitives::parachain::ValidatorId;
+use polkadot_primitives::{parachain::ValidatorId, Hash};
 
 /// Error type for validation
 #[derive(Debug, derive_more::Display, derive_more::From)]
@@ -25,6 +25,12 @@ pub enum Error {
 	Client(sp_blockchain::Error),
 	/// Consensus error
 	Consensus(consensus::error::Error),
+	/// A wasm-validation error.
+	WasmValidation(parachain::wasm_executor::Error),
+	/// An I/O error.
+	Io(std::io::Error),
+	/// An error in the availability erasure-coding.
+	ErasureCoding(polkadot_erasure_coding::Error),
 	#[display(fmt = "Invalid duty roster length: expected {}, got {}", expected, got)]
 	InvalidDutyRosterLength {
 		/// Expected roster length
@@ -48,6 +54,30 @@ pub enum Error {
 	DeadlineComputeFailure(std::time::Duration),
 	#[display(fmt = "Validation service is down.")]
 	ValidationServiceDown,
+	/// PoV-block in collation doesn't match provided.
+	#[display(fmt = "PoV hash mismatch. Expected {:?}, got {:?}", _0, _1)]
+	PoVHashMismatch(Hash, Hash),
+	/// Collator signature is invalid.
+	#[display(fmt = "Invalid collator signature on collation")]
+	InvalidCollatorSignature,
+	/// Head-data too large.
+	#[display(fmt = "Head data size of {} exceeded maximum of {}", _0, _1)]
+	HeadDataTooLarge(usize, usize),
+	/// Head-data mismatch after validation.
+	#[display(fmt = "Validation produced a different parachain header")]
+	HeadDataMismatch,
+	/// Relay parent of candidate not allowed.
+	#[display(fmt = "Relay parent {} of candidate not allowed in this context.", _0)]
+	DisallowedRelayParent(Hash),
+	/// Commitments in candidate match commitments produced by validation.
+	#[display(fmt = "Commitments in candidate receipt do not match those produced by validation")]
+	CommitmentsMismatch,
+	/// The parachain for which validation work is being done is not active.
+	#[display(fmt = "Parachain {:?} is not active", _0)]
+	InactiveParachain(polkadot_primitives::parachain::Id),
+	/// Block data is too big
+	#[display(fmt = "Block data is too big (maximum allowed size: {}, actual size: {})", size, max_size)]
+	BlockDataTooBig { size: u64, max_size: u64 },
 	Join(tokio::task::JoinError)
 }
 
@@ -56,6 +86,9 @@ impl std::error::Error for Error {
 		match self {
 			Error::Client(ref err) => Some(err),
 			Error::Consensus(ref err) => Some(err),
+			Error::WasmValidation(ref err) => Some(err),
+			Error::ErasureCoding(ref err) => Some(err),
+			Error::Io(ref err) => Some(err),
 			_ => None,
 		}
 	}

polkadot/validation/src/evaluation.rs

@@ -1,95 +0,0 @@
-// Copyright 2017-2020 Parity Technologies (UK) Ltd.
-// This file is part of Polkadot.
-
-// Polkadot is free software: you can redistribute it and/or modify
-// it under the terms of the GNU General Public License as published by
-// the Free Software Foundation, either version 3 of the License, or
-// (at your option) any later version.
-
-// Polkadot is distributed in the hope that it will be useful,
-// but WITHOUT ANY WARRANTY; without even the implied warranty of
-// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-// GNU General Public License for more details.
-
-// You should have received a copy of the GNU General Public License
-// along with Polkadot.  If not, see <http://www.gnu.org/licenses/>.
-
-//! Polkadot block evaluation and evaluation errors.
-
-// block size limit.
-pub(crate) const MAX_TRANSACTIONS_SIZE: usize = 4 * 1024 * 1024;
-
-use codec::Encode;
-use polkadot_primitives::{Block, Hash, BlockNumber};
-use polkadot_primitives::parachain::{Id as ParaId, CollatorId, Retriable};
-
-/// Result type alias for block evaluation
-pub type Result<T> = std::result::Result<T, Error>;
-
-/// Error type for block evaluation
-#[derive(Debug, derive_more::Display, derive_more::From)]
-pub enum Error {
-	/// Client error
-	Client(sp_blockchain::Error),
-	/// Too many parachain candidates in proposal
-	#[display(fmt = "Proposal included {} candidates for {} parachains", expected, got)]
-	TooManyCandidates { expected: usize, got: usize },
-	/// Proposal included unregistered parachain
-	#[display(fmt = "Proposal included unregistered parachain {:?}", _0)]
-	UnknownParachain(ParaId),
-	/// Proposal had wrong parent hash
-	#[display(fmt = "Proposal had wrong parent hash. Expected {:?}, got {:?}", expected, got)]
-	WrongParentHash { expected: Hash, got: Hash },
-	/// Proposal had wrong number
-	#[display(fmt = "Proposal had wrong number. Expected {:?}, got {:?}", expected, got)]
-	WrongNumber { expected: BlockNumber, got: BlockNumber },
-	/// Proposal exceeded the maximum size
-	#[display(
-		fmt = "Proposal exceeded the maximum size of {} by {} bytes.",
-		MAX_TRANSACTIONS_SIZE, MAX_TRANSACTIONS_SIZE.saturating_sub(*_0)
-	)]
-	ProposalTooLarge(usize),
-}
-
-impl std::error::Error for Error {
-	fn source(&self) -> Option<&(dyn std::error::Error + 'static)> {
-		match self {
-			Error::Client(ref err) => Some(err),
-			_ => None,
-		}
-	}
-}
-
-/// Attempt to evaluate a substrate block as a polkadot block, returning error
-/// upon any initial validity checks failing.
-pub fn evaluate_initial(
-	proposal: &Block,
-	_now: u64,
-	parent_hash: &Hash,
-	parent_number: BlockNumber,
-	_active_parachains: &[(ParaId, Option<(CollatorId, Retriable)>)],
-) -> Result<()> {
-	let transactions_size = proposal.extrinsics.iter().fold(0, |a, tx| {
-		a + Encode::encode(tx).len()
-	});
-
-	if transactions_size > MAX_TRANSACTIONS_SIZE {
-		return Err(Error::ProposalTooLarge(transactions_size))
-	}
-
-	if proposal.header.parent_hash != *parent_hash {
-		return Err(Error::WrongParentHash {
-			expected: *parent_hash,
-			got: proposal.header.parent_hash
-		});
-	}
-
-	if proposal.header.number != parent_number + 1 {
-		return Err(Error::WrongNumber {
-			expected: parent_number + 1,
-			got: proposal.header.number
-		});
-	}
-
-	Ok(())
-}

polkadot/validation/src/lib.rs

@@ -33,13 +33,12 @@ use std::{
 	collections::{HashMap, HashSet},
 	sync::Arc,
 };
-
 use codec::Encode;
 use polkadot_primitives::Hash;
 use polkadot_primitives::parachain::{
-	Id as ParaId, Chain, DutyRoster, CandidateReceipt,
+	Id as ParaId, Chain, DutyRoster, AbridgedCandidateReceipt,
 	Statement as PrimitiveStatement,
-	Collation, PoVBlock, ErasureChunk, ValidatorSignature, ValidatorIndex,
+	PoVBlock, ErasureChunk, ValidatorSignature, ValidatorIndex,
 	ValidatorPair, ValidatorId,
 };
 use primitives::Pair;
@@ -47,10 +46,7 @@ use primitives::Pair;
 use futures::prelude::*;
 
 pub use self::block_production::ProposerFactory;
-pub use self::collation::{
-	validate_collation, message_queue_root, Collators,
-	produce_receipt_and_chunks,
-};
+pub use self::collation::Collators;
 pub use self::error::Error;
 pub use self::shared_table::{
 	SharedTable, ParachainWork, PrimedParachainWork, Validated, Statement, SignedStatement,
@@ -62,13 +58,13 @@ pub use self::validation_service::{ServiceHandle, ServiceBuilder};
 pub use parachain::wasm_executor::{run_worker as run_validation_worker};
 
 mod dynamic_inclusion;
-mod evaluation;
 mod error;
 mod shared_table;
 
-pub mod collation;
-pub mod validation_service;
 pub mod block_production;
+pub mod collation;
+pub mod pipeline;
+pub mod validation_service;
 
 /// A handle to a statement table router.
 ///
@@ -83,12 +79,11 @@ pub trait TableRouter: Clone {
 	/// Future that resolves when candidate data is fetched.
 	type FetchValidationProof: Future<Output=Result<PoVBlock, Self::Error>>;
 
-	/// Call with local candidate data. This will make the data available on the network,
-	/// and sign, import, and broadcast a statement about the candidate.
+	/// Call with local candidate data. This will sign, import, and broadcast a statement about the candidate.
 	fn local_collation(
 		&self,
-		collation: Collation,
-		receipt: CandidateReceipt,
+		receipt: AbridgedCandidateReceipt,
+		pov_block: PoVBlock,
 		chunks: (ValidatorIndex, &[ErasureChunk]),
 	) -> Self::SendLocalCollation;
 
@@ -96,7 +91,7 @@ pub trait TableRouter: Clone {
 	///
 	/// This future must conclude once all `Clone`s of this `TableRouter` have
 	/// been cleaned up.
-	fn fetch_pov_block(&self, candidate: &CandidateReceipt) -> Self::FetchValidationProof;
+	fn fetch_pov_block(&self, candidate: &AbridgedCandidateReceipt) -> Self::FetchValidationProof;
 }
 
 /// A long-lived network which can create parachain statement and BFT message routing processes on demand.
@@ -141,10 +136,7 @@ pub struct GroupInfo {
 /// The actual message signed is the encoded statement concatenated with the
 /// parent hash.
 pub fn sign_table_statement(statement: &Statement, key: &ValidatorPair, parent_hash: &Hash) -> ValidatorSignature {
-	// we sign using the primitive statement type because that's what the runtime
-	// expects. These types probably encode the same way so this clone could be optimized
-	// out in the future.
-	let mut encoded = PrimitiveStatement::from(statement.clone()).encode();
+	let mut encoded = PrimitiveStatement::from(statement).encode();
 	encoded.extend(parent_hash.as_ref());
 
 	key.sign(&encoded)
@@ -159,7 +151,7 @@ pub fn check_statement(
 ) -> bool {
 	use runtime_primitives::traits::AppVerify;
 
-	let mut encoded = PrimitiveStatement::from(statement.clone()).encode();
+	let mut encoded = PrimitiveStatement::from(statement).encode();
 	encoded.extend(parent_hash.as_ref());
 
 	signature.verify(&encoded[..], &signer)
@@ -211,7 +203,6 @@ pub fn make_group_info(
 	});
 
 	Ok((map, local_duty))
-
 }
 
 #[cfg(test)]

polkadot/validation/src/pipeline.rs

@@ -0,0 +1,375 @@
+// Copyright 2020 Parity Technologies (UK) Ltd.
+// This file is part of Polkadot.
+
+// Polkadot is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// Polkadot is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with Polkadot.  If not, see <http://www.gnu.org/licenses/>.
+
+//! The pipeline of validation functions a parachain block must pass through before
+//! it can be voted for.
+
+use std::sync::Arc;
+
+use codec::Encode;
+use polkadot_erasure_coding as erasure;
+use polkadot_primitives::parachain::{
+	CollationInfo, PoVBlock, LocalValidationData, GlobalValidationSchedule, OmittedValidationData,
+	AvailableData, FeeSchedule, CandidateCommitments, ErasureChunk, HeadData, ParachainHost,
+	Id as ParaId, AbridgedCandidateReceipt,
+};
+use polkadot_primitives::{Block, BlockId, Balance, Hash};
+use parachain::{
+	wasm_executor::{self, ExecutionMode},
+	UpwardMessage, ValidationParams,
+};
+use runtime_primitives::traits::{BlakeTwo256, Hash as HashT};
+use sp_api::ProvideRuntimeApi;
+use parking_lot::Mutex;
+use crate::Error;
+
+/// Does basic checks of a collation. Provide the encoded PoV-block.
+pub fn basic_checks(
+	collation: &CollationInfo,
+	expected_relay_parent: &Hash,
+	max_block_data_size: Option<u64>,
+	encoded_pov: &[u8],
+) -> Result<(), Error> {
+	if &collation.relay_parent != expected_relay_parent {
+		return Err(Error::DisallowedRelayParent(collation.relay_parent));
+	}
+
+	if let Some(max_size) = max_block_data_size {
+		if encoded_pov.len() as u64 > max_size {
+			return Err(Error::BlockDataTooBig { size: encoded_pov.len() as _, max_size });
+		}
+	}
+
+	let hash = BlakeTwo256::hash(encoded_pov);
+	if hash != collation.pov_block_hash {
+		return Err(Error::PoVHashMismatch(collation.pov_block_hash, hash));
+	}
+
+	if let Err(()) = collation.check_signature() {
+		return Err(Error::InvalidCollatorSignature);
+	}
+
+	Ok(())
+}
+
+struct ExternalitiesInner {
+	upward: Vec<UpwardMessage>,
+	fees_charged: Balance,
+	free_balance: Balance,
+	fee_schedule: FeeSchedule,
+}
+
+impl wasm_executor::Externalities for ExternalitiesInner {
+	fn post_upward_message(&mut self, message: UpwardMessage) -> Result<(), String> {
+		self.apply_message_fee(message.data.len())?;
+
+		self.upward.push(message);
+
+		Ok(())
+	}
+}
+
+impl ExternalitiesInner {
+	fn new(free_balance: Balance, fee_schedule: FeeSchedule) -> Self {
+		Self {
+			free_balance,
+			fee_schedule,
+			fees_charged: 0,
+			upward: Vec::new(),
+		}
+	}
+
+	fn apply_message_fee(&mut self, message_len: usize) -> Result<(), String> {
+		let fee = self.fee_schedule.compute_fee(message_len);
+		let new_fees_charged = self.fees_charged.saturating_add(fee);
+		if new_fees_charged > self.free_balance {
+			Err("could not cover fee.".into())
+		} else {
+			self.fees_charged = new_fees_charged;
+			Ok(())
+		}
+	}
+
+	// Returns the noted outputs of execution so far - upward messages and balances.
+	fn outputs(self) -> (Vec<UpwardMessage>, Balance) {
+		(self.upward, self.fees_charged)
+	}
+}
+
+#[derive(Clone)]
+struct Externalities(Arc<Mutex<ExternalitiesInner>>);
+
+impl Externalities {
+	fn new(free_balance: Balance, fee_schedule: FeeSchedule) -> Self {
+		Self(Arc::new(Mutex::new(
+			ExternalitiesInner::new(free_balance, fee_schedule)
+		)))
+	}
+}
+
+impl wasm_executor::Externalities for Externalities {
+	fn post_upward_message(&mut self, message: UpwardMessage) -> Result<(), String> {
+		self.0.lock().post_upward_message(message)
+	}
+}
+
+/// Data from a fully-outputted validation of a parachain candidate. This contains
+/// all outputs and commitments of the validation as well as all additional data to make available.
+pub struct FullOutput {
+	/// Data about the candidate to keep available in the network.
+	pub available_data: AvailableData,
+	/// Commitments issued alongside the candidate to be placed on-chain.
+	pub commitments: CandidateCommitments,
+	/// All erasure-chunks associated with the available data. Each validator
+	/// should keep their chunk (by index). Other chunks do not need to be
+	/// kept available long-term, but should be distributed to other validators.
+	pub erasure_chunks: Vec<ErasureChunk>,
+}
+
+impl FullOutput {
+	/// Check consistency of the outputs produced by the validation pipeline against
+	/// data contained within a candidate receipt.
+	pub fn check_consistency(&self, receipt: &AbridgedCandidateReceipt) -> Result<(), Error> {
+		if self.commitments != receipt.commitments {
+			Err(Error::CommitmentsMismatch)
+		} else {
+			Ok(())
+		}
+	}
+}
+
+/// The successful result of validating a collation. If the full commitments of the
+/// validation are needed, call `full_output`. Otherwise, safely drop this value.
+pub struct ValidatedCandidate<'a> {
+	pov_block: &'a PoVBlock,
+	global_validation: &'a GlobalValidationSchedule,
+	parent_head: &'a HeadData,
+	balance: Balance,
+	upward_messages: Vec<UpwardMessage>,
+	fees: Balance,
+}
+
+impl<'a> ValidatedCandidate<'a> {
+	/// Fully-compute the commitments and outputs of the candidate. Provide the number
+	/// of validators. This computes the erasure-coding.
+	pub fn full_output(self, n_validators: usize) -> Result<FullOutput, Error> {
+		let ValidatedCandidate {
+			pov_block,
+			global_validation,
+			parent_head,
+			balance,
+			upward_messages,
+			fees,
+		} = self;
+
+		let omitted_validation = OmittedValidationData {
+			global_validation: global_validation.clone(),
+			local_validation: LocalValidationData {
+				parent_head: parent_head.clone(),
+				balance,
+			},
+		};
+
+		let available_data = AvailableData {
+			pov_block: pov_block.clone(),
+			omitted_validation,
+		};
+
+		let erasure_chunks = erasure::obtain_chunks(
+			n_validators,
+			&available_data,
+		)?;
+
+		let branches = erasure::branches(erasure_chunks.as_ref());
+		let erasure_root = branches.root();
+
+		let chunks: Vec<_> = erasure_chunks
+			.iter()
+			.zip(branches.map(|(proof, _)| proof))
+			.enumerate()
+			.map(|(index, (chunk, proof))| ErasureChunk {
+				// branches borrows the original chunks, but this clone could probably be dodged.
+				chunk: chunk.clone(),
+				index: index as u32,
+				proof,
+			})
+			.collect();
+
+		let commitments = CandidateCommitments {
+			upward_messages,
+			fees,
+			erasure_root,
+		};
+
+		Ok(FullOutput {
+			available_data,
+			commitments,
+			erasure_chunks: chunks,
+		})
+	}
+}
+
+/// Does full checks of a collation, with provided PoV-block and contextual data.
+pub fn validate<'a>(
+	collation: &'a CollationInfo,
+	pov_block: &'a PoVBlock,
+	local_validation: &'a LocalValidationData,
+	global_validation: &'a GlobalValidationSchedule,
+	validation_code: &[u8],
+) -> Result<ValidatedCandidate<'a>, Error> {
+	if collation.head_data.0.len() > global_validation.max_head_data_size as _ {
+		return Err(Error::HeadDataTooLarge(
+			collation.head_data.0.len(),
+			global_validation.max_head_data_size as _,
+		));
+	}
+
+	let params = ValidationParams {
+		parent_head: collation.head_data.0.clone(),
+		block_data: pov_block.block_data.0.clone(),
+	};
+
+	// TODO: remove when ext does not do this.
+	let fee_schedule = FeeSchedule {
+		base: 0,
+		per_byte: 0,
+	};
+
+	let ext = Externalities::new(local_validation.balance, fee_schedule);
+	match wasm_executor::validate_candidate(
+		&validation_code,
+		params,
+		ext.clone(),
+		ExecutionMode::Remote,
+	) {
+		Ok(result) => {
+			if result.head_data == collation.head_data.0 {
+				let (upward_messages, fees) = Arc::try_unwrap(ext.0)
+					.map_err(|_| "<non-unique>")
+					.expect("Wasm executor drops passed externalities on completion; \
+						call has concluded; qed")
+					.into_inner()
+					.outputs();
+
+				Ok(ValidatedCandidate {
+					pov_block,
+					global_validation,
+					parent_head: &local_validation.parent_head,
+					balance: local_validation.balance,
+					upward_messages,
+					fees,
+				})
+			} else {
+				Err(Error::HeadDataMismatch)
+			}
+		}
+		Err(e) => Err(e.into()),
+	}
+}
+
+/// Extracts validation parameters from a Polkadot runtime API for a specific parachain.
+pub fn validation_params<P>(api: &P, relay_parent: Hash, para_id: ParaId)
+	-> Result<(LocalValidationData, GlobalValidationSchedule, Vec<u8>), Error>
+where
+	P: ProvideRuntimeApi<Block>,
+	P::Api: ParachainHost<Block, Error = sp_blockchain::Error>,
+{
+	let api = api.runtime_api();
+	let relay_parent = BlockId::hash(relay_parent);
+
+	// fetch all necessary data from runtime.
+	let local_validation = api.local_validation_data(&relay_parent, para_id)?
+		.ok_or_else(|| Error::InactiveParachain(para_id))?;
+
+	let global_validation = api.global_validation_schedule(&relay_parent)?;
+	let validation_code = api.parachain_code(&relay_parent, para_id)?
+		.ok_or_else(|| Error::InactiveParachain(para_id))?;
+
+	Ok((local_validation, global_validation, validation_code))
+}
+
+/// Does full-pipeline validation of a collation with provided contextual parameters.
+pub fn full_output_validation_with_api<P>(
+	api: &P,
+	collation: &CollationInfo,
+	pov_block: &PoVBlock,
+	expected_relay_parent: &Hash,
+	max_block_data_size: Option<u64>,
+	n_validators: usize,
+) -> Result<FullOutput, Error> where
+	P: ProvideRuntimeApi<Block>,
+	P::Api: ParachainHost<Block, Error = sp_blockchain::Error>,
+{
+	let para_id = collation.parachain_index;
+	let (local_validation, global_validation, validation_code)
+		= validation_params(&*api, collation.relay_parent, para_id)?;
+
+	// put the parameters through the validation pipeline, producing
+	// erasure chunks.
+	let encoded_pov = pov_block.encode();
+	basic_checks(
+		&collation,
+		&expected_relay_parent,
+		max_block_data_size,
+		&encoded_pov,
+	)
+		.and_then(|()| validate(
+			&collation,
+			&pov_block,
+			&local_validation,
+			&global_validation,
+			&validation_code,
+		))
+		.and_then(|validated| validated.full_output(n_validators))
+}
+
+#[cfg(test)]
+mod tests {
+	use super::*;
+	use parachain::wasm_executor::Externalities as ExternalitiesTrait;
+	use parachain::ParachainDispatchOrigin;
+
+	#[test]
+	fn ext_checks_fees_and_updates_correctly() {
+		let mut ext = ExternalitiesInner {
+			upward: vec![
+				UpwardMessage{ data: vec![42], origin: ParachainDispatchOrigin::Parachain },
+			],
+			fees_charged: 0,
+			free_balance: 1_000_000,
+			fee_schedule: FeeSchedule {
+				base: 1000,
+				per_byte: 10,
+			},
+		};
+
+		ext.apply_message_fee(100).unwrap();
+		assert_eq!(ext.fees_charged, 2000);
+
+		ext.post_upward_message(UpwardMessage {
+			origin: ParachainDispatchOrigin::Signed,
+			data: vec![0u8; 100],
+		}).unwrap();
+		assert_eq!(ext.fees_charged, 4000);
+
+
+		ext.apply_message_fee((1_000_000 - 4000 - 1000) / 10).unwrap();
+		assert_eq!(ext.fees_charged, 1_000_000);
+
+		// cannot pay fee.
+		assert!(ext.apply_message_fee(1).is_err());
+	}
+}

polkadot/validation/src/shared_table/mod.rs

@@ -22,10 +22,10 @@ use std::sync::Arc;
 
 use availability_store::{Store as AvailabilityStore};
 use table::{self, Table, Context as TableContextTrait};
-use polkadot_primitives::{Block, BlockId, Hash};
+use polkadot_primitives::{Block, Hash};
 use polkadot_primitives::parachain::{
-	Id as ParaId, CandidateReceipt, ValidatorPair, ValidatorId,
-	AttestedCandidate, ParachainHost, PoVBlock, ValidatorIndex, ErasureChunk,
+	Id as ParaId, AbridgedCandidateReceipt, ValidatorPair, ValidatorId,
+	AttestedCandidate, ParachainHost, PoVBlock, ValidatorIndex,
 };
 
 use parking_lot::Mutex;
@@ -39,6 +39,9 @@ use self::includable::IncludabilitySender;
 use primitives::Pair;
 use sp_api::ProvideRuntimeApi;
 
+use crate::pipeline::FullOutput;
+use crate::Error;
+
 mod includable;
 
 pub use table::{SignedStatement, Statement};
@@ -188,8 +191,8 @@ impl SharedTableInner {
 			availability_store: self.availability_store.clone(),
 			relay_parent: context.parent_hash.clone(),
 			work,
-			local_index: local_index as usize,
 			max_block_data_size,
+			n_validators: context.validators.len(),
 		})
 	}
 
@@ -232,7 +235,7 @@ impl Validated {
 	/// Note that we've collated a candidate.
 	/// outgoing message required.
 	pub fn collated_local(
-		receipt: CandidateReceipt,
+		receipt: AbridgedCandidateReceipt,
 		collation: PoVBlock,
 	) -> Self {
 		Validated {
@@ -253,9 +256,9 @@ impl Validated {
 pub struct ParachainWork<Fetch> {
 	work: Work<Fetch>,
 	relay_parent: Hash,
-	local_index: usize,
 	availability_store: AvailabilityStore,
 	max_block_data_size: Option<u64>,
+	n_validators: usize,
 }
 
 impl<Fetch: Future + Unpin> ParachainWork<Fetch> {
@@ -264,48 +267,47 @@ impl<Fetch: Future + Unpin> ParachainWork<Fetch> {
 	pub fn prime<P: ProvideRuntimeApi<Block>>(self, api: Arc<P>)
 		-> PrimedParachainWork<
 			Fetch,
-			impl Send + FnMut(&BlockId, &PoVBlock, &CandidateReceipt) -> Result<ErasureChunk, ()> + Unpin,
+			impl Send + FnMut(&PoVBlock, &AbridgedCandidateReceipt)
+				-> Result<FullOutput, Error> + Unpin,
 		>
 		where
 			P: Send + Sync + 'static,
 			P::Api: ParachainHost<Block, Error = sp_blockchain::Error>,
 	{
 		let max_block_data_size = self.max_block_data_size;
-		let local_index = self.local_index;
+		let n_validators = self.n_validators;
+		let expected_relay_parent = self.relay_parent;
 
-		let validate = move |id: &_, pov_block: &_, receipt: &_| {
-			let res = crate::collation::validate_receipt(
+		let validate = move |pov_block: &PoVBlock, candidate: &AbridgedCandidateReceipt| {
+			let collation_info = candidate.to_collation_info();
+			let full_output = crate::pipeline::full_output_validation_with_api(
 				&*api,
-				id,
+				&collation_info,
 				pov_block,
-				receipt,
+				&expected_relay_parent,
 				max_block_data_size,
-			);
+				n_validators,
+			)?;
 
-			match res {
-				Ok(mut chunks) => {
-					Ok(chunks.swap_remove(local_index))
-				}
-				Err(e) => {
-					debug!(target: "validation", "Encountered bad collation: {}", e);
-					Err(())
-				}
-			}
+			full_output.check_consistency(candidate)?;
+			Ok(full_output)
 		};
 
 		PrimedParachainWork { inner: self, validate }
 	}
 
 	/// Prime the parachain work with a custom validation function.
+	#[cfg(test)]
 	pub fn prime_with<F>(self, validate: F) -> PrimedParachainWork<Fetch, F>
-		where F: FnMut(&BlockId, &PoVBlock, &CandidateReceipt) -> Result<ErasureChunk, ()>
+		where F: FnMut(&PoVBlock, &AbridgedCandidateReceipt)
+			-> Result<FullOutput, Error>
 	{
 		PrimedParachainWork { inner: self, validate }
 	}
 }
 
 struct Work<Fetch> {
-	candidate_receipt: CandidateReceipt,
+	candidate_receipt: AbridgedCandidateReceipt,
 	fetch: Fetch
 }
 
@@ -318,46 +320,62 @@ pub struct PrimedParachainWork<Fetch, F> {
 impl<Fetch, F, Err> PrimedParachainWork<Fetch, F>
 	where
 		Fetch: Future<Output=Result<PoVBlock,Err>> + Unpin,
-		F: FnMut(&BlockId, &PoVBlock, &CandidateReceipt) -> Result<ErasureChunk, ()> + Unpin,
-		Err: From<::std::io::Error>,
+		F: FnMut(&PoVBlock, &AbridgedCandidateReceipt) -> Result<FullOutput, Error> + Unpin,
+		Err: From<std::io::Error>,
 {
-	pub async fn validate(mut self) -> Result<(Validated, Option<ErasureChunk>), Err> {
-		let candidate = &self.inner.work.candidate_receipt;
+	pub async fn validate(self) -> Result<Validated, Err> {
+		let candidate = self.inner.work.candidate_receipt;
 		let pov_block = self.inner.work.fetch.await?;
 
-		let validation_res = (self.validate)(
-			&BlockId::hash(self.inner.relay_parent),
-			&pov_block,
-			&candidate,
-		);
+		let mut validate = self.validate;
+		let relay_parent = self.inner.relay_parent;
 
+		// create a wrapper around the custom validation function that does
+		// some more general pre and post checks.
+		let mut validate = move |pov_block: &_, candidate: &AbridgedCandidateReceipt| {
+			if candidate.relay_parent != relay_parent {
+				return Err(Error::DisallowedRelayParent(candidate.relay_parent));
+			}
+
+			let full_output = validate(pov_block, candidate)?;
+
+			if full_output.commitments != candidate.commitments {
+				return Err(Error::CommitmentsMismatch);
+			}
+
+			Ok(full_output)
+		};
+
+		let validation_res = (validate)(&pov_block, &candidate);
 		let candidate_hash = candidate.hash();
 
 		debug!(target: "validation", "Making validity statement about candidate {}: is_good? {:?}",
 			candidate_hash, validation_res.is_ok());
 
 		match validation_res {
-			Err(()) => Ok((
-				Validated {
+			Err(err) => {
+				debug!(target: "validation", "candidate is invalid: {}", err);
+				Ok(Validated {
 					statement: GenericStatement::Invalid(candidate_hash),
 					result: Validation::Invalid(pov_block),
-				},
-				None,
-			)),
-			Ok(our_chunk) => {
-				self.inner.availability_store.add_erasure_chunk(
-					self.inner.relay_parent,
-					candidate.clone(),
-					our_chunk.clone(),
+				})
+			}
+			Ok(full_output) => {
+				// make data and all erasure chunks available. The chunk
+				// must be fully available before we add the chunks.
+				self.inner.availability_store.make_available(
+					candidate_hash,
+					full_output.available_data,
+				).await?;
+				self.inner.availability_store.add_erasure_chunks(
+					candidate,
+					full_output.erasure_chunks,
 				).await?;
 
-				Ok((
-					Validated {
-						statement: GenericStatement::Valid(candidate_hash),
-						result: Validation::Valid(pov_block),
-					},
-					Some(our_chunk),
-				))
+				Ok(Validated {
+					statement: GenericStatement::Valid(candidate_hash),
+					result: Validation::Valid(pov_block),
+				})
 			}
 		}
 	}
@@ -461,7 +479,8 @@ impl SharedTable {
 	}
 
 	/// Sign and import the result of candidate validation. Returns `None` if the table
-	/// was instantiated without a local key.
+	/// was instantiated without a local key. Otherwise, returns a copy of the signed
+	/// statement.
 	pub fn import_validated(&self, validated: Validated)
 		-> Option<SignedStatement>
 	{
@@ -485,7 +504,7 @@ impl SharedTable {
 	///
 	/// Deadlocks if called recursively.
 	pub fn with_candidate<F, U>(&self, digest: &Hash, f: F) -> U
-		where F: FnOnce(Option<&CandidateReceipt>) -> U
+		where F: FnOnce(Option<&AbridgedCandidateReceipt>) -> U
 	{
 		let inner = self.inner.lock();
 		f(inner.table.get_candidate(digest))
@@ -570,8 +589,9 @@ impl SharedTable {
 mod tests {
 	use super::*;
 	use sp_keyring::Sr25519Keyring;
-	use primitives::crypto::UncheckedInto;
-	use polkadot_primitives::parachain::{BlockData, Collation, HeadData};
+	use polkadot_primitives::parachain::{
+		BlockData, ErasureChunk, AvailableData,
+	};
 	use polkadot_erasure_coding::{self as erasure};
 	use availability_store::ProvideGossipMessages;
 	use futures::future;
@@ -613,12 +633,12 @@ mod tests {
 
 		fn local_collation(
 			&self,
-			_collation: Collation,
-			_candidate: CandidateReceipt,
+			_candidate: AbridgedCandidateReceipt,
+			_pov_block: PoVBlock,
 			_chunks: (ValidatorIndex, &[ErasureChunk])
 		) -> Self::SendLocalCollation { future::ready(Ok(())) }
 
-		fn fetch_pov_block(&self, _candidate: &CandidateReceipt) -> Self::FetchValidationProof {
+		fn fetch_pov_block(&self, _candidate: &AbridgedCandidateReceipt) -> Self::FetchValidationProof {
 			future::ok(pov_block_with_data(vec![1, 2, 3, 4, 5]))
 		}
 	}
@@ -652,17 +672,9 @@ mod tests {
 			None,
 		);
 
-		let candidate = CandidateReceipt {
-			parachain_index: para_id,
-			collator: [1; 32].unchecked_into(),
-			signature: Default::default(),
-			head_data: HeadData(vec![1, 2, 3, 4]),
-			parent_head: HeadData(vec![]),
-			fees: 1_000_000,
-			block_data_hash: [2; 32].into(),
-			upward_messages: Vec::new(),
-			erasure_root: [1u8; 32].into(),
-		};
+		let mut candidate = AbridgedCandidateReceipt::default();
+		candidate.parachain_index = para_id;
+		candidate.relay_parent = parent_hash;
 
 		let candidate_statement = GenericStatement::Candidate(candidate);
 
@@ -708,17 +720,9 @@ mod tests {
 			None,
 		);
 
-		let candidate = CandidateReceipt {
-			parachain_index: para_id,
-			collator: [1; 32].unchecked_into(),
-			signature: Default::default(),
-			head_data: HeadData(vec![1, 2, 3, 4]),
-			parent_head: HeadData(vec![]),
-			fees: 1_000_000,
-			block_data_hash: [2; 32].into(),
-			upward_messages: Vec::new(),
-			erasure_root: [1u8; 32].into(),
-		};
+		let mut candidate = AbridgedCandidateReceipt::default();
+		candidate.parachain_index = para_id;
+		candidate.relay_parent = parent_hash;
 
 		let candidate_statement = GenericStatement::Candidate(candidate);
 
@@ -741,25 +745,18 @@ mod tests {
 		let relay_parent = [0; 32].into();
 		let para_id = 5.into();
 		let pov_block = pov_block_with_data(vec![1, 2, 3]);
-		let block_data_hash = [2; 32].into();
+		let pov_block_hash = pov_block.hash();
 		let local_index = 0;
 		let n_validators = 2;
 
-		let candidate = CandidateReceipt {
-			parachain_index: para_id,
-			collator: [1; 32].unchecked_into(),
-			signature: Default::default(),
-			head_data: HeadData(vec![1, 2, 3, 4]),
-			parent_head: HeadData(vec![]),
-			fees: 1_000_000,
-			block_data_hash,
-			upward_messages: Vec::new(),
-			erasure_root: [1u8; 32].into(),
-		};
+		let mut candidate = AbridgedCandidateReceipt::default();
+		candidate.parachain_index = para_id;
+		candidate.relay_parent = relay_parent;
+		candidate.pov_block_hash = pov_block_hash;
 
-		let hash = candidate.hash();
+		let candidate_hash = candidate.hash();
 
-		store.add_validator_index_and_n_validators(
+		store.note_validator_index_and_n_validators(
 			&relay_parent,
 			local_index as u32,
 			n_validators as u32,
@@ -770,25 +767,37 @@ mod tests {
 				candidate_receipt: candidate,
 				fetch: future::ok(pov_block.clone()),
 			},
-			local_index,
 			relay_parent,
 			availability_store: store.clone(),
 			max_block_data_size: None,
+			n_validators,
 		};
 
-		let validated = block_on(producer.prime_with(|_, _, _| Ok(
-			ErasureChunk {
-				chunk: vec![1, 2, 3],
-				index: local_index as u32,
-				proof: vec![],
+		for i in 0..n_validators {
+			assert!(store.get_erasure_chunk(&candidate_hash, i).is_none());
+		}
+
+		let validated = block_on(producer.prime_with(|_, _| Ok(
+			FullOutput {
+				available_data: AvailableData {
+					pov_block: pov_block.clone(),
+					omitted_validation: Default::default(),
+				},
+				erasure_chunks: (0..n_validators).map(|i| ErasureChunk {
+					chunk: vec![1, 2, 3],
+					index: i as u32,
+					proof: vec![],
+				}).collect(),
+				commitments: Default::default(),
 			}
 		)).validate()).unwrap();
 
-		assert_eq!(validated.0.pov_block(), &pov_block);
-		assert_eq!(validated.0.statement, GenericStatement::Valid(hash));
+		assert_eq!(validated.pov_block(), &pov_block);
+		assert_eq!(validated.statement, GenericStatement::Valid(candidate_hash));
 
-		assert!(store.get_erasure_chunk(&relay_parent, block_data_hash, local_index).is_some());
-		assert!(store.get_erasure_chunk(&relay_parent, block_data_hash, local_index + 1).is_none());
+		for i in 0..n_validators {
+			assert!(store.get_erasure_chunk(&candidate_hash, i).is_some());
+		}
 	}
 
 	#[test]
@@ -797,25 +806,25 @@ mod tests {
 		let relay_parent = [0; 32].into();
 		let para_id = 5.into();
 		let pov_block = pov_block_with_data(vec![1, 2, 3]);
-		let block_data_hash = pov_block.block_data.hash();
+		let pov_block_hash = pov_block.hash();
 		let local_index = 0;
 		let n_validators = 2;
 
-		let candidate = CandidateReceipt {
-			parachain_index: para_id,
-			collator: [1; 32].unchecked_into(),
-			signature: Default::default(),
-			head_data: HeadData(vec![1, 2, 3, 4]),
-			parent_head: HeadData(vec![]),
-			fees: 1_000_000,
-			block_data_hash: [2; 32].into(),
-			upward_messages: Vec::new(),
-			erasure_root: [1u8; 32].into(),
+		let mut candidate = AbridgedCandidateReceipt::default();
+		candidate.parachain_index = para_id;
+		candidate.pov_block_hash = pov_block_hash;
+		candidate.relay_parent = relay_parent;
+
+		let candidate_hash = candidate.hash();
+
+		let available_data = AvailableData {
+			pov_block: pov_block.clone(),
+			omitted_validation: Default::default(),
 		};
 
-		let chunks = erasure::obtain_chunks(n_validators, &pov_block.block_data).unwrap();
+		let chunks = erasure::obtain_chunks(n_validators, &available_data).unwrap();
 
-		store.add_validator_index_and_n_validators(
+		store.note_validator_index_and_n_validators(
 			&relay_parent,
 			local_index as u32,
 			n_validators as u32,
@@ -826,26 +835,30 @@ mod tests {
 				candidate_receipt: candidate,
 				fetch: future::ok::<_, ::std::io::Error>(pov_block.clone()),
 			},
-			local_index,
 			relay_parent,
 			availability_store: store.clone(),
 			max_block_data_size: None,
+			n_validators,
 		};
 
-		let validated = block_on(producer.prime_with(|_, _, _| Ok(
-				ErasureChunk {
-					chunk: chunks[local_index].clone(),
-					index: local_index as u32,
-					proof: vec![],
+		let validated = block_on(producer.prime_with(|_, _| Ok(
+			FullOutput {
+				available_data: AvailableData {
+					pov_block: pov_block.clone(),
+					omitted_validation: Default::default(),
 				},
-			)).validate()).unwrap();
+				erasure_chunks: (0..n_validators).map(|i| ErasureChunk {
+					chunk: chunks[i].clone(),
+					index: i as u32,
+					proof: vec![],
+				}).collect(),
+				commitments: Default::default(),
+			}
+		)).validate()).unwrap();
 
-		assert_eq!(validated.0.pov_block(), &pov_block);
+		assert_eq!(validated.pov_block(), &pov_block);
 
-		// This works since there are only two validators and one erasure chunk should be
-		// enough to reconstruct the block data.
-		assert_eq!(store.block_data(relay_parent, block_data_hash).unwrap(), pov_block.block_data);
-		// TODO: check that a message queue is included by root.
+		assert_eq!(store.execution_data(&candidate_hash).unwrap().pov_block, pov_block);
 	}
 
 	#[test]
@@ -877,19 +890,11 @@ mod tests {
 			None,
 		);
 
-		let candidate = CandidateReceipt {
-			parachain_index: para_id,
-			collator: [1; 32].unchecked_into(),
-			signature: Default::default(),
-			head_data: HeadData(vec![1, 2, 3, 4]),
-			parent_head: HeadData(vec![]),
-			fees: 1_000_000,
-			block_data_hash: [2; 32].into(),
-			upward_messages: Vec::new(),
-			erasure_root: [1u8; 32].into(),
-		};
+		let mut candidate = AbridgedCandidateReceipt::default();
+		candidate.parachain_index = para_id;
+		candidate.relay_parent = parent_hash;
 
-		let hash = candidate.hash();
+		let candidate_hash = candidate.hash();
 		let candidate_statement = GenericStatement::Candidate(candidate);
 
 		let signature = crate::sign_table_statement(&candidate_statement, &validity_other_key.into(), &parent_hash);
@@ -904,7 +909,8 @@ mod tests {
 			signed_statement.clone(),
 		).expect("should produce work");
 
-		assert!(shared_table.inner.lock().validated.get(&hash).expect("validation has started").is_in_progress());
+		assert!(shared_table.inner.lock().validated.get(&candidate_hash)
+			.expect("validation has started").is_in_progress());
 
 		let b = shared_table.import_remote_statement(
 			&DummyRouter,
@@ -943,25 +949,18 @@ mod tests {
 			None,
 		);
 
-		let candidate = CandidateReceipt {
-			parachain_index: para_id,
-			collator: [1; 32].unchecked_into(),
-			signature: Default::default(),
-			head_data: HeadData(vec![1, 2, 3, 4]),
-			parent_head: HeadData(vec![]),
-			fees: 1_000_000,
-			block_data_hash: [2; 32].into(),
-			upward_messages: Vec::new(),
-			erasure_root: [1u8; 32].into(),
-		};
+		let mut candidate = AbridgedCandidateReceipt::default();
+		candidate.parachain_index = para_id;
+		candidate.relay_parent = parent_hash;
 
-		let hash = candidate.hash();
+		let candidate_hash = candidate.hash();
 		let signed_statement = shared_table.import_validated(Validated::collated_local(
 			candidate,
 			pov_block,
 		)).unwrap();
 
-		assert!(shared_table.inner.lock().validated.get(&hash).expect("validation has started").is_done());
+		assert!(shared_table.inner.lock().validated.get(&candidate_hash)
+			.expect("validation has started").is_done());
 
 		let a = shared_table.import_remote_statement(
 			&DummyRouter,

polkadot/validation/src/validation_service/mod.rs

@@ -322,7 +322,7 @@ impl<C, N, P, SP> ParachainValidationInstances<C, N, P, SP> where
 		// If we are a validator, we need to store our index in this round in availability store.
 		// This will tell which erasure chunk we should store.
 		if let Some(ref local_duty) = local_duty {
-			if let Err(e) = self.availability_store.add_validator_index_and_n_validators(
+			if let Err(e) = self.availability_store.note_validator_index_and_n_validators(
 				&parent_hash,
 				local_duty.index,
 				validators.len() as u32,
@@ -374,7 +374,7 @@ impl<C, N, P, SP> ParachainValidationInstances<C, N, P, SP> where
 		validation_para: ParaId,
 		build_router: N::BuildTableRouter,
 		max_block_data_size: Option<u64>,
-		authorities_num: usize,
+		n_validators: usize,
 		local_id: ValidatorIndex,
 	) {
 		let (collators, client) = (self.collators.clone(), self.client.clone());
@@ -388,49 +388,54 @@ impl<C, N, P, SP> ParachainValidationInstances<C, N, P, SP> where
 				collators,
 				client.clone(),
 				max_block_data_size,
+				n_validators,
 			);
 
 			collation_work.then(move |result| match result {
-				Ok((collation, parent_head, fees_charged)) => {
-					match crate::collation::produce_receipt_and_chunks(
-						authorities_num,
-						parent_head,
-						&collation.pov,
-						fees_charged,
-						&collation.info,
-					) {
-						Ok((receipt, chunks)) => {
-							// Apparently the `async move` block is the only way to convince
-							// the compiler that we are not moving values out of borrowed context.
-							let av_clone = availability_store.clone();
-							let chunks_clone = chunks.clone();
-							let receipt_clone = receipt.clone();
+				Ok((collation_info, full_output)) => {
+					let crate::pipeline::FullOutput {
+						commitments,
+						erasure_chunks,
+						available_data,
+					} = full_output;
 
-							let res = async move {
-								if let Err(e) = av_clone.clone().add_erasure_chunks(
-									relay_parent.clone(),
-									receipt_clone,
-									chunks_clone,
-								).await {
-									warn!(target: "validation", "Failed to add erasure chunks: {}", e);
-								}
-							}
-							.unit_error()
-							.then(move |_| {
-								router.local_collation(
-									collation,
-									receipt,
-									(local_id, &chunks),
-								).map_err(|e| warn!(target: "validation", "Failed to send local collation: {:?}", e))
-							});
+					let receipt = collation_info.into_receipt(commitments);
 
-							res.boxed()
+					// Apparently the `async move` block is the only way to convince
+					// the compiler that we are not moving values out of borrowed context.
+					let av_clone = availability_store.clone();
+					let receipt_clone = receipt.clone();
+					let erasure_chunks_clone = erasure_chunks.clone();
+					let pov_block = available_data.pov_block.clone();
+
+					let res = async move {
+						if let Err(e) = av_clone.make_available(
+							receipt_clone.hash(),
+							available_data,
+						).await {
+							warn!(
+								target: "validation",
+								"Failed to make parachain block data available: {}",
+								e,
+							);
 						}
-						Err(e) => {
-							warn!(target: "validation", "Failed to produce a receipt: {:?}", e);
-							Box::pin(ready(Ok(())))
+						if let Err(e) = av_clone.clone().add_erasure_chunks(
+							receipt_clone,
+							erasure_chunks_clone,
+						).await {
+							warn!(target: "validation", "Failed to add erasure chunks: {}", e);
 						}
 					}
+					.unit_error()
+					.then(move |_| {
+						router.local_collation(
+							receipt,
+							pov_block,
+							(local_id, &erasure_chunks),
+						).map_err(|e| warn!(target: "validation", "Failed to send local collation: {:?}", e))
+					});
+
+					res.boxed()
 				}
 				Err(e) => {
 					warn!(target: "validation", "Failed to collate candidate: {:?}", e);