Erasure encoding availability (#345)

* Erasure encoding availability initial commit

 * Modifications to availability store to keep chunks as well as
   reconstructed blocks and extrinsics.
 * Gossip messages containig signed erasure chunks.
 * Requesting eraure chunks with polkadot-specific messages.
 * Validation of erasure chunk messages.

* Apply suggestions from code review

Co-Authored-By: Luke Schoen <ltfschoen@users.noreply.github.com>

* Fix build after a merge

* Gossip erasure chunk messages under their own topic

* erasure_chunks should use the appropriate topic

* Updates Cargo.lock

* Fixes after merge

* Removes a couple of leftover pieces of code

* Fixes simple stuff from review

* Updates erasure and storage for more flexible logic

* Changes validation and candidate receipt production.

* Adds add_erasure_chunks method

* Fixes most of the nits

* Better validate_collation and validate_receipt functions

* Fixes the tests

* Apply suggestions from code review

Co-Authored-By: Robert Habermeier <rphmeier@gmail.com>

* Removes unwrap() calls

* Removes ErasureChunks primitive

* Removes redundant fields from ErasureChunk struct

* AvailabilityStore should store CandidateReceipt

* Changes the way chunk messages are imported and validated.

 * Availability store now stores a validator_index and n_validators for
 each relay_parent.
 * Availability store now also stores candidate receipts.
 * Removes importing chunks in the table and moves it into network
 gossip validation.
 * Validation of erasure messages id done against receipts that are
 stored in the availability store.

* Correctly compute topics for erasure messages

* Removes an unused parameter

* Refactors availability db querying into a helper

* Adds the apis described in the writeup

* Adds a runtime api to extract erasure roots form raw extrinsics.

* Adds a barebone BlockImport impl for avalability store

* Adds the implementation of the availability worker

* Fix build after the merge with master.

* Make availability store API async

* Bring back the default wasmtime feature

* Lines width

* Bump runtime version

* Formatting and dead code elimination

* some style nits (#1)

* More nits and api cleanup

* Disable wasm CI for availability-store

* Another nit

* Formatting

polkadot/primitives/src/parachain.rs

@@ -30,6 +30,9 @@ use primitives::bytes;
 use primitives::RuntimeDebug;
 use application_crypto::KeyTypeId;
 
+#[cfg(feature = "std")]
+use trie::TrieConfiguration;
+
 pub use polkadot_parachain::{
 	Id, ParachainDispatchOrigin, LOWEST_USER_ID, UpwardMessage,
 };
@@ -227,9 +230,84 @@ impl OutgoingMessages {
 	}
 }
 
+/// Messages by queue root that are stored in the availability store.
+#[derive(PartialEq, Clone, Decode)]
+#[cfg_attr(feature = "std", derive(Serialize, Deserialize, Encode, Debug))]
+pub struct AvailableMessages(pub Vec<(Hash, Vec<Message>)>);
+
+
+/// Compute a trie root for a set of messages, given the raw message data.
+#[cfg(feature = "std")]
+pub fn message_queue_root<A, I: IntoIterator<Item=A>>(messages: I) -> Hash
+	where A: AsRef<[u8]>
+{
+	trie::trie_types::Layout::<primitives::Blake2Hasher>::ordered_trie_root(messages)
+}
+
+#[cfg(feature = "std")]
+impl From<OutgoingMessages> for AvailableMessages {
+	fn from(outgoing: OutgoingMessages) -> Self {
+		let queues = outgoing.message_queues().filter_map(|queue| {
+			let queue_root = message_queue_root(queue);
+			let queue_data = queue.iter().map(|msg| msg.clone().into()).collect();
+			Some((queue_root, queue_data))
+		}).collect();
+
+		AvailableMessages(queues)
+	}
+}
+
 /// Candidate receipt type.
 #[derive(PartialEq, Eq, Clone, Encode, Decode)]
 #[cfg_attr(feature = "std", derive(Debug))]
+pub struct CollationInfo {
+	/// The ID of the parachain this is a candidate for.
+	pub parachain_index: Id,
+	/// The collator's relay-chain account ID
+	pub collator: CollatorId,
+	/// Signature on blake2-256 of the block data by collator.
+	pub signature: CollatorSignature,
+	/// Egress queue roots. Must be sorted lexicographically (ascending)
+	/// by parachain ID.
+	pub egress_queue_roots: Vec<(Id, Hash)>,
+	/// The head-data
+	pub head_data: HeadData,
+	/// blake2-256 Hash of block data.
+	pub block_data_hash: Hash,
+	/// Messages destined to be interpreted by the Relay chain itself.
+	pub upward_messages: Vec<UpwardMessage>,
+}
+
+impl From<CandidateReceipt> for CollationInfo {
+	fn from(receipt: CandidateReceipt) -> Self {
+		CollationInfo {
+			parachain_index: receipt.parachain_index,
+			collator: receipt.collator,
+			signature: receipt.signature,
+			egress_queue_roots: receipt.egress_queue_roots,
+			head_data: receipt.head_data,
+			block_data_hash: receipt.block_data_hash,
+			upward_messages: receipt.upward_messages,
+		}
+	}
+}
+
+impl CollationInfo {
+	/// Check integrity vs. provided block data.
+	pub fn check_signature(&self) -> Result<(), ()> {
+		use runtime_primitives::traits::AppVerify;
+
+		if self.signature.verify(self.block_data_hash.as_ref(), &self.collator) {
+			Ok(())
+		} else {
+			Err(())
+		}
+	}
+}
+
+/// Candidate receipt type.
+#[derive(PartialEq, Eq, Clone, Encode, Decode)]
+#[cfg_attr(feature = "std", derive(Debug, Default))]
 pub struct CandidateReceipt {
 	/// The ID of the parachain this is a candidate for.
 	pub parachain_index: Id,
@@ -248,6 +326,8 @@ pub struct CandidateReceipt {
 	pub block_data_hash: Hash,
 	/// Messages destined to be interpreted by the Relay chain itself.
 	pub upward_messages: Vec<UpwardMessage>,
+	/// The root of a block's erasure encoding Merkle tree.
+	pub erasure_root: Hash,
 }
 
 impl CandidateReceipt {
@@ -275,6 +355,18 @@ impl PartialOrd for CandidateReceipt {
 	}
 }
 
+impl PartialEq<CollationInfo> for CandidateReceipt {
+	fn eq(&self, info: &CollationInfo) -> bool {
+		self.parachain_index == info.parachain_index &&
+		self.collator == info.collator &&
+		self.signature == info.signature &&
+		self.egress_queue_roots == info.egress_queue_roots &&
+		self.head_data == info.head_data &&
+		self.block_data_hash == info.block_data_hash &&
+		self.upward_messages == info.upward_messages
+	}
+}
+
 impl Ord for CandidateReceipt {
 	fn cmp(&self, other: &Self) -> Ordering {
 		// TODO: compare signatures or something more sane
@@ -289,7 +381,7 @@ impl Ord for CandidateReceipt {
 #[cfg_attr(feature = "std", derive(Debug, Encode, Decode))]
 pub struct Collation {
 	/// Candidate receipt itself.
-	pub receipt: CandidateReceipt,
+	pub info: CollationInfo,
 	/// A proof-of-validation for the receipt.
 	pub pov: PoVBlock,
 }
@@ -369,6 +461,18 @@ pub struct ConsolidatedIngress(pub Vec<(Id, Vec<Message>)>);
 #[cfg_attr(feature = "std", derive(Serialize, Deserialize, Debug))]
 pub struct BlockData(#[cfg_attr(feature = "std", serde(with="bytes"))] pub Vec<u8>);
 
+/// A chunk of erasure-encoded block data.
+#[derive(PartialEq, Eq, Clone, Encode, Decode, Default)]
+#[cfg_attr(feature = "std", derive(Serialize, Deserialize, Debug))]
+pub struct ErasureChunk {
+	/// The erasure-encoded chunk of data belonging to the candidate block.
+	pub chunk: Vec<u8>,
+	/// The index of this erasure-encoded chunk of data.
+	pub index: u32,
+	/// Proof for this chunk's branch in the Merkle tree.
+	pub proof: Vec<Vec<u8>>,
+}
+
 impl BlockData {
 	/// Compute hash of block data.
 	#[cfg(feature = "std")]
@@ -384,7 +488,7 @@ pub struct Header(#[cfg_attr(feature = "std", serde(with="bytes"))] pub Vec<u8>)
 
 /// Parachain head data included in the chain.
 #[derive(PartialEq, Eq, Clone, PartialOrd, Ord, Encode, Decode)]
-#[cfg_attr(feature = "std", derive(Serialize, Deserialize, Debug))]
+#[cfg_attr(feature = "std", derive(Serialize, Deserialize, Debug, Default))]
 pub struct HeadData(#[cfg_attr(feature = "std", serde(with="bytes"))] pub Vec<u8>);
 
 /// Parachain validation code.
@@ -483,6 +587,8 @@ pub struct Status {
 	pub fee_schedule: FeeSchedule,
 }
 
+use runtime_primitives::traits::{Block as BlockT};
+
 sp_api::decl_runtime_apis! {
 	/// The API for querying the state of parachains on-chain.
 	pub trait ParachainHost {
@@ -502,6 +608,8 @@ sp_api::decl_runtime_apis! {
 		/// If `since` is provided, only messages since (including those in) that block
 		/// will be included.
 		fn ingress(to: Id, since: Option<BlockNumber>) -> Option<StructuredUnroutedIngress>;
+		/// Extract the heads that were set by this set of extrinsics.
+		fn get_heads(extrinsics: Vec<<Block as BlockT>::Extrinsic>) -> Option<Vec<CandidateReceipt>>;
 	}
 }