Erasure encoding availability (#345)

* Erasure encoding availability initial commit * Modifications to availability store to keep chunks as well as reconstructed blocks and extrinsics. * Gossip messages containig signed erasure chunks. * Requesting eraure chunks with polkadot-specific messages. * Validation of erasure chunk messages. * Apply suggestions from code review Co-Authored-By: Luke Schoen <ltfschoen@users.noreply.github.com> * Fix build after a merge * Gossip erasure chunk messages under their own topic * erasure_chunks should use the appropriate topic * Updates Cargo.lock * Fixes after merge * Removes a couple of leftover pieces of code * Fixes simple stuff from review * Updates erasure and storage for more flexible logic * Changes validation and candidate receipt production. * Adds add_erasure_chunks method * Fixes most of the nits * Better validate_collation and validate_receipt functions * Fixes the tests * Apply suggestions from code review Co-Authored-By: Robert Habermeier <rphmeier@gmail.com> * Removes unwrap() calls * Removes ErasureChunks primitive * Removes redundant fields from ErasureChunk struct * AvailabilityStore should store CandidateReceipt * Changes the way chunk messages are imported and validated. * Availability store now stores a validator_index and n_validators for each relay_parent. * Availability store now also stores candidate receipts. * Removes importing chunks in the table and moves it into network gossip validation. * Validation of erasure messages id done against receipts that are stored in the availability store. * Correctly compute topics for erasure messages * Removes an unused parameter * Refactors availability db querying into a helper * Adds the apis described in the writeup * Adds a runtime api to extract erasure roots form raw extrinsics. * Adds a barebone BlockImport impl for avalability store * Adds the implementation of the availability worker * Fix build after the merge with master. * Make availability store API async * Bring back the default wasmtime feature * Lines width * Bump runtime version * Formatting and dead code elimination * some style nits (#1) * More nits and api cleanup * Disable wasm CI for availability-store * Another nit * Formatting
2026-07-10 04:07:27 +00:00 · 2019-12-03 17:49:07 +03:00
parent ec54d5b1e4
commit 99d164b5e7
29 changed files with 2957 additions and 572 deletions
@@ -27,12 +27,13 @@ pub mod gossip;

 use codec::{Decode, Encode};
 use futures::sync::oneshot;
+use futures::future::Either;
 use futures::prelude::*;
-use futures03::{channel::mpsc, compat::Compat, StreamExt};
+use futures03::{channel::mpsc, compat::{Compat, Stream01CompatExt}, FutureExt, StreamExt, TryFutureExt};
 use polkadot_primitives::{Block, Hash, Header};
 use polkadot_primitives::parachain::{
-	Id as ParaId, BlockData, CollatorId, CandidateReceipt, Collation, PoVBlock,
-	StructuredUnroutedIngress, ValidatorId, OutgoingMessages,
+	Id as ParaId, CollatorId, CandidateReceipt, Collation, PoVBlock,
+	StructuredUnroutedIngress, ValidatorId, OutgoingMessages, ErasureChunk,
 };
 use sc_network::{
 	PeerId, RequestId, Context, StatusMessage as GenericFullStatus,
@@ -48,7 +49,7 @@ use log::{trace, debug, warn};

 use std::collections::{HashMap, HashSet};

-use crate::gossip::{POLKADOT_ENGINE_ID, GossipMessage};
+use crate::gossip::{POLKADOT_ENGINE_ID, GossipMessage, ErasureChunkMessage};

 #[cfg(test)]
 mod tests;
@@ -98,6 +99,63 @@ pub trait NetworkService: Send + Sync + 'static {
 		where F: FnOnce(&mut PolkadotProtocol, &mut dyn Context<Block>);
 }

+/// This is a newtype that implements a [`ProvideGossipMessages`] shim trait.
+///
+/// For any wrapped [`NetworkService`] type it implements a [`ProvideGossipMessages`].
+/// For more details see documentation of [`ProvideGossipMessages`].
+///
+/// [`NetworkService`]: ./trait.NetworkService.html
+/// [`ProvideGossipMessages`]: ../polkadot_availability_store/trait.ProvideGossipMessages.html
+pub struct AvailabilityNetworkShim<T>(pub std::sync::Arc<T>);
+
+impl<T> av_store::ProvideGossipMessages for AvailabilityNetworkShim<T>
+	where T: NetworkService
+{
+	fn gossip_messages_for(&self, topic: Hash)
+		-> Box<dyn futures03::Stream<Item = (Hash, Hash, ErasureChunk)> + Unpin + Send>
+	{
+		Box::new(self.0.gossip_messages_for(topic)
+			.compat()
+			.filter_map(|msg| async move {
+				match msg {
+					Ok(msg) => match msg.0 {
+						GossipMessage::ErasureChunk(chunk) => {
+							Some((chunk.relay_parent, chunk.candidate_hash, chunk.chunk))
+						},
+						_ => None,
+					}
+					_ => None,
+				}
+			})
+			.boxed()
+		)
+	}
+
+	fn gossip_erasure_chunk(
+		&self,
+		relay_parent: Hash,
+		candidate_hash: Hash,
+		erasure_root: Hash,
+		chunk: ErasureChunk
+	) {
+		let topic = av_store::erasure_coding_topic(relay_parent, erasure_root, chunk.index);
+		self.0.gossip_message(
+			topic,
+			GossipMessage::ErasureChunk(ErasureChunkMessage {
+				chunk,
+				relay_parent,
+				candidate_hash,
+			})
+		)
+	}
+}
+
+impl<T> Clone for AvailabilityNetworkShim<T> {
+	fn clone(&self) -> Self {
+		AvailabilityNetworkShim(self.0.clone())
+	}
+}
+
 impl NetworkService for PolkadotNetworkService {
 	fn gossip_messages_for(&self, topic: Hash) -> GossipMessageStream {
 		let (tx, rx) = std::sync::mpsc::channel();
@@ -280,10 +338,6 @@ pub enum Message {
 	RequestPovBlock(RequestId, Hash, Hash),
 	/// Provide requested proof-of-validation block data by candidate hash or nothing if unknown.
 	PovBlock(RequestId, Option<PoVBlock>),
-	/// Request block data (relay_parent, candidate_hash)
-	RequestBlockData(RequestId, Hash, Hash),
-	/// Provide requested block data by candidate hash or nothing.
-	BlockData(RequestId, Option<BlockData>),
 	/// Tell a collator their role.
 	CollatorRole(Role),
 	/// A collation provided by a peer. Relay parent and collation.
@@ -444,24 +498,7 @@ impl PolkadotProtocol {

 				send_polkadot_message(ctx, who, Message::PovBlock(req_id, pov_block));
 			}
-			Message::RequestBlockData(req_id, relay_parent, candidate_hash) => {
-				let block_data = self.live_validation_leaves
-					.with_pov_block(
-						&relay_parent,
-						&candidate_hash,
-						|res| res.ok().map(|b| b.block_data.clone()),
-					)
-					.or_else(|| self.availability_store.as_ref()
-						.and_then(|s| s.block_data(relay_parent, candidate_hash))
-					);
-
-				send_polkadot_message(ctx, who, Message::BlockData(req_id, block_data));
-			}
 			Message::PovBlock(req_id, data) => self.on_pov_block(ctx, who, req_id, data),
-			Message::BlockData(_req_id, _data) => {
-				// current block data is never requested bare by the node.
-				ctx.report_peer(who, cost::UNEXPECTED_MESSAGE);
-			}
 			Message::Collation(relay_parent, collation) => self.on_collation(ctx, who, relay_parent, collation),
 			Message::CollatorRole(role) => self.on_new_role(ctx, who, role),
 		}
@@ -731,8 +768,8 @@ impl PolkadotProtocol {
 		relay_parent: Hash,
 		collation: Collation
 	) {
-		let collation_para = collation.receipt.parachain_index;
-		let collated_acc = collation.receipt.collator.clone();
+		let collation_para = collation.info.parachain_index;
+		let collated_acc = collation.info.collator.clone();

 		match self.peers.get(&from) {
 			None => ctx.report_peer(from, cost::UNKNOWN_PEER),
@@ -743,7 +780,7 @@ impl PolkadotProtocol {
 					Some((ref acc_id, ref para_id)) => {
 						ctx.report_peer(from.clone(), benefit::EXPECTED_MESSAGE);
 						let structurally_valid = para_id == &collation_para && acc_id == &collated_acc;
-						if structurally_valid && collation.receipt.check_signature().is_ok() {
+						if structurally_valid && collation.info.check_signature().is_ok() {
 							debug!(target: "p_net", "Received collation for parachain {:?} from peer {}", para_id, from);
 							ctx.report_peer(from, benefit::GOOD_COLLATION);
 							self.collators.on_collation(acc_id.clone(), relay_parent, collation)
@@ -798,23 +835,31 @@ impl PolkadotProtocol {
 		targets: HashSet<ValidatorId>,
 		collation: Collation,
 		outgoing_targeted: OutgoingMessages,
-	) -> std::io::Result<()> {
+	) -> impl futures::future::Future<Item = (), Error=()> {
 		debug!(target: "p_net", "Importing local collation on relay parent {:?} and parachain {:?}",
-			relay_parent, collation.receipt.parachain_index);
+			relay_parent, collation.info.parachain_index);

-		let outgoing_queues = polkadot_validation::outgoing_queues(&outgoing_targeted)
-			.map(|(_target, root, data)| (root, data))
-			.collect();
-
-		if let Some(ref availability_store) = self.availability_store {
-			availability_store.make_available(av_store::Data {
-				relay_parent,
-				parachain_id: collation.receipt.parachain_index,
-				candidate_hash: collation.receipt.hash(),
-				block_data: collation.pov.block_data.clone(),
-				outgoing_queues: Some(outgoing_queues),
-			})?;
-		}
+		let res = match self.availability_store {
+			Some(ref availability_store) => {
+				let availability_store_cloned = availability_store.clone();
+				let collation_cloned = collation.clone();
+				Either::A((async move {
+						let _ = availability_store_cloned.make_available(av_store::Data {
+							relay_parent,
+							parachain_id: collation_cloned.info.parachain_index,
+							block_data: collation_cloned.pov.block_data.clone(),
+							outgoing_queues: Some(outgoing_targeted.clone().into()),
+						}).await;
+					}
+				)
+					.unit_error()
+					.boxed()
+					.compat()
+					.then(|_| Ok(()))
+				)
+			}
+			None => Either::B(futures::future::ok::<(), ()>(())),
+		};

 		for (primary, cloned_collation) in self.local_collations.add_collation(relay_parent, targets, collation.clone()) {
 			match self.validators.get(&primary) {
@@ -831,7 +876,7 @@ impl PolkadotProtocol {
 			}
 		}

-		Ok(())
+		res
 	}

 	/// Give the network protocol a handle to an availability store, used for