Code, PoV compression and remove CompressedPoV struct (#2852)

* use compressed blob in candidate-validation

* add some tests for compressed code blobs

* remove CompressedPoV and apply compression in collation-generation

* decompress BlockData before executing

* don't produce oversized collations

* add test for PoV decompression failure

* fix tests and clean up

* fix test

* address review and fix CI

* take this )

polkadot/node/collation-generation/Cargo.toml

@@ -13,7 +13,9 @@ polkadot-node-subsystem = { path = "../subsystem" }
 polkadot-node-subsystem-util = { path = "../subsystem-util" }
 polkadot-primitives = { path = "../../primitives" }
 sp-core = { git = "https://github.com/paritytech/substrate", branch = "master" }
+sp-maybe-compressed-blob  = { git = "https://github.com/paritytech/substrate", branch = "master" }
 thiserror = "1.0.23"
+parity-scale-codec = { version = "2.0.0", default-features = false, features = ["bit-vec", "derive"] }
 
 [dev-dependencies]
 polkadot-node-subsystem-test-helpers = { path = "../subsystem-test-helpers" }

polkadot/node/collation-generation/src/lib.rs

@@ -26,7 +26,9 @@ use futures::{
 	sink::SinkExt,
 	stream::StreamExt,
 };
-use polkadot_node_primitives::{CollationGenerationConfig, AvailableData, PoV};
+use polkadot_node_primitives::{
+	CollationGenerationConfig, AvailableData, PoV,
+};
 use polkadot_node_subsystem::{
 	messages::{AllMessages, CollationGenerationMessage, CollatorProtocolMessage},
 	FromOverseer, SpawnedSubsystem, Subsystem, SubsystemContext, SubsystemResult,
@@ -41,6 +43,7 @@ use polkadot_primitives::v1::{
 	CandidateDescriptor, CandidateReceipt, CoreState, Hash, OccupiedCoreAssumption,
 	PersistedValidationData,
 };
+use parity_scale_codec::Encode;
 use sp_core::crypto::Pair;
 use std::sync::Arc;
 
@@ -313,7 +316,32 @@ async fn handle_new_activations<Context: SubsystemContext>(
 					}
 				};
 
-				let pov_hash = collation.proof_of_validity.hash();
+				// Apply compression to the block data.
+				let pov = {
+					let pov = polkadot_node_primitives::maybe_compress_pov(collation.proof_of_validity);
+					let encoded_size = pov.encoded_size();
+
+					// As long as `POV_BOMB_LIMIT` is at least `max_pov_size`, this ensures
+					// that honest collators never produce a PoV which is uncompressed.
+					//
+					// As such, honest collators never produce an uncompressed PoV which starts with
+					// a compression magic number, which would lead validators to reject the collation.
+					if encoded_size > validation_data.max_pov_size as usize {
+						tracing::debug!(
+							target: LOG_TARGET,
+							para_id = %scheduled_core.para_id,
+							size = encoded_size,
+							max_size = validation_data.max_pov_size,
+							"PoV exceeded maximum size"
+						);
+
+						return
+					}
+
+					pov
+				};
+
+				let pov_hash = pov.hash();
 
 				let signature_payload = collator_signature_payload(
 					&relay_parent,
@@ -326,7 +354,7 @@ async fn handle_new_activations<Context: SubsystemContext>(
 				let erasure_root = match erasure_root(
 					n_validators,
 					validation_data,
-					collation.proof_of_validity.clone(),
+					pov.clone(),
 				) {
 					Ok(erasure_root) => erasure_root,
 					Err(err) => {
@@ -375,7 +403,7 @@ async fn handle_new_activations<Context: SubsystemContext>(
 				metrics.on_collation_generated();
 
 				if let Err(err) = task_sender.send(AllMessages::CollatorProtocol(
-					CollatorProtocolMessage::DistributeCollation(ccr, collation.proof_of_validity, result_sender)
+					CollatorProtocolMessage::DistributeCollation(ccr, pov, result_sender)
 				)).await {
 					tracing::warn!(
 						target: LOG_TARGET,
@@ -492,7 +520,7 @@ mod tests {
 			task::{Context as FuturesContext, Poll},
 			Future,
 		};
-		use polkadot_node_primitives::{Collation, CollationResult, BlockData, PoV};
+		use polkadot_node_primitives::{Collation, CollationResult, BlockData, PoV, POV_BOMB_LIMIT};
 		use polkadot_node_subsystem::messages::{
 			AllMessages, RuntimeApiMessage, RuntimeApiRequest,
 		};
@@ -500,8 +528,7 @@ mod tests {
 			subsystem_test_harness, TestSubsystemContextHandle,
 		};
 		use polkadot_primitives::v1::{
-			BlockNumber, CollatorPair, Id as ParaId,
-			PersistedValidationData, ScheduledCore, ValidationCode,
+			CollatorPair, Id as ParaId, PersistedValidationData, ScheduledCore, ValidationCode,
 		};
 		use std::pin::Pin;
 
@@ -519,6 +546,24 @@ mod tests {
 			}
 		}
 
+		fn test_collation_compressed() -> Collation {
+			let mut collation = test_collation();
+			let compressed = PoV {
+				block_data: BlockData(sp_maybe_compressed_blob::compress(
+					&collation.proof_of_validity.block_data.0,
+					POV_BOMB_LIMIT,
+				).unwrap())
+			};
+			collation.proof_of_validity = compressed;
+			collation
+		}
+
+		fn test_validation_data() -> PersistedValidationData {
+			let mut persisted_validation_data: PersistedValidationData = Default::default();
+			persisted_validation_data.max_pov_size = 1024;
+			persisted_validation_data
+		}
+
 		// Box<dyn Future<Output = Collation> + Unpin + Send
 		struct TestCollator;
 
@@ -715,7 +760,7 @@ mod tests {
 								tx,
 							),
 						))) => {
-							tx.send(Ok(Some(Default::default()))).unwrap();
+							tx.send(Ok(Some(test_validation_data()))).unwrap();
 						}
 						Some(AllMessages::RuntimeApi(RuntimeApiMessage::Request(
 							_hash,
@@ -766,9 +811,8 @@ mod tests {
 			// we expect a single message to be sent, containing a candidate receipt.
 			// we don't care too much about the commitments_hash right now, but let's ensure that we've calculated the
 			// correct descriptor
-			let expect_pov_hash = test_collation().proof_of_validity.hash();
-			let expect_validation_data_hash
-				= PersistedValidationData::<Hash, BlockNumber>::default().hash();
+			let expect_pov_hash = test_collation_compressed().proof_of_validity.hash();
+			let expect_validation_data_hash = test_validation_data().hash();
 			let expect_relay_parent = Hash::repeat_byte(4);
 			let expect_validation_code_hash = ValidationCode(vec![1, 2, 3]).hash();
 			let expect_payload = collator_signature_payload(

polkadot/node/core/candidate-validation/Cargo.toml

@@ -9,6 +9,7 @@ futures = "0.3.12"
 tracing = "0.1.25"
 
 sp-core = { package = "sp-core", git = "https://github.com/paritytech/substrate", branch = "master" }
+sp-maybe-compressed-blob = { package = "sp-maybe-compressed-blob", git = "https://github.com/paritytech/substrate", branch = "master" }
 parity-scale-codec = { version = "2.0.0", default-features = false, features = ["bit-vec", "derive"] }
 
 polkadot-primitives = { path = "../../../primitives" }

polkadot/node/core/candidate-validation/src/lib.rs

@@ -33,7 +33,9 @@ use polkadot_subsystem::{
 };
 use polkadot_node_subsystem_util::metrics::{self, prometheus};
 use polkadot_subsystem::errors::RuntimeApiError;
-use polkadot_node_primitives::{ValidationResult, InvalidCandidate, PoV};
+use polkadot_node_primitives::{
+	VALIDATION_CODE_BOMB_LIMIT, POV_BOMB_LIMIT, ValidationResult, InvalidCandidate, PoV, BlockData,
+};
 use polkadot_primitives::v1::{
 	ValidationCode, CandidateDescriptor, PersistedValidationData,
 	OccupiedCoreAssumption, Hash, CandidateCommitments,
@@ -368,12 +370,12 @@ fn perform_basic_checks(
 	pov: &PoV,
 	validation_code: &ValidationCode,
 ) -> Result<(), InvalidCandidate> {
-	let encoded_pov = pov.encode();
 	let pov_hash = pov.hash();
 	let validation_code_hash = validation_code.hash();
 
-	if encoded_pov.len() > max_pov_size as usize {
-		return Err(InvalidCandidate::ParamsTooLarge(encoded_pov.len() as u64));
+	let encoded_pov_size = pov.encoded_size();
+	if encoded_pov_size > max_pov_size as usize {
+		return Err(InvalidCandidate::ParamsTooLarge(encoded_pov_size as u64));
 	}
 
 	if pov_hash != candidate.pov_hash {
@@ -396,7 +398,7 @@ trait ValidationBackend {
 
 	fn validate<S: SpawnNamed + 'static>(
 		arg: Self::Arg,
-		validation_code: &ValidationCode,
+		raw_validation_code: &[u8],
 		params: ValidationParams,
 		spawn: S,
 	) -> Result<WasmValidationResult, ValidationError>;
@@ -409,12 +411,12 @@ impl ValidationBackend for RealValidationBackend {
 
 	fn validate<S: SpawnNamed + 'static>(
 		isolation_strategy: IsolationStrategy,
-		validation_code: &ValidationCode,
+		raw_validation_code: &[u8],
 		params: ValidationParams,
 		spawn: S,
 	) -> Result<WasmValidationResult, ValidationError> {
 		wasm_executor::validate_candidate(
-			&validation_code.0,
+			&raw_validation_code,
 			params,
 			&isolation_strategy,
 			spawn,
@@ -446,14 +448,40 @@ fn validate_candidate_exhaustive<B: ValidationBackend, S: SpawnNamed + 'static>(
 		return Ok(ValidationResult::Invalid(e))
 	}
 
+	let raw_validation_code = match sp_maybe_compressed_blob::decompress(
+		&validation_code.0,
+		VALIDATION_CODE_BOMB_LIMIT,
+	) {
+		Ok(code) => code,
+		Err(e) => {
+			tracing::debug!(target: LOG_TARGET, err=?e, "Invalid validation code");
+
+			// If the validation code is invalid, the candidate certainly is.
+			return Ok(ValidationResult::Invalid(InvalidCandidate::CodeDecompressionFailure));
+		}
+	};
+
+	let raw_block_data = match sp_maybe_compressed_blob::decompress(
+		&pov.block_data.0,
+		POV_BOMB_LIMIT,
+	) {
+		Ok(block_data) => BlockData(block_data.to_vec()),
+		Err(e) => {
+			tracing::debug!(target: LOG_TARGET, err=?e, "Invalid PoV code");
+
+			// If the PoV is invalid, the candidate certainly is.
+			return Ok(ValidationResult::Invalid(InvalidCandidate::PoVDecompressionFailure));
+		}
+	};
+
 	let params = ValidationParams {
 		parent_head: persisted_validation_data.parent_head.clone(),
-		block_data: pov.block_data.clone(),
+		block_data: raw_block_data,
 		relay_parent_number: persisted_validation_data.relay_parent_number,
 		relay_parent_storage_root: persisted_validation_data.relay_parent_storage_root,
 	};
 
-	match B::validate(backend_arg, &validation_code, params, spawn) {
+	match B::validate(backend_arg, &raw_validation_code, params, spawn) {
 		Err(ValidationError::InvalidCandidate(WasmInvalidCandidate::Timeout)) =>
 			Ok(ValidationResult::Invalid(InvalidCandidate::Timeout)),
 		Err(ValidationError::InvalidCandidate(WasmInvalidCandidate::ParamsTooLarge(l, _))) =>
@@ -580,7 +608,6 @@ mod tests {
 	use super::*;
 	use polkadot_node_subsystem_test_helpers as test_helpers;
 	use polkadot_primitives::v1::{HeadData, UpwardMessage};
-	use polkadot_node_primitives::BlockData;
 	use sp_core::testing::TaskExecutor;
 	use futures::executor;
 	use assert_matches::assert_matches;
@@ -597,7 +624,7 @@ mod tests {
 
 		fn validate<S: SpawnNamed + 'static>(
 			arg: Self::Arg,
-			_validation_code: &ValidationCode,
+			_raw_validation_code: &[u8],
 			_params: ValidationParams,
 			_spawn: S,
 		) -> Result<WasmValidationResult, ValidationError> {
@@ -1059,4 +1086,139 @@ mod tests {
 		assert_matches!(v, ValidationResult::Invalid(InvalidCandidate::CodeHashMismatch));
 	}
 
+	#[test]
+	fn compressed_code_works() {
+		let validation_data = PersistedValidationData { max_pov_size: 1024, ..Default::default() };
+		let pov = PoV { block_data: BlockData(vec![1; 32]) };
+		let head_data = HeadData(vec![1, 1, 1]);
+
+		let raw_code = vec![2u8; 16];
+		let validation_code = sp_maybe_compressed_blob::compress(
+			&raw_code,
+			VALIDATION_CODE_BOMB_LIMIT,
+		)
+			.map(ValidationCode)
+			.unwrap();
+
+		let mut descriptor = CandidateDescriptor::default();
+		descriptor.pov_hash = pov.hash();
+		descriptor.para_head = head_data.hash();
+		descriptor.validation_code_hash = validation_code.hash();
+		collator_sign(&mut descriptor, Sr25519Keyring::Alice);
+
+		let validation_result = WasmValidationResult {
+			head_data,
+			new_validation_code: None,
+			upward_messages: Vec::new(),
+			horizontal_messages: Vec::new(),
+			processed_downward_messages: 0,
+			hrmp_watermark: 0,
+		};
+
+		let v = validate_candidate_exhaustive::<MockValidationBackend, _>(
+			MockValidationArg { result: Ok(validation_result) },
+			validation_data,
+			validation_code,
+			descriptor,
+			Arc::new(pov),
+			TaskExecutor::new(),
+			&Default::default(),
+		);
+
+		assert_matches!(v, Ok(ValidationResult::Valid(_, _)));
+	}
+
+	#[test]
+	fn code_decompression_failure_is_invalid() {
+		let validation_data = PersistedValidationData { max_pov_size: 1024, ..Default::default() };
+		let pov = PoV { block_data: BlockData(vec![1; 32]) };
+		let head_data = HeadData(vec![1, 1, 1]);
+
+		let raw_code = vec![2u8; VALIDATION_CODE_BOMB_LIMIT + 1];
+		let validation_code = sp_maybe_compressed_blob::compress(
+			&raw_code,
+			VALIDATION_CODE_BOMB_LIMIT + 1,
+		)
+			.map(ValidationCode)
+			.unwrap();
+
+		let mut descriptor = CandidateDescriptor::default();
+		descriptor.pov_hash = pov.hash();
+		descriptor.para_head = head_data.hash();
+		descriptor.validation_code_hash = validation_code.hash();
+		collator_sign(&mut descriptor, Sr25519Keyring::Alice);
+
+		let validation_result = WasmValidationResult {
+			head_data,
+			new_validation_code: None,
+			upward_messages: Vec::new(),
+			horizontal_messages: Vec::new(),
+			processed_downward_messages: 0,
+			hrmp_watermark: 0,
+		};
+
+		let v = validate_candidate_exhaustive::<MockValidationBackend, _>(
+			MockValidationArg { result: Ok(validation_result) },
+			validation_data,
+			validation_code,
+			descriptor,
+			Arc::new(pov),
+			TaskExecutor::new(),
+			&Default::default(),
+		);
+
+		assert_matches!(
+			v,
+			Ok(ValidationResult::Invalid(InvalidCandidate::CodeDecompressionFailure))
+		);
+	}
+
+	#[test]
+	fn pov_decompression_failure_is_invalid() {
+		let validation_data = PersistedValidationData {
+			max_pov_size: POV_BOMB_LIMIT as u32,
+			..Default::default()
+		 };
+		let head_data = HeadData(vec![1, 1, 1]);
+
+		let raw_block_data = vec![2u8; POV_BOMB_LIMIT + 1];
+		let pov = sp_maybe_compressed_blob::compress(
+			&raw_block_data,
+			POV_BOMB_LIMIT + 1,
+		)
+			.map(|raw| PoV { block_data: BlockData(raw) })
+			.unwrap();
+
+		let validation_code = ValidationCode(vec![2; 16]);
+
+		let mut descriptor = CandidateDescriptor::default();
+		descriptor.pov_hash = pov.hash();
+		descriptor.para_head = head_data.hash();
+		descriptor.validation_code_hash = validation_code.hash();
+		collator_sign(&mut descriptor, Sr25519Keyring::Alice);
+
+		let validation_result = WasmValidationResult {
+			head_data,
+			new_validation_code: None,
+			upward_messages: Vec::new(),
+			horizontal_messages: Vec::new(),
+			processed_downward_messages: 0,
+			hrmp_watermark: 0,
+		};
+
+		let v = validate_candidate_exhaustive::<MockValidationBackend, _>(
+			MockValidationArg { result: Ok(validation_result) },
+			validation_data,
+			validation_code,
+			descriptor,
+			Arc::new(pov),
+			TaskExecutor::new(),
+			&Default::default(),
+		);
+
+		assert_matches!(
+			v,
+			Ok(ValidationResult::Invalid(InvalidCandidate::PoVDecompressionFailure))
+		);
+	}
 }

polkadot/node/network/availability-distribution/src/error.rs

@@ -24,7 +24,6 @@ use futures::channel::oneshot;
 
 use polkadot_node_subsystem_util::Error as UtilError;
 use polkadot_primitives::v1::SessionIndex;
-use polkadot_node_primitives::CompressedPoVError;
 use polkadot_subsystem::{errors::RuntimeApiError, SubsystemError};
 
 use crate::LOG_TARGET;
@@ -79,10 +78,6 @@ pub enum Error {
 	#[error("There was no session with the given index")]
 	NoSuchSession(SessionIndex),
 
-	/// Decompressing PoV failed.
-	#[error("PoV could not be decompressed")]
-	PoVDecompression(CompressedPoVError),
-
 	/// Fetching PoV failed with `RequestError`.
 	#[error("FetchPoV request error")]
 	FetchPoV(#[source] RequestError),

polkadot/node/network/availability-distribution/src/pov_requester/mod.rs

@@ -152,9 +152,7 @@ async fn do_fetch_pov(
 {
 	let response = pending_response.await.map_err(Error::FetchPoV)?;
 	let pov = match response {
-		PoVFetchingResponse::PoV(compressed) => {
-			compressed.decompress().map_err(Error::PoVDecompression)?
-		}
+		PoVFetchingResponse::PoV(pov) => pov,
 		PoVFetchingResponse::NoSuchPoV => {
 			return Err(Error::NoSuchPoV)
 		}
@@ -244,7 +242,7 @@ mod tests {
 	use sp_core::testing::TaskExecutor;
 
 	use polkadot_primitives::v1::{CandidateHash, Hash, ValidatorIndex};
-	use polkadot_node_primitives::{BlockData, CompressedPoV};
+	use polkadot_node_primitives::BlockData;
 	use polkadot_subsystem_testhelpers as test_helpers;
 	use polkadot_subsystem::messages::{AvailabilityDistributionMessage, RuntimeApiMessage, RuntimeApiRequest};
 
@@ -315,9 +313,8 @@ mod tests {
 							reqs.pop(),
 							Some(Requests::PoVFetching(outgoing)) => {outgoing}
 						);
-						req.pending_response.send(Ok(PoVFetchingResponse::PoV(
-							CompressedPoV::compress(&pov).unwrap()).encode()
-						)).unwrap();
+						req.pending_response.send(Ok(PoVFetchingResponse::PoV(pov.clone()).encode()))
+							.unwrap();
 						break
 					},
 					msg => tracing::debug!(target: LOG_TARGET, msg = ?msg, "Received msg"),

polkadot/node/network/availability-distribution/src/responder.rs

@@ -16,11 +16,13 @@
 
 //! Answer requests for availability chunks.
 
+use std::sync::Arc;
+
 use futures::channel::oneshot;
 
 use polkadot_node_network_protocol::request_response::{request::IncomingRequest, v1};
 use polkadot_primitives::v1::{CandidateHash, ValidatorIndex};
-use polkadot_node_primitives::{AvailableData, CompressedPoV, ErasureChunk};
+use polkadot_node_primitives::{AvailableData, ErasureChunk};
 use polkadot_subsystem::{
 	messages::{AllMessages, AvailabilityStoreMessage},
 	SubsystemContext, jaeger,
@@ -100,18 +102,7 @@ where
 	let response = match av_data {
 		None => v1::PoVFetchingResponse::NoSuchPoV,
 		Some(av_data) => {
-			let pov = match CompressedPoV::compress(&av_data.pov) {
-				Ok(pov) => pov,
-				Err(error) => {
-					tracing::error!(
-						target: LOG_TARGET,
-						error = ?error,
-						"Failed to create `CompressedPov`",
-					);
-					// this should really not happen, let this request time out:
-					return Err(Error::PoVDecompression(error))
-				}
-			};
+			let pov = Arc::try_unwrap(av_data.pov).unwrap_or_else(|a| (&*a).clone());
 			v1::PoVFetchingResponse::PoV(pov)
 		}
 	};

polkadot/node/network/collator-protocol/src/collator_side.rs

@@ -42,7 +42,7 @@ use polkadot_node_subsystem_util::{
 	request_availability_cores,
 	metrics::{self, prometheus},
 };
-use polkadot_node_primitives::{SignedFullStatement, Statement, PoV, CompressedPoV};
+use polkadot_node_primitives::{SignedFullStatement, Statement, PoV};
 
 const COST_UNEXPECTED_MESSAGE: Rep = Rep::CostMinor("An unexpected message");
 
@@ -660,27 +660,6 @@ async fn send_collation(
 	receipt: CandidateReceipt,
 	pov: PoV,
 ) {
-	let pov = match CompressedPoV::compress(&pov) {
-		Ok(compressed) => {
-			tracing::trace!(
-				target: LOG_TARGET,
-				size = %pov.block_data.0.len(),
-				compressed = %compressed.len(),
-				peer_id = ?request.peer,
-				"Sending collation."
-			);
-			compressed
-		},
-		Err(error) => {
-			tracing::error!(
-				target: LOG_TARGET,
-				?error,
-				"Failed to create `CompressedPov`",
-			);
-			return
-		}
-	};
-
 	if let Err(_) = request.send_response(CollationFetchingResponse::Collation(receipt, pov)) {
 		tracing::warn!(
 			target: LOG_TARGET,
@@ -1519,7 +1498,7 @@ mod tests {
 					)
 					.expect("Decoding should work");
 					assert_eq!(receipt, candidate);
-					assert_eq!(pov.decompress().unwrap(), pov_block);
+					assert_eq!(pov, pov_block);
 				}
 			);
 

polkadot/node/network/collator-protocol/src/validator_side.rs

@@ -1158,49 +1158,33 @@ where
 
 				modify_reputation(ctx, *peer_id, COST_WRONG_PARA).await;
 			}
-			Ok(CollationFetchingResponse::Collation(receipt, compressed_pov)) => {
-				match compressed_pov.decompress() {
-					Ok(pov) => {
-						tracing::debug!(
-							target: LOG_TARGET,
-							para_id = %para_id,
-							hash = ?hash,
-							candidate_hash = ?receipt.hash(),
-							"Received collation",
-						);
+			Ok(CollationFetchingResponse::Collation(receipt, pov)) => {
+				tracing::debug!(
+					target: LOG_TARGET,
+					para_id = %para_id,
+					hash = ?hash,
+					candidate_hash = ?receipt.hash(),
+					"Received collation",
+				);
 
-						// Actual sending:
-						let _span = jaeger::Span::new(&pov, "received-collation");
-						let (mut tx, _) = oneshot::channel();
-						std::mem::swap(&mut tx, &mut (per_req.to_requester));
-						let result = tx.send((receipt, pov));
+				// Actual sending:
+				let _span = jaeger::Span::new(&pov, "received-collation");
+				let (mut tx, _) = oneshot::channel();
+				std::mem::swap(&mut tx, &mut (per_req.to_requester));
+				let result = tx.send((receipt, pov));
 
-						if let Err(_) = result  {
-							tracing::warn!(
-								target: LOG_TARGET,
-								hash = ?hash,
-								para_id = ?para_id,
-								peer_id = ?peer_id,
-								"Sending response back to requester failed (receiving side closed)"
-							);
-						} else {
-							metrics_result = Ok(());
-							success = "true";
-						}
-
-					}
-					Err(error) => {
-						tracing::warn!(
-							target: LOG_TARGET,
-							hash = ?hash,
-							para_id = ?para_id,
-							peer_id = ?peer_id,
-							?error,
-							"Failed to extract PoV",
-						);
-						modify_reputation(ctx, *peer_id, COST_CORRUPTED_MESSAGE).await;
-					}
-				};
+				if let Err(_) = result  {
+					tracing::warn!(
+						target: LOG_TARGET,
+						hash = ?hash,
+						para_id = ?para_id,
+						peer_id = ?peer_id,
+						"Sending response back to requester failed (receiving side closed)"
+					);
+				} else {
+					metrics_result = Ok(());
+					success = "true";
+				}
 			}
 		};
 		metrics.on_request(metrics_result);
@@ -1227,7 +1211,7 @@ mod tests {
 		CollatorPair, ValidatorId, ValidatorIndex, CoreState, CandidateDescriptor,
 		GroupRotationInfo, ScheduledCore, OccupiedCore, GroupIndex,
 	};
-	use polkadot_node_primitives::{BlockData, CompressedPoV};
+	use polkadot_node_primitives::BlockData;
 	use polkadot_node_subsystem_util::TimeoutExt;
 	use polkadot_subsystem_testhelpers as test_helpers;
 	use polkadot_subsystem::messages::{RuntimeApiMessage, RuntimeApiRequest};
@@ -1859,9 +1843,9 @@ mod tests {
 			response_channel.send(Ok(
 				CollationFetchingResponse::Collation(
 					candidate_a.clone(),
-					CompressedPoV::compress(&PoV {
+					PoV {
 						block_data: BlockData(vec![]),
-					}).unwrap(),
+					},
 				).encode()
 			)).expect("Sending response should succeed");
 
@@ -1889,9 +1873,9 @@ mod tests {
 			response_channel.send(Ok(
 				CollationFetchingResponse::Collation(
 					candidate_b.clone(),
-					CompressedPoV::compress(&PoV {
+					PoV {
 						block_data: BlockData(vec![1, 2, 3]),
-					}).unwrap(),
+					},
 				).encode()
 			)).expect("Sending response should succeed");
 

polkadot/node/network/protocol/src/request_response/mod.rs

@@ -36,7 +36,7 @@ use std::borrow::Cow;
 use std::time::Duration;
 
 use futures::channel::mpsc;
-use polkadot_node_primitives::MAX_COMPRESSED_POV_SIZE;
+use polkadot_node_primitives::MAX_POV_SIZE;
 use strum::EnumIter;
 
 pub use sc_network::config as network;
@@ -84,7 +84,7 @@ const MIN_BANDWIDTH_BYTES: u64  = 50 * 1024 * 1024;
 /// Timeout for PoV like data, 2 times what it should take, assuming we can fully utilize the
 /// bandwidth. This amounts to two seconds right now.
 const POV_REQUEST_TIMEOUT_CONNECTED: Duration =
-	Duration::from_millis(2 * 1000 * (MAX_COMPRESSED_POV_SIZE as u64)  / MIN_BANDWIDTH_BYTES);
+	Duration::from_millis(2 * 1000 * (MAX_POV_SIZE as u64)  / MIN_BANDWIDTH_BYTES);
 
 impl Protocol {
 	/// Get a configuration for a given Request response protocol.
@@ -114,7 +114,7 @@ impl Protocol {
 			Protocol::CollationFetching => RequestResponseConfig {
 				name: p_name,
 				max_request_size: 10_000,
-				max_response_size: MAX_COMPRESSED_POV_SIZE as u64,
+				max_response_size: MAX_POV_SIZE as u64,
 				// Taken from initial implementation in collator protocol:
 				request_timeout: POV_REQUEST_TIMEOUT_CONNECTED,
 				inbound_queue: Some(tx),
@@ -122,7 +122,7 @@ impl Protocol {
 			Protocol::PoVFetching => RequestResponseConfig {
 				name: p_name,
 				max_request_size: 1_000,
-				max_response_size: MAX_COMPRESSED_POV_SIZE as u64,
+				max_response_size: MAX_POV_SIZE as u64,
 				request_timeout: POV_REQUEST_TIMEOUT_CONNECTED,
 				inbound_queue: Some(tx),
 			},
@@ -130,7 +130,7 @@ impl Protocol {
 				name: p_name,
 				max_request_size: 1_000,
 				// Available data size is dominated by the PoV size.
-				max_response_size: MAX_COMPRESSED_POV_SIZE as u64,
+				max_response_size: MAX_POV_SIZE as u64,
 				request_timeout: POV_REQUEST_TIMEOUT_CONNECTED,
 				inbound_queue: Some(tx),
 			},

polkadot/node/network/protocol/src/request_response/v1.rs

@@ -23,7 +23,7 @@ use polkadot_primitives::v1::{
 	Hash,
 };
 use polkadot_primitives::v1::Id as ParaId;
-use polkadot_node_primitives::{AvailableData, CompressedPoV, ErasureChunk};
+use polkadot_node_primitives::{AvailableData, PoV, ErasureChunk};
 
 use super::request::IsRequest;
 use super::Protocol;
@@ -107,7 +107,7 @@ pub struct CollationFetchingRequest {
 pub enum CollationFetchingResponse {
 	/// Deliver requested collation.
 	#[codec(index = 0)]
-	Collation(CandidateReceipt, CompressedPoV),
+	Collation(CandidateReceipt, PoV),
 }
 
 impl IsRequest for CollationFetchingRequest {
@@ -127,7 +127,7 @@ pub struct PoVFetchingRequest {
 pub enum PoVFetchingResponse {
 	/// Deliver requested PoV.
 	#[codec(index = 0)]
-	PoV(CompressedPoV),
+	PoV(PoV),
 	/// PoV was not found in store.
 	#[codec(index = 1)]
 	NoSuchPoV,

polkadot/node/primitives/Cargo.toml

@@ -15,6 +15,7 @@ sp-core = { git = "https://github.com/paritytech/substrate", branch = "master" }
 sp-application-crypto = { git = "https://github.com/paritytech/substrate", branch = "master" }
 sp-consensus-vrf = { git = "https://github.com/paritytech/substrate", branch = "master" }
 sp-consensus-babe = { git = "https://github.com/paritytech/substrate", branch = "master" }
+sp-maybe-compressed-blob  = { git = "https://github.com/paritytech/substrate", branch = "master" }
 polkadot-parachain = { path = "../../parachain", default-features = false }
 schnorrkel = "0.9.1"
 thiserror = "1.0.22"

polkadot/node/primitives/src/lib.rs

@@ -36,9 +36,17 @@ pub use sp_consensus_babe::{
 use polkadot_primitives::v1::{CandidateCommitments, CandidateHash, CollatorPair, CommittedCandidateReceipt, CompactStatement, EncodeAs, Hash, HeadData, Id as ParaId, OutboundHrmpMessage, PersistedValidationData, Signed, UpwardMessage, ValidationCode, BlakeTwo256, HashT, ValidatorIndex};
 pub use polkadot_parachain::primitives::BlockData;
 
-
 pub mod approval;
 
+/// The bomb limit for decompressing code blobs.
+pub const VALIDATION_CODE_BOMB_LIMIT: usize = 16 * 1024 * 1024;
+
+/// Maximum PoV size we support right now.
+pub const MAX_POV_SIZE: u32 = 50 * 1024 * 1024;
+
+/// The bomb limit for decompressing PoV blobs.
+pub const POV_BOMB_LIMIT: usize = MAX_POV_SIZE as usize;
+
 /// A statement, where the candidate receipt is included in the `Seconded` variant.
 ///
 /// This is the committed candidate receipt instead of the bare candidate receipt. As such,
@@ -119,6 +127,10 @@ pub enum InvalidCandidate {
 	ParamsTooLarge(u64),
 	/// Code size is over the limit.
 	CodeTooLarge(u64),
+	/// Code does not decompress correctly.
+	CodeDecompressionFailure,
+	/// PoV does not decompress correctly.
+	PoVDecompressionFailure,
 	/// Validation function returned invalid data.
 	BadReturn,
 	/// Invalid relay chain parent.
@@ -143,17 +155,6 @@ pub enum ValidationResult {
 	Invalid(InvalidCandidate),
 }
 
-/// Maximum PoV size we support right now.
-pub const MAX_POV_SIZE: u32 = 50 * 1024 * 1024;
-
-/// Very conservative (compression ratio of 1).
-///
-/// Experiments showed that we have a typical compression ratio of 3.4.
-/// https://github.com/ordian/bench-compression-algorithms/
-///
-/// So this could be reduced if deemed necessary.
-pub const MAX_COMPRESSED_POV_SIZE: u32 = MAX_POV_SIZE;
-
 /// A Proof-of-Validity
 #[derive(PartialEq, Eq, Clone, Encode, Decode, Debug)]
 pub struct PoV {
@@ -168,77 +169,6 @@ impl PoV {
 	}
 }
 
-/// SCALE and Zstd encoded [`PoV`].
-#[derive(Clone, Encode, Decode, PartialEq, Eq)]
-pub struct CompressedPoV(Vec<u8>);
-
-#[derive(Debug, Clone, Copy, PartialEq, Eq, thiserror::Error)]
-#[allow(missing_docs)]
-pub enum CompressedPoVError {
-	#[error("Failed to compress a PoV")]
-	Compress,
-	#[error("Failed to decompress a PoV")]
-	Decompress,
-	#[error("Failed to decode the uncompressed PoV")]
-	Decode,
-	#[error("Architecture is not supported")]
-	NotSupported,
-}
-
-impl CompressedPoV {
-	/// Compress the given [`PoV`] and returns a [`CompressedPoV`].
-	#[cfg(not(target_os = "unknown"))]
-	pub fn compress(pov: &PoV) -> Result<Self, CompressedPoVError> {
-		zstd::encode_all(pov.encode().as_slice(), 3).map_err(|_| CompressedPoVError::Compress).map(Self)
-	}
-
-	/// Compress the given [`PoV`] and returns a [`CompressedPoV`].
-	#[cfg(target_os = "unknown")]
-	pub fn compress(_: &PoV) -> Result<Self, CompressedPoVError> {
-		Err(CompressedPoVError::NotSupported)
-	}
-
-	/// Decompress `self` and returns the [`PoV`] on success.
-	#[cfg(not(target_os = "unknown"))]
-	pub fn decompress(&self) -> Result<PoV, CompressedPoVError> {
-		use std::io::Read;
-
-		struct InputDecoder<'a, 'b: 'a, T: std::io::BufRead>(&'a mut zstd::Decoder<'b, T>, usize);
-		impl<'a, 'b, T: std::io::BufRead> parity_scale_codec::Input for InputDecoder<'a, 'b, T> {
-			fn read(&mut self, into: &mut [u8]) -> Result<(), parity_scale_codec::Error> {
-				self.1 = self.1.saturating_add(into.len());
-				if self.1 > MAX_POV_SIZE as usize {
-					return Err("pov block too big".into())
-				}
-				self.0.read_exact(into).map_err(Into::into)
-			}
-			fn remaining_len(&mut self) -> Result<Option<usize>, parity_scale_codec::Error> {
-				Ok(None)
-			}
-		}
-
-		let mut decoder = zstd::Decoder::new(self.0.as_slice()).map_err(|_| CompressedPoVError::Decompress)?;
-		PoV::decode(&mut InputDecoder(&mut decoder, 0)).map_err(|_| CompressedPoVError::Decode)
-	}
-
-	/// Decompress `self` and returns the [`PoV`] on success.
-	#[cfg(target_os = "unknown")]
-	pub fn decompress(&self) -> Result<PoV, CompressedPoVError> {
-		Err(CompressedPoVError::NotSupported)
-	}
-
-	/// Get compressed data size.
-	pub fn len(&self) -> usize {
-		self.0.len()
-	}
-}
-
-impl std::fmt::Debug for CompressedPoV {
-	fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
-		write!(f, "CompressedPoV({} bytes)", self.0.len())
-	}
-}
-
 /// The output of a collator.
 ///
 /// This differs from `CandidateCommitments` in two ways:
@@ -330,15 +260,13 @@ pub struct ErasureChunk {
 	pub proof: Vec<Vec<u8>>,
 }
 
-#[cfg(test)]
-mod test {
-	use super::{CompressedPoV, CompressedPoVError, PoV};
+/// Compress a PoV, unless it exceeds the [`POV_BOMB_LIMIT`].
+#[cfg(not(target_os = "unknown"))]
+pub fn maybe_compress_pov(pov: PoV) -> PoV {
+	let PoV { block_data: BlockData(raw) } = pov;
+	let raw = sp_maybe_compressed_blob::compress(&raw, POV_BOMB_LIMIT)
+		.unwrap_or(raw);
 
-	#[test]
-	fn decompress_huge_pov_block_fails() {
-		let pov = PoV { block_data: vec![0; 63 * 1024 * 1024].into() };
-
-		let compressed = CompressedPoV::compress(&pov).unwrap();
-		assert_eq!(CompressedPoVError::Decode, compressed.decompress().unwrap_err());
-	}
+	let pov = PoV { block_data: BlockData(raw) };
+	pov
 }

polkadot/parachain/test-parachains/adder/collator/src/lib.rs

@@ -176,13 +176,15 @@ impl Collator {
 				hrmp_watermark: validation_data.relay_parent_number,
 			};
 
+			let compressed_pov = polkadot_node_primitives::maybe_compress_pov(pov);
+
 			let (result_sender, recv) = oneshot::channel::<SignedFullStatement>();
 			let seconded_collations = seconded_collations.clone();
 			spawner.spawn("adder-collator-seconded", async move {
 				if let Ok(res) = recv.await {
 					if !matches!(
 						res.payload(),
-						Statement::Seconded(s) if s.descriptor.pov_hash == pov.hash(),
+						Statement::Seconded(s) if s.descriptor.pov_hash == compressed_pov.hash(),
 					) {
 						log::error!("Seconded statement should match our collation: {:?}", res.payload());
 						std::process::exit(-1);

polkadot/roadmap/implementers-guide/src/types/network.md

@@ -19,9 +19,6 @@ enum ObservedRole {
 	Full,
 	Light,
 }
-
-/// SCALE and zstd encoded `PoV`.
-struct CompressedPoV(Vec<u8>);
 ```
 
 ## V1 Network Subsystem Message Types
@@ -84,8 +81,8 @@ enum PoVDistributionV1Message {
 	/// specific relay-parent hash.
 	Awaiting(Hash, Vec<Hash>),
 	/// Notification of an awaited PoV, in a given relay-parent context.
-	/// (relay_parent, pov_hash, compressed_pov)
-	SendPoV(Hash, Hash, CompressedPoV),
+	/// (relay_parent, pov_hash, pov)
+	SendPoV(Hash, Hash, PoV),
 }
 ```