Compress the PoV block before sending it over the network (#2288)

* Compress the PoV block before sending it over the network This pr changes the way we send PoV blocks over the network. We now compress the PoV block before it is send over the network. This should reduce the size significant for PoVs which contain the runtime WASM for example. * Preallocate 1KB * Try something.. * Switch to zstd and some renamings * Make compression/decompression fail in browsers * Use some sane maximum value * Update roadmap/implementers-guide/src/types/network.md Co-authored-by: Andronik Ordian <write@reusable.software> * Fix and add test * add Co-authored-by: Andronik Ordian <write@reusable.software> Co-authored-by: Robert Habermeier <rphmeier@gmail.com>
2026-04-26 19:17:58 +00:00 · 2021-01-21 19:04:14 +01:00
parent 2c0982d75e
commit 1439558f14
9 changed files with 190 additions and 56 deletions
@@ -16,7 +16,7 @@

 //! Network protocol types for parachains.

-#![deny(unused_crate_dependencies, unused_results)]
+#![deny(unused_crate_dependencies)]
 #![warn(missing_docs)]

 use polkadot_primitives::v1::{Hash, BlockNumber};
@@ -286,8 +286,8 @@ pub mod v1 {
 	};
 	use polkadot_node_primitives::SignedFullStatement;
 	use parity_scale_codec::{Encode, Decode};
-	use std::convert::TryFrom;
 	use super::RequestId;
+	use std::convert::TryFrom;

 	/// Network messages used by the availability distribution subsystem
 	#[derive(Debug, Clone, Encode, Decode, PartialEq)]
@@ -323,9 +323,9 @@ pub mod v1 {
 		#[codec(index = "0")]
 		Awaiting(Hash, Vec<Hash>),
 		/// Notification of an awaited PoV, in a given relay-parent context.
-		/// (relay_parent, pov_hash, pov)
+		/// (relay_parent, pov_hash, compressed_pov)
 		#[codec(index = "1")]
-		SendPoV(Hash, Hash, PoV),
+		SendPoV(Hash, Hash, CompressedPoV),
 	}

 	/// Network messages used by the statement distribution subsystem.
@@ -336,6 +336,67 @@ pub mod v1 {
 		Statement(Hash, SignedFullStatement)
 	}

+	#[derive(Debug, Clone, Copy, PartialEq, thiserror::Error)]
+	#[allow(missing_docs)]
+	pub enum CompressedPoVError {
+		#[error("Failed to compress a PoV")]
+		Compress,
+		#[error("Failed to decompress a PoV")]
+		Decompress,
+		#[error("Failed to decode the uncompressed PoV")]
+		Decode,
+		#[error("Architecture is not supported")]
+		NotSupported,
+	}
+
+	/// SCALE and Zstd encoded [`PoV`].
+	#[derive(Debug, Clone, Encode, Decode, PartialEq)]
+	pub struct CompressedPoV(Vec<u8>);
+
+	impl CompressedPoV {
+		/// Compress the given [`PoV`] and returns a [`CompressedPoV`].
+		#[cfg(not(target_os = "unknown"))]
+		pub fn compress(pov: &PoV) -> Result<Self, CompressedPoVError> {
+			zstd::encode_all(pov.encode().as_slice(), 3).map_err(|_| CompressedPoVError::Compress).map(Self)
+		}
+
+		/// Compress the given [`PoV`] and returns a [`CompressedPoV`].
+		#[cfg(target_os = "unknown")]
+		pub fn compress(_: &PoV) -> Result<Self, CompressedPoVError> {
+			Err(CompressedPoVError::NotSupported)
+		}
+
+		/// Decompress `self` and returns the [`PoV`] on success.
+		#[cfg(not(target_os = "unknown"))]
+		pub fn decompress(&self) -> Result<PoV, CompressedPoVError> {
+			use std::io::Read;
+			const MAX_POV_BLOCK_SIZE: usize = 32 * 1024 * 1024;
+
+			struct InputDecoder<'a, T: std::io::BufRead>(&'a mut zstd::Decoder<T>, usize);
+			impl<'a, T: std::io::BufRead> parity_scale_codec::Input for InputDecoder<'a, T> {
+				fn read(&mut self, into: &mut [u8]) -> Result<(), parity_scale_codec::Error> {
+					self.1 = self.1.saturating_add(into.len());
+					if self.1 > MAX_POV_BLOCK_SIZE {
+						return Err("pov block too big".into())
+					}
+					self.0.read_exact(into).map_err(Into::into)
+				}
+				fn remaining_len(&mut self) -> Result<Option<usize>, parity_scale_codec::Error> {
+					Ok(None)
+				}
+			}
+
+			let mut decoder = zstd::Decoder::new(self.0.as_slice()).map_err(|_| CompressedPoVError::Decompress)?;
+			PoV::decode(&mut InputDecoder(&mut decoder, 0)).map_err(|_| CompressedPoVError::Decode)
+		}
+
+		/// Decompress `self` and returns the [`PoV`] on success.
+		#[cfg(target_os = "unknown")]
+		pub fn decompress(&self) -> Result<PoV, CompressedPoVError> {
+			Err(CompressedPoVError::NotSupported)
+		}
+	}
+
 	/// Network messages used by the collator protocol subsystem
 	#[derive(Debug, Clone, Encode, Decode, PartialEq)]
 	pub enum CollatorProtocolMessage {
@@ -351,7 +412,7 @@ pub mod v1 {
 		RequestCollation(RequestId, Hash, ParaId),
 		/// A requested collation.
 		#[codec(index = "3")]
-		Collation(RequestId, CandidateReceipt, PoV),
+		Collation(RequestId, CandidateReceipt, CompressedPoV),
 	}

 	/// All network messages on the validation peer-set.
@@ -389,3 +450,17 @@ pub mod v1 {

 	impl_try_from!(CollationProtocol, CollatorProtocol, CollatorProtocolMessage);
 }
+
+#[cfg(test)]
+mod tests {
+	use polkadot_primitives::v1::PoV;
+	use super::v1::{CompressedPoV, CompressedPoVError};
+
+	#[test]
+	fn decompress_huge_pov_block_fails() {
+		let pov = PoV { block_data: vec![0; 63 * 1024 * 1024].into() };
+
+		let compressed = CompressedPoV::compress(&pov).unwrap();
+		assert_eq!(CompressedPoVError::Decode, compressed.decompress().unwrap_err());
+	}
+}