pezkuwichain/pezkuwi-subxt
1
0
Fork 0
mirror of https://github.com/pezkuwichain/pezkuwi-subxt.git synced 2026-04-27 04:37:57 +00:00
Code Issues Packages Projects Releases Wiki Activity

Code, PoV compression and remove CompressedPoV struct (#2852)

Browse Source 
* use compressed blob in candidate-validation

* add some tests for compressed code blobs

* remove CompressedPoV and apply compression in collation-generation

* decompress BlockData before executing

* don't produce oversized collations

* add test for PoV decompression failure

* fix tests and clean up

* fix test

* address review and fix CI

* take this )
This commit is contained in:
Robert Habermeier
2021-04-08 22:09:36 +02:00
committed by GitHub
parent bb48c47fbf
commit 896ec8dbc3
16 changed files with 489 additions and 380 deletions
Download Patch File Download Diff File Expand all files Collapse all files
polkadot/node/primitives/src/lib.rs
+21 -93
View File
@@ -36,9 +36,17 @@ pub use sp_consensus_babe::{
use polkadot_primitives::v1::{CandidateCommitments, CandidateHash, CollatorPair, CommittedCandidateReceipt, CompactStatement, EncodeAs, Hash, HeadData, Id as ParaId, OutboundHrmpMessage, PersistedValidationData, Signed, UpwardMessage, ValidationCode, BlakeTwo256, HashT, ValidatorIndex};
pub use polkadot_parachain::primitives::BlockData;
pub mod approval;
/// The bomb limit for decompressing code blobs.
pub const VALIDATION_CODE_BOMB_LIMIT: usize = 16 * 1024 * 1024;
/// Maximum PoV size we support right now.
pub const MAX_POV_SIZE: u32 = 50 * 1024 * 1024;
/// The bomb limit for decompressing PoV blobs.
pub const POV_BOMB_LIMIT: usize = MAX_POV_SIZE as usize;
/// A statement, where the candidate receipt is included in the `Seconded` variant.
///
/// This is the committed candidate receipt instead of the bare candidate receipt. As such,
@@ -119,6 +127,10 @@ pub enum InvalidCandidate {
ParamsTooLarge(u64),
/// Code size is over the limit.
CodeTooLarge(u64),
/// Code does not decompress correctly.
CodeDecompressionFailure,
/// PoV does not decompress correctly.
PoVDecompressionFailure,
/// Validation function returned invalid data.
BadReturn,
/// Invalid relay chain parent.
@@ -143,17 +155,6 @@ pub enum ValidationResult {
Invalid(InvalidCandidate),
}
/// Maximum PoV size we support right now.
pub const MAX_POV_SIZE: u32 = 50 * 1024 * 1024;
/// Very conservative (compression ratio of 1).
///
/// Experiments showed that we have a typical compression ratio of 3.4.
/// https://github.com/ordian/bench-compression-algorithms/
///
/// So this could be reduced if deemed necessary.
pub const MAX_COMPRESSED_POV_SIZE: u32 = MAX_POV_SIZE;
/// A Proof-of-Validity
#[derive(PartialEq, Eq, Clone, Encode, Decode, Debug)]
pub struct PoV {
@@ -168,77 +169,6 @@ impl PoV {
}
}
/// SCALE and Zstd encoded [`PoV`].
#[derive(Clone, Encode, Decode, PartialEq, Eq)]
pub struct CompressedPoV(Vec<u8>);
#[derive(Debug, Clone, Copy, PartialEq, Eq, thiserror::Error)]
#[allow(missing_docs)]
pub enum CompressedPoVError {
#[error("Failed to compress a PoV")]
Compress,
#[error("Failed to decompress a PoV")]
Decompress,
#[error("Failed to decode the uncompressed PoV")]
Decode,
#[error("Architecture is not supported")]
NotSupported,
}
impl CompressedPoV {
/// Compress the given [`PoV`] and returns a [`CompressedPoV`].
#[cfg(not(target_os = "unknown"))]
pub fn compress(pov: &PoV) -> Result<Self, CompressedPoVError> {
zstd::encode_all(pov.encode().as_slice(), 3).map_err(|_| CompressedPoVError::Compress).map(Self)
}
/// Compress the given [`PoV`] and returns a [`CompressedPoV`].
#[cfg(target_os = "unknown")]
pub fn compress(_: &PoV) -> Result<Self, CompressedPoVError> {
Err(CompressedPoVError::NotSupported)
}
/// Decompress `self` and returns the [`PoV`] on success.
#[cfg(not(target_os = "unknown"))]
pub fn decompress(&self) -> Result<PoV, CompressedPoVError> {
use std::io::Read;
struct InputDecoder<'a, 'b: 'a, T: std::io::BufRead>(&'a mut zstd::Decoder<'b, T>, usize);
impl<'a, 'b, T: std::io::BufRead> parity_scale_codec::Input for InputDecoder<'a, 'b, T> {
fn read(&mut self, into: &mut [u8]) -> Result<(), parity_scale_codec::Error> {
self.1 = self.1.saturating_add(into.len());
if self.1 > MAX_POV_SIZE as usize {
return Err("pov block too big".into())
}
self.0.read_exact(into).map_err(Into::into)
}
fn remaining_len(&mut self) -> Result<Option<usize>, parity_scale_codec::Error> {
Ok(None)
}
}
let mut decoder = zstd::Decoder::new(self.0.as_slice()).map_err(|_| CompressedPoVError::Decompress)?;
PoV::decode(&mut InputDecoder(&mut decoder, 0)).map_err(|_| CompressedPoVError::Decode)
}
/// Decompress `self` and returns the [`PoV`] on success.
#[cfg(target_os = "unknown")]
pub fn decompress(&self) -> Result<PoV, CompressedPoVError> {
Err(CompressedPoVError::NotSupported)
}
/// Get compressed data size.
pub fn len(&self) -> usize {
self.0.len()
}
}
impl std::fmt::Debug for CompressedPoV {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
write!(f, "CompressedPoV({} bytes)", self.0.len())
}
}
/// The output of a collator.
///
/// This differs from `CandidateCommitments` in two ways:
@@ -330,15 +260,13 @@ pub struct ErasureChunk {
pub proof: Vec<Vec<u8>>,
}
#[cfg(test)]
mod test {
use super::{CompressedPoV, CompressedPoVError, PoV};
/// Compress a PoV, unless it exceeds the [`POV_BOMB_LIMIT`].
#[cfg(not(target_os = "unknown"))]
pub fn maybe_compress_pov(pov: PoV) -> PoV {
let PoV { block_data: BlockData(raw) } = pov;
let raw = sp_maybe_compressed_blob::compress(&raw, POV_BOMB_LIMIT)
.unwrap_or(raw);
#[test]
fn decompress_huge_pov_block_fails() {
let pov = PoV { block_data: vec![0; 63 * 1024 * 1024].into() };
let compressed = CompressedPoV::compress(&pov).unwrap();
assert_eq!(CompressedPoVError::Decode, compressed.decompress().unwrap_err());
}
let pov = PoV { block_data: BlockData(raw) };
pov
}