Optimize encoded-size of SignedCommitment (#10409)

* Include code from paritytech/grandpa-bridge-gadget#186 * Cargo fmt commitment.rs * Make changes suggested by @tomusdrw Co-authored-by: Tomasz Drwięga <tomasz@parity.io>
2026-10-24 20:55:03 +00:00 · 2021-12-03 21:48:11 +00:00
1 parent 2804b56d78
commit b2f1374487
@@ -15,7 +15,7 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.

-use codec::{Decode, Encode};
+use codec::{Decode, Encode, Error, Input};
 use sp_std::{cmp, prelude::*};

 use crate::{crypto::Signature, ValidatorSetId};
@@ -80,7 +80,7 @@ impl Payload {
 /// height [block_number](Commitment::block_number).
 /// GRANDPA validators collect signatures on commitments and a stream of such signed commitments
 /// (see [SignedCommitment]) forms the BEEFY protocol.
-#[derive(Clone, Debug, PartialEq, Eq, codec::Encode, codec::Decode)]
+#[derive(Clone, Debug, PartialEq, Eq, Encode, Decode)]
 pub struct Commitment<TBlockNumber> {
 	///  A collection of payloads to be signed, see [`Payload`] for details.
 	///
@@ -134,7 +134,11 @@ where
 }

 /// A commitment with matching GRANDPA validators' signatures.
-#[derive(Clone, Debug, PartialEq, Eq, codec::Encode, codec::Decode)]
+///
+/// Note that SCALE-encoding of the structure is optimized for size efficiency over the wire,
+/// please take a look at custom [`Encode`] and [`Decode`] implementations and
+/// [`CompactSignedCommitment`] struct.
+#[derive(Clone, Debug, PartialEq, Eq)]
 pub struct SignedCommitment<TBlockNumber> {
 	/// The commitment signatures are collected for.
 	pub commitment: Commitment<TBlockNumber>,
@@ -152,6 +156,126 @@ impl<TBlockNumber> SignedCommitment<TBlockNumber> {
 	}
 }

+/// Type to be used to denote placement of signatures
+type BitField = Vec<u8>;
+/// Compress 8 bit values into a single u8 Byte
+const CONTAINER_BIT_SIZE: usize = 8;
+
+/// Compressed representation of [`SignedCommitment`], used for encoding efficiency.
+#[derive(Clone, Debug, PartialEq, Eq, Encode, Decode)]
+struct CompactSignedCommitment<TCommitment> {
+	/// The commitment, unchanged compared to regular [`SignedCommitment`].
+	commitment: TCommitment,
+	/// A bitfield representing presence of a signature coming from a validator at some index.
+	///
+	/// The bit at index `0` is set to `1` in case we have a signature coming from a validator at
+	/// index `0` in in the original validator set. In case the [`SignedCommitment`] does not
+	/// contain that signature the `bit` will be set to `0`. Bits are packed into `Vec<u8>`
+	signatures_from: BitField,
+	/// Number of validators in the Validator Set and hence number of significant bits in the
+	/// [`signatures_from`] collection.
+	///
+	/// Note this might be smaller than the size of `signatures_compact` in case some signatures
+	/// are missing.
+	validator_set_len: u32,
+	/// A `Vec` containing all `Signature`s present in the original [`SignedCommitment`].
+	///
+	/// Note that in order to associate a `Signature` from this `Vec` with a validator, one needs
+	/// to look at the `signatures_from` bitfield, since some validators might have not produced a
+	/// signature.
+	signatures_compact: Vec<Signature>,
+}
+
+impl<'a, TBlockNumber> CompactSignedCommitment<&'a Commitment<TBlockNumber>> {
+	/// Packs a `SignedCommitment` into the compressed `CompactSignedCommitment` format for
+	/// efficient network transport.
+	fn pack(signed_commitment: &'a SignedCommitment<TBlockNumber>) -> Self {
+		let SignedCommitment { commitment, signatures } = signed_commitment;
+		let validator_set_len = signatures.len() as u32;
+		let mut signatures_from: BitField = vec![];
+		let mut signatures_compact: Vec<Signature> = vec![];
+
+		for signature in signatures {
+			match signature {
+				Some(value) => signatures_compact.push(value.clone()),
+				None => (),
+			}
+		}
+
+		let mut bits: Vec<u8> =
+			signatures.iter().map(|x| if x.is_some() { 1 } else { 0 }).collect();
+
+		// Resize with excess bits for placement purposes
+		let excess_bits_len =
+			CONTAINER_BIT_SIZE - (validator_set_len as usize % CONTAINER_BIT_SIZE);
+		bits.resize(bits.len() + excess_bits_len, 0);
+
+		let chunks = bits.chunks(CONTAINER_BIT_SIZE);
+		for chunk in chunks {
+			let mut iter = chunk.iter().copied();
+			let mut v = iter.next().unwrap() as u8;
+
+			for bit in iter {
+				v <<= 1;
+				v |= bit as u8;
+			}
+
+			signatures_from.push(v);
+		}
+
+		Self { commitment, signatures_from, validator_set_len, signatures_compact }
+	}
+
+	/// Unpacks a `CompactSignedCommitment` into the uncompressed `SignedCommitment` form.
+	fn unpack(
+		temporary_signatures: CompactSignedCommitment<Commitment<TBlockNumber>>,
+	) -> SignedCommitment<TBlockNumber> {
+		let CompactSignedCommitment {
+			commitment,
+			signatures_from,
+			validator_set_len,
+			signatures_compact,
+		} = temporary_signatures;
+		let mut bits: Vec<u8> = vec![];
+
+		for block in signatures_from {
+			for bit in 0..CONTAINER_BIT_SIZE {
+				bits.push((block >> (CONTAINER_BIT_SIZE - bit - 1)) & 1);
+			}
+		}
+
+		bits.truncate(validator_set_len as usize);
+
+		let mut next_signature = signatures_compact.into_iter();
+		let signatures: Vec<Option<Signature>> = bits
+			.iter()
+			.map(|&x| if x == 1 { next_signature.next() } else { None })
+			.collect();
+
+		SignedCommitment { commitment, signatures }
+	}
+}
+
+impl<TBlockNumber> Encode for SignedCommitment<TBlockNumber>
+where
+	TBlockNumber: Encode,
+{
+	fn using_encoded<R, F: FnOnce(&[u8]) -> R>(&self, f: F) -> R {
+		let temp = CompactSignedCommitment::pack(self);
+		temp.using_encoded(f)
+	}
+}
+
+impl<TBlockNumber> Decode for SignedCommitment<TBlockNumber>
+where
+	TBlockNumber: Decode,
+{
+	fn decode<I: Input>(input: &mut I) -> Result<Self, Error> {
+		let temp = CompactSignedCommitment::decode(input)?;
+		Ok(CompactSignedCommitment::unpack(temp))
+	}
+}
+
 /// A [SignedCommitment] with a version number. This variant will be appended
 /// to the block justifications for the block for which the signed commitment
 /// has been generated.
@@ -244,10 +368,10 @@ mod tests {
 			encoded,
 			hex_literal::hex!(
 				"046d68343048656c6c6f20576f726c6421050000000000000000000000000000000000000000000000
-				10000001558455ad81279df0795cc985580e4fb75d72d948d1107b2ac80a09abed4da8480c746cc321
-				f2319a5e99a830e314d10dd3cd68ce3dc0c33c86e99bcb7816f9ba01012d6e1f8105c337a86cdd9aaa
-				cdc496577f3db8c55ef9e6fd48f2c5c05a2274707491635d8ba3df64f324575b7b2a34487bca2324b6a
-				0046395a71681be3d0c2a00"
+				04300400000008558455ad81279df0795cc985580e4fb75d72d948d1107b2ac80a09abed4da8480c746
+				cc321f2319a5e99a830e314d10dd3cd68ce3dc0c33c86e99bcb7816f9ba012d6e1f8105c337a86cdd9a
+				aacdc496577f3db8c55ef9e6fd48f2c5c05a2274707491635d8ba3df64f324575b7b2a34487bca2324b
+				6a0046395a71681be3d0c2a00"
 			)
 		);
 	}
@@ -322,4 +446,78 @@ mod tests {

 		assert_eq!(decoded, Ok(versioned));
 	}
+
+	#[test]
+	fn large_signed_commitment_encode_decode() {
+		// given
+		let payload = Payload::new(known_payload_ids::MMR_ROOT_ID, "Hello World!".encode());
+		let commitment: TestCommitment =
+			Commitment { payload, block_number: 5, validator_set_id: 0 };
+
+		let sigs = mock_signatures();
+
+		let signatures: Vec<Option<_>> = (0..1024)
+			.into_iter()
+			.map(|x| if x < 340 { None } else { Some(sigs.0.clone()) })
+			.collect();
+		let signed = SignedCommitment { commitment, signatures };
+
+		// when
+		let encoded = codec::Encode::encode(&signed);
+		let decoded = TestSignedCommitment::decode(&mut &*encoded);
+
+		// then
+		assert_eq!(decoded, Ok(signed));
+		assert_eq!(
+			encoded,
+			hex_literal::hex!(
+				"046d68343048656c6c6f20576f726c6421050000000000000000000000000000000000000000000000
+				05020000000000000000000000000000000000000000000000000000000000000000000000000000000
+				000000fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff
+				fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff
+				fffffffffff0000040000b10a558455ad81279df0795cc985580e4fb75d72d948d1107b2ac80a09abed
+				4da8480c746cc321f2319a5e99a830e314d10dd3cd68ce3dc0c33c86e99bcb7816f9ba01558455ad812
+				79df0795cc985580e4fb75d72d948d1107b2ac80a09abed4da8480c746cc321f2319a5e99a830e314d1
+				0dd3cd68ce3dc0c33c86e99bcb7816f9ba01558455ad81279df0795cc985580e4fb75d72d948d1107b2
+				ac80a09abed4da8480c746cc321f2319a5e99a830e314d10dd3cd68ce3dc0c33c86e99bcb7816f9ba01
+				558455ad81279df0795cc985580e4fb75d72d948d1107b2ac80a09abed4da8480c746cc321f2319a5e9
+				9a830e314d10dd3cd68ce3dc0c33c86e99bcb7816f9ba01558455ad81279df0795cc985580e4fb75d72
+				d948d1107b2ac80a09abed4da8480c746cc321f2319a5e99a830e314d10dd3cd68ce3dc0c33c86e99bc
+				b7816f9ba01558455ad81279df0795cc985580e4fb75d72d948d1107b2ac80a09abed4da8480c746cc3
+				21f2319a5e99a830e314d10dd3cd68ce3dc0c33c86e99bcb7816f9ba01558455ad81279df0795cc9855
+				80e4fb75d72d948d1107b2ac80a09abed4da8480c746cc321f2319a5e99a830e314d10dd3cd68ce3dc0
+				c33c86e99bcb7816f9ba01558455ad81279df0795cc985580e4fb75d72d948d1107b2ac80a09abed4da
+				8480c746cc321f2319a5e99a830e314d10dd3cd68ce3dc0c33c86e99bcb7816f9ba01558455ad81279d
+				f0795cc985580e4fb75d72d948d1107b2ac80a09abed4da8480c746cc321f2319a5e99a830e314d10dd
+				3cd68ce3dc0c33c86e99bcb7816f9ba01558455ad81279df0795cc985580e4fb75d72d948d1107b2ac8
+				0a09abed4da8480c746cc321f2319a5e99a830e314d10dd3cd68ce3dc0c33c86e99bcb7816f9ba01558
+				455ad81279df0795cc985580e4fb75d72d948d1107b2ac80a09abed4da8480c746cc321f2319a5e99a8
+				30e314d10dd3cd68ce3dc0c33c86e99bcb7816f9ba01558455ad81279df0795cc985580e4fb75d72d94
+				8d1107b2ac80a09abed4da8480c746cc321f2319a5e99a830e314d10dd3cd68ce3dc0c33c86e99bcb78
+				16f9ba01558455ad81279df0795cc985580e4fb75d72d948d1107b2ac80a09abed4da8480c746cc321f
+				2319a5e99a830e314d10dd3cd68ce3dc0c33c86e99bcb7816f9ba01558455ad81279df0795cc985580e
+				4fb75d72d948d1107b2ac80a09abed4da8480c746cc321f2319a5e99a830e314d10dd3cd68ce3dc0c33
+				c86e99bcb7816f9ba01558455ad81279df0795cc985580e4fb75d72d948d1107b2ac80a09abed4da848
+				0c746cc321f2319a5e99a830e314d10dd3cd68ce3dc0c33c86e99bcb7816f9ba01558455ad81279df07
+				95cc985580e4fb75d72d948d1107b2ac80a09abed4da8480c746cc321f2319a5e99a830e314d10dd3cd
+				68ce3dc0c33c86e99bcb7816f9ba01558455ad81279df0795cc985580e4fb75d72d948d1107b2ac80a0
+				9abed4da8480c746cc321f2319a5e99a830e314d10dd3cd68ce3dc0c33c86e99bcb7816f9ba01558455
+				ad81279df0795cc985580e4fb75d72d948d1107b2ac80a09abed4da8480c746cc321f2319a5e99a830e
+				314d10dd3cd68ce3dc0c33c86e99bcb7816f9ba01558455ad81279df0795cc985580e4fb75d72d948d1
+				107b2ac80a09abed4da8480c746cc321f2319a5e99a830e314d10dd3cd68ce3dc0c33c86e99bcb7816f
+				9ba01558455ad81279df0795cc985580e4fb75d72d948d1107b2ac80a09abed4da8480c746cc321f231
+				9a5e99a830e314d10dd3cd68ce3dc0c33c86e99bcb7816f9ba01558455ad81279df0795cc985580e4fb
+				75d72d948d1107b2ac80a09abed4da8480c746cc321f2319a5e99a830e314d10dd3cd68ce3dc0c33c86
+				e99bcb7816f9ba01558455ad81279df0795cc985580e4fb75d72d948d1107b2ac80a09abed4da8480c7
+				46cc321f2319a5e99a830e314d10dd3cd68ce3dc0c33c86e99bcb7816f9ba01558455ad81279df0795c
+				c985580e4fb75d72d948d1107b2ac80a09abed4da8480c746cc321f2319a5e99a830e314d10dd3cd68c
+				e3dc0c33c86e99bcb7816f9ba01558455ad81279df0795cc985580e4fb75d72d948d1107b2ac80a09ab
+				ed4da8480c746cc321f2319a5e99a830e314d10dd3cd68ce3dc0c33c86e99bcb7816f9ba01558455ad8
+				1279df0795cc985580e4fb75d72d948d1107b2ac80a09abed4da8480c746cc321f2319a5e99a830e314
+				d10dd3cd68ce3dc0c33c86e99bcb7816f9ba01558455ad81279df0795cc985580e4fb75d72d948d1107
+				b2ac80a09abed4da8480c746cc321f2319a5e99a830e314d10dd3cd68ce3dc0c33c86e99bcb7816f9ba
+				01558455ad81279df0795cc985580e4fb75d72d948d1107b2ac80a09abed4da8480c746cc321f2319a5e99a830e314d10dd3cd68ce3dc0c33c86e99bcb7816f9ba01558455ad81279df0795cc985580e4fb75d72d948d1107b2ac80a09abed4da8480c746cc321f2319a5e99a830e314d10dd3cd68ce3dc0c33c86e99bcb7816f9ba01558455ad81279df0795cc985580e4fb75d72d948d1107b2ac80a09abed4da8480c746cc321f2319a5e99a830e314d10dd3cd68ce3dc0c33c86e99bcb7816f9ba01558455ad81279df0795cc985580e4fb75d72d948d1107b2ac80a09abed4da8480c746cc321f2319a5e99a830e314d10dd3cd68ce3dc0c33c86e99bcb7816f9ba01558455ad81279df0795cc985580e4fb75d72d948d1107b2ac80a09abed4da8480c746cc321f2319a5e99a830e314d10dd3cd68ce3dc0c33c86e99bcb7816f9ba01558455ad81279df0795cc985580e4fb75d72d948d1107b2ac80a09abed4da8480c746cc321f2319a5e99a830e314d10dd3cd68ce3dc0c33c86e99bcb7816f9ba01558455ad81279df0795cc985580e4fb75d72d948d1107b2ac80a09abed4da8480c746cc321f2319a5e99a830e314d10dd3cd68ce3dc0c33c86e99bcb7816f9ba01558455ad81279df0795cc985580e4fb75d72d948d1107b2ac80a09abed4da8480c746cc321f2319a5e99a830e314d10dd3cd68ce3dc0c33c86e99bcb7816f9ba01558455ad81279df0795cc985580e4fb75d72d948d1107b2ac80a09abed4da8480c746cc321f2319a5e99a830e314d10dd3cd68ce3dc0c33c86e99bcb7816f9ba01558455ad81279df0795cc985580e4fb75d72d948d1107b2ac80a09abed4da8480c746cc321f2319a5e99a830e314d10dd3cd68ce3dc0c33c86e99bcb7816f9ba01558455ad81279df0795cc985580e4fb75d72d948d1107b2ac80a09abed4da8480c746cc321f2319a5e99a830e314d10dd3cd68ce3dc0c33c86e99bcb7816f9ba01558455ad81279df0795cc985580e4fb75d72d948d1107b2ac80a09abed4da8480c746cc321f2319a5e99a830e314d10dd3cd68ce3dc0c33c86e99bcb7816f9ba01558455ad81279df0795cc985580e4fb75d72d948d1107b2ac80a09abed4da8480c746cc321f2319a5e99a830e314d10dd3cd68ce3dc0c33c86e99bcb7816f9ba01558455ad81279df0795cc985580e4fb75d72d948d1107b2ac80a09abed4da8480c746cc321f2319a5e99a830e314d10dd3cd68ce3dc0c33c86e99bcb7816f9ba01558455ad81279df0795cc985580e4fb75d72d948d1107b2ac80a09abed4da8480c746cc321f2319a5e99a830e314d10dd3cd68ce3dc0c33c86e99bcb7816f9ba01558455ad81279df0795cc985580e4fb75d72d948d1107b2ac80a09abed4da8480c746cc321f2319a5e99a830e314d10dd3cd68ce3dc0c33c86e99bcb7816f9ba01558455ad81279df0795cc985580e4fb75d72d948d1107b2ac80a09abed4da8480c746cc321f2319a5e99a830e314d10dd3cd68ce3dc0c33c86e99bcb7816f9ba01558455ad81279df0795cc985580e4fb75d72d948d1107b2ac80a09abed4da8480c746cc321f2319a5e99a830e314d10dd3cd68ce3dc0c33c86e99bcb7816f9ba01558455ad81279df0795cc985580e4fb75d72d948d1107b2ac80a09abed4da8480c746cc321f2319a5e99a830e314d10dd3cd68ce3dc0c33c86e99bcb7816f9ba01558455ad81279df0795cc985580e4fb75d72d948d1107b2ac80a09abed4da8480c746cc321f2319a5e99a830e314d10dd3cd68ce3dc0c33c86e99bcb7816f9ba01558455ad81279df0795cc985580e4fb75d72d948d1107b2ac80a09abed4da8480c746cc321f2319a5e99a830e314d10dd3cd68ce3dc0c33c86e99bcb7816f9ba01558455ad81279df0795cc985580e4fb75d72d948d1107b2ac80a09abed4da8480c746cc321f2319a5e99a830e314d10dd3cd68ce3dc0c33c86e99bcb7816f9ba01558455ad81279df0795cc985580e4fb75d72d948d1107b2ac80a09abed4da8480c746cc321f2319a5e99a830e314d10dd3cd68ce3dc0c33c86e99bcb7816f9ba01558455ad81279df0795cc985580e4fb75d72d948d1107b2ac80a09abed4da8480c746cc321f2319a5e99a830e314d10dd3cd68ce3dc0c33c86e99bcb7816f9ba01558455ad81279df0795cc985580e4fb75d72d948d1107b2ac80a09abed4da8480c746cc321f2319a5e99a830e314d10dd3cd68ce3dc0c33c86e99bcb7816f9ba01558455ad81279df0795cc985580e4fb75d72d948d1107b2ac80a09abed4da8480c746cc321f2319a5e99a830e314d10dd3cd68ce3dc0c33c86e99bcb7816f9ba01558455ad81279df0795cc985580e4fb75d72d948d1107b2ac80a09abed4da8480c746cc321f2319a5e99a830e314d10dd3cd68ce3dc0c33c86e99bcb7816f9ba01558455ad81279df0795cc985580e4fb75d72d948d1107b2ac80a09abed4da8480c746cc321f2319a5e99a830e314d10dd3cd68ce3dc0c33c86e99bcb7816f9ba01558455ad81279df0795cc985580e4fb75d72d948d1107b2ac80a09abed4da8480c746cc321f2319a5e99a830e314d10dd3cd68ce3dc0c33c86e99bcb7816f9ba01558455ad81279df0795cc985580e4fb75d72d948d1107b2ac80a09abed4da8480c746cc321f2319a5e99a830e314d10dd3cd68ce3dc0c33c86e99bcb7816f9ba01558455ad81279df0795cc985580e4fb75d72d948d1107b2ac80a09abed4da8480c746cc321f2319a5e99a830e314d10dd3cd68ce3dc0c33c86e99bcb7816f9ba01558455ad81279df0795cc985580e4fb75d72d948d1107b2ac80a09abed4da8480c746cc321f2319a5e99a830e314d10dd3cd68ce3dc0c33c86e99bcb7816f9ba01558455ad81279df0795cc985580e4fb75d72d948d1107b2ac80a09abed4da8480c746cc321f2319a5e99a830e314d10dd3cd68ce3dc0c33c86e99bcb7816f9ba01558455ad81279df0795cc985580e4fb75d72d948d1107b2ac80a09abed4da8480c746cc321f2319a5e99a830e314d10dd3cd68ce3dc0c33c86e99bcb7816f9ba01558455ad81279df0795cc985580e4fb75d72d948d1107b2ac80a09abed4da8480c746cc321f2319a5e99a830e314d10dd3cd68ce3dc0c33c86e99bcb7816f9ba01558455ad81279df0795cc985580e4fb75d72d948d1107b2ac80a09abed4da8480c746cc321f2319a5e99a830e314d10dd3cd68ce3dc0c33c86e99bcb7816f9ba01558455ad81279df0795cc985580e4fb75d72d948d1107b2ac80a09abed4da8480c746cc321f2319a5e99a830e314d10dd3cd68ce3dc0c33c86e99bcb7816f9ba01558455ad81279df0795cc985580e4fb75d72d948d1107b2ac80a09abed4da8480c746cc321f2319a5e99a830e314d10dd3cd68ce3dc0c33c86e99bcb7816f9ba01558455ad81279df0795cc985580e4fb75d72d948d1107b2ac80a0Line truncated
+			)
+		);
+	}
 }