Companion for pallet-mmr: generate historical proofs (#6061)

* BEEFY: generate historical proofs

Signed-off-by: Serban Iorga <serban@parity.io>

* cargo update -p sp-io

* Properly set max proof size for runtimes

* Properly set max proof size for mocks

* cargo fmt

* Set appropriate UMP service total proof size weight

* Disable zombienet-tests-parachains-disputes CI

* Add comment explaining weight math

* Use MAX_POV_SIZE for max proof size

* Cast to u64

* Remove comment

Signed-off-by: Serban Iorga <serban@parity.io>
Co-authored-by: Keith Yeung <kungfukeith11@gmail.com>

polkadot/node/service/src/chain_spec.rs

@@ -190,7 +190,8 @@ fn default_parachains_host_configuration(
 		max_upward_queue_count: 8,
 		max_upward_queue_size: 1024 * 1024,
 		max_downward_message_size: 1024 * 1024,
-		ump_service_total_weight: Weight::from_ref_time(100_000_000_000),
+		ump_service_total_weight: Weight::from_ref_time(100_000_000_000)
+			.set_proof_size(MAX_POV_SIZE as u64),
 		max_upward_message_size: 50 * 1024,
 		max_upward_message_num_per_candidate: 5,
 		hrmp_sender_deposit: 0,

polkadot/runtime/common/src/impls.rs

@@ -97,7 +97,7 @@ mod tests {
 				weight.base_extrinsic = Weight::from_ref_time(100);
 			})
 			.for_class(DispatchClass::non_mandatory(), |weight| {
-				weight.max_total = Some(Weight::from_ref_time(1024));
+				weight.max_total = Some(Weight::from_ref_time(1024).set_proof_size(u64::MAX));
 			})
 			.build_or_panic();
 		pub BlockLength: limits::BlockLength = limits::BlockLength::max(2 * 1024);

polkadot/runtime/common/src/integration_tests.rs

@@ -104,7 +104,9 @@ use crate::{auctions::Error as AuctionsError, crowdloan::Error as CrowdloanError
 parameter_types! {
 	pub const BlockHashCount: u32 = 250;
 	pub BlockWeights: frame_system::limits::BlockWeights =
-		frame_system::limits::BlockWeights::simple_max(Weight::from_ref_time(4 * 1024 * 1024));
+		frame_system::limits::BlockWeights::simple_max(
+			Weight::from_ref_time(4 * 1024 * 1024).set_proof_size(u64::MAX),
+		);
 }
 
 impl frame_system::Config for Test {

polkadot/runtime/common/src/lib.rs

@@ -43,7 +43,7 @@ use frame_support::{
 	weights::{constants::WEIGHT_PER_SECOND, Weight},
 };
 use frame_system::limits;
-use primitives::v2::{AssignmentId, Balance, BlockNumber, ValidatorId};
+use primitives::v2::{AssignmentId, Balance, BlockNumber, ValidatorId, MAX_POV_SIZE};
 use sp_runtime::{FixedPointNumber, Perbill, Perquintill};
 use static_assertions::const_assert;
 
@@ -68,8 +68,9 @@ pub const AVERAGE_ON_INITIALIZE_RATIO: Perbill = Perbill::from_percent(1);
 /// We allow `Normal` extrinsics to fill up the block up to 75%, the rest can be used
 /// by  Operational  extrinsics.
 pub const NORMAL_DISPATCH_RATIO: Perbill = Perbill::from_percent(75);
-/// We allow for 2 seconds of compute with a 6 second average block time.
+/// The storage proof size is not limited so far.
-pub const MAXIMUM_BLOCK_WEIGHT: Weight = WEIGHT_PER_SECOND.saturating_mul(2);
+pub const MAXIMUM_BLOCK_WEIGHT: Weight =
+	WEIGHT_PER_SECOND.saturating_mul(2).set_proof_size(MAX_POV_SIZE as u64);
 
 const_assert!(NORMAL_DISPATCH_RATIO.deconstruct() >= AVERAGE_ON_INITIALIZE_RATIO.deconstruct());
 

polkadot/runtime/common/src/paras_registrar.rs

@@ -642,7 +642,9 @@ mod tests {
 	parameter_types! {
 		pub const BlockHashCount: u32 = 250;
 		pub BlockWeights: limits::BlockWeights =
-			frame_system::limits::BlockWeights::simple_max(Weight::from_ref_time(1024));
+			frame_system::limits::BlockWeights::simple_max(
+				Weight::from_ref_time(1024).set_proof_size(u64::MAX),
+			);
 		pub BlockLength: limits::BlockLength =
 			limits::BlockLength::max_with_normal_ratio(4 * 1024 * 1024, NORMAL_RATIO);
 	}

polkadot/runtime/kusama/src/lib.rs

@@ -1740,6 +1740,13 @@ sp_api::impl_runtime_apis! {
 			Err(mmr::Error::PalletNotIncluded)
 		}
 
+		fn generate_historical_batch_proof(
+			_leaf_indices: Vec<u64>,
+			_leaves_count: u64,
+		) -> Result<(Vec<mmr::EncodableOpaqueLeaf>, mmr::BatchProof<Hash>), mmr::Error> {
+			Err(mmr::Error::PalletNotIncluded)
+		}
+
 		fn verify_batch_proof(_leaves: Vec<mmr::EncodableOpaqueLeaf>, _proof: mmr::BatchProof<Hash>)
 			-> Result<(), mmr::Error>
 		{
@@ -2101,7 +2108,7 @@ mod multiplier_tests {
 			BlockWeights::get().get(DispatchClass::Normal).max_total.unwrap();
 		// if the min is too small, then this will not change, and we are doomed forever.
 		// the weight is 1/100th bigger than target.
-		run_with_system_weight(target * 101 / 100, || {
+		run_with_system_weight(target.saturating_mul(101) / 100, || {
 			let next = SlowAdjustingFeeUpdate::<Runtime>::convert(minimum_multiplier);
 			assert!(next > minimum_multiplier, "{:?} !>= {:?}", next, minimum_multiplier);
 		})

polkadot/runtime/parachains/src/mock.rs

@@ -82,7 +82,9 @@ where
 parameter_types! {
 	pub const BlockHashCount: u32 = 250;
 	pub BlockWeights: frame_system::limits::BlockWeights =
-		frame_system::limits::BlockWeights::simple_max(Weight::from_ref_time(4 * 1024 * 1024));
+		frame_system::limits::BlockWeights::simple_max(
+			Weight::from_ref_time(4 * 1024 * 1024).set_proof_size(u64::MAX),
+		);
 }
 
 pub type AccountId = u64;

polkadot/runtime/parachains/src/ump.rs

@@ -524,7 +524,10 @@ impl<T: Config> Pallet<T> {
 			let max_weight = if weight_used == Weight::zero() {
 				// we increase the amount of weight that we're allowed to use on the first message to try to prevent
 				// the possibility of blockage of the queue.
-				config.ump_service_total_weight * T::FirstMessageFactorPercent::get() / 100
+				config
+					.ump_service_total_weight
+					.saturating_mul(T::FirstMessageFactorPercent::get()) /
+					100
 			} else {
 				config.ump_service_total_weight - weight_used
 			};

polkadot/runtime/parachains/src/ump/benchmarking.rs

@@ -117,7 +117,7 @@ frame_benchmarking::benchmarks! {
 		let msg = create_message_overweight::<T>();
 
 		// This just makes sure that 0 is not a valid index and we can use it later on.
-		let _ = Ump::<T>::service_overweight(RawOrigin::Root.into(), 0, Weight::from_ref_time(1000));
+		let _ = Ump::<T>::service_overweight(RawOrigin::Root.into(), 0, Weight::from_ref_time(1000).set_proof_size(u64::MAX));
 		// Start with the block number 1. This is needed because should an event be
 		// emitted during the genesis block they will be implicitly wiped.
 		frame_system::Pallet::<T>::set_block_number(1u32.into());

polkadot/runtime/parachains/src/ump/tests.rs

@@ -38,8 +38,8 @@ impl Default for GenesisConfigBuilder {
 			max_upward_message_num_per_candidate: 2,
 			max_upward_queue_count: 4,
 			max_upward_queue_size: 64,
-			ump_service_total_weight: Weight::from_ref_time(1000),
+			ump_service_total_weight: Weight::from_ref_time(1000).set_proof_size(1000),
-			ump_max_individual_weight: Weight::from_ref_time(100),
+			ump_max_individual_weight: Weight::from_ref_time(100).set_proof_size(100),
 		}
 	}
 }
@@ -156,7 +156,7 @@ fn dispatch_resume_after_exceeding_dispatch_stage_weight() {
 
 	new_test_ext(
 		GenesisConfigBuilder {
-			ump_service_total_weight: Weight::from_ref_time(500),
+			ump_service_total_weight: Weight::from_ref_time(500).set_proof_size(500),
 			..Default::default()
 		}
 		.build(),
@@ -203,8 +203,8 @@ fn dispatch_keeps_message_after_weight_exhausted() {
 
 	new_test_ext(
 		GenesisConfigBuilder {
-			ump_service_total_weight: Weight::from_ref_time(500),
+			ump_service_total_weight: Weight::from_ref_time(500).set_proof_size(500),
-			ump_max_individual_weight: Weight::from_ref_time(300),
+			ump_max_individual_weight: Weight::from_ref_time(300).set_proof_size(300),
 			..Default::default()
 		}
 		.build(),
@@ -243,7 +243,7 @@ fn dispatch_correctly_handle_remove_of_latest() {
 
 	new_test_ext(
 		GenesisConfigBuilder {
-			ump_service_total_weight: Weight::from_ref_time(900),
+			ump_service_total_weight: Weight::from_ref_time(900).set_proof_size(900),
 			..Default::default()
 		}
 		.build(),
@@ -312,8 +312,8 @@ fn overweight_queue_works() {
 
 	new_test_ext(
 		GenesisConfigBuilder {
-			ump_service_total_weight: Weight::from_ref_time(900),
+			ump_service_total_weight: Weight::from_ref_time(900).set_proof_size(900),
-			ump_max_individual_weight: Weight::from_ref_time(300),
+			ump_max_individual_weight: Weight::from_ref_time(300).set_proof_size(300),
 			..Default::default()
 		}
 		.build(),

polkadot/runtime/polkadot/src/lib.rs

@@ -1827,6 +1827,13 @@ sp_api::impl_runtime_apis! {
 			Err(mmr::Error::PalletNotIncluded)
 		}
 
+		fn generate_historical_batch_proof(
+			_leaf_indices: Vec<u64>,
+			_leaves_count: u64,
+		) -> Result<(Vec<mmr::EncodableOpaqueLeaf>, mmr::BatchProof<Hash>), mmr::Error> {
+			Err(mmr::Error::PalletNotIncluded)
+		}
+
 		fn verify_batch_proof(_leaves: Vec<mmr::EncodableOpaqueLeaf>, _proof: mmr::BatchProof<Hash>)
 			-> Result<(), mmr::Error>
 		{
@@ -2284,7 +2291,7 @@ mod multiplier_tests {
 			BlockWeights::get().get(DispatchClass::Normal).max_total.unwrap();
 		// if the min is too small, then this will not change, and we are doomed forever.
 		// the weight is 1/100th bigger than target.
-		run_with_system_weight(target * 101 / 100, || {
+		run_with_system_weight(target.saturating_mul(101) / 100, || {
 			let next = SlowAdjustingFeeUpdate::<Runtime>::convert(minimum_multiplier);
 			assert!(next > minimum_multiplier, "{:?} !>= {:?}", next, minimum_multiplier);
 		})

polkadot/runtime/rococo/src/lib.rs

@@ -1723,6 +1723,23 @@ sp_api::impl_runtime_apis! {
 				.map(|(leaves, proof)| (leaves.into_iter().map(|leaf| mmr::EncodableOpaqueLeaf::from_leaf(&leaf)).collect(), proof))
 		}
 
+		fn generate_historical_batch_proof(
+			leaf_indices: Vec<mmr::LeafIndex>,
+			leaves_count: mmr::LeafIndex,
+		) -> Result<(Vec<mmr::EncodableOpaqueLeaf>, mmr::BatchProof<Hash>), mmr::Error> {
+			Mmr::generate_historical_batch_proof(leaf_indices, leaves_count).map(
+				|(leaves, proof)| {
+					(
+						leaves
+							.into_iter()
+							.map(|leaf| mmr::EncodableOpaqueLeaf::from_leaf(&leaf))
+							.collect(),
+						proof,
+					)
+				},
+			)
+		}
+
 		fn verify_batch_proof(leaves: Vec<mmr::EncodableOpaqueLeaf>, proof: mmr::BatchProof<Hash>)
 			-> Result<(), mmr::Error>
 		{

polkadot/runtime/test-runtime/src/lib.rs

@@ -947,6 +947,13 @@ sp_api::impl_runtime_apis! {
 			Err(mmr::Error::PalletNotIncluded)
 		}
 
+		fn generate_historical_batch_proof(
+			_leaf_indices: Vec<u64>,
+			_leaves_count: u64,
+		) -> Result<(Vec<mmr::EncodableOpaqueLeaf>, mmr::BatchProof<Hash>), mmr::Error> {
+			Err(mmr::Error::PalletNotIncluded)
+		}
+
 		fn verify_batch_proof(_leaves: Vec<mmr::EncodableOpaqueLeaf>, _proof: mmr::BatchProof<Hash>)
 			-> Result<(), mmr::Error>
 		{

polkadot/runtime/westend/src/lib.rs

@@ -1473,7 +1473,13 @@ sp_api::impl_runtime_apis! {
 		fn generate_batch_proof(_leaf_indices: Vec<u64>)
 			-> Result<(Vec<mmr::EncodableOpaqueLeaf>, mmr::BatchProof<Hash>), mmr::Error>
 		{
+			Err(mmr::Error::PalletNotIncluded)
+		}
 
+		fn generate_historical_batch_proof(
+			_leaf_indices: Vec<u64>,
+			_leaves_count: u64,
+		) -> Result<(Vec<mmr::EncodableOpaqueLeaf>, mmr::BatchProof<Hash>), mmr::Error> {
 			Err(mmr::Error::PalletNotIncluded)
 		}
 

polkadot/scripts/ci/gitlab/pipeline/zombienet.yml

@@ -88,7 +88,7 @@ zombienet-tests-parachains-disputes:
     - /home/nonroot/zombie-net/scripts/ci/run-test-env-manager.sh
         --github-remote-dir="${GH_DIR}"
         --test="0002-parachains-disputes.zndsl"
-  allow_failure:                   false
+  allow_failure:                   true
   retry: 2
   tags:
     - zombienet-polkadot-integration-test

polkadot/xcm/pallet-xcm-benchmarks/src/fungible/mock.rs

@@ -47,7 +47,9 @@ frame_support::construct_runtime!(
 parameter_types! {
 	pub const BlockHashCount: u64 = 250;
 	pub BlockWeights: frame_system::limits::BlockWeights =
-		frame_system::limits::BlockWeights::simple_max(Weight::from_ref_time(1024));
+		frame_system::limits::BlockWeights::simple_max(
+			Weight::from_ref_time(1024).set_proof_size(u64::MAX),
+		);
 }
 impl frame_system::Config for Test {
 	type BaseCallFilter = Everything;

polkadot/xcm/pallet-xcm-benchmarks/src/generic/mock.rs

@@ -52,7 +52,9 @@ frame_support::construct_runtime!(
 parameter_types! {
 	pub const BlockHashCount: u64 = 250;
 	pub BlockWeights: frame_system::limits::BlockWeights =
-		frame_system::limits::BlockWeights::simple_max(Weight::from_ref_time(1024));
+		frame_system::limits::BlockWeights::simple_max(
+			Weight::from_ref_time(1024).set_proof_size(u64::MAX),
+		);
 }
 
 impl frame_system::Config for Test {