move paras inherent filtering to runtime (#4028)

* move things around, add filter methods

* validator keys, modify availability bitfields according to disputes

* simplify, keep the filter -> sanitize generic for both usecases

* minor

* assure tests still work, reduce changeset

* integration

* start entropy passing

* fixins

* compile, 1 failing test

* filter with coverage

* fixins

* Update runtime/parachains/src/paras_inherent.rs

Co-authored-by: Robert Habermeier <rphmeier@gmail.com>

* slip of the pen

* improve test cases

* misc

* fix

* fixins

* test avoid extra into() calls in assert_noop!

* chores

* ff

* test fixup superfluous into call

* chore: pfmt

* improve apply_block_weight_limit to try to maximize the number of sufficiently backed

blocks and add extra bitfields in a round-robin fashion

* new code treats the lack of backed candidates as ok

* Use vrf based entropy

* fixup vrf random

* add warn

* slip of the pen

* fixup

* assure ordering

* rethink apply_weights

* mock

* use a closure as predicate check

* extract and use DisputedBitfield

* chore: simplify

* remove stray dbg

* chore: fmt

* address feedback

* fix test, halfway there

* stage1

* dbg stuff

* make group selection align

* fix session index

* fix wrongly returned candidates

* cleanup

* chore fmt

* fix ensure check

* make good case test work

* more tests for bitfields

* create sanitize_backed_candidates

* fixup tests

* update guide

* add check referenced in the guide

* improve weights code

* fmt

* fixins

* Update roadmap/implementers-guide/src/runtime/inclusion.md

Co-authored-by: Zeke Mostov <32168567+emostov@users.noreply.github.com>

* compiling + address review

* add comments

* fix weight calc

* address review comments and test failure

* fix

* fix: condition

* Fix random_sel function

* Fix overlength block check

* Zeke + Ladi commit for disputes filtering + integration test builder + runtime benchmarks + integration tests

* Add benchmarks for code upgrades

* Code upgrade bench; Feature gate TestWeightInfo

* Try and make CI happier

* Feature gate enter test to not(benchmarks)

* Make sure no unused imports/fn

* refactor, re-use, the beginning

* Fix issue with frame benchmarking dep compilation

* More precise feature gating for some derives

* integrate piece-wise

* foo

* fixins

* chore fmt

* fixins

* rename const generic

* Update runtime/parachains/src/paras_inherent.rs

Co-authored-by: Zeke Mostov <z.mostov@gmail.com>

* Fix compilation

* limit to test

* remove unused spam slots

* spellcheck

* remove a tick, fix a typo

* Add Code upgrade weights

* comment improvements + >=

Co-authored-by: Zeke Mostov <z.mostov@gmail.com>

* remove another tick

* Update runtime/parachains/src/paras_inherent/benchmarking.rs

Co-authored-by: Zeke Mostov <z.mostov@gmail.com>

* saturating fixins + some spaces

* fix

* benchmarking - preliminary results

* Add training wheels

* Refactor some early exit logic for enter

* Gracefully handle filtering bitfields & candidates (#4280)

This updates the logic for sanitize_bitfields and sanitize_backed_candidates to never error when there is an issue, but instead to simply skip the problematic items.

* Refactor inherent data weight limiting logic (#4287)

* Apply suggestions from code review

* Update runtime/parachains/src/builder.rs

Co-authored-by: Zeke Mostov <z.mostov@gmail.com>

* Update runtime/parachains/src/builder.rs

* Update runtime/parachains/src/paras_inherent.rs

* final pass

* Run cargo +nightly-2021-10-29 fmt

* Update implementors guide with `sanitize_*` & `enter` (#4294)

* Make spell check happier

* Make wasm runtimes compile with benchmarks enabled (#4303)

* comment stuff out, use old toml

* Seems to be working?

* Remove feature gating from builder

* Remove commented out stuff

* Remove generic from digest

* Update weight files for runtime

Co-authored-by: Robert Habermeier <rphmeier@gmail.com>
Co-authored-by: Zeke Mostov <32168567+emostov@users.noreply.github.com>
Co-authored-by: Lldenaurois <Ljdenaurois@gmail.com>
Co-authored-by: Zeke Mostov <z.mostov@gmail.com>
Co-authored-by: Bastian Köcher <info@kchr.de>

polkadot/runtime/parachains/src/mock.rs

@@ -22,15 +22,24 @@ use crate::{
 	ump::{self, MessageId, UmpSink},
 	ParaId,
 };
-use frame_support::{parameter_types, traits::GenesisBuild, weights::Weight};
+
+use frame_support::{
+	parameter_types,
+	traits::{GenesisBuild, KeyOwnerProofSystem},
+	weights::Weight,
+};
 use frame_support_test::TestRandomness;
 use parity_scale_codec::Decode;
 use primitives::v1::{
-	AuthorityDiscoveryId, Balance, BlockNumber, Header, SessionIndex, UpwardMessage, ValidatorIndex,
+	AuthorityDiscoveryId, Balance, BlockNumber, Header, Moment, SessionIndex, UpwardMessage,
+	ValidatorIndex,
 };
 use sp_core::H256;
 use sp_io::TestExternalities;
-use sp_runtime::traits::{BlakeTwo256, IdentityLookup};
+use sp_runtime::{
+	traits::{BlakeTwo256, IdentityLookup},
+	KeyTypeId,
+};
 use std::{cell::RefCell, collections::HashMap};
 
 type UncheckedExtrinsic = frame_system::mocking::MockUncheckedExtrinsic<Test>;
@@ -56,6 +65,7 @@ frame_support::construct_runtime!(
 		Hrmp: hrmp::{Pallet, Call, Storage, Event<T>},
 		SessionInfo: session_info::{Pallet, Storage},
 		Disputes: disputes::{Pallet, Storage, Event<T>},
+		Babe: pallet_babe::{Pallet, Call, Storage, Config, ValidateUnsigned},
 	}
 );
 
@@ -109,6 +119,52 @@ impl pallet_balances::Config for Test {
 	type WeightInfo = ();
 }
 
+parameter_types! {
+	pub const EpochDuration: u64 = 10;
+	pub const ExpectedBlockTime: Moment = 6_000;
+	pub const ReportLongevity: u64 = 10;
+	pub const MaxAuthorities: u32 = 100_000;
+}
+
+impl pallet_babe::Config for Test {
+	type EpochDuration = EpochDuration;
+	type ExpectedBlockTime = ExpectedBlockTime;
+
+	// session module is the trigger
+	type EpochChangeTrigger = pallet_babe::ExternalTrigger;
+
+	type DisabledValidators = ();
+
+	type KeyOwnerProof = <Self::KeyOwnerProofSystem as KeyOwnerProofSystem<(
+		KeyTypeId,
+		pallet_babe::AuthorityId,
+	)>>::Proof;
+
+	type KeyOwnerIdentification = <Self::KeyOwnerProofSystem as KeyOwnerProofSystem<(
+		KeyTypeId,
+		pallet_babe::AuthorityId,
+	)>>::IdentificationTuple;
+
+	type KeyOwnerProofSystem = ();
+
+	type HandleEquivocation = ();
+
+	type WeightInfo = ();
+
+	type MaxAuthorities = MaxAuthorities;
+}
+
+parameter_types! {
+	pub const MinimumPeriod: Moment = 6_000 / 2;
+}
+
+impl pallet_timestamp::Config for Test {
+	type Moment = Moment;
+	type OnTimestampSet = ();
+	type MinimumPeriod = MinimumPeriod;
+	type WeightInfo = ();
+}
+
 impl crate::initializer::Config for Test {
 	type Randomness = TestRandomness<Self>;
 	type ForceOrigin = frame_system::EnsureRoot<u64>;
@@ -203,7 +259,9 @@ impl crate::inclusion::Config for Test {
 	type RewardValidators = TestRewardValidators;
 }
 
-impl crate::paras_inherent::Config for Test {}
+impl crate::paras_inherent::Config for Test {
+	type WeightInfo = crate::paras_inherent::TestWeightInfo;
+}
 
 impl crate::session_info::Config for Test {}
 
@@ -299,6 +357,8 @@ impl inclusion::RewardValidators for TestRewardValidators {
 
 /// Create a new set of test externalities.
 pub fn new_test_ext(state: MockGenesisConfig) -> TestExternalities {
+	use sp_keystore::{testing::KeyStore, KeystoreExt, SyncCryptoStorePtr};
+	use sp_std::sync::Arc;
 	BACKING_REWARDS.with(|r| r.borrow_mut().clear());
 	AVAILABILITY_REWARDS.with(|r| r.borrow_mut().clear());
 
@@ -306,7 +366,10 @@ pub fn new_test_ext(state: MockGenesisConfig) -> TestExternalities {
 	state.configuration.assimilate_storage(&mut t).unwrap();
 	GenesisBuild::<Test>::assimilate_storage(&state.paras, &mut t).unwrap();
 
-	t.into()
+	let mut ext: TestExternalities = t.into();
+	ext.register_extension(KeystoreExt(Arc::new(KeyStore::new()) as SyncCryptoStorePtr));
+
+	ext
 }
 
 #[derive(Default)]