Fuzz testing for nomination pools (#12002)

* some additional tests and stuff

* make sanity public

* add some sort of fuzz test for pools

* breaks every now and then

* breaks every now and then

* IT WORKS AND PASSES 100k TESTS

* cleanup

* safe id addition

* fix assert_eq_error_rate

* Update frame/nomination-pools/src/tests.rs

Co-authored-by: Oliver Tale-Yazdi <oliver.tale-yazdi@parity.io>

* Update frame/nomination-pools/src/tests.rs

Co-authored-by: Oliver Tale-Yazdi <oliver.tale-yazdi@parity.io>

* add some doc

* Fix

* ".git/.scripts/fmt.sh" 1

Co-authored-by: Oliver Tale-Yazdi <oliver.tale-yazdi@parity.io>
Co-authored-by: command-bot <>

substrate/frame/nomination-pools/src/lib.rs

@@ -1453,7 +1453,7 @@ pub mod pallet {
 		PartialUnbondNotAllowedPermissionlessly,
 	}
 
-	#[derive(Encode, Decode, PartialEq, TypeInfo, frame_support::PalletError)]
+	#[derive(Encode, Decode, PartialEq, TypeInfo, frame_support::PalletError, RuntimeDebug)]
 	pub enum DefensiveError {
 		/// There isn't enough space in the unbond pool.
 		NotEnoughSpaceInUnbondPool,
@@ -1758,8 +1758,8 @@ pub mod pallet {
 
 			let bonded_pool = BondedPool::<T>::get(member.pool_id)
 				.defensive_ok_or::<Error<T>>(DefensiveError::PoolNotFound.into())?;
-			let mut sub_pools = SubPoolsStorage::<T>::get(member.pool_id)
-				.defensive_ok_or::<Error<T>>(DefensiveError::SubPoolsNotFound.into())?;
+			let mut sub_pools =
+				SubPoolsStorage::<T>::get(member.pool_id).ok_or(Error::<T>::SubPoolsNotFound)?;
 
 			bonded_pool.ok_to_withdraw_unbonded_with(&caller, &member_account)?;
 
@@ -1887,10 +1887,10 @@ pub mod pallet {
 			);
 			ensure!(!PoolMembers::<T>::contains_key(&who), Error::<T>::AccountBelongsToOtherPool);
 
-			let pool_id = LastPoolId::<T>::mutate(|id| {
-				*id += 1;
-				*id
-			});
+			let pool_id = LastPoolId::<T>::try_mutate::<_, Error<T>, _>(|id| {
+				*id = id.checked_add(1).ok_or(Error::<T>::OverflowRisk)?;
+				Ok(*id)
+			})?;
 			let mut bonded_pool = BondedPool::<T>::new(
 				pool_id,
 				PoolRoles {
@@ -2416,16 +2416,46 @@ impl<T: Config> Pallet<T> {
 
 		for id in reward_pools {
 			let account = Self::create_reward_account(id);
-			assert!(T::Currency::free_balance(&account) >= T::Currency::minimum_balance());
+			assert!(
+				T::Currency::free_balance(&account) >= T::Currency::minimum_balance(),
+				"reward pool of {id}: {:?} (ed = {:?})",
+				T::Currency::free_balance(&account),
+				T::Currency::minimum_balance()
+			);
 		}
 
 		let mut pools_members = BTreeMap::<PoolId, u32>::new();
+		let mut pools_members_pending_rewards = BTreeMap::<PoolId, BalanceOf<T>>::new();
 		let mut all_members = 0u32;
 		PoolMembers::<T>::iter().for_each(|(_, d)| {
-			assert!(BondedPools::<T>::contains_key(d.pool_id));
+			let bonded_pool = BondedPools::<T>::get(d.pool_id).unwrap();
 			assert!(!d.total_points().is_zero(), "no member should have zero points: {:?}", d);
 			*pools_members.entry(d.pool_id).or_default() += 1;
 			all_members += 1;
+
+			let reward_pool = RewardPools::<T>::get(d.pool_id).unwrap();
+			if !bonded_pool.points.is_zero() {
+				let current_rc =
+					reward_pool.current_reward_counter(d.pool_id, bonded_pool.points).unwrap();
+				*pools_members_pending_rewards.entry(d.pool_id).or_default() +=
+					d.pending_rewards(current_rc).unwrap();
+			} // else this pool has been heavily slashed and cannot have any rewards anymore.
+		});
+
+		RewardPools::<T>::iter_keys().for_each(|id| {
+			// the sum of the pending rewards must be less than the leftover balance. Since the
+			// reward math rounds down, we might accumulate some dust here.
+			log!(
+				trace,
+				"pool {:?}, sum pending rewards = {:?}, remaining balance = {:?}",
+				id,
+				pools_members_pending_rewards.get(&id),
+				RewardPool::<T>::current_balance(id)
+			);
+			assert!(
+				RewardPool::<T>::current_balance(id) >=
+					pools_members_pending_rewards.get(&id).map(|x| *x).unwrap_or_default()
+			)
 		});
 
 		BondedPools::<T>::iter().for_each(|(id, inner)| {
@@ -2470,6 +2500,7 @@ impl<T: Config> Pallet<T> {
 				sum_unbonding_balance
 			);
 		}
+
 		Ok(())
 	}
 

substrate/frame/nomination-pools/src/mock.rs

@@ -4,6 +4,7 @@ use frame_support::{assert_ok, parameter_types, PalletId};
 use frame_system::RawOrigin;
 use sp_runtime::FixedU128;
 
+pub type BlockNumber = u64;
 pub type AccountId = u128;
 pub type Balance = u128;
 pub type RewardCounter = FixedU128;
@@ -129,7 +130,7 @@ impl frame_system::Config for Runtime {
 	type BaseCallFilter = frame_support::traits::Everything;
 	type Origin = Origin;
 	type Index = u64;
-	type BlockNumber = u64;
+	type BlockNumber = BlockNumber;
 	type Call = Call;
 	type Hash = sp_core::H256;
 	type Hashing = sp_runtime::traits::BlakeTwo256;

substrate/frame/nomination-pools/src/tests.rs

@@ -1774,6 +1774,54 @@ mod claim_payout {
 		})
 	}
 
+	#[test]
+	fn bond_extra_pending_rewards_works() {
+		ExtBuilder::default().add_members(vec![(20, 20)]).build_and_execute(|| {
+			MaxPoolMembers::<Runtime>::set(None);
+			MaxPoolMembersPerPool::<Runtime>::set(None);
+
+			// pool receives some rewards.
+			Balances::mutate_account(&default_reward_account(), |f| f.free += 30).unwrap();
+			System::reset_events();
+
+			// 10 cashes it out, and bonds it.
+			{
+				assert_ok!(Pools::claim_payout(Origin::signed(10)));
+				let (member, _, reward_pool) = Pools::get_member_with_pools(&10).unwrap();
+				// there is 30 points and 30 reward points in the system RC is 1.
+				assert_eq!(member.last_recorded_reward_counter, 1.into());
+				assert_eq!(reward_pool.total_rewards_claimed, 10);
+				// these two are not updated -- only updated when the points change.
+				assert_eq!(reward_pool.last_recorded_total_payouts, 0);
+				assert_eq!(reward_pool.last_recorded_reward_counter, 0.into());
+
+				assert_eq!(
+					pool_events_since_last_call(),
+					vec![Event::PaidOut { member: 10, pool_id: 1, payout: 10 }]
+				);
+			}
+
+			// 20 re-bonds it.
+			{
+				assert_ok!(Pools::bond_extra(Origin::signed(20), BondExtra::Rewards));
+				let (member, _, reward_pool) = Pools::get_member_with_pools(&10).unwrap();
+				assert_eq!(member.last_recorded_reward_counter, 1.into());
+				assert_eq!(reward_pool.total_rewards_claimed, 30);
+				// since points change, these two are updated.
+				assert_eq!(reward_pool.last_recorded_total_payouts, 30);
+				assert_eq!(reward_pool.last_recorded_reward_counter, 1.into());
+
+				assert_eq!(
+					pool_events_since_last_call(),
+					vec![
+						Event::PaidOut { member: 20, pool_id: 1, payout: 20 },
+						Event::Bonded { member: 20, pool_id: 1, bonded: 20, joined: false }
+					]
+				);
+			}
+		})
+	}
+
 	#[test]
 	fn unbond_updates_recorded_data() {
 		ExtBuilder::default()
@@ -3733,6 +3781,170 @@ mod withdraw_unbonded {
 		})
 	}
 
+	#[test]
+	fn out_of_sync_unbonding_chunks() {
+		// the unbonding_eras in pool member are always fixed to the era at which they are unlocked,
+		// but the actual unbonding pools get pruned and might get combined in the no_era pool.
+		// Pools are only merged when one unbonds, so we unbond a little bit on every era to
+		// simulate this.
+		ExtBuilder::default()
+			.add_members(vec![(20, 100), (30, 100)])
+			.build_and_execute(|| {
+				System::reset_events();
+
+				// when
+				assert_ok!(Pools::unbond(Origin::signed(20), 20, 5));
+				assert_ok!(Pools::unbond(Origin::signed(30), 30, 5));
+
+				// then member-local unbonding is pretty much in sync with the global pools.
+				assert_eq!(
+					PoolMembers::<Runtime>::get(20).unwrap().unbonding_eras,
+					member_unbonding_eras!(3 => 5)
+				);
+				assert_eq!(
+					PoolMembers::<Runtime>::get(30).unwrap().unbonding_eras,
+					member_unbonding_eras!(3 => 5)
+				);
+				assert_eq!(
+					SubPoolsStorage::<Runtime>::get(1).unwrap(),
+					SubPools {
+						no_era: Default::default(),
+						with_era: unbonding_pools_with_era! {
+							3 => UnbondPool { points: 10, balance: 10 }
+						}
+					}
+				);
+				assert_eq!(
+					pool_events_since_last_call(),
+					vec![
+						Event::Unbonded { member: 20, pool_id: 1, points: 5, balance: 5, era: 3 },
+						Event::Unbonded { member: 30, pool_id: 1, points: 5, balance: 5, era: 3 },
+					]
+				);
+
+				// when
+				CurrentEra::set(1);
+				assert_ok!(Pools::unbond(Origin::signed(20), 20, 5));
+
+				// then still member-local unbonding is pretty much in sync with the global pools.
+				assert_eq!(
+					PoolMembers::<Runtime>::get(20).unwrap().unbonding_eras,
+					member_unbonding_eras!(3 => 5, 4 => 5)
+				);
+				assert_eq!(
+					SubPoolsStorage::<Runtime>::get(1).unwrap(),
+					SubPools {
+						no_era: Default::default(),
+						with_era: unbonding_pools_with_era! {
+							3 => UnbondPool { points: 10, balance: 10 },
+							4 => UnbondPool { points: 5, balance: 5 }
+						}
+					}
+				);
+				assert_eq!(
+					pool_events_since_last_call(),
+					vec![Event::Unbonded { member: 20, pool_id: 1, points: 5, balance: 5, era: 4 }]
+				);
+
+				// when
+				CurrentEra::set(2);
+				assert_ok!(Pools::unbond(Origin::signed(20), 20, 5));
+
+				// then still member-local unbonding is pretty much in sync with the global pools.
+				assert_eq!(
+					PoolMembers::<Runtime>::get(20).unwrap().unbonding_eras,
+					member_unbonding_eras!(3 => 5, 4 => 5, 5 => 5)
+				);
+				assert_eq!(
+					SubPoolsStorage::<Runtime>::get(1).unwrap(),
+					SubPools {
+						no_era: Default::default(),
+						with_era: unbonding_pools_with_era! {
+							3 => UnbondPool { points: 10, balance: 10 },
+							4 => UnbondPool { points: 5, balance: 5 },
+							5 => UnbondPool { points: 5, balance: 5 }
+						}
+					}
+				);
+				assert_eq!(
+					pool_events_since_last_call(),
+					vec![Event::Unbonded { member: 20, pool_id: 1, points: 5, balance: 5, era: 5 }]
+				);
+
+				// when
+				CurrentEra::set(5);
+				assert_ok!(Pools::unbond(Origin::signed(20), 20, 5));
+
+				// then
+				assert_eq!(
+					PoolMembers::<Runtime>::get(20).unwrap().unbonding_eras,
+					member_unbonding_eras!(3 => 5, 4 => 5, 5 => 5, 8 => 5)
+				);
+				assert_eq!(
+					SubPoolsStorage::<Runtime>::get(1).unwrap(),
+					SubPools {
+						// era 3 is merged into no_era.
+						no_era: UnbondPool { points: 10, balance: 10 },
+						with_era: unbonding_pools_with_era! {
+							4 => UnbondPool { points: 5, balance: 5 },
+							5 => UnbondPool { points: 5, balance: 5 },
+							8 => UnbondPool { points: 5, balance: 5 }
+						}
+					}
+				);
+				assert_eq!(
+					pool_events_since_last_call(),
+					vec![Event::Unbonded { member: 20, pool_id: 1, points: 5, balance: 5, era: 8 }]
+				);
+
+				// now we start withdrawing unlocked bonds.
+
+				// when
+				assert_ok!(Pools::withdraw_unbonded(Origin::signed(20), 20, 0));
+				// then
+				assert_eq!(
+					PoolMembers::<Runtime>::get(20).unwrap().unbonding_eras,
+					member_unbonding_eras!(8 => 5)
+				);
+				assert_eq!(
+					SubPoolsStorage::<Runtime>::get(1).unwrap(),
+					SubPools {
+						// era 3 is merged into no_era.
+						no_era: UnbondPool { points: 5, balance: 5 },
+						with_era: unbonding_pools_with_era! {
+							8 => UnbondPool { points: 5, balance: 5 }
+						}
+					}
+				);
+				assert_eq!(
+					pool_events_since_last_call(),
+					vec![Event::Withdrawn { member: 20, pool_id: 1, points: 15, balance: 15 }]
+				);
+
+				// when
+				assert_ok!(Pools::withdraw_unbonded(Origin::signed(30), 30, 0));
+				// then
+				assert_eq!(
+					PoolMembers::<Runtime>::get(30).unwrap().unbonding_eras,
+					member_unbonding_eras!()
+				);
+				assert_eq!(
+					SubPoolsStorage::<Runtime>::get(1).unwrap(),
+					SubPools {
+						// era 3 is merged into no_era.
+						no_era: Default::default(),
+						with_era: unbonding_pools_with_era! {
+							8 => UnbondPool { points: 5, balance: 5 }
+						}
+					}
+				);
+				assert_eq!(
+					pool_events_since_last_call(),
+					vec![Event::Withdrawn { member: 30, pool_id: 1, points: 5, balance: 5 }]
+				);
+			})
+	}
+
 	#[test]
 	fn full_multi_step_withdrawing_depositor() {
 		ExtBuilder::default().ed(1).build_and_execute(|| {
@@ -4768,3 +4980,328 @@ mod reward_counter_precision {
 			});
 	}
 }
+
+// NOTE: run this with debug_assertions, but in release mode.
+#[cfg(feature = "fuzz-test")]
+mod fuzz_test {
+	use super::*;
+	use crate::pallet::{Call as PoolsCall, Event as PoolsEvents};
+	use frame_support::traits::UnfilteredDispatchable;
+	use rand::{seq::SliceRandom, thread_rng, Rng};
+	use sp_runtime::{assert_eq_error_rate, Perquintill};
+
+	const ERA: BlockNumber = 1000;
+	const MAX_ED_MULTIPLE: Balance = 10_000;
+	const MIN_ED_MULTIPLE: Balance = 10;
+
+	// not quite elegant, just to make it available in random_signed_origin.
+	const REWARD_AGENT_ACCOUNT: AccountId = 42;
+
+	/// Grab random accounts, either known ones, or new ones.
+	fn random_signed_origin<R: Rng>(rng: &mut R) -> (Origin, AccountId) {
+		let count = PoolMembers::<T>::count();
+		if rng.gen::<bool>() && count > 0 {
+			// take an existing account.
+			let skip = rng.gen_range(0..count as usize);
+
+			// this is tricky: the account might be our reward agent, which we never want to be
+			// randomly chosen here. Try another one, or, if it is only our agent, return a random
+			// one nonetheless.
+			let candidate = PoolMembers::<T>::iter_keys().skip(skip).take(1).next().unwrap();
+			let acc =
+				if candidate == REWARD_AGENT_ACCOUNT { rng.gen::<AccountId>() } else { candidate };
+
+			(Origin::signed(acc), acc)
+		} else {
+			// create a new account
+			let acc = rng.gen::<AccountId>();
+			(Origin::signed(acc), acc)
+		}
+	}
+
+	fn random_ed_multiple<R: Rng>(rng: &mut R) -> Balance {
+		let multiple = rng.gen_range(MIN_ED_MULTIPLE..MAX_ED_MULTIPLE);
+		ExistentialDeposit::get() * multiple
+	}
+
+	fn fund_account<R: Rng>(rng: &mut R, account: &AccountId) {
+		let target_amount = random_ed_multiple(rng);
+		if let Some(top_up) = target_amount.checked_sub(Balances::free_balance(account)) {
+			let _ = Balances::deposit_creating(account, top_up);
+		}
+		assert!(Balances::free_balance(account) >= target_amount);
+	}
+
+	fn random_existing_pool<R: Rng>(mut rng: &mut R) -> Option<PoolId> {
+		BondedPools::<T>::iter_keys().collect::<Vec<_>>().choose(&mut rng).map(|x| *x)
+	}
+
+	fn random_call<R: Rng>(mut rng: &mut R) -> (crate::pallet::Call<T>, Origin) {
+		let op = rng.gen::<usize>();
+		let mut op_count =
+			<crate::pallet::Call<T> as frame_support::dispatch::GetCallName>::get_call_names()
+				.len();
+		// Exclude set_state, set_metadata, set_configs, update_roles and chill.
+		op_count -= 5;
+
+		match op % op_count {
+			0 => {
+				// join
+				let pool_id = random_existing_pool(&mut rng).unwrap_or_default();
+				let (origin, who) = random_signed_origin(&mut rng);
+				fund_account(&mut rng, &who);
+				let amount = random_ed_multiple(&mut rng);
+				(PoolsCall::<T>::join { amount, pool_id }, origin)
+			},
+			1 => {
+				// bond_extra
+				let (origin, who) = random_signed_origin(&mut rng);
+				let extra = if rng.gen::<bool>() {
+					BondExtra::Rewards
+				} else {
+					fund_account(&mut rng, &who);
+					let amount = random_ed_multiple(&mut rng);
+					BondExtra::FreeBalance(amount)
+				};
+				(PoolsCall::<T>::bond_extra { extra }, origin)
+			},
+			2 => {
+				// claim_payout
+				let (origin, _) = random_signed_origin(&mut rng);
+				(PoolsCall::<T>::claim_payout {}, origin)
+			},
+			3 => {
+				// unbond
+				let (origin, who) = random_signed_origin(&mut rng);
+				let amount = random_ed_multiple(&mut rng);
+				(PoolsCall::<T>::unbond { member_account: who, unbonding_points: amount }, origin)
+			},
+			4 => {
+				// pool_withdraw_unbonded
+				let pool_id = random_existing_pool(&mut rng).unwrap_or_default();
+				let (origin, _) = random_signed_origin(&mut rng);
+				(PoolsCall::<T>::pool_withdraw_unbonded { pool_id, num_slashing_spans: 0 }, origin)
+			},
+			5 => {
+				// withdraw_unbonded
+				let (origin, who) = random_signed_origin(&mut rng);
+				(
+					PoolsCall::<T>::withdraw_unbonded {
+						member_account: who,
+						num_slashing_spans: 0,
+					},
+					origin,
+				)
+			},
+			6 => {
+				// create
+				let (origin, who) = random_signed_origin(&mut rng);
+				let amount = random_ed_multiple(&mut rng);
+				fund_account(&mut rng, &who);
+				let root = who.clone();
+				let state_toggler = who.clone();
+				let nominator = who.clone();
+				(PoolsCall::<T>::create { amount, root, state_toggler, nominator }, origin)
+			},
+			7 => {
+				// nominate
+				let (origin, _) = random_signed_origin(&mut rng);
+				let pool_id = random_existing_pool(&mut rng).unwrap_or_default();
+				let validators = Default::default();
+				(PoolsCall::<T>::nominate { pool_id, validators }, origin)
+			},
+			_ => unreachable!(),
+		}
+	}
+
+	#[derive(Default)]
+	struct RewardAgent {
+		who: AccountId,
+		pool_id: Option<PoolId>,
+		expected_reward: Balance,
+	}
+
+	// TODO: inject some slashes into the game.
+	impl RewardAgent {
+		fn new(who: AccountId) -> Self {
+			Self { who, ..Default::default() }
+		}
+
+		fn join(&mut self) {
+			if self.pool_id.is_some() {
+				return
+			}
+			let pool_id = LastPoolId::<T>::get();
+			let amount = 10 * ExistentialDeposit::get();
+			let origin = Origin::signed(self.who);
+			let _ = Balances::deposit_creating(&self.who, 10 * amount);
+			self.pool_id = Some(pool_id);
+			log::info!(target: "reward-agent", "🤖 reward agent joining in {} with {}", pool_id, amount);
+			assert_ok!(PoolsCall::join::<T> { amount, pool_id }.dispatch_bypass_filter(origin));
+		}
+
+		fn claim_payout(&mut self) {
+			// 10 era later, we claim our payout. We expect our income to be roughly what we
+			// calculated.
+			if !PoolMembers::<T>::contains_key(&self.who) {
+				log!(warn, "reward agent is not in the pool yet, cannot claim");
+				return
+			}
+			let pre = Balances::free_balance(&42);
+			let origin = Origin::signed(42);
+			assert_ok!(PoolsCall::<T>::claim_payout {}.dispatch_bypass_filter(origin));
+			let post = Balances::free_balance(&42);
+
+			let income = post - pre;
+			log::info!(
+				target: "reward-agent", "🤖 CLAIM: actual: {}, expected: {}",
+				income,
+				self.expected_reward,
+			);
+			assert_eq_error_rate!(income, self.expected_reward, 10);
+			self.expected_reward = 0;
+		}
+	}
+
+	#[test]
+	fn fuzz_test() {
+		let mut reward_agent = RewardAgent::new(42);
+		sp_tracing::try_init_simple();
+		// NOTE: use this to get predictable (non)randomness:
+		// use::{rngs::SmallRng, SeedableRng};
+		// let mut rng = SmallRng::from_seed([0u8; 32]);
+		let mut rng = thread_rng();
+		let mut ext = sp_io::TestExternalities::new_empty();
+		// NOTE: sadly events don't fulfill the requirements of hashmap or btreemap.
+		let mut events_histogram = Vec::<(PoolsEvents<T>, u32)>::default();
+		let mut iteration = 0 as BlockNumber;
+		let mut ok = 0;
+		let mut err = 0;
+
+		ext.execute_with(|| {
+			MaxPoolMembers::<T>::set(Some(10_000));
+			MaxPoolMembersPerPool::<T>::set(Some(1000));
+			MaxPools::<T>::set(Some(1_000));
+
+			MinCreateBond::<T>::set(10 * ExistentialDeposit::get());
+			MinJoinBond::<T>::set(5 * ExistentialDeposit::get());
+			System::set_block_number(1);
+		});
+
+		ExistentialDeposit::set(10u128.pow(12u32));
+		BondingDuration::set(8);
+
+		loop {
+			ext.execute_with(|| {
+				iteration += 1;
+				let (call, origin) = random_call(&mut rng);
+				let outcome = call.clone().dispatch_bypass_filter(origin.clone());
+
+				match outcome {
+					Ok(_) => ok += 1,
+					Err(_) => err += 1,
+				};
+
+				log!(
+					debug,
+					"iteration {}, call {:?}, origin {:?}, outcome: {:?}, so far {} ok {} err",
+					iteration,
+					call,
+					origin,
+					outcome,
+					ok,
+					err,
+				);
+
+				// possibly join the reward_agent
+				if iteration > ERA / 2 && BondedPools::<T>::count() > 0 {
+					reward_agent.join();
+				}
+				// and possibly roughly every 4 era, trigger payout for the agent. Doing this more
+				// frequent is also harmless.
+				if rng.gen_range(0..(4 * ERA)) == 0 {
+					reward_agent.claim_payout();
+				}
+
+				// execute sanity checks at a fixed interval, possibly on every block.
+				if iteration %
+					(std::env::var("SANITY_CHECK_INTERVAL")
+						.ok()
+						.and_then(|x| x.parse::<u64>().ok()))
+					.unwrap_or(1) == 0
+				{
+					log!(info, "running sanity checks at {}", iteration);
+					Pools::do_try_state(u8::MAX).unwrap();
+				}
+
+				// collect and reset events.
+				System::events()
+					.into_iter()
+					.map(|r| r.event)
+					.filter_map(
+						|e| if let mock::Event::Pools(inner) = e { Some(inner) } else { None },
+					)
+					.for_each(|e| {
+						if let Some((_, c)) = events_histogram
+							.iter_mut()
+							.find(|(x, _)| std::mem::discriminant(x) == std::mem::discriminant(&e))
+						{
+							*c += 1;
+						} else {
+							events_histogram.push((e, 1))
+						}
+					});
+				System::reset_events();
+
+				// trigger an era change, and check the status of the reward agent.
+				if iteration % ERA == 0 {
+					CurrentEra::mutate(|c| *c += 1);
+					BondedPools::<T>::iter().for_each(|(id, _)| {
+						let amount = random_ed_multiple(&mut rng);
+						let _ =
+							Balances::deposit_creating(&Pools::create_reward_account(id), amount);
+						// if we just paid out the reward agent, let's calculate how much we expect
+						// our reward agent to have earned.
+						if reward_agent.pool_id.map_or(false, |mid| mid == id) {
+							let all_points = BondedPool::<T>::get(id).map(|p| p.points).unwrap();
+							let member_points =
+								PoolMembers::<T>::get(reward_agent.who).map(|m| m.points).unwrap();
+							let agent_share = Perquintill::from_rational(member_points, all_points);
+							log::info!(
+								target: "reward-agent",
+								"🤖 REWARD = amount = {:?}, ratio: {:?}, share {:?}",
+								amount,
+								agent_share,
+								agent_share * amount,
+							);
+							reward_agent.expected_reward += agent_share * amount;
+						}
+					});
+
+					log!(
+						info,
+						"iteration {}, {} pools, {} members, {} ok {} err, events = {:?}",
+						iteration,
+						BondedPools::<T>::count(),
+						PoolMembers::<T>::count(),
+						ok,
+						err,
+						events_histogram
+							.iter()
+							.map(|(x, c)| (
+								format!("{:?}", x)
+									.split(" ")
+									.map(|x| x.to_string())
+									.collect::<Vec<_>>()
+									.first()
+									.cloned()
+									.unwrap(),
+								c,
+							))
+							.collect::<Vec<_>>(),
+					);
+				}
+			});
+		}
+	}
+}