Delay reputation updates (#7214)

* Add futures-timer

* Make cost_or_benefit public

* Update ReportPeer message format

* Add delay to reputation updates (dirtywork)

* Update ReputationAggregator

* Update tests

* Fix flucky tests

* Move reputation to state

* Use the main loop for handling reputation sendings

* Update

* Move reputation to utils

* Update reputation sending

* Fix arguments order

* Update state

* Remove new from state

* Add constant

* Add failing test for delay

* Change mocking approach

* Fix type errors

* Fix comments

* Add message handling to select

* Fix bitfields-distribution tests

* Add docs to reputation aggregator

* Replace .into_base_rep

* Use one REPUTATION_CHANGE_INTERVAL by default

* Add reputation change to statement-distribution

* Update polkadot-availability-bitfield-distribution

* Update futures selecting in subsystems

* Update reputation adding

* Send malicious changes right away without adding to state

* Add reputation to StatementDistributionSubsystem

* Handle reputation in statement distribution

* Add delay test for polkadot-statement-distribution

* Fix collator-protocol tests before applying reputation delay

* Remove into_base_rep

* Add reputation to State

* Fix failed tests

* Add reputation delay

* Update tests

* Add batched network message for peer reporting

* Update approval-distribution tests

* Update bitfield-distribution tests

* Update statement-distribution tests

* Update collator-protocol tests

* Remove levels in matching

* Address clippy errors

* Fix overseer test

* Add a metric for original count of rep changes

* Update Reputation

* Revert "Add a metric for original count of rep changes"

This reverts commit 6c9b0c1ec34491d16e562bdcba8db6b9dcf484db.

* Update node/subsystem-util/src/reputation.rs

Co-authored-by: Vsevolod Stakhov <vsevolod.stakhov@parity.io>

* Remove redundant vec

---------

Co-authored-by: Vsevolod Stakhov <vsevolod.stakhov@parity.io>

polkadot/node/network/collator-protocol/src/collator_side/mod.rs

@@ -44,6 +44,7 @@ use polkadot_node_subsystem::{
 	overseer, FromOrchestra, OverseerSignal, PerLeafSpan,
 };
 use polkadot_node_subsystem_util::{
+	reputation::{ReputationAggregator, REPUTATION_CHANGE_INTERVAL},
 	runtime::{get_availability_cores, get_group_rotation_info, RuntimeInfo},
 	TimeoutExt,
 };
@@ -53,7 +54,10 @@ use polkadot_primitives::{
 };
 
 use super::LOG_TARGET;
-use crate::error::{log_error, Error, FatalError, Result};
+use crate::{
+	error::{log_error, Error, FatalError, Result},
+	modify_reputation,
+};
 use fatality::Split;
 
 mod metrics;
@@ -245,12 +249,20 @@ struct State {
 	///
 	/// Each future returns the relay parent of the finished collation fetch.
 	active_collation_fetches: ActiveCollationFetches,
+
+	/// Aggregated reputation change
+	reputation: ReputationAggregator,
 }
 
 impl State {
 	/// Creates a new `State` instance with the given parameters and setting all remaining
 	/// state fields to their default values (i.e. empty).
-	fn new(local_peer_id: PeerId, collator_pair: CollatorPair, metrics: Metrics) -> State {
+	fn new(
+		local_peer_id: PeerId,
+		collator_pair: CollatorPair,
+		metrics: Metrics,
+		reputation: ReputationAggregator,
+	) -> State {
 		State {
 			local_peer_id,
 			collator_pair,
@@ -267,6 +279,7 @@ impl State {
 			last_connected_at: None,
 			waiting_collation_fetches: Default::default(),
 			active_collation_fetches: Default::default(),
+			reputation,
 		}
 	}
 
@@ -707,7 +720,7 @@ async fn handle_incoming_peer_message<Context>(
 				"AdvertiseCollation message is not expected on the collator side of the protocol",
 			);
 
-			ctx.send_message(NetworkBridgeTxMessage::ReportPeer(origin, COST_UNEXPECTED_MESSAGE))
+			modify_reputation(&mut state.reputation, ctx.sender(), origin, COST_UNEXPECTED_MESSAGE)
 				.await;
 
 			// If we are advertised to, this is another collator, and we should disconnect.
@@ -794,8 +807,13 @@ async fn handle_incoming_request<Context>(
 					target: LOG_TARGET,
 					"Dropping incoming request as peer has a request in flight already."
 				);
-				ctx.send_message(NetworkBridgeTxMessage::ReportPeer(req.peer, COST_APPARENT_FLOOD))
-					.await;
+				modify_reputation(
+					&mut state.reputation,
+					ctx.sender(),
+					req.peer,
+					COST_APPARENT_FLOOD.into(),
+				)
+				.await;
 				return Ok(())
 			}
 
@@ -940,15 +958,40 @@ async fn handle_our_view_change(state: &mut State, view: OurView) -> Result<()>
 /// The collator protocol collator side main loop.
 #[overseer::contextbounds(CollatorProtocol, prefix = crate::overseer)]
 pub(crate) async fn run<Context>(
+	ctx: Context,
+	local_peer_id: PeerId,
+	collator_pair: CollatorPair,
+	req_receiver: IncomingRequestReceiver<request_v1::CollationFetchingRequest>,
+	metrics: Metrics,
+) -> std::result::Result<(), FatalError> {
+	run_inner(
+		ctx,
+		local_peer_id,
+		collator_pair,
+		req_receiver,
+		metrics,
+		ReputationAggregator::default(),
+		REPUTATION_CHANGE_INTERVAL,
+	)
+	.await
+}
+
+#[overseer::contextbounds(CollatorProtocol, prefix = crate::overseer)]
+async fn run_inner<Context>(
 	mut ctx: Context,
 	local_peer_id: PeerId,
 	collator_pair: CollatorPair,
 	mut req_receiver: IncomingRequestReceiver<request_v1::CollationFetchingRequest>,
 	metrics: Metrics,
+	reputation: ReputationAggregator,
+	reputation_interval: Duration,
 ) -> std::result::Result<(), FatalError> {
 	use OverseerSignal::*;
 
-	let mut state = State::new(local_peer_id, collator_pair, metrics);
+	let new_reputation_delay = || futures_timer::Delay::new(reputation_interval).fuse();
+	let mut reputation_delay = new_reputation_delay();
+
+	let mut state = State::new(local_peer_id, collator_pair, metrics, reputation);
 	let mut runtime = RuntimeInfo::new(None);
 
 	let reconnect_stream = super::tick_stream(RECONNECT_POLL);
@@ -958,6 +1001,10 @@ pub(crate) async fn run<Context>(
 		let recv_req = req_receiver.recv(|| vec![COST_INVALID_REQUEST]).fuse();
 		pin_mut!(recv_req);
 		select! {
+			_ = reputation_delay => {
+				state.reputation.send(ctx.sender()).await;
+				reputation_delay = new_reputation_delay();
+			},
 			msg = ctx.recv().fuse() => match msg.map_err(FatalError::SubsystemReceive)? {
 				FromOrchestra::Communication { msg } => {
 					log_error(

polkadot/node/network/collator-protocol/src/lib.rs

@@ -28,6 +28,7 @@ use futures::{
 	FutureExt, TryFutureExt,
 };
 
+use polkadot_node_subsystem_util::reputation::ReputationAggregator;
 use sp_keystore::KeystorePtr;
 
 use polkadot_node_network_protocol::{
@@ -36,9 +37,7 @@ use polkadot_node_network_protocol::{
 };
 use polkadot_primitives::CollatorPair;
 
-use polkadot_node_subsystem::{
-	errors::SubsystemError, messages::NetworkBridgeTxMessage, overseer, SpawnedSubsystem,
-};
+use polkadot_node_subsystem::{errors::SubsystemError, overseer, SpawnedSubsystem};
 
 mod error;
 
@@ -124,6 +123,7 @@ impl<Context> CollatorProtocolSubsystem {
 
 /// Modify the reputation of a peer based on its behavior.
 async fn modify_reputation(
+	reputation: &mut ReputationAggregator,
 	sender: &mut impl overseer::CollatorProtocolSenderTrait,
 	peer: PeerId,
 	rep: Rep,
@@ -135,7 +135,7 @@ async fn modify_reputation(
 		"reputation change for peer",
 	);
 
-	sender.send_message(NetworkBridgeTxMessage::ReportPeer(peer, rep)).await;
+	reputation.modify(sender, peer, rep).await;
 }
 
 /// Wait until tick and return the timestamp for the following one.

polkadot/node/network/collator-protocol/src/validator_side/mod.rs

@@ -52,7 +52,10 @@ use polkadot_node_subsystem::{
 	},
 	overseer, FromOrchestra, OverseerSignal, PerLeafSpan, SubsystemSender,
 };
-use polkadot_node_subsystem_util::metrics::{self, prometheus};
+use polkadot_node_subsystem_util::{
+	metrics::{self, prometheus},
+	reputation::{ReputationAggregator, REPUTATION_CHANGE_INTERVAL},
+};
 use polkadot_primitives::{CandidateReceipt, CollatorId, Hash, Id as ParaId};
 
 use crate::error::Result;
@@ -612,6 +615,9 @@ struct State {
 
 	/// Keep track of all pending candidate collations
 	pending_candidates: HashMap<Hash, CollationEvent>,
+
+	/// Aggregated reputation change
+	reputation: ReputationAggregator,
 }
 
 // O(n) search for collator ID by iterating through the peers map. This should be fast enough
@@ -675,28 +681,31 @@ async fn fetch_collation(
 
 /// Report a collator for some malicious actions.
 async fn report_collator(
+	reputation: &mut ReputationAggregator,
 	sender: &mut impl overseer::CollatorProtocolSenderTrait,
 	peer_data: &HashMap<PeerId, PeerData>,
 	id: CollatorId,
 ) {
 	if let Some(peer_id) = collator_peer_id(peer_data, &id) {
-		modify_reputation(sender, peer_id, COST_REPORT_BAD).await;
+		modify_reputation(reputation, sender, peer_id, COST_REPORT_BAD).await;
 	}
 }
 
 /// Some other subsystem has reported a collator as a good one, bump reputation.
 async fn note_good_collation(
+	reputation: &mut ReputationAggregator,
 	sender: &mut impl overseer::CollatorProtocolSenderTrait,
 	peer_data: &HashMap<PeerId, PeerData>,
 	id: CollatorId,
 ) {
 	if let Some(peer_id) = collator_peer_id(peer_data, &id) {
-		modify_reputation(sender, peer_id, BENEFIT_NOTIFY_GOOD).await;
+		modify_reputation(reputation, sender, peer_id, BENEFIT_NOTIFY_GOOD).await;
 	}
 }
 
 /// Notify a collator that its collation got seconded.
 async fn notify_collation_seconded(
+	reputation: &mut ReputationAggregator,
 	sender: &mut impl overseer::CollatorProtocolSenderTrait,
 	peer_id: PeerId,
 	relay_parent: Hash,
@@ -711,7 +720,7 @@ async fn notify_collation_seconded(
 		))
 		.await;
 
-	modify_reputation(sender, peer_id, BENEFIT_NOTIFY_GOOD).await;
+	modify_reputation(reputation, sender, peer_id, BENEFIT_NOTIFY_GOOD).await;
 }
 
 /// A peer's view has changed. A number of things should be done:
@@ -813,7 +822,13 @@ async fn process_incoming_peer_message<Context>(
 	match msg {
 		Declare(collator_id, para_id, signature) => {
 			if collator_peer_id(&state.peer_data, &collator_id).is_some() {
-				modify_reputation(ctx.sender(), origin, COST_UNEXPECTED_MESSAGE).await;
+				modify_reputation(
+					&mut state.reputation,
+					ctx.sender(),
+					origin,
+					COST_UNEXPECTED_MESSAGE,
+				)
+				.await;
 				return
 			}
 
@@ -826,7 +841,13 @@ async fn process_incoming_peer_message<Context>(
 						?para_id,
 						"Unknown peer",
 					);
-					modify_reputation(ctx.sender(), origin, COST_UNEXPECTED_MESSAGE).await;
+					modify_reputation(
+						&mut state.reputation,
+						ctx.sender(),
+						origin,
+						COST_UNEXPECTED_MESSAGE,
+					)
+					.await;
 					return
 				},
 			};
@@ -838,7 +859,13 @@ async fn process_incoming_peer_message<Context>(
 					?para_id,
 					"Peer is not in the collating state",
 				);
-				modify_reputation(ctx.sender(), origin, COST_UNEXPECTED_MESSAGE).await;
+				modify_reputation(
+					&mut state.reputation,
+					ctx.sender(),
+					origin,
+					COST_UNEXPECTED_MESSAGE,
+				)
+				.await;
 				return
 			}
 
@@ -849,7 +876,13 @@ async fn process_incoming_peer_message<Context>(
 					?para_id,
 					"Signature verification failure",
 				);
-				modify_reputation(ctx.sender(), origin, COST_INVALID_SIGNATURE).await;
+				modify_reputation(
+					&mut state.reputation,
+					ctx.sender(),
+					origin,
+					COST_INVALID_SIGNATURE,
+				)
+				.await;
 				return
 			}
 
@@ -872,7 +905,13 @@ async fn process_incoming_peer_message<Context>(
 					"Declared as collator for unneeded para",
 				);
 
-				modify_reputation(ctx.sender(), origin, COST_UNNEEDED_COLLATOR).await;
+				modify_reputation(
+					&mut state.reputation,
+					ctx.sender(),
+					origin,
+					COST_UNNEEDED_COLLATOR,
+				)
+				.await;
 				gum::trace!(target: LOG_TARGET, "Disconnecting unneeded collator");
 				disconnect_peer(ctx.sender(), origin).await;
 			}
@@ -890,7 +929,13 @@ async fn process_incoming_peer_message<Context>(
 					"Advertise collation out of view",
 				);
 
-				modify_reputation(ctx.sender(), origin, COST_UNEXPECTED_MESSAGE).await;
+				modify_reputation(
+					&mut state.reputation,
+					ctx.sender(),
+					origin,
+					COST_UNEXPECTED_MESSAGE,
+				)
+				.await;
 				return
 			}
 
@@ -902,7 +947,13 @@ async fn process_incoming_peer_message<Context>(
 						?relay_parent,
 						"Advertise collation message has been received from an unknown peer",
 					);
-					modify_reputation(ctx.sender(), origin, COST_UNEXPECTED_MESSAGE).await;
+					modify_reputation(
+						&mut state.reputation,
+						ctx.sender(),
+						origin,
+						COST_UNEXPECTED_MESSAGE,
+					)
+					.await;
 					return
 				},
 				Some(p) => p,
@@ -961,7 +1012,13 @@ async fn process_incoming_peer_message<Context>(
 						"Invalid advertisement",
 					);
 
-					modify_reputation(ctx.sender(), origin, COST_UNEXPECTED_MESSAGE).await;
+					modify_reputation(
+						&mut state.reputation,
+						ctx.sender(),
+						origin,
+						COST_UNEXPECTED_MESSAGE,
+					)
+					.await;
 				},
 			}
 		},
@@ -1106,7 +1163,7 @@ async fn process_msg<Context>(
 			);
 		},
 		ReportCollator(id) => {
-			report_collator(ctx.sender(), &state.peer_data, id).await;
+			report_collator(&mut state.reputation, ctx.sender(), &state.peer_data, id).await;
 		},
 		NetworkBridgeUpdate(event) => {
 			if let Err(e) = handle_network_msg(ctx, state, keystore, event).await {
@@ -1121,8 +1178,21 @@ async fn process_msg<Context>(
 			if let Some(collation_event) = state.pending_candidates.remove(&parent) {
 				let (collator_id, pending_collation) = collation_event;
 				let PendingCollation { relay_parent, peer_id, .. } = pending_collation;
-				note_good_collation(ctx.sender(), &state.peer_data, collator_id).await;
-				notify_collation_seconded(ctx.sender(), peer_id, relay_parent, stmt).await;
+				note_good_collation(
+					&mut state.reputation,
+					ctx.sender(),
+					&state.peer_data,
+					collator_id,
+				)
+				.await;
+				notify_collation_seconded(
+					&mut state.reputation,
+					ctx.sender(),
+					peer_id,
+					relay_parent,
+					stmt,
+				)
+				.await;
 
 				if let Some(collations) = state.collations_per_relay_parent.get_mut(&parent) {
 					collations.status = CollationStatus::Seconded;
@@ -1153,7 +1223,8 @@ async fn process_msg<Context>(
 				Entry::Vacant(_) => return,
 			};
 
-			report_collator(ctx.sender(), &state.peer_data, id.clone()).await;
+			report_collator(&mut state.reputation, ctx.sender(), &state.peer_data, id.clone())
+				.await;
 
 			dequeue_next_collation_and_fetch(ctx, state, parent, id).await;
 		},
@@ -1163,12 +1234,35 @@ async fn process_msg<Context>(
 /// The main run loop.
 #[overseer::contextbounds(CollatorProtocol, prefix = self::overseer)]
 pub(crate) async fn run<Context>(
-	mut ctx: Context,
+	ctx: Context,
 	keystore: KeystorePtr,
 	eviction_policy: crate::CollatorEvictionPolicy,
 	metrics: Metrics,
 ) -> std::result::Result<(), crate::error::FatalError> {
-	let mut state = State { metrics, ..Default::default() };
+	run_inner(
+		ctx,
+		keystore,
+		eviction_policy,
+		metrics,
+		ReputationAggregator::default(),
+		REPUTATION_CHANGE_INTERVAL,
+	)
+	.await
+}
+
+#[overseer::contextbounds(CollatorProtocol, prefix = self::overseer)]
+async fn run_inner<Context>(
+	mut ctx: Context,
+	keystore: KeystorePtr,
+	eviction_policy: crate::CollatorEvictionPolicy,
+	metrics: Metrics,
+	reputation: ReputationAggregator,
+	reputation_interval: Duration,
+) -> std::result::Result<(), crate::error::FatalError> {
+	let new_reputation_delay = || futures_timer::Delay::new(reputation_interval).fuse();
+	let mut reputation_delay = new_reputation_delay();
+
+	let mut state = State { metrics, reputation, ..Default::default() };
 
 	let next_inactivity_stream = tick_stream(ACTIVITY_POLL);
 	futures::pin_mut!(next_inactivity_stream);
@@ -1178,6 +1272,10 @@ pub(crate) async fn run<Context>(
 
 	loop {
 		select! {
+			_ = reputation_delay => {
+				state.reputation.send(ctx.sender()).await;
+				reputation_delay = new_reputation_delay();
+			},
 			res = ctx.recv().fuse() => {
 				match res {
 					Ok(FromOrchestra::Communication { msg }) => {
@@ -1217,7 +1315,7 @@ pub(crate) async fn run<Context>(
 				).await;
 
 				for (peer_id, rep) in reputation_changes {
-					modify_reputation(ctx.sender(), peer_id, rep).await;
+					modify_reputation(&mut state.reputation,ctx.sender(), peer_id, rep).await;
 				}
 			},
 		}

polkadot/node/network/collator-protocol/src/validator_side/tests.rs

@@ -29,9 +29,11 @@ use polkadot_node_network_protocol::{
 	ObservedRole,
 };
 use polkadot_node_primitives::BlockData;
-use polkadot_node_subsystem::messages::{AllMessages, RuntimeApiMessage, RuntimeApiRequest};
+use polkadot_node_subsystem::messages::{
+	AllMessages, ReportPeerMessage, RuntimeApiMessage, RuntimeApiRequest,
+};
 use polkadot_node_subsystem_test_helpers as test_helpers;
-use polkadot_node_subsystem_util::TimeoutExt;
+use polkadot_node_subsystem_util::{reputation::add_reputation, TimeoutExt};
 use polkadot_primitives::{
 	CollatorPair, CoreState, GroupIndex, GroupRotationInfo, OccupiedCore, ScheduledCore,
 	ValidatorId, ValidatorIndex,
@@ -42,6 +44,7 @@ use polkadot_primitives_test_helpers::{
 
 const ACTIVITY_TIMEOUT: Duration = Duration::from_millis(500);
 const DECLARE_TIMEOUT: Duration = Duration::from_millis(25);
+const REPUTATION_CHANGE_TEST_INTERVAL: Duration = Duration::from_millis(10);
 
 #[derive(Clone)]
 struct TestState {
@@ -119,7 +122,10 @@ struct TestHarness {
 	virtual_overseer: VirtualOverseer,
 }
 
-fn test_harness<T: Future<Output = VirtualOverseer>>(test: impl FnOnce(TestHarness) -> T) {
+fn test_harness<T: Future<Output = VirtualOverseer>>(
+	reputation: ReputationAggregator,
+	test: impl FnOnce(TestHarness) -> T,
+) {
 	let _ = env_logger::builder()
 		.is_test(true)
 		.filter(Some("polkadot_collator_protocol"), log::LevelFilter::Trace)
@@ -138,7 +144,7 @@ fn test_harness<T: Future<Output = VirtualOverseer>>(test: impl FnOnce(TestHarne
 		)
 		.unwrap();
 
-	let subsystem = run(
+	let subsystem = run_inner(
 		context,
 		Arc::new(keystore),
 		crate::CollatorEvictionPolicy {
@@ -146,6 +152,8 @@ fn test_harness<T: Future<Output = VirtualOverseer>>(test: impl FnOnce(TestHarne
 			undeclared: DECLARE_TIMEOUT,
 		},
 		Metrics::default(),
+		reputation,
+		REPUTATION_CHANGE_TEST_INTERVAL,
 	);
 
 	let test_fut = test(TestHarness { virtual_overseer });
@@ -348,7 +356,7 @@ async fn advertise_collation(
 fn act_on_advertisement() {
 	let test_state = TestState::default();
 
-	test_harness(|test_harness| async move {
+	test_harness(ReputationAggregator::new(|_| true), |test_harness| async move {
 		let TestHarness { mut virtual_overseer } = test_harness;
 
 		let pair = CollatorPair::generate().0;
@@ -392,7 +400,7 @@ fn act_on_advertisement() {
 fn collator_reporting_works() {
 	let test_state = TestState::default();
 
-	test_harness(|test_harness| async move {
+	test_harness(ReputationAggregator::new(|_| true), |test_harness| async move {
 		let TestHarness { mut virtual_overseer } = test_harness;
 
 		overseer_send(
@@ -433,10 +441,10 @@ fn collator_reporting_works() {
 		assert_matches!(
 			overseer_recv(&mut virtual_overseer).await,
 			AllMessages::NetworkBridgeTx(
-				NetworkBridgeTxMessage::ReportPeer(peer, rep),
+				NetworkBridgeTxMessage::ReportPeer(ReportPeerMessage::Single(peer, rep)),
 			) => {
 				assert_eq!(peer, peer_b);
-				assert_eq!(rep, COST_REPORT_BAD);
+				assert_eq!(rep.value, COST_REPORT_BAD.cost_or_benefit());
 			}
 		);
 
@@ -449,7 +457,7 @@ fn collator_reporting_works() {
 fn collator_authentication_verification_works() {
 	let test_state = TestState::default();
 
-	test_harness(|test_harness| async move {
+	test_harness(ReputationAggregator::new(|_| true), |test_harness| async move {
 		let TestHarness { mut virtual_overseer } = test_harness;
 
 		let peer_b = PeerId::random();
@@ -483,10 +491,10 @@ fn collator_authentication_verification_works() {
 		assert_matches!(
 			overseer_recv(&mut virtual_overseer).await,
 			AllMessages::NetworkBridgeTx(
-				NetworkBridgeTxMessage::ReportPeer(peer, rep),
+				NetworkBridgeTxMessage::ReportPeer(ReportPeerMessage::Single(peer, rep)),
 			) => {
 				assert_eq!(peer, peer_b);
-				assert_eq!(rep, COST_INVALID_SIGNATURE);
+				assert_eq!(rep.value, COST_INVALID_SIGNATURE.cost_or_benefit());
 			}
 		);
 		virtual_overseer
@@ -500,7 +508,7 @@ fn collator_authentication_verification_works() {
 fn fetch_one_collation_at_a_time() {
 	let test_state = TestState::default();
 
-	test_harness(|test_harness| async move {
+	test_harness(ReputationAggregator::new(|_| true), |test_harness| async move {
 		let TestHarness { mut virtual_overseer } = test_harness;
 
 		let second = Hash::random();
@@ -585,7 +593,7 @@ fn fetch_one_collation_at_a_time() {
 fn fetches_next_collation() {
 	let test_state = TestState::default();
 
-	test_harness(|test_harness| async move {
+	test_harness(ReputationAggregator::new(|_| true), |test_harness| async move {
 		let TestHarness { mut virtual_overseer } = test_harness;
 
 		let second = Hash::random();
@@ -683,7 +691,7 @@ fn fetches_next_collation() {
 fn reject_connection_to_next_group() {
 	let test_state = TestState::default();
 
-	test_harness(|test_harness| async move {
+	test_harness(ReputationAggregator::new(|_| true), |test_harness| async move {
 		let TestHarness { mut virtual_overseer } = test_harness;
 
 		overseer_send(
@@ -709,11 +717,10 @@ fn reject_connection_to_next_group() {
 		assert_matches!(
 			overseer_recv(&mut virtual_overseer).await,
 			AllMessages::NetworkBridgeTx(NetworkBridgeTxMessage::ReportPeer(
-				peer,
-				rep,
+				ReportPeerMessage::Single(peer, rep),
 			)) => {
 				assert_eq!(peer, peer_b);
-				assert_eq!(rep, COST_UNNEEDED_COLLATOR);
+				assert_eq!(rep.value, COST_UNNEEDED_COLLATOR.cost_or_benefit());
 			}
 		);
 
@@ -728,7 +735,7 @@ fn reject_connection_to_next_group() {
 fn fetch_next_collation_on_invalid_collation() {
 	let test_state = TestState::default();
 
-	test_harness(|test_harness| async move {
+	test_harness(ReputationAggregator::new(|_| true), |test_harness| async move {
 		let TestHarness { mut virtual_overseer } = test_harness;
 
 		let second = Hash::random();
@@ -802,11 +809,10 @@ fn fetch_next_collation_on_invalid_collation() {
 		assert_matches!(
 			overseer_recv(&mut virtual_overseer).await,
 			AllMessages::NetworkBridgeTx(NetworkBridgeTxMessage::ReportPeer(
-				peer,
-				rep,
+				ReportPeerMessage::Single(peer, rep),
 			)) => {
 				assert_eq!(peer, peer_b);
-				assert_eq!(rep, COST_REPORT_BAD);
+				assert_eq!(rep.value, COST_REPORT_BAD.cost_or_benefit());
 			}
 		);
 
@@ -826,7 +832,7 @@ fn fetch_next_collation_on_invalid_collation() {
 fn inactive_disconnected() {
 	let test_state = TestState::default();
 
-	test_harness(|test_harness| async move {
+	test_harness(ReputationAggregator::new(|_| true), |test_harness| async move {
 		let TestHarness { mut virtual_overseer } = test_harness;
 
 		let pair = CollatorPair::generate().0;
@@ -872,7 +878,7 @@ fn inactive_disconnected() {
 fn activity_extends_life() {
 	let test_state = TestState::default();
 
-	test_harness(|test_harness| async move {
+	test_harness(ReputationAggregator::new(|_| true), |test_harness| async move {
 		let TestHarness { mut virtual_overseer } = test_harness;
 
 		let pair = CollatorPair::generate().0;
@@ -937,7 +943,7 @@ fn activity_extends_life() {
 fn disconnect_if_no_declare() {
 	let test_state = TestState::default();
 
-	test_harness(|test_harness| async move {
+	test_harness(ReputationAggregator::new(|_| true), |test_harness| async move {
 		let TestHarness { mut virtual_overseer } = test_harness;
 
 		overseer_send(
@@ -973,7 +979,7 @@ fn disconnect_if_no_declare() {
 fn disconnect_if_wrong_declare() {
 	let test_state = TestState::default();
 
-	test_harness(|test_harness| async move {
+	test_harness(ReputationAggregator::new(|_| true), |test_harness| async move {
 		let TestHarness { mut virtual_overseer } = test_harness;
 
 		let pair = CollatorPair::generate().0;
@@ -1017,11 +1023,10 @@ fn disconnect_if_wrong_declare() {
 		assert_matches!(
 			overseer_recv(&mut virtual_overseer).await,
 			AllMessages::NetworkBridgeTx(NetworkBridgeTxMessage::ReportPeer(
-				peer,
-				rep,
+				ReportPeerMessage::Single(peer, rep),
 			)) => {
 				assert_eq!(peer, peer_b);
-				assert_eq!(rep, COST_UNNEEDED_COLLATOR);
+				assert_eq!(rep.value, COST_UNNEEDED_COLLATOR.cost_or_benefit());
 			}
 		);
 
@@ -1031,11 +1036,96 @@ fn disconnect_if_wrong_declare() {
 	})
 }
 
+#[test]
+fn delay_reputation_change() {
+	let test_state = TestState::default();
+
+	test_harness(ReputationAggregator::new(|_| false), |test_harness| async move {
+		let TestHarness { mut virtual_overseer } = test_harness;
+
+		let pair = CollatorPair::generate().0;
+
+		overseer_send(
+			&mut virtual_overseer,
+			CollatorProtocolMessage::NetworkBridgeUpdate(NetworkBridgeEvent::OurViewChange(
+				our_view![test_state.relay_parent],
+			)),
+		)
+		.await;
+
+		respond_to_core_info_queries(&mut virtual_overseer, &test_state).await;
+
+		let peer_b = PeerId::random();
+
+		overseer_send(
+			&mut virtual_overseer,
+			CollatorProtocolMessage::NetworkBridgeUpdate(NetworkBridgeEvent::PeerConnected(
+				peer_b.clone(),
+				ObservedRole::Full,
+				CollationVersion::V1.into(),
+				None,
+			)),
+		)
+		.await;
+
+		overseer_send(
+			&mut virtual_overseer,
+			CollatorProtocolMessage::NetworkBridgeUpdate(NetworkBridgeEvent::PeerMessage(
+				peer_b.clone(),
+				Versioned::V1(protocol_v1::CollatorProtocolMessage::Declare(
+					pair.public(),
+					ParaId::from(69),
+					pair.sign(&protocol_v1::declare_signature_payload(&peer_b)),
+				)),
+			)),
+		)
+		.await;
+
+		overseer_send(
+			&mut virtual_overseer,
+			CollatorProtocolMessage::NetworkBridgeUpdate(NetworkBridgeEvent::PeerMessage(
+				peer_b.clone(),
+				Versioned::V1(protocol_v1::CollatorProtocolMessage::Declare(
+					pair.public(),
+					ParaId::from(69),
+					pair.sign(&protocol_v1::declare_signature_payload(&peer_b)),
+				)),
+			)),
+		)
+		.await;
+
+		// Wait enough to fire reputation delay
+		futures_timer::Delay::new(REPUTATION_CHANGE_TEST_INTERVAL).await;
+
+		loop {
+			match overseer_recv(&mut virtual_overseer).await {
+				AllMessages::NetworkBridgeTx(NetworkBridgeTxMessage::DisconnectPeer(_, _)) => {
+					gum::trace!("`Disconnecting inactive peer` message skipped");
+					continue
+				},
+				AllMessages::NetworkBridgeTx(NetworkBridgeTxMessage::ReportPeer(
+					ReportPeerMessage::Batch(v),
+				)) => {
+					let mut expected_change = HashMap::new();
+					for rep in vec![COST_UNNEEDED_COLLATOR, COST_UNNEEDED_COLLATOR] {
+						add_reputation(&mut expected_change, peer_b, rep);
+					}
+					assert_eq!(v, expected_change);
+					break
+				},
+				_ => panic!("Message should be either `DisconnectPeer` or `ReportPeer`"),
+			}
+		}
+
+		virtual_overseer
+	})
+}
+
 #[test]
 fn view_change_clears_old_collators() {
 	let mut test_state = TestState::default();
 
-	test_harness(|test_harness| async move {
+	test_harness(ReputationAggregator::new(|_| true), |test_harness| async move {
 		let TestHarness { mut virtual_overseer } = test_harness;
 
 		let pair = CollatorPair::generate().0;