Try to fix out of view statements (#5177)

This issue happens when some peer sends a good but already known Seconded statement and the statement-distribution code does not update the statements_received field in the peer_knowledge structure. Subsequently, a Valid statement causes out-of-view message that is incorrectly emitted and causes reputation lose.

This PR also introduces a concept of passing the specific pseudo-random generator to subsystems to make it easier to write deterministic tests. This functionality is not really necessary for the specific issue and unit test but it can be useful for other tests and subsystems.

polkadot/node/subsystem-util/src/lib.rs

@@ -55,6 +55,7 @@ use polkadot_primitives::v2::{
 	PersistedValidationData, SessionIndex, SessionInfo, Signed, SigningContext, ValidationCode,
 	ValidationCodeHash, ValidatorId, ValidatorIndex, ValidatorSignature,
 };
+pub use rand;
 use sp_application_crypto::AppKey;
 use sp_core::{traits::SpawnNamed, ByteArray};
 use sp_keystore::{CryptoStore, Error as KeystoreError, SyncCryptoStorePtr};
@@ -276,33 +277,41 @@ pub fn find_validator_group(
 
 /// Choose a random subset of `min` elements.
 /// But always include `is_priority` elements.
-pub fn choose_random_subset<T, F: FnMut(&T) -> bool>(
+pub fn choose_random_subset<T, F: FnMut(&T) -> bool>(is_priority: F, v: &mut Vec<T>, min: usize) {
+	choose_random_subset_with_rng(is_priority, v, &mut rand::thread_rng(), min)
+}
+
+/// Choose a random subset of `min` elements using a specific Random Generator `Rng`
+/// But always include `is_priority` elements.
+pub fn choose_random_subset_with_rng<T, F: FnMut(&T) -> bool, R: rand::Rng>(
 	is_priority: F,
-	mut v: Vec<T>,
+	v: &mut Vec<T>,
+	rng: &mut R,
 	min: usize,
-) -> Vec<T> {
+) {
 	use rand::seq::SliceRandom as _;
 
 	// partition the elements into priority first
 	// the returned index is when non_priority elements start
-	let i = itertools::partition(&mut v, is_priority);
+	let i = itertools::partition(v.iter_mut(), is_priority);
 
 	if i >= min || v.len() <= i {
 		v.truncate(i);
-		return v
+		return
 	}
 
-	let mut rng = rand::thread_rng();
-	v[i..].shuffle(&mut rng);
+	v[i..].shuffle(rng);
 
 	v.truncate(min);
-	v
 }
 
 /// Returns a `bool` with a probability of `a / b` of being true.
 pub fn gen_ratio(a: usize, b: usize) -> bool {
-	use rand::Rng as _;
-	let mut rng = rand::thread_rng();
+	gen_ratio_rng(a, b, &mut rand::thread_rng())
+}
+
+/// Returns a `bool` with a probability of `a / b` of being true.
+pub fn gen_ratio_rng<R: rand::Rng>(a: usize, b: usize, rng: &mut R) -> bool {
 	rng.gen_ratio(a as u32, b as u32)
 }
 

polkadot/node/subsystem-util/src/tests.rs

@@ -25,7 +25,7 @@ use polkadot_node_subsystem::{
 };
 use polkadot_node_subsystem_test_helpers::{self as test_helpers, make_subsystem_context};
 use polkadot_primitives::v2::Hash;
-use polkadot_primitives_test_helpers::{dummy_candidate_receipt, dummy_hash};
+use polkadot_primitives_test_helpers::{dummy_candidate_receipt, dummy_hash, AlwaysZeroRng};
 use std::{
 	pin::Pin,
 	sync::{
@@ -248,11 +248,23 @@ fn tick_tack_metronome() {
 
 #[test]
 fn subset_generation_check() {
-	let values = (0_u8..=25).collect::<Vec<_>>();
+	let mut values = (0_u8..=25).collect::<Vec<_>>();
 	// 12 even numbers exist
-	let mut chosen = choose_random_subset::<u8, _>(|v| v & 0x01 == 0, values, 12);
-	chosen.sort();
-	for (idx, v) in dbg!(chosen).into_iter().enumerate() {
+	choose_random_subset::<u8, _>(|v| v & 0x01 == 0, &mut values, 12);
+	values.sort();
+	for (idx, v) in dbg!(values).into_iter().enumerate() {
 		assert_eq!(v as usize, idx * 2);
 	}
 }
+
+#[test]
+fn subset_predefined_generation_check() {
+	let mut values = (0_u8..=25).collect::<Vec<_>>();
+	choose_random_subset_with_rng::<u8, _, _>(|_| false, &mut values, &mut AlwaysZeroRng, 12);
+	assert_eq!(values.len(), 12);
+	for (idx, v) in dbg!(values).into_iter().enumerate() {
+		// Since shuffle actually shuffles the indexes from 1..len, then
+		// our PRG that returns zeroes will shuffle 0 and 1, 1 and 2, ... len-2 and len-1
+		assert_eq!(v as usize, idx + 1);
+	}
+}