Use a more typesafe approach for managing indexed data (#6150)

* Fix for issue #2403

* Nightly fmt

* Quick documentation fixes

* Default Implementation

* iter() function integrated

* Implemented iter functionalities

* Fmt

* small change

* updates node-network

* updates in dispute-coordinator

* Updates

* benchmarking fix

* minor fix

* test fixes in runtime api

* Update primitives/src/v2/mod.rs

Co-authored-by: Andronik <write@reusable.software>

* Update primitives/src/v2/mod.rs

Co-authored-by: Andronik <write@reusable.software>

* Update primitives/src/v2/mod.rs

Co-authored-by: Andronik <write@reusable.software>

* Update primitives/src/v2/mod.rs

Co-authored-by: Andronik <write@reusable.software>

* Update primitives/src/v2/mod.rs

Co-authored-by: Andronik <write@reusable.software>

* Removal of [index], shorting of FromIterator, Renaming of GroupValidators to ValidatorGroups

* Removal of ops import

* documentation fixes for spell check

* implementation of generic type

* Refactoring

* Test and documentation fixes

* minor test fix

* minor test fix

* minor test fix

* Update node/network/statement-distribution/src/lib.rs

Co-authored-by: Andronik <write@reusable.software>

* Update primitives/src/v2/mod.rs

Co-authored-by: Andronik <write@reusable.software>

* Update primitives/src/v2/mod.rs

Co-authored-by: Andronik <write@reusable.software>

* removed IterMut

* Update node/core/dispute-coordinator/src/import.rs

Co-authored-by: Andronik <write@reusable.software>

* Update node/core/dispute-coordinator/src/initialized.rs

Co-authored-by: Andronik <write@reusable.software>

* Update primitives/src/v2/mod.rs

Co-authored-by: Andronik <write@reusable.software>

* fmt

* IterMut

* documentation update

Co-authored-by: Andronik <write@reusable.software>

* minor adjustments and new TypeIndex trait

* spelling fix

* TypeIndex fix

Co-authored-by: Andronik <write@reusable.software>

polkadot/node/network/availability-distribution/src/tests/mock.rs

@@ -24,8 +24,8 @@ use polkadot_erasure_coding::{branches, obtain_chunks_v1 as obtain_chunks};
 use polkadot_node_primitives::{AvailableData, BlockData, ErasureChunk, PoV, Proof};
 use polkadot_primitives::v2::{
 	CandidateCommitments, CandidateDescriptor, CandidateHash, CommittedCandidateReceipt,
-	GroupIndex, Hash, HeadData, Id as ParaId, OccupiedCore, PersistedValidationData, SessionInfo,
-	ValidatorIndex,
+	GroupIndex, Hash, HeadData, Id as ParaId, IndexedVec, OccupiedCore, PersistedValidationData,
+	SessionInfo, ValidatorIndex,
 };
 use polkadot_primitives_test_helpers::{
 	dummy_collator, dummy_collator_signature, dummy_hash, dummy_validation_code,
@@ -43,10 +43,11 @@ pub fn make_session_info() -> SessionInfo {
 		Sr25519Keyring::One,
 	];
 
-	let validator_groups: Vec<Vec<ValidatorIndex>> = [vec![5, 0, 3], vec![1, 6, 2, 4]]
-		.iter()
-		.map(|g| g.into_iter().map(|v| ValidatorIndex(*v)).collect())
-		.collect();
+	let validator_groups: IndexedVec<GroupIndex, Vec<ValidatorIndex>> =
+		[vec![5, 0, 3], vec![1, 6, 2, 4]]
+			.iter()
+			.map(|g| g.into_iter().map(|v| ValidatorIndex(*v)).collect())
+			.collect();
 
 	SessionInfo {
 		discovery_keys: validators.iter().map(|k| k.public().into()).collect(),

polkadot/node/network/availability-recovery/src/lib.rs

@@ -58,7 +58,7 @@ use polkadot_node_subsystem::{
 use polkadot_node_subsystem_util::request_session_info;
 use polkadot_primitives::v2::{
 	AuthorityDiscoveryId, BlakeTwo256, BlockNumber, CandidateHash, CandidateReceipt, GroupIndex,
-	Hash, HashT, SessionIndex, SessionInfo, ValidatorId, ValidatorIndex,
+	Hash, HashT, IndexedVec, SessionIndex, SessionInfo, ValidatorId, ValidatorIndex,
 };
 
 mod error;
@@ -134,7 +134,7 @@ struct RecoveryParams {
 	validator_authority_keys: Vec<AuthorityDiscoveryId>,
 
 	/// Validators relevant to this `RecoveryTask`.
-	validators: Vec<ValidatorId>,
+	validators: IndexedVec<ValidatorIndex, ValidatorId>,
 
 	/// The number of pieces needed.
 	threshold: usize,
@@ -871,7 +871,7 @@ async fn launch_recovery_task<Context>(
 	};
 
 	let phase = backing_group
-		.and_then(|g| session_info.validator_groups.get(g.0 as usize))
+		.and_then(|g| session_info.validator_groups.get(g))
 		.map(|group| Source::RequestFromBackers(RequestFromBackers::new(group.clone())))
 		.unwrap_or_else(|| {
 			Source::RequestChunks(RequestChunksFromValidators::new(params.validators.len() as _))

polkadot/node/network/availability-recovery/src/tests.rs

@@ -36,7 +36,9 @@ use polkadot_node_subsystem::{
 };
 use polkadot_node_subsystem_test_helpers::{make_subsystem_context, TestSubsystemContextHandle};
 use polkadot_node_subsystem_util::TimeoutExt;
-use polkadot_primitives::v2::{AuthorityDiscoveryId, Hash, HeadData, PersistedValidationData};
+use polkadot_primitives::v2::{
+	AuthorityDiscoveryId, Hash, HeadData, IndexedVec, PersistedValidationData, ValidatorId,
+};
 use polkadot_primitives_test_helpers::{dummy_candidate_receipt, dummy_hash};
 
 type VirtualOverseer = TestSubsystemContextHandle<AvailabilityRecoveryMessage>;
@@ -179,7 +181,7 @@ impl Has {
 #[derive(Clone)]
 struct TestState {
 	validators: Vec<Sr25519Keyring>,
-	validator_public: Vec<ValidatorId>,
+	validator_public: IndexedVec<ValidatorIndex, ValidatorId>,
 	validator_authority_id: Vec<AuthorityDiscoveryId>,
 	current: Hash,
 	candidate: CandidateReceipt,
@@ -218,7 +220,7 @@ impl TestState {
 					validators: self.validator_public.clone(),
 					discovery_keys: self.validator_authority_id.clone(),
 					// all validators in the same group.
-					validator_groups: vec![(0..self.validators.len()).map(|i| ValidatorIndex(i as _)).collect()],
+					validator_groups: IndexedVec::<GroupIndex,Vec<ValidatorIndex>>::from(vec![(0..self.validators.len()).map(|i| ValidatorIndex(i as _)).collect()]),
 					assignment_keys: vec![],
 					n_cores: 0,
 					zeroth_delay_tranche_width: 0,
@@ -402,7 +404,7 @@ impl TestState {
 	}
 }
 
-fn validator_pubkeys(val_ids: &[Sr25519Keyring]) -> Vec<ValidatorId> {
+fn validator_pubkeys(val_ids: &[Sr25519Keyring]) -> IndexedVec<ValidatorIndex, ValidatorId> {
 	val_ids.iter().map(|v| v.public().into()).collect()
 }
 

polkadot/node/network/collator-protocol/src/collator_side/mod.rs

@@ -448,10 +448,8 @@ async fn determine_our_validators<Context>(
 	let rotation_info = get_group_rotation_info(ctx.sender(), relay_parent).await?;
 
 	let current_group_index = rotation_info.group_for_core(core_index, cores);
-	let current_validators = groups
-		.get(current_group_index.0 as usize)
-		.map(|v| v.as_slice())
-		.unwrap_or_default();
+	let current_validators =
+		groups.get(current_group_index).map(|v| v.as_slice()).unwrap_or_default();
 
 	let validators = &info.discovery_keys;
 

polkadot/node/network/collator-protocol/src/collator_side/tests.rs

@@ -44,8 +44,8 @@ use polkadot_node_subsystem::{
 use polkadot_node_subsystem_test_helpers as test_helpers;
 use polkadot_node_subsystem_util::TimeoutExt;
 use polkadot_primitives::v2::{
-	AuthorityDiscoveryId, CollatorPair, GroupRotationInfo, ScheduledCore, SessionIndex,
-	SessionInfo, ValidatorId, ValidatorIndex,
+	AuthorityDiscoveryId, CollatorPair, GroupIndex, GroupRotationInfo, IndexedVec, ScheduledCore,
+	SessionIndex, SessionInfo, ValidatorId, ValidatorIndex,
 };
 use polkadot_primitives_test_helpers::TestCandidateBuilder;
 
@@ -62,7 +62,7 @@ struct TestState {
 	session_index: SessionIndex,
 }
 
-fn validator_pubkeys(val_ids: &[Sr25519Keyring]) -> Vec<ValidatorId> {
+fn validator_pubkeys(val_ids: &[Sr25519Keyring]) -> IndexedVec<ValidatorIndex, ValidatorId> {
 	val_ids.iter().map(|v| v.public().into()).collect()
 }
 
@@ -135,7 +135,7 @@ impl TestState {
 	fn current_group_validator_indices(&self) -> &[ValidatorIndex] {
 		let core_num = self.availability_cores.len();
 		let GroupIndex(group_idx) = self.group_rotation_info.group_for_core(CoreIndex(0), core_num);
-		&self.session_info.validator_groups[group_idx as usize]
+		&self.session_info.validator_groups.get(GroupIndex::from(group_idx)).unwrap()
 	}
 
 	fn current_session_index(&self) -> SessionIndex {
@@ -367,7 +367,7 @@ async fn distribute_collation(
 			)) => {
 				assert_eq!(relay_parent, test_state.relay_parent);
 				tx.send(Ok((
-					test_state.session_info.validator_groups.clone(),
+					test_state.session_info.validator_groups.to_vec(),
 					test_state.group_rotation_info.clone(),
 				)))
 				.unwrap();

polkadot/node/network/dispute-distribution/src/sender/mod.rs

@@ -299,7 +299,7 @@ impl DisputeSender {
 		let valid_public = info
 			.session_info
 			.validators
-			.get(valid_index.0 as usize)
+			.get(*valid_index)
 			.ok_or(JfyiError::InvalidStatementFromCoordinator)?;
 		let valid_signed = SignedDisputeStatement::new_checked(
 			DisputeStatement::Valid(kind.clone()),
@@ -314,7 +314,7 @@ impl DisputeSender {
 		let invalid_public = info
 			.session_info
 			.validators
-			.get(invalid_index.0 as usize)
+			.get(*invalid_index)
 			.ok_or(JfyiError::InvalidValidatorIndexFromCoordinator)?;
 		let invalid_signed = SignedDisputeStatement::new_checked(
 			DisputeStatement::Invalid(kind.clone()),

polkadot/node/network/dispute-distribution/src/tests/mock.rs

@@ -84,7 +84,7 @@ pub static ref MOCK_SESSION_INFO: SessionInfo =
 			.map(|k| MOCK_VALIDATORS_DISCOVERY_KEYS.get(&k).unwrap().clone())
 			.collect(),
 		assignment_keys: vec![],
-		validator_groups: vec![],
+		validator_groups: Default::default(),
 		n_cores: 0,
 		zeroth_delay_tranche_width: 0,
 		relay_vrf_modulo_samples: 0,
@@ -104,9 +104,9 @@ pub static ref MOCK_NEXT_SESSION_INFO: SessionInfo =
 				.iter()
 				.map(|k| MOCK_VALIDATORS_DISCOVERY_KEYS.get(&k).unwrap().clone())
 				.collect(),
-		validators: vec![],
+		validators: Default::default(),
 		assignment_keys: vec![],
-		validator_groups: vec![],
+		validator_groups: Default::default(),
 		n_cores: 0,
 		zeroth_delay_tranche_width: 0,
 		relay_vrf_modulo_samples: 0,

polkadot/node/network/gossip-support/src/tests.rs

@@ -37,6 +37,7 @@ use polkadot_node_subsystem::{
 };
 use polkadot_node_subsystem_test_helpers as test_helpers;
 use polkadot_node_subsystem_util::TimeoutExt as _;
+use polkadot_primitives::v2::{GroupIndex, IndexedVec};
 use test_helpers::mock::make_ferdie_keystore;
 
 use super::*;
@@ -219,7 +220,9 @@ fn make_session_info() -> SessionInfo {
 		validators: AUTHORITY_KEYRINGS.iter().map(|k| k.public().into()).collect(),
 		discovery_keys: AUTHORITIES.clone(),
 		assignment_keys: AUTHORITY_KEYRINGS.iter().map(|k| k.public().into()).collect(),
-		validator_groups: vec![all_validator_indices],
+		validator_groups: IndexedVec::<GroupIndex, Vec<ValidatorIndex>>::from(vec![
+			all_validator_indices,
+		]),
 		n_cores: 1,
 		zeroth_delay_tranche_width: 1,
 		relay_vrf_modulo_samples: 1,

polkadot/node/network/statement-distribution/src/lib.rs

@@ -47,8 +47,8 @@ use polkadot_node_subsystem::{
 };
 use polkadot_primitives::v2::{
 	AuthorityDiscoveryId, CandidateHash, CommittedCandidateReceipt, CompactStatement, Hash,
-	SignedStatement, SigningContext, UncheckedSignedStatement, ValidatorId, ValidatorIndex,
-	ValidatorSignature,
+	IndexedVec, SignedStatement, SigningContext, UncheckedSignedStatement, ValidatorId,
+	ValidatorIndex, ValidatorSignature,
 };
 
 use futures::{
@@ -665,7 +665,7 @@ struct ActiveHeadData {
 	/// Large statements we are waiting for with associated meta data.
 	waiting_large_statements: HashMap<CandidateHash, LargeStatementStatus>,
 	/// The parachain validators at the head's child session index.
-	validators: Vec<ValidatorId>,
+	validators: IndexedVec<ValidatorIndex, ValidatorId>,
 	/// The current session index of this fork.
 	session_index: sp_staking::SessionIndex,
 	/// How many `Seconded` statements we've seen per validator.
@@ -676,7 +676,7 @@ struct ActiveHeadData {
 
 impl ActiveHeadData {
 	fn new(
-		validators: Vec<ValidatorId>,
+		validators: IndexedVec<ValidatorIndex, ValidatorId>,
 		session_index: sp_staking::SessionIndex,
 		span: PerLeafSpan,
 	) -> Self {
@@ -878,7 +878,7 @@ fn check_statement_signature(
 		SigningContext { session_index: head.session_index, parent_hash: relay_parent };
 
 	head.validators
-		.get(statement.unchecked_validator_index().0 as usize)
+		.get(statement.unchecked_validator_index())
 		.ok_or_else(|| statement.clone())
 		.and_then(|v| statement.try_into_checked(&signing_context, v))
 }
@@ -2072,7 +2072,10 @@ impl<R: rand::Rng> StatementDistributionSubsystem<R> {
 					// directly:
 					let group_peers = {
 						if let Some(our_group) = validator_info.our_group {
-							let our_group = &session_info.validator_groups[our_group.0 as usize];
+							let our_group = &session_info
+								.validator_groups
+								.get(our_group)
+								.expect("`our_group` is derived from `validator_groups`; qed");
 
 							our_group
 								.into_iter()

polkadot/node/network/statement-distribution/src/tests.rs

@@ -35,7 +35,9 @@ use polkadot_node_subsystem::{
 	ActivatedLeaf, LeafStatus,
 };
 use polkadot_node_subsystem_test_helpers::mock::make_ferdie_keystore;
-use polkadot_primitives::v2::{Hash, Id as ParaId, SessionInfo, ValidationCode};
+use polkadot_primitives::v2::{
+	GroupIndex, Hash, Id as ParaId, IndexedVec, SessionInfo, ValidationCode, ValidatorId,
+};
 use polkadot_primitives_test_helpers::{
 	dummy_committed_candidate_receipt, dummy_hash, AlwaysZeroRng,
 };
@@ -83,7 +85,7 @@ fn active_head_accepts_only_2_seconded_per_validator() {
 	};
 
 	let mut head_data = ActiveHeadData::new(
-		validators,
+		IndexedVec::<ValidatorIndex, ValidatorId>::from(validators),
 		session_index,
 		PerLeafSpan::new(Arc::new(jaeger::Span::Disabled), "test"),
 	);
@@ -429,7 +431,7 @@ fn peer_view_update_sends_messages() {
 
 	let new_head_data = {
 		let mut data = ActiveHeadData::new(
-			validators,
+			IndexedVec::<ValidatorIndex, ValidatorId>::from(validators),
 			session_index,
 			PerLeafSpan::new(Arc::new(jaeger::Span::Disabled), "test"),
 		);
@@ -2319,7 +2321,7 @@ fn handle_multiple_seconded_statements() {
 }
 
 fn make_session_info(validators: Vec<Pair>, groups: Vec<Vec<u32>>) -> SessionInfo {
-	let validator_groups: Vec<Vec<ValidatorIndex>> = groups
+	let validator_groups: IndexedVec<GroupIndex, Vec<ValidatorIndex>> = groups
 		.iter()
 		.map(|g| g.into_iter().map(|v| ValidatorIndex(*v)).collect())
 		.collect();