Blocks carry full attestations for candidates (#42)

* statement table yields fully-attested candidates

* attestation types in polkadot-primitives

* propose block with fully-attested candidates in consensus

* some signature-checking logic in the runtime

* fix runtime compilation

* ensure attestations are full and without duplicate when checking

* fix consensus-service compilation

* add some tests

* use bitvec from crates.io now that it's published

* sign statements based on primitive statement's encoding

* remove some serialize bounds

* Fix error message with duplicate availability attestations

Co-Authored-By: rphmeier <rphmeier@gmail.com>

polkadot/runtime/src/parachains.rs

@@ -19,9 +19,11 @@
 use rstd::prelude::*;
 use codec::Decode;
 
-use sr_primitives::{RuntimeString, traits::{Extrinsic, Block as BlockT,
-	Hash, BlakeTwo256, ProvideInherent}};
-use primitives::parachain::{Id, Chain, DutyRoster, CandidateReceipt};
+use bitvec::BigEndian;
+use sr_primitives::{RuntimeString, traits::{
+	Extrinsic, Block as BlockT, Hash as HashT, BlakeTwo256, ProvideInherent,
+}};
+use primitives::parachain::{Id, Chain, DutyRoster, AttestedCandidate, Statement};
 use {system, session};
 
 use srml_support::{StorageValue, StorageMap};
@@ -77,7 +79,7 @@ decl_module! {
 	/// Parachains module.
 	pub struct Module<T: Trait> for enum Call where origin: T::Origin {
 		/// Provide candidate receipts for parachains, in ascending order by id.
-		fn set_heads(origin, heads: Vec<CandidateReceipt>) -> Result {
+		fn set_heads(origin, heads: Vec<AttestedCandidate>) -> Result {
 			ensure_inherent(origin)?;
 			ensure!(!<DidUpdate<T>>::exists(), "Parachain heads must be updated only once in the block");
 			ensure!(
@@ -98,23 +100,26 @@ decl_module! {
 				for head in &heads {
 					// proposed heads must be ascending order by parachain ID without duplicate.
 					ensure!(
-						last_id.as_ref().map_or(true, |x| x < &head.parachain_index),
+						last_id.as_ref().map_or(true, |x| x < &head.parachain_index()),
 						"Parachain candidates out of order by ID"
 					);
 
 					// must be unknown since active parachains are always sorted.
 					ensure!(
-						iter.find(|x| x == &&head.parachain_index).is_some(),
+						iter.find(|x| x == &&head.parachain_index()).is_some(),
 						"Submitted candidate for unregistered or out-of-order parachain {}"
 					);
 
-					last_id = Some(head.parachain_index);
+					last_id = Some(head.parachain_index());
 				}
 			}
 
+			Self::check_attestations(&heads)?;
+
+
 			for head in heads {
-				let id = head.parachain_index.clone();
-				<Heads<T>>::insert(id, head.head_data.0);
+				let id = head.parachain_index();
+				<Heads<T>>::insert(id, head.candidate.head_data.0);
 			}
 
 			<DidUpdate<T>>::put(true);
@@ -158,6 +163,18 @@ decl_module! {
 	}
 }
 
+fn majority_of(list_len: usize) -> usize {
+	list_len / 2 + list_len % 2
+}
+
+fn localized_payload(statement: Statement, parent_hash: ::primitives::Hash) -> Vec<u8> {
+	use codec::Encode;
+
+	let mut encoded = statement.encode();
+	encoded.extend(parent_hash.as_ref());
+	encoded
+}
+
 impl<T: Trait> Module<T> {
 	/// Calculate the current block's duty roster using system's random seed.
 	pub fn calculate_duty_roster() -> DutyRoster {
@@ -208,6 +225,179 @@ impl<T: Trait> Module<T> {
 		}
 	}
 
+	// check the attestations on these candidates. The candidates should have been checked
+	// that each candidates' chain ID is valid.
+	fn check_attestations(attested_candidates: &[AttestedCandidate]) -> Result {
+		use primitives::parachain::ValidityAttestation;
+		use sr_primitives::traits::Verify;
+
+		// returns groups of slices that have the same chain ID.
+		// assumes the inner slice is sorted by id.
+		struct GroupedDutyIter<'a> {
+			next_idx: usize,
+			inner: &'a [(usize, Id)],
+		}
+
+		impl<'a> GroupedDutyIter<'a> {
+			fn new(inner: &'a [(usize, Id)]) -> Self {
+				GroupedDutyIter { next_idx: 0, inner }
+			}
+
+			fn group_for(&mut self, wanted_id: Id) -> Option<&'a [(usize, Id)]> {
+				while let Some((id, keys)) = self.next() {
+					if wanted_id == id {
+						return Some(keys)
+					}
+				}
+
+				None
+			}
+		}
+
+		impl<'a> Iterator for GroupedDutyIter<'a> {
+			type Item = (Id, &'a [(usize, Id)]);
+
+			fn next(&mut self) -> Option<Self::Item> {
+				if self.next_idx == self.inner.len() { return None }
+				let start_idx = self.next_idx;
+				self.next_idx += 1;
+				let start_id = self.inner[start_idx].1;
+
+				while self.inner.get(self.next_idx).map_or(false, |&(_, ref id)| id == &start_id) {
+					self.next_idx += 1;
+				}
+
+				Some((start_id, &self.inner[start_idx..self.next_idx]))
+			}
+		}
+
+		let authorities = super::Consensus::authorities();
+		let duty_roster = Self::calculate_duty_roster();
+
+		// convert a duty roster, which is originally a Vec<Chain>, where each
+		// item corresponds to the same position in the session keys, into
+		// a list containing (index, parachain duty) where indices are into the session keys.
+		// this list is sorted ascending by parachain duty, just like the
+		// parachain candidates are.
+		let make_sorted_duties = |duty: &[Chain]| {
+			let mut sorted_duties = Vec::with_capacity(duty.len());
+			for (val_idx, duty) in duty.iter().enumerate() {
+				let id = match duty {
+					Chain::Relay => continue,
+					Chain::Parachain(id) => id,
+				};
+
+				let idx = sorted_duties.binary_search_by_key(&id, |&(_, ref id)| id)
+					.unwrap_or_else(|idx| idx);
+
+				sorted_duties.insert(idx, (val_idx, *id));
+			}
+
+			sorted_duties
+		};
+
+		let sorted_validators = make_sorted_duties(&duty_roster.validator_duty);
+		let sorted_guarantors = make_sorted_duties(&duty_roster.guarantor_duty);
+
+		let parent_hash = super::System::parent_hash();
+		let localized_payload = |statement: Statement| localized_payload(statement, parent_hash);
+
+		let mut validator_groups = GroupedDutyIter::new(&sorted_validators[..]);
+		let mut guarantor_groups = GroupedDutyIter::new(&sorted_guarantors[..]);
+
+		for candidate in attested_candidates {
+			let validator_group = validator_groups.group_for(candidate.parachain_index())
+				.ok_or("no validator group for parachain")?;
+
+			let availability_group = guarantor_groups.group_for(candidate.parachain_index())
+				.ok_or("no availability group for parachain")?;
+
+			ensure!(
+				candidate.validity_votes.len() >= majority_of(validator_group.len()),
+				"Not enough validity attestations"
+			);
+
+			ensure!(
+				candidate.availability_votes.len() >= majority_of(availability_group.len()),
+				"Not enough availability attestations"
+			);
+
+			let mut candidate_hash = None;
+			let mut encoded_implicit = None;
+			let mut encoded_explicit = None;
+
+			// track which voters have voted already. the first `authorities.len()`
+			// bits is for validity, the next are for availability.
+			let mut track_voters = bitvec![0; authorities.len() * 2];
+			for (auth_id, validity_attestation) in &candidate.validity_votes {
+				// protect against double-votes.
+				match validator_group.iter().find(|&(idx, _)| &authorities[*idx] == auth_id) {
+					None => return Err("Attesting validator not on this chain's validation duty."),
+					Some(&(idx, _)) => {
+						if track_voters.get(idx) {
+							return Err("Voter already attested validity once")
+						}
+						track_voters.set(idx, true)
+					}
+				}
+
+				let (payload, sig) = match validity_attestation {
+					ValidityAttestation::Implicit(sig) => {
+						let payload = encoded_implicit.get_or_insert_with(|| localized_payload(
+							Statement::Candidate(candidate.candidate.clone()),
+						));
+
+						(payload, sig)
+					}
+					ValidityAttestation::Explicit(sig) => {
+						let hash = candidate_hash
+							.get_or_insert_with(|| candidate.candidate.hash())
+							.clone();
+
+						let payload = encoded_explicit.get_or_insert_with(|| localized_payload(
+							Statement::Valid(hash),
+						));
+
+						(payload, sig)
+					}
+				};
+
+				ensure!(
+					sig.verify(&payload[..], &auth_id.0.into()),
+					"Candidate validity attestation signature is bad."
+				);
+			}
+
+			let mut encoded_available = None;
+			for (auth_id, sig) in &candidate.availability_votes {
+				match availability_group.iter().find(|&(idx, _)| &authorities[*idx] == auth_id) {
+					None => return Err("Attesting validator not on this chain's availability duty."),
+					Some(&(idx, _)) => {
+						if track_voters.get(authorities.len() + idx) {
+							return Err("Voter already attested availability once")
+						}
+						track_voters.set(authorities.len() + idx, true)
+					}
+				}
+
+				let hash = candidate_hash
+					.get_or_insert_with(|| candidate.candidate.hash())
+					.clone();
+
+				let payload = encoded_available.get_or_insert_with(|| localized_payload(
+					Statement::Available(hash),
+				));
+
+				ensure!(
+					sig.verify(&payload[..], &auth_id.0.into()),
+					"Candidate availability attestation signature is bad."
+				)
+			}
+		}
+
+		Ok(())
+	}
+
 /*
 	// TODO: Consider integrating if needed.
 	/// Extract the parachain heads from the block.
@@ -226,7 +416,7 @@ impl<T: Trait> Module<T> {
 }
 
 impl<T: Trait> ProvideInherent for Module<T> {
-	type Inherent = Vec<CandidateReceipt>;
+	type Inherent = Vec<AttestedCandidate>;
 	type Call = Call<T>;
 	type Error = RuntimeString;
 
@@ -259,9 +449,10 @@ mod tests {
 	use rstd::marker::PhantomData;
 	use sr_io::{TestExternalities, with_externalities};
 	use substrate_primitives::{H256, Blake2Hasher};
-	use sr_primitives::BuildStorage;
-	use sr_primitives::traits::{Identity, BlakeTwo256};
-	use sr_primitives::testing::{Digest, Header, DigestItem};
+	use sr_primitives::{generic, BuildStorage};
+	use sr_primitives::traits::BlakeTwo256;
+	use primitives::{parachain::{CandidateReceipt, HeadData, ValidityAttestation}, SessionKey};
+	use keyring::Keyring;
 	use {consensus, timestamp};
 
 	impl_outer_origin! {
@@ -272,24 +463,24 @@ mod tests {
 	pub struct Test;
 	impl consensus::Trait for Test {
 		const NOTE_OFFLINE_POSITION: u32 = 1;
-		type SessionKey = u64;
+		type SessionKey = SessionKey;
 		type OnOfflineValidator = ();
-		type Log = DigestItem;
+		type Log = ::Log;
 	}
 	impl system::Trait for Test {
 		type Origin = Origin;
-		type Index = u64;
+		type Index = ::Index;
 		type BlockNumber = u64;
 		type Hash = H256;
 		type Hashing = BlakeTwo256;
-		type Digest = Digest;
-		type AccountId = u64;
-		type Header = Header;
+		type Digest = generic::Digest<::Log>;
+		type AccountId = ::AccountId;
+		type Header = ::Header;
 		type Event = ();
-		type Log = DigestItem;
+		type Log = ::Log;
 	}
 	impl session::Trait for Test {
-		type ConvertAccountIdToSessionKey = Identity;
+		type ConvertAccountIdToSessionKey = ::SessionKeyConversion;
 		type OnSessionChange = ();
 		type Event = ();
 	}
@@ -305,14 +496,25 @@ mod tests {
 
 	fn new_test_ext(parachains: Vec<(Id, Vec<u8>, Vec<u8>)>) -> TestExternalities<Blake2Hasher> {
 		let mut t = system::GenesisConfig::<Test>::default().build_storage().unwrap().0;
+		let authority_keys = [
+			Keyring::Alice,
+			Keyring::Bob,
+			Keyring::Charlie,
+			Keyring::Dave,
+			Keyring::Eve,
+			Keyring::Ferdie,
+			Keyring::One,
+			Keyring::Two,
+		];
+
 		t.extend(consensus::GenesisConfig::<Test>{
 			code: vec![],
-			authorities: vec![1, 2, 3],
+			authorities: authority_keys.iter().map(|k| k.to_raw_public().into()).collect(),
 			_genesis_phantom_data: PhantomData,
 		}.build_storage().unwrap().0);
 		t.extend(session::GenesisConfig::<Test>{
 			session_length: 1000,
-			validators: vec![1, 2, 3, 4, 5, 6, 7, 8],
+			validators: authority_keys.iter().map(|k| k.to_raw_public().into()).collect(),
 			_genesis_phantom_data: PhantomData,
 		}.build_storage().unwrap().0);
 		t.extend(GenesisConfig::<Test>{
@@ -322,6 +524,55 @@ mod tests {
 		t.into()
 	}
 
+	fn make_attestations(candidate: &mut AttestedCandidate) {
+		let mut vote_implicit = false;
+		let parent_hash = ::System::parent_hash();
+
+		let duty_roster = Parachains::calculate_duty_roster();
+		let candidate_hash = candidate.candidate.hash();
+
+		let authorities = ::Consensus::authorities();
+		let extract_key = |public: SessionKey| {
+			Keyring::from_raw_public(public.0).unwrap()
+		};
+
+		let validation_entries = duty_roster.validator_duty.iter()
+			.enumerate()
+			.map(|(i, d)| (i, d, true));
+
+		let availability_entries = duty_roster.guarantor_duty.iter()
+			.enumerate()
+			.map(|(i, d)| (i, d, false));
+
+		for (idx, &duty, is_validation) in validation_entries.chain(availability_entries) {
+			if duty != Chain::Parachain(candidate.parachain_index()) { continue }
+			if is_validation { vote_implicit = !vote_implicit };
+
+			let key = extract_key(authorities[idx]);
+
+			let statement = if is_validation && vote_implicit {
+				Statement::Candidate(candidate.candidate.clone())
+			} else if is_validation {
+				Statement::Valid(candidate_hash.clone())
+			} else {
+				Statement::Available(candidate_hash.clone())
+			};
+
+			let payload = localized_payload(statement, parent_hash);
+			let signature = key.sign(&payload[..]).into();
+
+			if is_validation {
+				candidate.validity_votes.push((authorities[idx], if vote_implicit {
+					ValidityAttestation::Implicit(signature)
+				} else {
+					ValidityAttestation::Explicit(signature)
+				}));
+			} else {
+				candidate.availability_votes.push((authorities[idx], signature));
+			}
+		}
+	}
+
 	#[test]
 	fn active_parachains_should_work() {
 		let parachains = vec![
@@ -397,4 +648,130 @@ mod tests {
 			assert!(duty_roster_1 != duty_roster_2);
 		});
 	}
+
+	#[test]
+	fn unattested_candidate_is_rejected() {
+		let parachains = vec![
+			(0u32.into(), vec![], vec![]),
+			(1u32.into(), vec![], vec![]),
+		];
+
+		with_externalities(&mut new_test_ext(parachains), || {
+			system::Module::<Test>::set_random_seed([0u8; 32].into());
+			let candidate = AttestedCandidate {
+				validity_votes: vec![],
+				availability_votes: vec![],
+				candidate: CandidateReceipt {
+					parachain_index: 0.into(),
+					collator: Default::default(),
+					signature: Default::default(),
+					head_data: HeadData(vec![1, 2, 3]),
+					balance_uploads: vec![],
+					egress_queue_roots: vec![],
+					fees: 0,
+					block_data_hash: Default::default(),
+				}
+			};
+
+			assert!(Parachains::dispatch(Call::set_heads(vec![candidate]), Origin::INHERENT).is_err());
+		})
+	}
+
+	#[test]
+	fn attested_candidates_accepted_in_order() {
+		let parachains = vec![
+			(0u32.into(), vec![], vec![]),
+			(1u32.into(), vec![], vec![]),
+		];
+
+		with_externalities(&mut new_test_ext(parachains), || {
+			system::Module::<Test>::set_random_seed([0u8; 32].into());
+			let mut candidate_a = AttestedCandidate {
+				validity_votes: vec![],
+				availability_votes: vec![],
+				candidate: CandidateReceipt {
+					parachain_index: 0.into(),
+					collator: Default::default(),
+					signature: Default::default(),
+					head_data: HeadData(vec![1, 2, 3]),
+					balance_uploads: vec![],
+					egress_queue_roots: vec![],
+					fees: 0,
+					block_data_hash: Default::default(),
+				}
+			};
+
+			let mut candidate_b = AttestedCandidate {
+				validity_votes: vec![],
+				availability_votes: vec![],
+				candidate: CandidateReceipt {
+					parachain_index: 1.into(),
+					collator: Default::default(),
+					signature: Default::default(),
+					head_data: HeadData(vec![2, 3, 4]),
+					balance_uploads: vec![],
+					egress_queue_roots: vec![],
+					fees: 0,
+					block_data_hash: Default::default(),
+				}
+			};
+
+			make_attestations(&mut candidate_a);
+			make_attestations(&mut candidate_b);
+
+			assert!(Parachains::dispatch(
+				Call::set_heads(vec![candidate_b.clone(), candidate_a.clone()]),
+				Origin::INHERENT,
+			).is_err());
+
+			assert!(Parachains::dispatch(
+				Call::set_heads(vec![candidate_a.clone(), candidate_b.clone()]),
+				Origin::INHERENT,
+			).is_ok());
+		});
+	}
+
+	#[test]
+	fn duplicate_vote_is_rejected() {
+		let parachains = vec![
+			(0u32.into(), vec![], vec![]),
+			(1u32.into(), vec![], vec![]),
+		];
+
+		with_externalities(&mut new_test_ext(parachains), || {
+			system::Module::<Test>::set_random_seed([0u8; 32].into());
+			let mut candidate = AttestedCandidate {
+				validity_votes: vec![],
+				availability_votes: vec![],
+				candidate: CandidateReceipt {
+					parachain_index: 0.into(),
+					collator: Default::default(),
+					signature: Default::default(),
+					head_data: HeadData(vec![1, 2, 3]),
+					balance_uploads: vec![],
+					egress_queue_roots: vec![],
+					fees: 0,
+					block_data_hash: Default::default(),
+				}
+			};
+
+			make_attestations(&mut candidate);
+
+			let mut double_validity = candidate.clone();
+			double_validity.validity_votes.push(candidate.validity_votes[0].clone());
+
+			assert!(Parachains::dispatch(
+				Call::set_heads(vec![double_validity]),
+				Origin::INHERENT,
+			).is_err());
+
+			let mut double_availability = candidate.clone();
+			double_availability.availability_votes.push(candidate.availability_votes[0].clone());
+
+			assert!(Parachains::dispatch(
+				Call::set_heads(vec![double_availability]),
+				Origin::INHERENT,
+			).is_err());
+		});
+	}
 }