babe: secondary blocks with VRF (#5501)

* babe: secondary blocks with VRF

* Fix node runtime compile

* Fix test-utils runtime interface

* Fix babe tests

* typo: v == 2

* babe: support online configuration upgrades

* Fix rpc tests

* Fix runtime version tests

* Switch to use NextConfigDescriptor instead of changing runtime interface

* Fix tests

* epoch-changes: map function that allows converting with different epoch types

* Add migration script for the epoch config change

* Fix docs for PrimaryAndSecondaryVRFSlots

* Add docs of `SecondaryVRF` in babe crate

* babe-primitives: Secondary -> SecondaryPlain

* babe-client: Secondary -> SecondaryPlain

* Fix migration tests

* test-utils-runtime: Secondary -> SecondaryPlain

* Fix missing name change in test-utils-runtime

* Fix migration: Epoch should be EpochV0

* Update client/consensus/babe/src/lib.rs

Co-Authored-By: André Silva <123550+andresilva@users.noreply.github.com>

* Fix new epochChanges version

* Fix babe-primitives naming changes

* Fix merge issues in babe-client

Co-authored-by: André Silva <123550+andresilva@users.noreply.github.com>
Co-authored-by: André Silva <andre.beat@gmail.com>

substrate/client/consensus/babe/src/authorship.rs

@@ -21,7 +21,9 @@ use sp_consensus_babe::{
 	AuthorityId, BabeAuthorityWeight, BABE_ENGINE_ID, BABE_VRF_PREFIX,
 	SlotNumber, AuthorityPair,
 };
-use sp_consensus_babe::digests::{PreDigest, PrimaryPreDigest, SecondaryPreDigest};
+use sp_consensus_babe::digests::{
+	PreDigest, PrimaryPreDigest, SecondaryPlainPreDigest, SecondaryVRFPreDigest,
+};
 use sp_consensus_vrf::schnorrkel::{VRFOutput, VRFProof};
 use sp_core::{U256, blake2_256};
 use codec::Encode;
@@ -105,10 +107,12 @@ pub(super) fn make_transcript(
 /// to propose.
 fn claim_secondary_slot(
 	slot_number: SlotNumber,
-	authorities: &[(AuthorityId, BabeAuthorityWeight)],
+	epoch: &Epoch,
 	keystore: &KeyStorePtr,
-	randomness: [u8; 32],
+	author_secondary_vrf: bool,
 ) -> Option<(PreDigest, AuthorityPair)> {
+	let Epoch { authorities, randomness, epoch_index, .. } = epoch;
+
 	if authorities.is_empty() {
 		return None;
 	}
@@ -116,7 +120,7 @@ fn claim_secondary_slot(
 	let expected_author = super::authorship::secondary_slot_author(
 		slot_number,
 		authorities,
-		randomness,
+		*randomness,
 	)?;
 
 	let keystore = keystore.read();
@@ -128,10 +132,27 @@ fn claim_secondary_slot(
 		})
 	{
 		if pair.public() == *expected_author {
-			let pre_digest = PreDigest::Secondary(SecondaryPreDigest {
-				slot_number,
-				authority_index: authority_index as u32,
-			});
+			let pre_digest = if author_secondary_vrf {
+				let transcript = super::authorship::make_transcript(
+					randomness,
+					slot_number,
+					*epoch_index,
+				);
+
+				let s = get_keypair(&pair).vrf_sign(transcript);
+
+				PreDigest::SecondaryVRF(SecondaryVRFPreDigest {
+					slot_number,
+					vrf_output: VRFOutput(s.0.to_output()),
+					vrf_proof: VRFProof(s.1),
+					authority_index: authority_index as u32,
+				})
+			} else {
+				PreDigest::SecondaryPlain(SecondaryPlainPreDigest {
+					slot_number,
+					authority_index: authority_index as u32,
+				})
+			};
 
 			return Some((pre_digest, pair));
 		}
@@ -151,12 +172,14 @@ pub fn claim_slot(
 ) -> Option<(PreDigest, AuthorityPair)> {
 	claim_primary_slot(slot_number, epoch, epoch.config.c, keystore)
 		.or_else(|| {
-			if epoch.config.secondary_slots {
+			if epoch.config.allowed_slots.is_secondary_plain_slots_allowed() ||
+				epoch.config.allowed_slots.is_secondary_vrf_slots_allowed()
+			{
 				claim_secondary_slot(
 					slot_number,
-					&epoch.authorities,
+					&epoch,
 					keystore,
-					epoch.randomness,
+					epoch.config.allowed_slots.is_secondary_vrf_slots_allowed(),
 				)
 			} else {
 				None

substrate/client/consensus/babe/src/aux_schema.rs

@@ -141,7 +141,7 @@ mod test {
 	use substrate_test_runtime_client;
 	use sp_core::H256;
 	use sp_runtime::traits::NumberFor;
-	use sp_consensus_babe::BabeGenesisConfiguration;
+	use sp_consensus_babe::{AllowedSlots, BabeGenesisConfiguration};
 	use sc_consensus_epochs::{PersistedEpoch, PersistedEpochHeader, EpochHeader};
 	use sp_consensus::Error as ConsensusError;
 	use sc_network_test::Block as TestBlock;
@@ -182,7 +182,7 @@ mod test {
 				c: (3, 10),
 				genesis_authorities: Vec::new(),
 				randomness: Default::default(),
-				secondary_slots: true,
+				allowed_slots: AllowedSlots::PrimaryAndSecondaryPlainSlots,
 			},
 		).unwrap();
 

substrate/client/consensus/babe/src/lib.rs

@@ -49,6 +49,11 @@
 //!
 //! `blake2_256(epoch_randomness ++ slot_number) % authorities_len`.
 //!
+//! The secondary slots supports either a `SecondaryPlain` or `SecondaryVRF`
+//! variant. Comparing with `SecondaryPlain` variant, the `SecondaryVRF` variant
+//! generates an additional VRF output. The output is not included in beacon
+//! randomness, but can be consumed by parachains.
+//!
 //! The fork choice rule is weight-based, where weight equals the number of
 //! primary blocks in the chain. We will pick the heaviest chain (more primary
 //! blocks) and will go with the longest one in case of a tie.
@@ -64,8 +69,8 @@ pub use sp_consensus_babe::{
 	AuthorityId, AuthorityPair, AuthoritySignature,
 	BabeAuthorityWeight, VRF_OUTPUT_LENGTH,
 	digests::{
-		CompatibleDigestItem, NextEpochDescriptor, NextConfigDescriptor,
-		PreDigest, PrimaryPreDigest, SecondaryPreDigest,
+		CompatibleDigestItem, NextEpochDescriptor, NextConfigDescriptor, PreDigest,
+		PrimaryPreDigest, SecondaryPlainPreDigest,
 	},
 };
 pub use sp_consensus::SyncOracle;
@@ -184,7 +189,7 @@ impl Epoch {
 			randomness: genesis_config.randomness.clone(),
 			config: BabeEpochConfiguration {
 				c: genesis_config.c,
-				secondary_slots: genesis_config.secondary_slots,
+				allowed_slots: genesis_config.allowed_slots,
 			},
 		}
 	}
@@ -279,7 +284,26 @@ impl Config {
 		C: AuxStore + ProvideRuntimeApi<B>, C::Api: BabeApi<B, Error = sp_blockchain::Error>,
 	{
 		trace!(target: "babe", "Getting slot duration");
-		match sc_consensus_slots::SlotDuration::get_or_compute(client, |a, b| a.configuration(b)).map(Self) {
+		match sc_consensus_slots::SlotDuration::get_or_compute(client, |a, b| {
+			let has_api_v1 = a.has_api_with::<dyn BabeApi<B, Error = sp_blockchain::Error>, _>(
+				&b, |v| v == 1,
+			)?;
+			let has_api_v2 = a.has_api_with::<dyn BabeApi<B, Error = sp_blockchain::Error>, _>(
+				&b, |v| v == 2,
+			)?;
+
+			if has_api_v1 {
+				#[allow(deprecated)] {
+					Ok(a.configuration_before_version_2(b)?.into())
+				}
+			} else if has_api_v2 {
+				a.configuration(b)
+			} else {
+				Err(sp_blockchain::Error::VersionInvalid(
+					"Unsupported or invalid BabeApi version".to_string()
+				))
+			}
+		}).map(Self) {
 			Ok(s) => Ok(s),
 			Err(s) => {
 				warn!(target: "babe", "Failed to get slot duration");
@@ -583,7 +607,7 @@ fn find_pre_digest<B: BlockT>(header: &B::Header) -> Result<PreDigest, Error<B>>
 	// genesis block doesn't contain a pre digest so let's generate a
 	// dummy one to not break any invariants in the rest of the code
 	if header.number().is_zero() {
-		return Ok(PreDigest::Secondary(SecondaryPreDigest {
+		return Ok(PreDigest::SecondaryPlain(SecondaryPlainPreDigest {
 			slot_number: 0,
 			authority_index: 0,
 		}));
@@ -1044,7 +1068,7 @@ impl<Block, Client, Inner> BlockImport<Block> for BabeBlockImport<Block, Client,
 			let epoch_config = next_config_digest.unwrap_or_else(
 				|| viable_epoch.as_ref().config.clone()
 			);
-      
+
 			// restrict info logging during initial sync to avoid spam
 			let log_level = if block.origin == BlockOrigin::NetworkInitialSync {
 				log::Level::Debug
@@ -1053,7 +1077,7 @@ impl<Block, Client, Inner> BlockImport<Block> for BabeBlockImport<Block, Client,
 			};
 
 			log!(target: "babe",
-				log_level, 
+				log_level,
 				"👶 New epoch {} launching at block {} (block slot {} >= start slot {}).",
 				viable_epoch.as_ref().epoch_index,
 				hash,

substrate/client/consensus/babe/src/migration.rs

@@ -57,7 +57,7 @@ impl EpochV0 {
 			randomness: self.randomness,
 			config: BabeEpochConfiguration {
 				c: config.c,
-				secondary_slots: config.secondary_slots,
+				allowed_slots: config.allowed_slots,
 			},
 		}
 	}

substrate/client/consensus/babe/src/tests.rs

@@ -22,7 +22,7 @@
 use super::*;
 use authorship::claim_slot;
 
-use sp_consensus_babe::{AuthorityPair, SlotNumber};
+use sp_consensus_babe::{AuthorityPair, SlotNumber, AllowedSlots};
 use sc_block_builder::{BlockBuilder, BlockBuilderProvider};
 use sp_consensus::{
 	NoNetwork as DummyOracle, Proposal, RecordProof,
@@ -507,7 +507,7 @@ fn can_author_block() {
 		duration: 100,
 		config: BabeEpochConfiguration {
 			c: (3, 10),
-			secondary_slots: true,
+			allowed_slots: AllowedSlots::PrimaryAndSecondaryPlainSlots,
 		},
 	};
 
@@ -517,7 +517,7 @@ fn can_author_block() {
 		c: (3, 10),
 		genesis_authorities: Vec::new(),
 		randomness: [0; 32],
-		secondary_slots: true,
+		allowed_slots: AllowedSlots::PrimaryAndSecondaryPlainSlots,
 	};
 
 	// with secondary slots enabled it should never be empty
@@ -528,7 +528,7 @@ fn can_author_block() {
 
 	// otherwise with only vrf-based primary slots we might need to try a couple
 	// of times.
-	config.secondary_slots = false;
+	config.allowed_slots = AllowedSlots::PrimarySlots;
 	loop {
 		match claim_slot(i, &epoch, &keystore) {
 			None => i += 1,
@@ -557,7 +557,7 @@ fn propose_and_import_block<Transaction>(
 	let pre_digest = sp_runtime::generic::Digest {
 		logs: vec![
 			Item::babe_pre_digest(
-				PreDigest::Secondary(SecondaryPreDigest {
+				PreDigest::SecondaryPlain(SecondaryPlainPreDigest {
 					authority_index: 0,
 					slot_number,
 				}),

substrate/client/consensus/babe/src/verification.rs

@@ -19,7 +19,8 @@ use sp_runtime::{traits::Header, traits::DigestItemFor};
 use sp_core::{Pair, Public};
 use sp_consensus_babe::{AuthoritySignature, SlotNumber, AuthorityPair, AuthorityId};
 use sp_consensus_babe::digests::{
-	PreDigest, PrimaryPreDigest, SecondaryPreDigest, CompatibleDigestItem
+	PreDigest, PrimaryPreDigest, SecondaryPlainPreDigest, SecondaryVRFPreDigest,
+	CompatibleDigestItem
 };
 use sc_consensus_slots::CheckedHeader;
 use log::{debug, trace};
@@ -28,15 +29,15 @@ use super::authorship::{make_transcript, calculate_primary_threshold, check_prim
 
 /// BABE verification parameters
 pub(super) struct VerificationParams<'a, B: 'a + BlockT> {
-	/// the header being verified.
+	/// The header being verified.
 	pub(super) header: B::Header,
-	/// the pre-digest of the header being verified. this is optional - if prior
+	/// The pre-digest of the header being verified. this is optional - if prior
 	/// verification code had to read it, it can be included here to avoid duplicate
 	/// work.
 	pub(super) pre_digest: Option<PreDigest>,
-	/// the slot number of the current time.
+	/// The slot number of the current time.
 	pub(super) slot_now: SlotNumber,
-	/// epoch descriptor of the epoch this block _should_ be under, if it's valid.
+	/// Epoch descriptor of the epoch this block _should_ be under, if it's valid.
 	pub(super) epoch: &'a Epoch,
 }
 
@@ -102,10 +103,18 @@ pub(super) fn check_header<B: BlockT + Sized>(
 				epoch.config.c,
 			)?;
 		},
-		PreDigest::Secondary(secondary) if epoch.config.secondary_slots => {
-			debug!(target: "babe", "Verifying Secondary block");
-
-			check_secondary_header::<B>(
+		PreDigest::SecondaryPlain(secondary) if epoch.config.allowed_slots.is_secondary_plain_slots_allowed() => {
+			debug!(target: "babe", "Verifying Secondary plain block");
+			check_secondary_plain_header::<B>(
+				pre_hash,
+				secondary,
+				sig,
+				&epoch,
+			)?;
+		},
+		PreDigest::SecondaryVRF(secondary) if epoch.config.allowed_slots.is_secondary_vrf_slots_allowed() => {
+			debug!(target: "babe", "Verifying Secondary VRF block");
+			check_secondary_vrf_header::<B>(
 				pre_hash,
 				secondary,
 				sig,
@@ -179,9 +188,9 @@ fn check_primary_header<B: BlockT + Sized>(
 /// properly signed by the expected authority, which we have a deterministic way
 /// of computing. Additionally, the weight of this block must stay the same
 /// compared to its parent since it is a secondary block.
-fn check_secondary_header<B: BlockT>(
+fn check_secondary_plain_header<B: BlockT>(
 	pre_hash: B::Hash,
-	pre_digest: &SecondaryPreDigest,
+	pre_digest: &SecondaryPlainPreDigest,
 	signature: AuthoritySignature,
 	epoch: &Epoch,
 ) -> Result<(), Error<B>> {
@@ -205,3 +214,43 @@ fn check_secondary_header<B: BlockT>(
 		Err(Error::BadSignature(pre_hash))
 	}
 }
+
+/// Check a secondary VRF slot proposal header.
+fn check_secondary_vrf_header<B: BlockT>(
+	pre_hash: B::Hash,
+	pre_digest: &SecondaryVRFPreDigest,
+	signature: AuthoritySignature,
+	epoch: &Epoch,
+) -> Result<(), Error<B>> {
+	// check the signature is valid under the expected authority and
+	// chain state.
+	let expected_author = secondary_slot_author(
+		pre_digest.slot_number,
+		&epoch.authorities,
+		epoch.randomness,
+	).ok_or_else(|| Error::NoSecondaryAuthorExpected)?;
+
+	let author = &epoch.authorities[pre_digest.authority_index as usize].0;
+
+	if expected_author != author {
+		return Err(Error::InvalidAuthor(expected_author.clone(), author.clone()));
+	}
+
+	if AuthorityPair::verify(&signature, pre_hash.as_ref(), author) {
+		let transcript = make_transcript(
+			&epoch.randomness,
+			pre_digest.slot_number,
+			epoch.epoch_index,
+		);
+
+		schnorrkel::PublicKey::from_bytes(author.as_slice()).and_then(|p| {
+			p.vrf_verify(transcript, &pre_digest.vrf_output, &pre_digest.vrf_proof)
+		}).map_err(|s| {
+			babe_err(Error::VRFVerificationFailed(s))
+		})?;
+
+		Ok(())
+	} else {
+		Err(Error::BadSignature(pre_hash))
+	}
+}