VRF refactory (#13889)

* First iteration to encapsulate schnorrkel and merlin usage

* Remove schnorkel direct dependency from BABE pallet

* Remove schnorrkel direct dependency from BABE client

* Trivial renaming for VrfTranscript data and value

* Better errors

* Expose a function to get a schnorrkel friendly transcript

* Keep the vrf signature stuff together (preventing some clones around)

* Fix tests

* Remove vrf agnostic transcript and define it as an associated type for VrfSigner and VrfVerifier

* Fix babe pallet mock

* Inner types are required to be public for polkadot

* Update client/consensus/babe/src/verification.rs

Co-authored-by: Koute <koute@users.noreply.github.com>

* Nit

* Remove Deref implementations

* make_bytes as a method

* Trigger CI

---------

Co-authored-by: Koute <koute@users.noreply.github.com>

substrate/frame/babe/src/lib.rs

@@ -29,13 +29,13 @@ use frame_support::{
 	weights::Weight,
 	BoundedVec, WeakBoundedVec,
 };
-use sp_application_crypto::ByteArray;
 use sp_consensus_babe::{
 	digests::{NextConfigDescriptor, NextEpochDescriptor, PreDigest},
 	AllowedSlots, BabeAuthorityWeight, BabeEpochConfiguration, ConsensusLog, Epoch,
-	EquivocationProof, Slot, BABE_ENGINE_ID,
+	EquivocationProof, Randomness as BabeRandomness, Slot, BABE_ENGINE_ID, RANDOMNESS_LENGTH,
+	RANDOMNESS_VRF_CONTEXT,
 };
-use sp_consensus_vrf::schnorrkel;
+use sp_core::crypto::Wraps;
 use sp_runtime::{
 	generic::DigestItem,
 	traits::{IsMember, One, SaturatedConversion, Saturating, Zero},
@@ -45,7 +45,7 @@ use sp_session::{GetSessionNumber, GetValidatorCount};
 use sp_staking::{offence::OffenceReportSystem, SessionIndex};
 use sp_std::prelude::*;
 
-pub use sp_consensus_babe::{AuthorityId, PUBLIC_KEY_LENGTH, RANDOMNESS_LENGTH, VRF_OUTPUT_LENGTH};
+pub use sp_consensus_babe::AuthorityId;
 
 const LOG_TARGET: &str = "runtime::babe";
 
@@ -218,7 +218,7 @@ pub mod pallet {
 	// variable to its underlying value.
 	#[pallet::storage]
 	#[pallet::getter(fn randomness)]
-	pub type Randomness<T> = StorageValue<_, schnorrkel::Randomness, ValueQuery>;
+	pub type Randomness<T> = StorageValue<_, BabeRandomness, ValueQuery>;
 
 	/// Pending epoch configuration change that will be applied when the next epoch is enacted.
 	#[pallet::storage]
@@ -226,7 +226,7 @@ pub mod pallet {
 
 	/// Next epoch randomness.
 	#[pallet::storage]
-	pub(super) type NextRandomness<T> = StorageValue<_, schnorrkel::Randomness, ValueQuery>;
+	pub(super) type NextRandomness<T> = StorageValue<_, BabeRandomness, ValueQuery>;
 
 	/// Next epoch authorities.
 	#[pallet::storage]
@@ -254,7 +254,7 @@ pub mod pallet {
 		_,
 		Twox64Concat,
 		u32,
-		BoundedVec<schnorrkel::Randomness, ConstU32<UNDER_CONSTRUCTION_SEGMENT_LENGTH>>,
+		BoundedVec<BabeRandomness, ConstU32<UNDER_CONSTRUCTION_SEGMENT_LENGTH>>,
 		ValueQuery,
 	>;
 
@@ -270,8 +270,7 @@ pub mod pallet {
 	/// It is set in `on_finalize`, before it will contain the value from the last block.
 	#[pallet::storage]
 	#[pallet::getter(fn author_vrf_randomness)]
-	pub(super) type AuthorVrfRandomness<T> =
-		StorageValue<_, Option<schnorrkel::Randomness>, ValueQuery>;
+	pub(super) type AuthorVrfRandomness<T> = StorageValue<_, Option<BabeRandomness>, ValueQuery>;
 
 	/// The block numbers when the last and current epoch have started, respectively `N-1` and
 	/// `N`.
@@ -358,31 +357,33 @@ pub mod pallet {
 					);
 				}
 
-				if let Some((vrf_output, vrf_proof)) = pre_digest.vrf() {
-					let randomness: Option<schnorrkel::Randomness> = Authorities::<T>::get()
+				if let Some(vrf_signature) = pre_digest.vrf_signature() {
+					let randomness: Option<BabeRandomness> = Authorities::<T>::get()
 						.get(authority_index as usize)
 						.and_then(|(authority, _)| {
-							schnorrkel::PublicKey::from_bytes(authority.as_slice()).ok()
-						})
-						.and_then(|pubkey| {
-							let current_slot = CurrentSlot::<T>::get();
-
+							let public = authority.as_inner_ref();
 							let transcript = sp_consensus_babe::make_transcript(
 								&Self::randomness(),
-								current_slot,
+								CurrentSlot::<T>::get(),
 								EpochIndex::<T>::get(),
 							);
 
 							// NOTE: this is verified by the client when importing the block, before
-							// execution. we don't run the verification again here to avoid slowing
+							// execution. We don't run the verification again here to avoid slowing
 							// down the runtime.
-							debug_assert!(pubkey
-								.vrf_verify(transcript.clone(), vrf_output, vrf_proof)
-								.is_ok());
+							debug_assert!({
+								use sp_core::crypto::VrfVerifier;
+								public.vrf_verify(&transcript, &vrf_signature)
+							});
 
-							vrf_output.0.attach_input_hash(&pubkey, transcript).ok()
-						})
-						.map(|inout| inout.make_bytes(sp_consensus_babe::BABE_VRF_INOUT_CONTEXT));
+							public
+								.make_bytes(
+									RANDOMNESS_VRF_CONTEXT,
+									&transcript,
+									&vrf_signature.output,
+								)
+								.ok()
+						});
 
 					if let Some(randomness) = pre_digest.is_primary().then(|| randomness).flatten()
 					{
@@ -484,9 +485,6 @@ pub mod pallet {
 	}
 }
 
-/// A BABE public key
-pub type BabeKey = [u8; PUBLIC_KEY_LENGTH];
-
 impl<T: Config> FindAuthor<u32> for Pallet<T> {
 	fn find_author<'a, I>(digests: I) -> Option<u32>
 	where
@@ -737,7 +735,7 @@ impl<T: Config> Pallet<T> {
 		<frame_system::Pallet<T>>::deposit_log(log)
 	}
 
-	fn deposit_randomness(randomness: &schnorrkel::Randomness) {
+	fn deposit_randomness(randomness: &BabeRandomness) {
 		let segment_idx = SegmentIndex::<T>::get();
 		let mut segment = UnderConstruction::<T>::get(&segment_idx);
 		if segment.try_push(*randomness).is_ok() {
@@ -831,7 +829,7 @@ impl<T: Config> Pallet<T> {
 
 	/// Call this function exactly once when an epoch changes, to update the
 	/// randomness. Returns the new randomness.
-	fn randomness_change_epoch(next_epoch_index: u64) -> schnorrkel::Randomness {
+	fn randomness_change_epoch(next_epoch_index: u64) -> BabeRandomness {
 		let this_randomness = NextRandomness::<T>::get();
 		let segment_idx: u32 = SegmentIndex::<T>::mutate(|s| sp_std::mem::replace(s, 0));
 
@@ -990,12 +988,12 @@ where
 //
 // an optional size hint as to how many VRF outputs there were may be provided.
 fn compute_randomness(
-	last_epoch_randomness: schnorrkel::Randomness,
+	last_epoch_randomness: BabeRandomness,
 	epoch_index: u64,
-	rho: impl Iterator<Item = schnorrkel::Randomness>,
+	rho: impl Iterator<Item = BabeRandomness>,
 	rho_size_hint: Option<usize>,
-) -> schnorrkel::Randomness {
-	let mut s = Vec::with_capacity(40 + rho_size_hint.unwrap_or(0) * VRF_OUTPUT_LENGTH);
+) -> BabeRandomness {
+	let mut s = Vec::with_capacity(40 + rho_size_hint.unwrap_or(0) * RANDOMNESS_LENGTH);
 	s.extend_from_slice(&last_epoch_randomness);
 	s.extend_from_slice(&epoch_index.to_le_bytes());
 

substrate/frame/babe/src/mock.rs

@@ -25,10 +25,9 @@ use frame_support::{
 	traits::{ConstU128, ConstU32, ConstU64, GenesisBuild, KeyOwnerProofSystem, OnInitialize},
 };
 use pallet_session::historical as pallet_session_historical;
-use sp_consensus_babe::{AuthorityId, AuthorityPair, Slot};
-use sp_consensus_vrf::schnorrkel::{VRFOutput, VRFProof};
+use sp_consensus_babe::{AuthorityId, AuthorityPair, Randomness, Slot, VrfSignature};
 use sp_core::{
-	crypto::{IsWrappedBy, KeyTypeId, Pair},
+	crypto::{KeyTypeId, Pair, VrfSigner},
 	H256, U256,
 };
 use sp_io;
@@ -283,16 +282,10 @@ pub fn start_era(era_index: EraIndex) {
 pub fn make_primary_pre_digest(
 	authority_index: sp_consensus_babe::AuthorityIndex,
 	slot: sp_consensus_babe::Slot,
-	vrf_output: VRFOutput,
-	vrf_proof: VRFProof,
+	vrf_signature: VrfSignature,
 ) -> Digest {
 	let digest_data = sp_consensus_babe::digests::PreDigest::Primary(
-		sp_consensus_babe::digests::PrimaryPreDigest {
-			authority_index,
-			slot,
-			vrf_output,
-			vrf_proof,
-		},
+		sp_consensus_babe::digests::PrimaryPreDigest { authority_index, slot, vrf_signature },
 	);
 	let log = DigestItem::PreRuntime(sp_consensus_babe::BABE_ENGINE_ID, digest_data.encode());
 	Digest { logs: vec![log] }
@@ -312,16 +305,10 @@ pub fn make_secondary_plain_pre_digest(
 pub fn make_secondary_vrf_pre_digest(
 	authority_index: sp_consensus_babe::AuthorityIndex,
 	slot: sp_consensus_babe::Slot,
-	vrf_output: VRFOutput,
-	vrf_proof: VRFProof,
+	vrf_signature: VrfSignature,
 ) -> Digest {
 	let digest_data = sp_consensus_babe::digests::PreDigest::SecondaryVRF(
-		sp_consensus_babe::digests::SecondaryVRFPreDigest {
-			authority_index,
-			slot,
-			vrf_output,
-			vrf_proof,
-		},
+		sp_consensus_babe::digests::SecondaryVRFPreDigest { authority_index, slot, vrf_signature },
 	);
 	let log = DigestItem::PreRuntime(sp_consensus_babe::BABE_ENGINE_ID, digest_data.encode());
 	Digest { logs: vec![log] }
@@ -330,16 +317,16 @@ pub fn make_secondary_vrf_pre_digest(
 pub fn make_vrf_output(
 	slot: Slot,
 	pair: &sp_consensus_babe::AuthorityPair,
-) -> (VRFOutput, VRFProof, [u8; 32]) {
-	let pair = sp_core::sr25519::Pair::from_ref(pair).as_ref();
+) -> (VrfSignature, Randomness) {
 	let transcript = sp_consensus_babe::make_transcript(&Babe::randomness(), slot, 0);
-	let vrf_inout = pair.vrf_sign(transcript);
-	let vrf_randomness: sp_consensus_vrf::schnorrkel::Randomness =
-		vrf_inout.0.make_bytes::<[u8; 32]>(&sp_consensus_babe::BABE_VRF_INOUT_CONTEXT);
-	let vrf_output = VRFOutput(vrf_inout.0.to_output());
-	let vrf_proof = VRFProof(vrf_inout.1);
 
-	(vrf_output, vrf_proof, vrf_randomness)
+	let signature = pair.as_ref().vrf_sign(&transcript);
+
+	let randomness = pair
+		.as_ref()
+		.make_bytes::<Randomness>(sp_consensus_babe::RANDOMNESS_VRF_CONTEXT, &transcript);
+
+	(signature, randomness)
 }
 
 pub fn new_test_ext(authorities_len: usize) -> sp_io::TestExternalities {

substrate/frame/babe/src/randomness.rs

@@ -19,7 +19,7 @@
 //! randomness collected from VRF outputs.
 
 use super::{
-	AuthorVrfRandomness, Config, EpochStart, NextRandomness, Randomness, VRF_OUTPUT_LENGTH,
+	AuthorVrfRandomness, Config, EpochStart, NextRandomness, Randomness, RANDOMNESS_LENGTH,
 };
 use frame_support::traits::Randomness as RandomnessT;
 use sp_runtime::traits::{Hash, One, Saturating};
@@ -132,7 +132,7 @@ pub struct CurrentBlockRandomness<T>(sp_std::marker::PhantomData<T>);
 impl<T: Config> RandomnessT<T::Hash, T::BlockNumber> for RandomnessFromTwoEpochsAgo<T> {
 	fn random(subject: &[u8]) -> (T::Hash, T::BlockNumber) {
 		let mut subject = subject.to_vec();
-		subject.reserve(VRF_OUTPUT_LENGTH);
+		subject.reserve(RANDOMNESS_LENGTH);
 		subject.extend_from_slice(&Randomness::<T>::get()[..]);
 
 		(T::Hashing::hash(&subject[..]), EpochStart::<T>::get().0)
@@ -142,7 +142,7 @@ impl<T: Config> RandomnessT<T::Hash, T::BlockNumber> for RandomnessFromTwoEpochs
 impl<T: Config> RandomnessT<T::Hash, T::BlockNumber> for RandomnessFromOneEpochAgo<T> {
 	fn random(subject: &[u8]) -> (T::Hash, T::BlockNumber) {
 		let mut subject = subject.to_vec();
-		subject.reserve(VRF_OUTPUT_LENGTH);
+		subject.reserve(RANDOMNESS_LENGTH);
 		subject.extend_from_slice(&NextRandomness::<T>::get()[..]);
 
 		(T::Hashing::hash(&subject[..]), EpochStart::<T>::get().1)
@@ -153,7 +153,7 @@ impl<T: Config> RandomnessT<Option<T::Hash>, T::BlockNumber> for ParentBlockRand
 	fn random(subject: &[u8]) -> (Option<T::Hash>, T::BlockNumber) {
 		let random = AuthorVrfRandomness::<T>::get().map(|random| {
 			let mut subject = subject.to_vec();
-			subject.reserve(VRF_OUTPUT_LENGTH);
+			subject.reserve(RANDOMNESS_LENGTH);
 			subject.extend_from_slice(&random);
 
 			T::Hashing::hash(&subject[..])

substrate/frame/babe/src/tests.rs

@@ -25,11 +25,12 @@ use frame_support::{
 };
 use mock::*;
 use pallet_session::ShouldEndSession;
-use sp_consensus_babe::{AllowedSlots, BabeEpochConfiguration, Slot};
-use sp_consensus_vrf::schnorrkel::{VRFOutput, VRFProof};
+use sp_consensus_babe::{
+	AllowedSlots, BabeEpochConfiguration, Slot, VrfSignature, RANDOMNESS_LENGTH,
+};
 use sp_core::crypto::Pair;
 
-const EMPTY_RANDOMNESS: [u8; 32] = [
+const EMPTY_RANDOMNESS: [u8; RANDOMNESS_LENGTH] = [
 	74, 25, 49, 128, 53, 97, 244, 49, 222, 202, 176, 2, 231, 66, 95, 10, 133, 49, 213, 228, 86,
 	161, 164, 127, 217, 153, 138, 37, 48, 192, 248, 0,
 ];
@@ -62,10 +63,9 @@ fn first_block_epoch_zero_start() {
 
 	ext.execute_with(|| {
 		let genesis_slot = Slot::from(100);
-		let (vrf_output, vrf_proof, vrf_randomness) = make_vrf_output(genesis_slot, &pairs[0]);
+		let (vrf_signature, vrf_randomness) = make_vrf_output(genesis_slot, &pairs[0]);
 
-		let first_vrf = vrf_output;
-		let pre_digest = make_primary_pre_digest(0, genesis_slot, first_vrf.clone(), vrf_proof);
+		let pre_digest = make_primary_pre_digest(0, genesis_slot, vrf_signature);
 
 		assert_eq!(Babe::genesis_slot(), Slot::from(0));
 		System::reset_events();
@@ -111,8 +111,8 @@ fn current_slot_is_processed_on_initialization() {
 
 	ext.execute_with(|| {
 		let genesis_slot = Slot::from(10);
-		let (vrf_output, vrf_proof, vrf_randomness) = make_vrf_output(genesis_slot, &pairs[0]);
-		let pre_digest = make_primary_pre_digest(0, genesis_slot, vrf_output, vrf_proof);
+		let (vrf_signature, vrf_randomness) = make_vrf_output(genesis_slot, &pairs[0]);
+		let pre_digest = make_primary_pre_digest(0, genesis_slot, vrf_signature);
 
 		System::reset_events();
 		System::initialize(&1, &Default::default(), &pre_digest);
@@ -134,14 +134,14 @@ fn current_slot_is_processed_on_initialization() {
 
 fn test_author_vrf_output<F>(make_pre_digest: F)
 where
-	F: Fn(sp_consensus_babe::AuthorityIndex, Slot, VRFOutput, VRFProof) -> sp_runtime::Digest,
+	F: Fn(sp_consensus_babe::AuthorityIndex, Slot, VrfSignature) -> sp_runtime::Digest,
 {
 	let (pairs, mut ext) = new_test_ext_with_pairs(1);
 
 	ext.execute_with(|| {
 		let genesis_slot = Slot::from(10);
-		let (vrf_output, vrf_proof, vrf_randomness) = make_vrf_output(genesis_slot, &pairs[0]);
-		let pre_digest = make_pre_digest(0, genesis_slot, vrf_output, vrf_proof);
+		let (vrf_signature, vrf_randomness) = make_vrf_output(genesis_slot, &pairs[0]);
+		let pre_digest = make_pre_digest(0, genesis_slot, vrf_signature);
 
 		System::reset_events();
 		System::initialize(&1, &Default::default(), &pre_digest);