VRF refactory (#13889)

* First iteration to encapsulate schnorrkel and merlin usage

* Remove schnorkel direct dependency from BABE pallet

* Remove schnorrkel direct dependency from BABE client

* Trivial renaming for VrfTranscript data and value

* Better errors

* Expose a function to get a schnorrkel friendly transcript

* Keep the vrf signature stuff together (preventing some clones around)

* Fix tests

* Remove vrf agnostic transcript and define it as an associated type for VrfSigner and VrfVerifier

* Fix babe pallet mock

* Inner types are required to be public for polkadot

* Update client/consensus/babe/src/verification.rs

Co-authored-by: Koute <koute@users.noreply.github.com>

* Nit

* Remove Deref implementations

* make_bytes as a method

* Trigger CI

---------

Co-authored-by: Koute <koute@users.noreply.github.com>

substrate/primitives/consensus/babe/Cargo.toml

@@ -15,14 +15,12 @@ targets = ["x86_64-unknown-linux-gnu"]
 [dependencies]
 async-trait = { version = "0.1.57", optional = true }
 codec = { package = "parity-scale-codec", version = "3.2.2", default-features = false }
-merlin = { version = "2.0", default-features = false }
 scale-info = { version = "2.5.0", default-features = false, features = ["derive"] }
 serde = { version = "1.0.136", features = ["derive"], optional = true }
 sp-api = { version = "4.0.0-dev", default-features = false, path = "../../api" }
 sp-application-crypto = { version = "7.0.0", default-features = false, path = "../../application-crypto" }
 sp-consensus = { version = "0.10.0-dev", optional = true, path = "../common" }
 sp-consensus-slots = { version = "0.10.0-dev", default-features = false, path = "../slots" }
-sp-consensus-vrf = { version = "0.10.0-dev", default-features = false, path = "../vrf" }
 sp-core = { version = "7.0.0", default-features = false, path = "../../core" }
 sp-inherents = { version = "4.0.0-dev", default-features = false, path = "../../inherents" }
 sp-keystore = { version = "0.13.0", default-features = false, optional = true, path = "../../keystore" }
@@ -35,14 +33,12 @@ default = ["std"]
 std = [
 	"async-trait",
 	"codec/std",
-	"merlin/std",
 	"scale-info/std",
 	"serde",
 	"sp-api/std",
 	"sp-application-crypto/std",
 	"sp-consensus",
 	"sp-consensus-slots/std",
-	"sp-consensus-vrf/std",
 	"sp-core/std",
 	"sp-inherents/std",
 	"sp-keystore",

substrate/primitives/consensus/babe/src/digests.rs

@@ -19,14 +19,15 @@
 
 use super::{
 	AllowedSlots, AuthorityId, AuthorityIndex, AuthoritySignature, BabeAuthorityWeight,
-	BabeEpochConfiguration, Slot, BABE_ENGINE_ID,
+	BabeEpochConfiguration, Randomness, Slot, BABE_ENGINE_ID,
 };
-use codec::{Decode, Encode, MaxEncodedLen};
-use scale_info::TypeInfo;
+
+use sp_core::sr25519::vrf::VrfSignature;
 use sp_runtime::{DigestItem, RuntimeDebug};
 use sp_std::vec::Vec;
 
-use sp_consensus_vrf::schnorrkel::{Randomness, VRFOutput, VRFProof};
+use codec::{Decode, Encode, MaxEncodedLen};
+use scale_info::TypeInfo;
 
 /// Raw BABE primary slot assignment pre-digest.
 #[derive(Clone, RuntimeDebug, Encode, Decode, MaxEncodedLen, TypeInfo)]
@@ -35,10 +36,8 @@ pub struct PrimaryPreDigest {
 	pub authority_index: super::AuthorityIndex,
 	/// Slot
 	pub slot: Slot,
-	/// VRF output
-	pub vrf_output: VRFOutput,
-	/// VRF proof
-	pub vrf_proof: VRFProof,
+	/// VRF signature
+	pub vrf_signature: VrfSignature,
 }
 
 /// BABE secondary slot assignment pre-digest.
@@ -62,10 +61,8 @@ pub struct SecondaryVRFPreDigest {
 	pub authority_index: super::AuthorityIndex,
 	/// Slot
 	pub slot: Slot,
-	/// VRF output
-	pub vrf_output: VRFOutput,
-	/// VRF proof
-	pub vrf_proof: VRFProof,
+	/// VRF signature
+	pub vrf_signature: VrfSignature,
 }
 
 /// A BABE pre-runtime digest. This contains all data required to validate a
@@ -118,11 +115,10 @@ impl PreDigest {
 	}
 
 	/// Returns the VRF output and proof, if they exist.
-	pub fn vrf(&self) -> Option<(&VRFOutput, &VRFProof)> {
+	pub fn vrf_signature(&self) -> Option<&VrfSignature> {
 		match self {
-			PreDigest::Primary(primary) => Some((&primary.vrf_output, &primary.vrf_proof)),
-			PreDigest::SecondaryVRF(secondary) =>
-				Some((&secondary.vrf_output, &secondary.vrf_proof)),
+			PreDigest::Primary(primary) => Some(&primary.vrf_signature),
+			PreDigest::SecondaryVRF(secondary) => Some(&secondary.vrf_signature),
 			PreDigest::SecondaryPlain(_) => None,
 		}
 	}

substrate/primitives/consensus/babe/src/lib.rs

@@ -23,22 +23,17 @@
 pub mod digests;
 pub mod inherents;
 
-pub use merlin::Transcript;
-pub use sp_consensus_vrf::schnorrkel::{
-	Randomness, RANDOMNESS_LENGTH, VRF_OUTPUT_LENGTH, VRF_PROOF_LENGTH,
-};
-
 use codec::{Decode, Encode, MaxEncodedLen};
 use scale_info::TypeInfo;
 #[cfg(feature = "std")]
 use serde::{Deserialize, Serialize};
-#[cfg(feature = "std")]
-use sp_keystore::vrf::{VRFTranscriptData, VRFTranscriptValue};
 use sp_runtime::{traits::Header, ConsensusEngineId, RuntimeDebug};
 use sp_std::vec::Vec;
 
 use crate::digests::{NextConfigDescriptor, NextEpochDescriptor};
 
+pub use sp_core::sr25519::vrf::{VrfOutput, VrfProof, VrfSignature, VrfTranscript};
+
 /// Key type for BABE module.
 pub const KEY_TYPE: sp_core::crypto::KeyTypeId = sp_application_crypto::key_types::BABE;
 
@@ -47,11 +42,14 @@ mod app {
 	app_crypto!(sr25519, BABE);
 }
 
-/// The prefix used by BABE for its VRF keys.
-pub const BABE_VRF_PREFIX: &[u8] = b"substrate-babe-vrf";
+/// VRF context used for per-slot randomness generation.
+pub const RANDOMNESS_VRF_CONTEXT: &[u8] = b"BabeVRFInOutContext";
 
-/// BABE VRFInOut context.
-pub static BABE_VRF_INOUT_CONTEXT: &[u8] = b"BabeVRFInOutContext";
+/// VRF output length for per-slot randomness.
+pub const RANDOMNESS_LENGTH: usize = 32;
+
+/// Randomness type required by BABE operations.
+pub type Randomness = [u8; RANDOMNESS_LENGTH];
 
 /// A Babe authority keypair. Necessarily equivalent to the schnorrkel public key used in
 /// the main Babe module. If that ever changes, then this must, too.
@@ -96,26 +94,16 @@ pub type BabeAuthorityWeight = u64;
 /// of 0 (regardless of whether they are plain or vrf secondary blocks).
 pub type BabeBlockWeight = u32;
 
-/// Make a VRF transcript from given randomness, slot number and epoch.
-pub fn make_transcript(randomness: &Randomness, slot: Slot, epoch: u64) -> Transcript {
-	let mut transcript = Transcript::new(&BABE_ENGINE_ID);
-	transcript.append_u64(b"slot number", *slot);
-	transcript.append_u64(b"current epoch", epoch);
-	transcript.append_message(b"chain randomness", &randomness[..]);
-	transcript
-}
-
 /// Make a VRF transcript data container
-#[cfg(feature = "std")]
-pub fn make_transcript_data(randomness: &Randomness, slot: Slot, epoch: u64) -> VRFTranscriptData {
-	VRFTranscriptData {
-		label: &BABE_ENGINE_ID,
-		items: vec![
-			("slot number", VRFTranscriptValue::U64(*slot)),
-			("current epoch", VRFTranscriptValue::U64(epoch)),
-			("chain randomness", VRFTranscriptValue::Bytes(randomness.to_vec())),
+pub fn make_transcript(randomness: &Randomness, slot: Slot, epoch: u64) -> VrfTranscript {
+	VrfTranscript::new(
+		&BABE_ENGINE_ID,
+		&[
+			(b"slot number", &slot.to_le_bytes()),
+			(b"current epoch", &epoch.to_le_bytes()),
+			(b"chain randomness", randomness),
 		],
-	}
+	)
 }
 
 /// An consensus log item for BABE.
@@ -355,7 +343,7 @@ pub struct Epoch {
 	/// The authorities and their weights.
 	pub authorities: Vec<(AuthorityId, BabeAuthorityWeight)>,
 	/// Randomness for this epoch.
-	pub randomness: [u8; VRF_OUTPUT_LENGTH],
+	pub randomness: Randomness,
 	/// Configuration of the epoch.
 	pub config: BabeEpochConfiguration,
 }

substrate/primitives/consensus/vrf/Cargo.toml

@@ -1,32 +0,0 @@
-[package]
-name = "sp-consensus-vrf"
-version = "0.10.0-dev"
-authors = ["Parity Technologies <admin@parity.io>"]
-description = "Primitives for VRF based consensus"
-edition = "2021"
-license = "Apache-2.0"
-repository = "https://github.com/paritytech/substrate/"
-homepage = "https://substrate.io"
-readme = "README.md"
-
-[package.metadata.docs.rs]
-targets = ["x86_64-unknown-linux-gnu"]
-
-[dependencies]
-codec = { package = "parity-scale-codec", version = "3.2.2", default-features = false }
-scale-info = { version = "2.5.0", default-features = false }
-schnorrkel = { version = "0.9.1", default-features = false, features = ["preaudit_deprecated", "u64_backend"] }
-sp-core = { version = "7.0.0", default-features = false, path = "../../core" }
-sp-runtime = { version = "7.0.0", default-features = false, path = "../../runtime" }
-sp-std = { version = "5.0.0", default-features = false, path = "../../std" }
-
-[features]
-default = ["std"]
-std = [
-	"codec/std",
-	"scale-info/std",
-	"schnorrkel/std",
-	"sp-core/std",
-	"sp-runtime/std",
-	"sp-std/std",
-]

substrate/primitives/consensus/vrf/README.md

@@ -1,3 +0,0 @@
-Primitives for VRF-based consensus engines.
-
-License: Apache-2.0

substrate/primitives/consensus/vrf/src/lib.rs

@@ -1,21 +0,0 @@
-// This file is part of Substrate.
-
-// Copyright (C) Parity Technologies (UK) Ltd.
-// SPDX-License-Identifier: Apache-2.0
-
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-// 	http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-//! Primitives for VRF-based consensus engines.
-#![cfg_attr(not(feature = "std"), no_std)]
-
-pub mod schnorrkel;

substrate/primitives/consensus/vrf/src/schnorrkel.rs

@@ -1,188 +0,0 @@
-// This file is part of Substrate.
-
-// Copyright (C) Parity Technologies (UK) Ltd.
-// SPDX-License-Identifier: Apache-2.0
-
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-// 	http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-//! Schnorrkel-based VRF.
-
-use codec::{Decode, Encode, EncodeLike, MaxEncodedLen};
-use scale_info::TypeInfo;
-use schnorrkel::errors::MultiSignatureStage;
-use sp_core::U512;
-use sp_std::{
-	ops::{Deref, DerefMut},
-	prelude::*,
-};
-
-pub use schnorrkel::{
-	vrf::{VRF_OUTPUT_LENGTH, VRF_PROOF_LENGTH},
-	PublicKey, SignatureError,
-};
-
-/// The length of the Randomness.
-pub const RANDOMNESS_LENGTH: usize = VRF_OUTPUT_LENGTH;
-
-/// VRF output type available for `std` environment, suitable for schnorrkel operations.
-#[derive(Clone, Debug, PartialEq, Eq)]
-pub struct VRFOutput(pub schnorrkel::vrf::VRFOutput);
-
-impl Deref for VRFOutput {
-	type Target = schnorrkel::vrf::VRFOutput;
-	fn deref(&self) -> &Self::Target {
-		&self.0
-	}
-}
-
-impl DerefMut for VRFOutput {
-	fn deref_mut(&mut self) -> &mut Self::Target {
-		&mut self.0
-	}
-}
-
-impl Encode for VRFOutput {
-	fn encode(&self) -> Vec<u8> {
-		self.0.as_bytes().encode()
-	}
-}
-
-impl EncodeLike for VRFOutput {}
-
-impl Decode for VRFOutput {
-	fn decode<R: codec::Input>(i: &mut R) -> Result<Self, codec::Error> {
-		let decoded = <[u8; VRF_OUTPUT_LENGTH]>::decode(i)?;
-		Ok(Self(schnorrkel::vrf::VRFOutput::from_bytes(&decoded).map_err(convert_error)?))
-	}
-}
-
-impl MaxEncodedLen for VRFOutput {
-	fn max_encoded_len() -> usize {
-		<[u8; VRF_OUTPUT_LENGTH]>::max_encoded_len()
-	}
-}
-
-impl TypeInfo for VRFOutput {
-	type Identity = [u8; VRF_OUTPUT_LENGTH];
-
-	fn type_info() -> scale_info::Type {
-		Self::Identity::type_info()
-	}
-}
-
-impl TryFrom<[u8; VRF_OUTPUT_LENGTH]> for VRFOutput {
-	type Error = SignatureError;
-
-	fn try_from(raw: [u8; VRF_OUTPUT_LENGTH]) -> Result<Self, Self::Error> {
-		schnorrkel::vrf::VRFOutput::from_bytes(&raw).map(VRFOutput)
-	}
-}
-
-/// VRF proof type available for `std` environment, suitable for schnorrkel operations.
-#[derive(Clone, Debug, PartialEq, Eq)]
-pub struct VRFProof(pub schnorrkel::vrf::VRFProof);
-
-impl PartialOrd for VRFProof {
-	fn partial_cmp(&self, other: &Self) -> Option<core::cmp::Ordering> {
-		Some(self.cmp(other))
-	}
-}
-
-impl Ord for VRFProof {
-	fn cmp(&self, other: &Self) -> core::cmp::Ordering {
-		U512::from(self.0.to_bytes()).cmp(&U512::from(other.0.to_bytes()))
-	}
-}
-
-impl Deref for VRFProof {
-	type Target = schnorrkel::vrf::VRFProof;
-	fn deref(&self) -> &Self::Target {
-		&self.0
-	}
-}
-
-impl DerefMut for VRFProof {
-	fn deref_mut(&mut self) -> &mut Self::Target {
-		&mut self.0
-	}
-}
-
-impl Encode for VRFProof {
-	fn encode(&self) -> Vec<u8> {
-		self.0.to_bytes().encode()
-	}
-}
-
-impl EncodeLike for VRFProof {}
-
-impl Decode for VRFProof {
-	fn decode<R: codec::Input>(i: &mut R) -> Result<Self, codec::Error> {
-		let decoded = <[u8; VRF_PROOF_LENGTH]>::decode(i)?;
-		Ok(Self(schnorrkel::vrf::VRFProof::from_bytes(&decoded).map_err(convert_error)?))
-	}
-}
-
-impl MaxEncodedLen for VRFProof {
-	fn max_encoded_len() -> usize {
-		<[u8; VRF_PROOF_LENGTH]>::max_encoded_len()
-	}
-}
-
-impl TypeInfo for VRFProof {
-	type Identity = [u8; VRF_PROOF_LENGTH];
-
-	fn type_info() -> scale_info::Type {
-		Self::Identity::type_info()
-	}
-}
-
-impl TryFrom<[u8; VRF_PROOF_LENGTH]> for VRFProof {
-	type Error = SignatureError;
-
-	fn try_from(raw: [u8; VRF_PROOF_LENGTH]) -> Result<Self, Self::Error> {
-		schnorrkel::vrf::VRFProof::from_bytes(&raw).map(VRFProof)
-	}
-}
-
-fn convert_error(e: SignatureError) -> codec::Error {
-	use MultiSignatureStage::*;
-	use SignatureError::*;
-	match e {
-		EquationFalse => "Signature error: `EquationFalse`".into(),
-		PointDecompressionError => "Signature error: `PointDecompressionError`".into(),
-		ScalarFormatError => "Signature error: `ScalarFormatError`".into(),
-		NotMarkedSchnorrkel => "Signature error: `NotMarkedSchnorrkel`".into(),
-		BytesLengthError { .. } => "Signature error: `BytesLengthError`".into(),
-		MuSigAbsent { musig_stage: Commitment } =>
-			"Signature error: `MuSigAbsent` at stage `Commitment`".into(),
-		MuSigAbsent { musig_stage: Reveal } =>
-			"Signature error: `MuSigAbsent` at stage `Reveal`".into(),
-		MuSigAbsent { musig_stage: Cosignature } =>
-			"Signature error: `MuSigAbsent` at stage `Commitment`".into(),
-		MuSigInconsistent { musig_stage: Commitment, duplicate: true } =>
-			"Signature error: `MuSigInconsistent` at stage `Commitment` on duplicate".into(),
-		MuSigInconsistent { musig_stage: Commitment, duplicate: false } =>
-			"Signature error: `MuSigInconsistent` at stage `Commitment` on not duplicate".into(),
-		MuSigInconsistent { musig_stage: Reveal, duplicate: true } =>
-			"Signature error: `MuSigInconsistent` at stage `Reveal` on duplicate".into(),
-		MuSigInconsistent { musig_stage: Reveal, duplicate: false } =>
-			"Signature error: `MuSigInconsistent` at stage `Reveal` on not duplicate".into(),
-		MuSigInconsistent { musig_stage: Cosignature, duplicate: true } =>
-			"Signature error: `MuSigInconsistent` at stage `Cosignature` on duplicate".into(),
-		MuSigInconsistent { musig_stage: Cosignature, duplicate: false } =>
-			"Signature error: `MuSigInconsistent` at stage `Cosignature` on not duplicate".into(),
-	}
-}
-
-/// Schnorrkel randomness value. Same size as `VRFOutput`.
-pub type Randomness = [u8; RANDOMNESS_LENGTH];