Correct BABE randomness by calculating InOut bytes directly in pallet (#5876)

* vrf: remove Raw* types

* babe: remove Raw* types

* pallet-babe: switch representation of RawVRFOutput to Randomness

* pallet-babe: calculate inout within the pallet

* Remove make_transcript duplication

* Bump spec version

* Fix frame tests

* and_then -> map

* Always enable u64_backend

* Fix nostd compile

* fix import: should not use std

* Remove unused definition of RawVRFOutput

* Remove unused import of RuntimeDebug

Co-authored-by: Gavin Wood <gavin@parity.io>

substrate/primitives/consensus/babe/Cargo.toml

@@ -14,6 +14,7 @@ targets = ["x86_64-unknown-linux-gnu"]
 [dependencies]
 sp-application-crypto = { version = "2.0.0-dev", default-features = false, path = "../../application-crypto" }
 codec = { package = "parity-scale-codec", version = "1.3.0", default-features = false }
+merlin = { version = "2.0", default-features = false }
 sp-std = { version = "2.0.0-dev", default-features = false, path = "../../std" }
 sp-api = { version = "2.0.0-dev", default-features = false, path = "../../api" }
 sp-consensus = { version = "0.8.0-dev", optional = true, path = "../common" }
@@ -27,6 +28,7 @@ default = ["std"]
 std = [
 	"sp-application-crypto/std",
 	"codec/std",
+	"merlin/std",
 	"sp-std/std",
 	"sp-api/std",
 	"sp-consensus",

substrate/primitives/consensus/babe/src/digests.rs

@@ -22,19 +22,17 @@ use super::{AuthorityId, AuthorityIndex, SlotNumber, BabeAuthorityWeight, BabeEp
 #[cfg(feature = "std")]
 use sp_runtime::{DigestItem, generic::OpaqueDigestItemId};
 #[cfg(feature = "std")]
-use std::{fmt::Debug, convert::{TryFrom, TryInto}};
+use std::fmt::Debug;
 use codec::{Decode, Encode};
 #[cfg(feature = "std")]
 use codec::Codec;
 use sp_std::vec::Vec;
 use sp_runtime::RuntimeDebug;
-use sp_consensus_vrf::schnorrkel::{self, Randomness};
-#[cfg(feature = "std")]
-use sp_consensus_vrf::schnorrkel::SignatureError;
+use sp_consensus_vrf::schnorrkel::{Randomness, VRFOutput, VRFProof};
 
 /// Raw BABE primary slot assignment pre-digest.
 #[derive(Clone, RuntimeDebug, Encode, Decode)]
-pub struct RawPrimaryPreDigest<VRFOutput=schnorrkel::RawVRFOutput, VRFProof=schnorrkel::RawVRFProof> {
+pub struct PrimaryPreDigest {
 	/// Authority index
 	pub authority_index: super::AuthorityIndex,
 	/// Slot number
@@ -45,24 +43,6 @@ pub struct RawPrimaryPreDigest<VRFOutput=schnorrkel::RawVRFOutput, VRFProof=schn
 	pub vrf_proof: VRFProof,
 }
 
-#[cfg(feature = "std")]
-/// BABE primary slot assignment pre-digest for std environment.
-pub type PrimaryPreDigest = RawPrimaryPreDigest<schnorrkel::VRFOutput, schnorrkel::VRFProof>;
-
-#[cfg(feature = "std")]
-impl TryFrom<RawPrimaryPreDigest> for PrimaryPreDigest {
-	type Error = SignatureError;
-
-	fn try_from(raw: RawPrimaryPreDigest) -> Result<PrimaryPreDigest, SignatureError> {
-		Ok(PrimaryPreDigest {
-			authority_index: raw.authority_index,
-			slot_number: raw.slot_number,
-			vrf_output: raw.vrf_output.try_into()?,
-			vrf_proof: raw.vrf_proof.try_into()?,
-		})
-	}
-}
-
 /// BABE secondary slot assignment pre-digest.
 #[derive(Clone, RuntimeDebug, Encode, Decode)]
 pub struct SecondaryPlainPreDigest {
@@ -79,7 +59,7 @@ pub struct SecondaryPlainPreDigest {
 
 /// BABE secondary deterministic slot assignment with VRF outputs.
 #[derive(Clone, RuntimeDebug, Encode, Decode)]
-pub struct RawSecondaryVRFPreDigest<VRFOutput=schnorrkel::RawVRFOutput, VRFProof=schnorrkel::RawVRFProof> {
+pub struct SecondaryVRFPreDigest {
 	/// Authority index
 	pub authority_index: super::AuthorityIndex,
 	/// Slot number
@@ -90,60 +70,38 @@ pub struct RawSecondaryVRFPreDigest<VRFOutput=schnorrkel::RawVRFOutput, VRFProof
 	pub vrf_proof: VRFProof,
 }
 
-#[cfg(feature = "std")]
-/// BABE secondary slot assignment with VRF outputs pre-digest, for std environment.
-pub type SecondaryVRFPreDigest = RawSecondaryVRFPreDigest<schnorrkel::VRFOutput, schnorrkel::VRFProof>;
-
-#[cfg(feature = "std")]
-impl TryFrom<RawSecondaryVRFPreDigest> for SecondaryVRFPreDigest {
-	type Error = SignatureError;
-
-	fn try_from(raw: RawSecondaryVRFPreDigest) -> Result<SecondaryVRFPreDigest, SignatureError> {
-		Ok(SecondaryVRFPreDigest {
-			authority_index: raw.authority_index,
-			slot_number: raw.slot_number,
-			vrf_output: raw.vrf_output.try_into()?,
-			vrf_proof: raw.vrf_proof.try_into()?,
-		})
-	}
-}
-
 /// A BABE pre-runtime digest. This contains all data required to validate a
 /// block and for the BABE runtime module. Slots can be assigned to a primary
 /// (VRF based) and to a secondary (slot number based).
 #[derive(Clone, RuntimeDebug, Encode, Decode)]
-pub enum RawPreDigest<VRFOutput=schnorrkel::RawVRFOutput, VRFProof=schnorrkel::RawVRFProof> {
+pub enum PreDigest {
 	/// A primary VRF-based slot assignment.
 	#[codec(index = "1")]
-	Primary(RawPrimaryPreDigest<VRFOutput, VRFProof>),
+	Primary(PrimaryPreDigest),
 	/// A secondary deterministic slot assignment.
 	#[codec(index = "2")]
 	SecondaryPlain(SecondaryPlainPreDigest),
 	/// A secondary deterministic slot assignment with VRF outputs.
 	#[codec(index = "3")]
-	SecondaryVRF(RawSecondaryVRFPreDigest<VRFOutput, VRFProof>),
+	SecondaryVRF(SecondaryVRFPreDigest),
 }
 
-#[cfg(feature = "std")]
-/// A BABE pre-runtime digest for std.
-pub type PreDigest = RawPreDigest<schnorrkel::VRFOutput, schnorrkel::VRFProof>;
-
-impl<VRFOutput, VRFProof> RawPreDigest<VRFOutput, VRFProof> {
+impl PreDigest {
 	/// Returns the slot number of the pre digest.
 	pub fn authority_index(&self) -> AuthorityIndex {
 		match self {
-			RawPreDigest::Primary(primary) => primary.authority_index,
-			RawPreDigest::SecondaryPlain(secondary) => secondary.authority_index,
-			RawPreDigest::SecondaryVRF(secondary) => secondary.authority_index,
+			PreDigest::Primary(primary) => primary.authority_index,
+			PreDigest::SecondaryPlain(secondary) => secondary.authority_index,
+			PreDigest::SecondaryVRF(secondary) => secondary.authority_index,
 		}
 	}
 
 	/// Returns the slot number of the pre digest.
 	pub fn slot_number(&self) -> SlotNumber {
 		match self {
-			RawPreDigest::Primary(primary) => primary.slot_number,
-			RawPreDigest::SecondaryPlain(secondary) => secondary.slot_number,
-			RawPreDigest::SecondaryVRF(secondary) => secondary.slot_number,
+			PreDigest::Primary(primary) => primary.slot_number,
+			PreDigest::SecondaryPlain(secondary) => secondary.slot_number,
+			PreDigest::SecondaryVRF(secondary) => secondary.slot_number,
 		}
 	}
 
@@ -151,25 +109,12 @@ impl<VRFOutput, VRFProof> RawPreDigest<VRFOutput, VRFProof> {
 	/// of the chain.
 	pub fn added_weight(&self) -> crate::BabeBlockWeight {
 		match self {
-			RawPreDigest::Primary(_) => 1,
-			RawPreDigest::SecondaryPlain(_) | RawPreDigest::SecondaryVRF(_) => 0,
+			PreDigest::Primary(_) => 1,
+			PreDigest::SecondaryPlain(_) | PreDigest::SecondaryVRF(_) => 0,
 		}
 	}
 }
 
-#[cfg(feature = "std")]
-impl TryFrom<RawPreDigest> for PreDigest {
-	type Error = SignatureError;
-
-	fn try_from(raw: RawPreDigest) -> Result<PreDigest, SignatureError> {
-		Ok(match raw {
-			RawPreDigest::Primary(primary) => PreDigest::Primary(primary.try_into()?),
-			RawPreDigest::SecondaryPlain(secondary) => PreDigest::SecondaryPlain(secondary),
-			RawPreDigest::SecondaryVRF(secondary) => PreDigest::SecondaryVRF(secondary.try_into()?),
-		})
-	}
-}
-
 /// Information about the next epoch. This is broadcast in the first block
 /// of the epoch.
 #[derive(Decode, Encode, PartialEq, Eq, Clone, RuntimeDebug)]

substrate/primitives/consensus/babe/src/lib.rs

@@ -25,6 +25,7 @@ pub mod inherents;
 pub use sp_consensus_vrf::schnorrkel::{
 	Randomness, VRF_PROOF_LENGTH, VRF_OUTPUT_LENGTH, RANDOMNESS_LENGTH
 };
+pub use merlin::Transcript;
 
 use codec::{Encode, Decode};
 use sp_std::vec::Vec;
@@ -39,6 +40,9 @@ mod app {
 /// The prefix used by BABE for its VRF keys.
 pub const BABE_VRF_PREFIX: &[u8] = b"substrate-babe-vrf";
 
+/// BABE VRFInOut context.
+pub static BABE_VRF_INOUT_CONTEXT: &[u8] = b"BabeVRFInOutContext";
+
 /// A Babe authority keypair. Necessarily equivalent to the schnorrkel public key used in
 /// the main Babe module. If that ever changes, then this must, too.
 #[cfg(feature = "std")]
@@ -76,6 +80,19 @@ pub type BabeAuthorityWeight = u64;
 /// The weight of a BABE block.
 pub type BabeBlockWeight = u32;
 
+/// Make a VRF transcript from given randomness, slot number and epoch.
+pub fn make_transcript(
+	randomness: &Randomness,
+	slot_number: u64,
+	epoch: u64,
+) -> Transcript {
+	let mut transcript = Transcript::new(&BABE_ENGINE_ID);
+	transcript.append_u64(b"slot number", slot_number);
+	transcript.append_u64(b"current epoch", epoch);
+	transcript.append_message(b"chain randomness", &randomness[..]);
+	transcript
+}
+
 /// An consensus log item for BABE.
 #[derive(Decode, Encode, Clone, PartialEq, Eq)]
 pub enum ConsensusLog {

substrate/primitives/consensus/vrf/Cargo.toml

@@ -13,7 +13,7 @@ targets = ["x86_64-unknown-linux-gnu"]
 
 [dependencies]
 codec = { version = "1.0.0", package = "parity-scale-codec", default-features = false }
-schnorrkel = { version = "0.9.1", features = ["preaudit_deprecated"], optional = true }
+schnorrkel = { version = "0.9.1", features = ["preaudit_deprecated", "u64_backend"], default-features = false }
 sp-std = { version = "2.0.0-dev", path = "../../std", default-features = false }
 sp-core = { version = "2.0.0-dev", path = "../../core", default-features = false }
 sp-runtime = { version = "2.0.0-dev", default-features = false, path = "../../runtime" }
@@ -22,7 +22,7 @@ sp-runtime = { version = "2.0.0-dev", default-features = false, path = "../../ru
 default = ["std"]
 std = [
 	"codec/std",
-	"schnorrkel",
+	"schnorrkel/std",
 	"sp-std/std",
 	"sp-core/std",
 	"sp-runtime/std",

substrate/primitives/consensus/vrf/src/schnorrkel.rs

@@ -16,72 +16,38 @@
 
 //! Schnorrkel-based VRF.
 
-use codec::{Encode, Decode};
-use sp_runtime::RuntimeDebug;
-use sp_std::ops::{Deref, DerefMut};
-#[cfg(feature = "std")]
-use std::convert::TryFrom;
-#[cfg(feature = "std")]
-use codec::EncodeLike;
-#[cfg(feature = "std")]
-use schnorrkel::errors::MultiSignatureStage;
-#[cfg(feature = "std")]
+use codec::{Encode, Decode, EncodeLike};
+use sp_std::{convert::TryFrom, prelude::*};
 use sp_core::U512;
+use sp_std::ops::{Deref, DerefMut};
+use schnorrkel::errors::MultiSignatureStage;
 
-#[cfg(feature = "std")]
-pub use schnorrkel::{SignatureError, vrf::{VRF_PROOF_LENGTH, VRF_OUTPUT_LENGTH}};
-
-/// The length of the VRF proof.
-#[cfg(not(feature = "std"))]
-pub const VRF_PROOF_LENGTH: usize = 64;
-
-/// The length of the VRF output.
-#[cfg(not(feature = "std"))]
-pub const VRF_OUTPUT_LENGTH: usize = 32;
+pub use schnorrkel::{SignatureError, PublicKey, vrf::{VRF_PROOF_LENGTH, VRF_OUTPUT_LENGTH}};
 
 /// The length of the Randomness.
 pub const RANDOMNESS_LENGTH: usize = VRF_OUTPUT_LENGTH;
 
-/// Raw VRF output.
-#[derive(Clone, Copy, Eq, PartialEq, RuntimeDebug, Encode, Decode)]
-pub struct RawVRFOutput(pub [u8; VRF_OUTPUT_LENGTH]);
-
-impl Deref for RawVRFOutput {
-	type Target = [u8; VRF_OUTPUT_LENGTH];
-	fn deref(&self) -> &Self::Target { &self.0 }
-}
-
-impl DerefMut for RawVRFOutput {
-	fn deref_mut(&mut self) -> &mut Self::Target { &mut self.0 }
-}
-
 /// VRF output type available for `std` environment, suitable for schnorrkel operations.
-#[cfg(feature = "std")]
 #[derive(Clone, Debug, PartialEq, Eq)]
 pub struct VRFOutput(pub schnorrkel::vrf::VRFOutput);
 
-#[cfg(feature = "std")]
 impl Deref for VRFOutput {
 	type Target = schnorrkel::vrf::VRFOutput;
 	fn deref(&self) -> &Self::Target { &self.0 }
 }
 
-#[cfg(feature = "std")]
 impl DerefMut for VRFOutput {
 	fn deref_mut(&mut self) -> &mut Self::Target { &mut self.0 }
 }
 
-#[cfg(feature = "std")]
 impl Encode for VRFOutput {
 	fn encode(&self) -> Vec<u8> {
 		self.0.as_bytes().encode()
 	}
 }
 
-#[cfg(feature = "std")]
 impl EncodeLike for VRFOutput { }
 
-#[cfg(feature = "std")]
 impl Decode for VRFOutput {
 	fn decode<R: codec::Input>(i: &mut R) -> Result<Self, codec::Error> {
 		let decoded = <[u8; VRF_OUTPUT_LENGTH]>::decode(i)?;
@@ -89,7 +55,6 @@ impl Decode for VRFOutput {
 	}
 }
 
-#[cfg(feature = "std")]
 impl TryFrom<[u8; VRF_OUTPUT_LENGTH]> for VRFOutput {
 	type Error = SignatureError;
 
@@ -98,91 +63,39 @@ impl TryFrom<[u8; VRF_OUTPUT_LENGTH]> for VRFOutput {
 	}
 }
 
-#[cfg(feature = "std")]
-impl TryFrom<RawVRFOutput> for VRFOutput {
-	type Error = SignatureError;
-
-	fn try_from(raw: RawVRFOutput) -> Result<VRFOutput, Self::Error> {
-		schnorrkel::vrf::VRFOutput::from_bytes(&raw.0).map(VRFOutput)
-	}
-}
-
-#[cfg(feature = "std")]
-impl From<VRFOutput> for RawVRFOutput {
-	fn from(output: VRFOutput) -> RawVRFOutput {
-		RawVRFOutput(output.to_bytes())
-	}
-}
-
-/// Raw VRF proof.
-#[derive(Clone, Copy, Encode, Decode)]
-pub struct RawVRFProof(pub [u8; VRF_PROOF_LENGTH]);
-
-impl Deref for RawVRFProof {
-	type Target = [u8; VRF_PROOF_LENGTH];
-	fn deref(&self) -> &Self::Target { &self.0 }
-}
-
-impl DerefMut for RawVRFProof {
-	fn deref_mut(&mut self) -> &mut Self::Target { &mut self.0 }
-}
-
-#[cfg(feature = "std")]
-impl std::fmt::Debug for RawVRFProof {
-	fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
-		write!(f, "{:?}", &self)
-	}
-}
-
-impl core::cmp::PartialEq for RawVRFProof {
-	fn eq(&self, other: &Self) -> bool {
-		self == other
-	}
-}
-
-impl core::cmp::Eq for RawVRFProof { }
-
 /// VRF proof type available for `std` environment, suitable for schnorrkel operations.
-#[cfg(feature = "std")]
 #[derive(Clone, Debug, PartialEq, Eq)]
 pub struct VRFProof(pub schnorrkel::vrf::VRFProof);
 
-#[cfg(feature = "std")]
 impl PartialOrd for VRFProof {
 	fn partial_cmp(&self, other: &Self) -> Option<core::cmp::Ordering> {
 		Some(self.cmp(other))
 	}
 }
 
-#[cfg(feature = "std")]
 impl Ord for VRFProof {
 	fn cmp(&self, other: &Self) -> core::cmp::Ordering {
 		U512::from(self.0.to_bytes()).cmp(&U512::from(other.0.to_bytes()))
 	}
 }
 
-#[cfg(feature = "std")]
 impl Deref for VRFProof {
 	type Target = schnorrkel::vrf::VRFProof;
 	fn deref(&self) -> &Self::Target { &self.0 }
 }
 
-#[cfg(feature = "std")]
 impl DerefMut for VRFProof {
 	fn deref_mut(&mut self) -> &mut Self::Target { &mut self.0 }
 }
 
-#[cfg(feature = "std")]
 impl Encode for VRFProof {
 	fn encode(&self) -> Vec<u8> {
 		self.0.to_bytes().encode()
 	}
 }
 
-#[cfg(feature = "std")]
 impl EncodeLike for VRFProof { }
 
-#[cfg(feature = "std")]
 impl Decode for VRFProof {
 	fn decode<R: codec::Input>(i: &mut R) -> Result<Self, codec::Error> {
 		let decoded = <[u8; VRF_PROOF_LENGTH]>::decode(i)?;
@@ -190,7 +103,6 @@ impl Decode for VRFProof {
 	}
 }
 
-#[cfg(feature = "std")]
 impl TryFrom<[u8; VRF_PROOF_LENGTH]> for VRFProof {
 	type Error = SignatureError;
 
@@ -199,23 +111,6 @@ impl TryFrom<[u8; VRF_PROOF_LENGTH]> for VRFProof {
 	}
 }
 
-#[cfg(feature = "std")]
-impl TryFrom<RawVRFProof> for VRFProof {
-	type Error = SignatureError;
-
-	fn try_from(raw: RawVRFProof) -> Result<VRFProof, Self::Error> {
-		schnorrkel::vrf::VRFProof::from_bytes(&raw.0).map(VRFProof)
-	}
-}
-
-#[cfg(feature = "std")]
-impl From<VRFProof> for RawVRFProof {
-	fn from(output: VRFProof) -> RawVRFProof {
-		RawVRFProof(output.to_bytes())
-	}
-}
-
-#[cfg(feature = "std")]
 fn convert_error(e: SignatureError) -> codec::Error {
 	use SignatureError::*;
 	use MultiSignatureStage::*;