Add pluggable BEEFY payload constructors (#12428)

* primitives/beefy: move Payload to its own file

* primitives/beefy: add Payload tests

* primitives/beefy: add MmrRootProvider as custom BEEFY payload provider

* client/beefy: use generic BEEFY 'PayloadProvider'

* primitives/beefy: rename Payload::new to Payload::from_single_entry for clarity

* fix visibility

* fix cargo doc

substrate/primitives/beefy/src/commitment.rs

@@ -19,61 +19,7 @@ use codec::{Decode, Encode, Error, Input};
 use scale_info::TypeInfo;
 use sp_std::{cmp, prelude::*};
 
-use crate::ValidatorSetId;
-
-/// Id of different payloads in the [`Commitment`] data
-pub type BeefyPayloadId = [u8; 2];
-
-/// Registry of all known [`BeefyPayloadId`].
-pub mod known_payload_ids {
-	use crate::BeefyPayloadId;
-
-	/// A [`Payload`](super::Payload) identifier for Merkle Mountain Range root hash.
-	///
-	/// Encoded value should contain a [`crate::MmrRootHash`] type (i.e. 32-bytes hash).
-	pub const MMR_ROOT_ID: BeefyPayloadId = *b"mh";
-}
-
-/// A BEEFY payload type allowing for future extensibility of adding additional kinds of payloads.
-///
-/// The idea is to store a vector of SCALE-encoded values with an extra identifier.
-/// Identifiers MUST be sorted by the [`BeefyPayloadId`] to allow efficient lookup of expected
-/// value. Duplicated identifiers are disallowed. It's okay for different implementations to only
-/// support a subset of possible values.
-#[derive(Decode, Encode, Debug, PartialEq, Eq, Clone, Ord, PartialOrd, Hash, TypeInfo)]
-pub struct Payload(Vec<(BeefyPayloadId, Vec<u8>)>);
-
-impl Payload {
-	/// Construct a new payload given an initial vallue
-	pub fn new(id: BeefyPayloadId, value: Vec<u8>) -> Self {
-		Self(vec![(id, value)])
-	}
-
-	/// Returns a raw payload under given `id`.
-	///
-	/// If the [`BeefyPayloadId`] is not found in the payload `None` is returned.
-	pub fn get_raw(&self, id: &BeefyPayloadId) -> Option<&Vec<u8>> {
-		let index = self.0.binary_search_by(|probe| probe.0.cmp(id)).ok()?;
-		Some(&self.0[index].1)
-	}
-
-	/// Returns a decoded payload value under given `id`.
-	///
-	/// In case the value is not there or it cannot be decoded does not match `None` is returned.
-	pub fn get_decoded<T: Decode>(&self, id: &BeefyPayloadId) -> Option<T> {
-		self.get_raw(id).and_then(|raw| T::decode(&mut &raw[..]).ok())
-	}
-
-	/// Push a `Vec<u8>` with a given id into the payload vec.
-	/// This method will internally sort the payload vec after every push.
-	///
-	/// Returns self to allow for daisy chaining.
-	pub fn push_raw(mut self, id: BeefyPayloadId, value: Vec<u8>) -> Self {
-		self.0.push((id, value));
-		self.0.sort_by_key(|(id, _)| *id);
-		self
-	}
-}
+use crate::{Payload, ValidatorSetId};
 
 /// A commitment signed by GRANDPA validators as part of BEEFY protocol.
 ///
@@ -302,14 +248,12 @@ impl<N, S> From<SignedCommitment<N, S>> for VersionedFinalityProof<N, S> {
 
 #[cfg(test)]
 mod tests {
+	use super::*;
+	use crate::{crypto, known_payloads, KEY_TYPE};
+	use codec::Decode;
 	use sp_core::{keccak_256, Pair};
 	use sp_keystore::{testing::KeyStore, SyncCryptoStore, SyncCryptoStorePtr};
 
-	use super::*;
-	use codec::Decode;
-
-	use crate::{crypto, KEY_TYPE};
-
 	type TestCommitment = Commitment<u128>;
 	type TestSignedCommitment = SignedCommitment<u128, crypto::Signature>;
 	type TestVersionedFinalityProof = VersionedFinalityProof<u128, crypto::Signature>;
@@ -341,7 +285,8 @@ mod tests {
 	#[test]
 	fn commitment_encode_decode() {
 		// given
-		let payload = Payload::new(known_payload_ids::MMR_ROOT_ID, "Hello World!".encode());
+		let payload =
+			Payload::from_single_entry(known_payloads::MMR_ROOT_ID, "Hello World!".encode());
 		let commitment: TestCommitment =
 			Commitment { payload, block_number: 5, validator_set_id: 0 };
 
@@ -362,7 +307,8 @@ mod tests {
 	#[test]
 	fn signed_commitment_encode_decode() {
 		// given
-		let payload = Payload::new(known_payload_ids::MMR_ROOT_ID, "Hello World!".encode());
+		let payload =
+			Payload::from_single_entry(known_payloads::MMR_ROOT_ID, "Hello World!".encode());
 		let commitment: TestCommitment =
 			Commitment { payload, block_number: 5, validator_set_id: 0 };
 
@@ -396,7 +342,8 @@ mod tests {
 	#[test]
 	fn signed_commitment_count_signatures() {
 		// given
-		let payload = Payload::new(known_payload_ids::MMR_ROOT_ID, "Hello World!".encode());
+		let payload =
+			Payload::from_single_entry(known_payloads::MMR_ROOT_ID, "Hello World!".encode());
 		let commitment: TestCommitment =
 			Commitment { payload, block_number: 5, validator_set_id: 0 };
 
@@ -421,7 +368,8 @@ mod tests {
 			block_number: u128,
 			validator_set_id: crate::ValidatorSetId,
 		) -> TestCommitment {
-			let payload = Payload::new(known_payload_ids::MMR_ROOT_ID, "Hello World!".encode());
+			let payload =
+				Payload::from_single_entry(known_payloads::MMR_ROOT_ID, "Hello World!".encode());
 			Commitment { payload, block_number, validator_set_id }
 		}
 
@@ -441,7 +389,8 @@ mod tests {
 
 	#[test]
 	fn versioned_commitment_encode_decode() {
-		let payload = Payload::new(known_payload_ids::MMR_ROOT_ID, "Hello World!".encode());
+		let payload =
+			Payload::from_single_entry(known_payloads::MMR_ROOT_ID, "Hello World!".encode());
 		let commitment: TestCommitment =
 			Commitment { payload, block_number: 5, validator_set_id: 0 };
 
@@ -467,7 +416,8 @@ mod tests {
 	#[test]
 	fn large_signed_commitment_encode_decode() {
 		// given
-		let payload = Payload::new(known_payload_ids::MMR_ROOT_ID, "Hello World!".encode());
+		let payload =
+			Payload::from_single_entry(known_payloads::MMR_ROOT_ID, "Hello World!".encode());
 		let commitment: TestCommitment =
 			Commitment { payload, block_number: 5, validator_set_id: 0 };
 

substrate/primitives/beefy/src/lib.rs

@@ -33,12 +33,11 @@
 
 mod commitment;
 pub mod mmr;
+mod payload;
 pub mod witness;
 
-pub use commitment::{
-	known_payload_ids, BeefyPayloadId, Commitment, Payload, SignedCommitment,
-	VersionedFinalityProof,
-};
+pub use commitment::{Commitment, SignedCommitment, VersionedFinalityProof};
+pub use payload::{known_payloads, BeefyPayloadId, Payload, PayloadProvider};
 
 use codec::{Codec, Decode, Encode};
 use scale_info::TypeInfo;

substrate/primitives/beefy/src/mmr.rs

@@ -17,8 +17,8 @@
 
 //! BEEFY + MMR utilties.
 //!
-//! While BEEFY can be used completely indepentently as an additional consensus gadget,
-//! it is designed around a main use case of making bridging standalone networks together.
+//! While BEEFY can be used completely independently as an additional consensus gadget,
+//! it is designed around a main use case of bridging standalone networks together.
 //! For that use case it's common to use some aggregated data structure (like MMR) to be
 //! used in conjunction with BEEFY, to be able to efficiently prove any past blockchain data.
 //!
@@ -26,9 +26,13 @@
 //! but we imagine they will be useful for other chains that either want to bridge with Polkadot
 //! or are completely standalone, but heavily inspired by Polkadot.
 
-use crate::Vec;
+use crate::{crypto::AuthorityId, ConsensusLog, MmrRootHash, Vec, BEEFY_ENGINE_ID};
 use codec::{Decode, Encode, MaxEncodedLen};
 use scale_info::TypeInfo;
+use sp_runtime::{
+	generic::OpaqueDigestItemId,
+	traits::{Block, Header},
+};
 
 /// A provider for extra data that gets added to the Mmr leaf
 pub trait BeefyDataProvider<ExtraData> {
@@ -121,9 +125,78 @@ pub struct BeefyAuthoritySet<MerkleRoot> {
 /// Details of the next BEEFY authority set.
 pub type BeefyNextAuthoritySet<MerkleRoot> = BeefyAuthoritySet<MerkleRoot>;
 
+/// Extract the MMR root hash from a digest in the given header, if it exists.
+pub fn find_mmr_root_digest<B: Block>(header: &B::Header) -> Option<MmrRootHash> {
+	let id = OpaqueDigestItemId::Consensus(&BEEFY_ENGINE_ID);
+
+	let filter = |log: ConsensusLog<AuthorityId>| match log {
+		ConsensusLog::MmrRoot(root) => Some(root),
+		_ => None,
+	};
+	header.digest().convert_first(|l| l.try_to(id).and_then(filter))
+}
+
+#[cfg(feature = "std")]
+pub use mmr_root_provider::MmrRootProvider;
+#[cfg(feature = "std")]
+mod mmr_root_provider {
+	use super::*;
+	use crate::{known_payloads, payload::PayloadProvider, Payload};
+	use sp_api::ProvideRuntimeApi;
+	use sp_mmr_primitives::MmrApi;
+	use sp_runtime::generic::BlockId;
+	use sp_std::{marker::PhantomData, sync::Arc};
+
+	/// A [`crate::Payload`] provider where payload is Merkle Mountain Range root hash.
+	///
+	/// Encoded payload contains a [`crate::MmrRootHash`] type (i.e. 32-bytes hash).
+	pub struct MmrRootProvider<B, R> {
+		runtime: Arc<R>,
+		_phantom: PhantomData<B>,
+	}
+
+	impl<B, R> MmrRootProvider<B, R>
+	where
+		B: Block,
+		R: ProvideRuntimeApi<B>,
+		R::Api: MmrApi<B, MmrRootHash>,
+	{
+		/// Create new BEEFY Payload provider with MMR Root as payload.
+		pub fn new(runtime: Arc<R>) -> Self {
+			Self { runtime, _phantom: PhantomData }
+		}
+
+		/// Simple wrapper that gets MMR root from header digests or from client state.
+		fn mmr_root_from_digest_or_runtime(&self, header: &B::Header) -> Option<MmrRootHash> {
+			find_mmr_root_digest::<B>(header).or_else(|| {
+				self.runtime
+					.runtime_api()
+					.mmr_root(&BlockId::hash(header.hash()))
+					.ok()
+					.and_then(|r| r.ok())
+			})
+		}
+	}
+
+	impl<B: Block, R> PayloadProvider<B> for MmrRootProvider<B, R>
+	where
+		B: Block,
+		R: ProvideRuntimeApi<B>,
+		R::Api: MmrApi<B, MmrRootHash>,
+	{
+		fn payload(&self, header: &B::Header) -> Option<Payload> {
+			self.mmr_root_from_digest_or_runtime(header).map(|mmr_root| {
+				Payload::from_single_entry(known_payloads::MMR_ROOT_ID, mmr_root.encode())
+			})
+		}
+	}
+}
+
 #[cfg(test)]
 mod tests {
 	use super::*;
+	use crate::H256;
+	use sp_runtime::{traits::BlakeTwo256, Digest, DigestItem, OpaqueExtrinsic};
 
 	#[test]
 	fn should_construct_version_correctly() {
@@ -147,4 +220,30 @@ mod tests {
 	fn should_panic_if_minor_too_large() {
 		MmrLeafVersion::new(0, 32);
 	}
+
+	#[test]
+	fn extract_mmr_root_digest() {
+		type Header = sp_runtime::generic::Header<u64, BlakeTwo256>;
+		type Block = sp_runtime::generic::Block<Header, OpaqueExtrinsic>;
+		let mut header = Header::new(
+			1u64,
+			Default::default(),
+			Default::default(),
+			Default::default(),
+			Digest::default(),
+		);
+
+		// verify empty digest shows nothing
+		assert!(find_mmr_root_digest::<Block>(&header).is_none());
+
+		let mmr_root_hash = H256::random();
+		header.digest_mut().push(DigestItem::Consensus(
+			BEEFY_ENGINE_ID,
+			ConsensusLog::<AuthorityId>::MmrRoot(mmr_root_hash).encode(),
+		));
+
+		// verify validator set is correctly extracted from digest
+		let extracted = find_mmr_root_digest::<Block>(&header);
+		assert_eq!(extracted, Some(mmr_root_hash));
+	}
 }

substrate/primitives/beefy/src/payload.rs

@@ -0,0 +1,105 @@
+// This file is part of Substrate.
+
+// Copyright (C) 2022 Parity Technologies (UK) Ltd.
+// SPDX-License-Identifier: Apache-2.0
+
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// 	http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use codec::{Decode, Encode};
+use scale_info::TypeInfo;
+use sp_runtime::traits::Block;
+use sp_std::prelude::*;
+
+/// Id of different payloads in the [`crate::Commitment`] data.
+pub type BeefyPayloadId = [u8; 2];
+
+/// Registry of all known [`BeefyPayloadId`].
+pub mod known_payloads {
+	use crate::BeefyPayloadId;
+
+	/// A [`Payload`](super::Payload) identifier for Merkle Mountain Range root hash.
+	///
+	/// Encoded value should contain a [`crate::MmrRootHash`] type (i.e. 32-bytes hash).
+	pub const MMR_ROOT_ID: BeefyPayloadId = *b"mh";
+}
+
+/// A BEEFY payload type allowing for future extensibility of adding additional kinds of payloads.
+///
+/// The idea is to store a vector of SCALE-encoded values with an extra identifier.
+/// Identifiers MUST be sorted by the [`BeefyPayloadId`] to allow efficient lookup of expected
+/// value. Duplicated identifiers are disallowed. It's okay for different implementations to only
+/// support a subset of possible values.
+#[derive(Decode, Encode, Debug, PartialEq, Eq, Clone, Ord, PartialOrd, Hash, TypeInfo)]
+pub struct Payload(Vec<(BeefyPayloadId, Vec<u8>)>);
+
+impl Payload {
+	/// Construct a new payload given an initial vallue
+	pub fn from_single_entry(id: BeefyPayloadId, value: Vec<u8>) -> Self {
+		Self(vec![(id, value)])
+	}
+
+	/// Returns a raw payload under given `id`.
+	///
+	/// If the [`BeefyPayloadId`] is not found in the payload `None` is returned.
+	pub fn get_raw(&self, id: &BeefyPayloadId) -> Option<&Vec<u8>> {
+		let index = self.0.binary_search_by(|probe| probe.0.cmp(id)).ok()?;
+		Some(&self.0[index].1)
+	}
+
+	/// Returns a decoded payload value under given `id`.
+	///
+	/// In case the value is not there or it cannot be decoded does not match `None` is returned.
+	pub fn get_decoded<T: Decode>(&self, id: &BeefyPayloadId) -> Option<T> {
+		self.get_raw(id).and_then(|raw| T::decode(&mut &raw[..]).ok())
+	}
+
+	/// Push a `Vec<u8>` with a given id into the payload vec.
+	/// This method will internally sort the payload vec after every push.
+	///
+	/// Returns self to allow for daisy chaining.
+	pub fn push_raw(mut self, id: BeefyPayloadId, value: Vec<u8>) -> Self {
+		self.0.push((id, value));
+		self.0.sort_by_key(|(id, _)| *id);
+		self
+	}
+}
+
+/// Trait for custom BEEFY payload providers.
+pub trait PayloadProvider<B: Block> {
+	/// Provide BEEFY payload if available for `header`.
+	fn payload(&self, header: &B::Header) -> Option<Payload>;
+}
+
+#[cfg(test)]
+mod tests {
+	use super::*;
+
+	#[test]
+	fn payload_methods_work_as_expected() {
+		let id1: BeefyPayloadId = *b"hw";
+		let msg1: String = "1. Hello World!".to_string();
+		let id2: BeefyPayloadId = *b"yb";
+		let msg2: String = "2. Yellow Board!".to_string();
+		let id3: BeefyPayloadId = *b"cs";
+		let msg3: String = "3. Cello Cord!".to_string();
+
+		let payload = Payload::from_single_entry(id1, msg1.encode())
+			.push_raw(id2, msg2.encode())
+			.push_raw(id3, msg3.encode());
+
+		assert_eq!(payload.get_decoded(&id1), Some(msg1));
+		assert_eq!(payload.get_decoded(&id2), Some(msg2));
+		assert_eq!(payload.get_raw(&id3), Some(&msg3.encode()));
+		assert_eq!(payload.get_raw(&known_payloads::MMR_ROOT_ID), None);
+	}
+}

substrate/primitives/beefy/src/witness.rs

@@ -81,7 +81,7 @@ mod tests {
 	use super::*;
 	use codec::Decode;
 
-	use crate::{crypto, known_payload_ids, Payload, KEY_TYPE};
+	use crate::{crypto, known_payloads, Payload, KEY_TYPE};
 
 	type TestCommitment = Commitment<u128>;
 	type TestSignedCommitment = SignedCommitment<u128, crypto::Signature>;
@@ -111,8 +111,10 @@ mod tests {
 	}
 
 	fn signed_commitment() -> TestSignedCommitment {
-		let payload =
-			Payload::new(known_payload_ids::MMR_ROOT_ID, "Hello World!".as_bytes().to_vec());
+		let payload = Payload::from_single_entry(
+			known_payloads::MMR_ROOT_ID,
+			"Hello World!".as_bytes().to_vec(),
+		);
 		let commitment: TestCommitment =
 			Commitment { payload, block_number: 5, validator_set_id: 0 };