Add pluggable BEEFY payload constructors (#12428)

* primitives/beefy: move Payload to its own file

* primitives/beefy: add Payload tests

* primitives/beefy: add MmrRootProvider as custom BEEFY payload provider

* client/beefy: use generic BEEFY 'PayloadProvider'

* primitives/beefy: rename Payload::new to Payload::from_single_entry for clarity

* fix visibility

* fix cargo doc

substrate/Cargo.lock

@@ -539,6 +539,7 @@ dependencies = [
  "sp-application-crypto",
  "sp-core",
  "sp-keystore",
+ "sp-mmr-primitives",
  "sp-runtime",
  "sp-std",
 ]

substrate/client/beefy/rpc/src/lib.rs

@@ -170,7 +170,7 @@ mod tests {
 		communication::notification::BeefyVersionedFinalityProofSender,
 		justification::BeefyVersionedFinalityProof,
 	};
-	use beefy_primitives::{known_payload_ids, Payload, SignedCommitment};
+	use beefy_primitives::{known_payloads, Payload, SignedCommitment};
 	use codec::{Decode, Encode};
 	use jsonrpsee::{types::EmptyParams, RpcModule};
 	use sp_runtime::traits::{BlakeTwo256, Hash};
@@ -266,7 +266,8 @@ mod tests {
 	}
 
 	fn create_finality_proof() -> BeefyVersionedFinalityProof<Block> {
-		let payload = Payload::new(known_payload_ids::MMR_ROOT_ID, "Hello World!".encode());
+		let payload =
+			Payload::from_single_entry(known_payloads::MMR_ROOT_ID, "Hello World!".encode());
 		BeefyVersionedFinalityProof::<Block>::V1(SignedCommitment {
 			commitment: beefy_primitives::Commitment {
 				payload,

substrate/client/beefy/src/communication/gossip.rs

@@ -237,8 +237,7 @@ mod tests {
 
 	use crate::keystore::{tests::Keyring, BeefyKeystore};
 	use beefy_primitives::{
-		crypto::Signature, known_payload_ids, Commitment, MmrRootHash, Payload, VoteMessage,
-		KEY_TYPE,
+		crypto::Signature, known_payloads, Commitment, MmrRootHash, Payload, VoteMessage, KEY_TYPE,
 	};
 
 	use super::*;
@@ -348,7 +347,10 @@ mod tests {
 	}
 
 	fn dummy_vote(block_number: u64) -> VoteMessage<u64, Public, Signature> {
-		let payload = Payload::new(known_payload_ids::MMR_ROOT_ID, MmrRootHash::default().encode());
+		let payload = Payload::from_single_entry(
+			known_payloads::MMR_ROOT_ID,
+			MmrRootHash::default().encode(),
+		);
 		let commitment = Commitment { payload, block_number, validator_set_id: 0 };
 		let signature = sign_commitment(&Keyring::Alice, &commitment);
 

substrate/client/beefy/src/justification.rs

@@ -81,7 +81,7 @@ fn verify_with_validator_set<Block: BlockT>(
 #[cfg(test)]
 pub(crate) mod tests {
 	use beefy_primitives::{
-		known_payload_ids, Commitment, Payload, SignedCommitment, VersionedFinalityProof,
+		known_payloads, Commitment, Payload, SignedCommitment, VersionedFinalityProof,
 	};
 	use substrate_test_runtime_client::runtime::Block;
 
@@ -94,7 +94,7 @@ pub(crate) mod tests {
 		keys: &[Keyring],
 	) -> BeefyVersionedFinalityProof<Block> {
 		let commitment = Commitment {
-			payload: Payload::new(known_payload_ids::MMR_ROOT_ID, vec![]),
+			payload: Payload::from_single_entry(known_payloads::MMR_ROOT_ID, vec![]),
 			block_number: block_num,
 			validator_set_id: validator_set.id(),
 		};

substrate/client/beefy/src/lib.rs

@@ -16,7 +16,7 @@
 // You should have received a copy of the GNU General Public License
 // along with this program. If not, see <https://www.gnu.org/licenses/>.
 
-use beefy_primitives::{BeefyApi, MmrRootHash};
+use beefy_primitives::{BeefyApi, MmrRootHash, PayloadProvider};
 use parking_lot::Mutex;
 use prometheus::Registry;
 use sc_client_api::{Backend, BlockBackend, BlockchainEvents, Finalizer};
@@ -167,11 +167,13 @@ pub struct BeefyNetworkParams<B: Block, N> {
 }
 
 /// BEEFY gadget initialization parameters.
-pub struct BeefyParams<B: Block, BE, C, N, R> {
+pub struct BeefyParams<B: Block, BE, C, N, P, R> {
 	/// BEEFY client
 	pub client: Arc<C>,
 	/// Client Backend
 	pub backend: Arc<BE>,
+	/// BEEFY Payload provider
+	pub payload_provider: P,
 	/// Runtime Api Provider
 	pub runtime: Arc<R>,
 	/// Local key store
@@ -191,11 +193,12 @@ pub struct BeefyParams<B: Block, BE, C, N, R> {
 /// Start the BEEFY gadget.
 ///
 /// This is a thin shim around running and awaiting a BEEFY worker.
-pub async fn start_beefy_gadget<B, BE, C, N, R>(beefy_params: BeefyParams<B, BE, C, N, R>)
+pub async fn start_beefy_gadget<B, BE, C, N, P, R>(beefy_params: BeefyParams<B, BE, C, N, P, R>)
 where
 	B: Block,
 	BE: Backend<B>,
 	C: Client<B, BE> + BlockBackend<B>,
+	P: PayloadProvider<B>,
 	R: ProvideRuntimeApi<B>,
 	R::Api: BeefyApi<B> + MmrApi<B, MmrRootHash>,
 	N: GossipNetwork<B> + NetworkRequest + SyncOracle + Send + Sync + 'static,
@@ -203,6 +206,7 @@ where
 	let BeefyParams {
 		client,
 		backend,
+		payload_provider,
 		runtime,
 		key_store,
 		network_params,
@@ -249,6 +253,7 @@ where
 	let worker_params = worker::WorkerParams {
 		client,
 		backend,
+		payload_provider,
 		runtime,
 		network,
 		key_store: key_store.into(),
@@ -261,7 +266,7 @@ where
 		min_block_delta,
 	};
 
-	let worker = worker::BeefyWorker::<_, _, _, _, _>::new(worker_params);
+	let worker = worker::BeefyWorker::<_, _, _, _, _, _>::new(worker_params);
 
 	futures::future::join(worker.run(), on_demand_justifications_handler.run()).await;
 }

substrate/client/beefy/src/tests.rs

@@ -38,6 +38,7 @@ use sc_utils::notification::NotificationReceiver;
 
 use beefy_primitives::{
 	crypto::{AuthorityId, Signature},
+	mmr::MmrRootProvider,
 	BeefyApi, ConsensusLog, MmrRootHash, ValidatorSet, VersionedFinalityProof, BEEFY_ENGINE_ID,
 	KEY_TYPE as BeefyKeyType,
 };
@@ -372,10 +373,12 @@ where
 			justifications_protocol_name: on_demand_justif_handler.protocol_name(),
 			_phantom: PhantomData,
 		};
+		let payload_provider = MmrRootProvider::new(api.clone());
 
 		let beefy_params = crate::BeefyParams {
 			client: peer.client().as_client(),
 			backend: peer.client().as_backend(),
+			payload_provider,
 			runtime: api.clone(),
 			key_store: Some(keystore),
 			network_params,
@@ -384,7 +387,7 @@ where
 			prometheus_registry: None,
 			on_demand_justifications_handler: on_demand_justif_handler,
 		};
-		let task = crate::start_beefy_gadget::<_, _, _, _, _>(beefy_params);
+		let task = crate::start_beefy_gadget::<_, _, _, _, _, _>(beefy_params);
 
 		fn assert_send<T: Send>(_: &T) {}
 		assert_send(&task);

substrate/client/beefy/src/worker.rs

@@ -48,7 +48,7 @@ use sp_runtime::{
 
 use beefy_primitives::{
 	crypto::{AuthorityId, Signature},
-	known_payload_ids, BeefyApi, Commitment, ConsensusLog, MmrRootHash, Payload, SignedCommitment,
+	BeefyApi, Commitment, ConsensusLog, MmrRootHash, Payload, PayloadProvider, SignedCommitment,
 	ValidatorSet, VersionedFinalityProof, VoteMessage, BEEFY_ENGINE_ID, GENESIS_AUTHORITY_SET_ID,
 };
 
@@ -194,9 +194,10 @@ impl<B: Block> VoterOracle<B> {
 	}
 }
 
-pub(crate) struct WorkerParams<B: Block, BE, C, R, N> {
+pub(crate) struct WorkerParams<B: Block, BE, C, P, R, N> {
 	pub client: Arc<C>,
 	pub backend: Arc<BE>,
+	pub payload_provider: P,
 	pub runtime: Arc<R>,
 	pub network: N,
 	pub key_store: BeefyKeystore,
@@ -210,10 +211,11 @@ pub(crate) struct WorkerParams<B: Block, BE, C, R, N> {
 }
 
 /// A BEEFY worker plays the BEEFY protocol
-pub(crate) struct BeefyWorker<B: Block, BE, C, R, N> {
+pub(crate) struct BeefyWorker<B: Block, BE, C, P, R, N> {
 	// utilities
 	client: Arc<C>,
 	backend: Arc<BE>,
+	payload_provider: P,
 	runtime: Arc<R>,
 	network: N,
 	key_store: BeefyKeystore,
@@ -243,11 +245,12 @@ pub(crate) struct BeefyWorker<B: Block, BE, C, R, N> {
 	voting_oracle: VoterOracle<B>,
 }
 
-impl<B, BE, C, R, N> BeefyWorker<B, BE, C, R, N>
+impl<B, BE, C, P, R, N> BeefyWorker<B, BE, C, P, R, N>
 where
 	B: Block + Codec,
 	BE: Backend<B>,
 	C: Client<B, BE>,
+	P: PayloadProvider<B>,
 	R: ProvideRuntimeApi<B>,
 	R::Api: BeefyApi<B> + MmrApi<B, MmrRootHash>,
 	N: NetworkEventStream + NetworkRequest + SyncOracle + Send + Sync + Clone + 'static,
@@ -258,10 +261,11 @@ where
 	/// BEEFY pallet has been deployed on-chain.
 	///
 	/// The BEEFY pallet is needed in order to keep track of the BEEFY authority set.
-	pub(crate) fn new(worker_params: WorkerParams<B, BE, C, R, N>) -> Self {
+	pub(crate) fn new(worker_params: WorkerParams<B, BE, C, P, R, N>) -> Self {
 		let WorkerParams {
 			client,
 			backend,
+			payload_provider,
 			runtime,
 			key_store,
 			network,
@@ -282,6 +286,7 @@ where
 		BeefyWorker {
 			client: client.clone(),
 			backend,
+			payload_provider,
 			runtime,
 			network,
 			known_peers,
@@ -299,17 +304,6 @@ where
 		}
 	}
 
-	/// Simple wrapper that gets MMR root from header digests or from client state.
-	fn get_mmr_root_digest(&self, header: &B::Header) -> Option<MmrRootHash> {
-		find_mmr_root_digest::<B>(header).or_else(|| {
-			self.runtime
-				.runtime_api()
-				.mmr_root(&BlockId::hash(header.hash()))
-				.ok()
-				.and_then(|r| r.ok())
-		})
-	}
-
 	/// Verify `active` validator set for `block` against the key store
 	///
 	/// We want to make sure that we have _at least one_ key in our keystore that
@@ -621,13 +615,12 @@ where
 		};
 		let target_hash = target_header.hash();
 
-		let mmr_root = if let Some(hash) = self.get_mmr_root_digest(&target_header) {
+		let payload = if let Some(hash) = self.payload_provider.payload(&target_header) {
 			hash
 		} else {
 			warn!(target: "beefy", "🥩 No MMR root digest found for: {:?}", target_hash);
 			return Ok(())
 		};
-		let payload = Payload::new(known_payload_ids::MMR_ROOT_ID, mmr_root.encode());
 
 		let rounds = self.voting_oracle.rounds_mut().ok_or(Error::UninitSession)?;
 		if !rounds.should_self_vote(&(payload.clone(), target_number)) {
@@ -917,20 +910,6 @@ where
 	}
 }
 
-/// Extract the MMR root hash from a digest in the given header, if it exists.
-fn find_mmr_root_digest<B>(header: &B::Header) -> Option<MmrRootHash>
-where
-	B: Block,
-{
-	let id = OpaqueDigestItemId::Consensus(&BEEFY_ENGINE_ID);
-
-	let filter = |log: ConsensusLog<AuthorityId>| match log {
-		ConsensusLog::MmrRoot(root) => Some(root),
-		_ => None,
-	};
-	header.digest().convert_first(|l| l.try_to(id).and_then(filter))
-}
-
 /// Scan the `header` digest log for a BEEFY validator set change. Return either the new
 /// validator set or `None` in case no validator set change has been signaled.
 fn find_authorities_change<B>(header: &B::Header) -> Option<ValidatorSet<AuthorityId>>
@@ -1016,8 +995,8 @@ pub(crate) mod tests {
 		BeefyRPCLinks,
 	};
 
+	use beefy_primitives::{known_payloads, mmr::MmrRootProvider};
 	use futures::{executor::block_on, future::poll_fn, task::Poll};
-
 	use sc_client_api::{Backend as BackendT, HeaderBackend};
 	use sc_network::NetworkService;
 	use sc_network_test::{PeersFullClient, TestNetFactory};
@@ -1032,7 +1011,14 @@ pub(crate) mod tests {
 		peer: &BeefyPeer,
 		key: &Keyring,
 		min_block_delta: u32,
-	) -> BeefyWorker<Block, Backend, PeersFullClient, TestApi, Arc<NetworkService<Block, H256>>> {
+	) -> BeefyWorker<
+		Block,
+		Backend,
+		PeersFullClient,
+		MmrRootProvider<Block, TestApi>,
+		TestApi,
+		Arc<NetworkService<Block, H256>>,
+	> {
 		let keystore = create_beefy_keystore(*key);
 
 		let (to_rpc_justif_sender, from_voter_justif_stream) =
@@ -1064,9 +1050,11 @@ pub(crate) mod tests {
 			"/beefy/justifs/1".into(),
 			known_peers.clone(),
 		);
+		let payload_provider = MmrRootProvider::new(api.clone());
 		let worker_params = crate::worker::WorkerParams {
 			client: peer.client().as_client(),
 			backend: peer.client().as_backend(),
+			payload_provider,
 			runtime: api,
 			key_store: Some(keystore).into(),
 			known_peers,
@@ -1078,7 +1066,7 @@ pub(crate) mod tests {
 			network,
 			on_demand_justifications,
 		};
-		BeefyWorker::<_, _, _, _, _>::new(worker_params)
+		BeefyWorker::<_, _, _, _, _, _>::new(worker_params)
 	}
 
 	#[test]
@@ -1300,30 +1288,6 @@ pub(crate) mod tests {
 		assert_eq!(extracted, Some(validator_set));
 	}
 
-	#[test]
-	fn extract_mmr_root_digest() {
-		let mut header = Header::new(
-			1u32.into(),
-			Default::default(),
-			Default::default(),
-			Default::default(),
-			Digest::default(),
-		);
-
-		// verify empty digest shows nothing
-		assert!(find_mmr_root_digest::<Block>(&header).is_none());
-
-		let mmr_root_hash = H256::random();
-		header.digest_mut().push(DigestItem::Consensus(
-			BEEFY_ENGINE_ID,
-			ConsensusLog::<AuthorityId>::MmrRoot(mmr_root_hash).encode(),
-		));
-
-		// verify validator set is correctly extracted from digest
-		let extracted = find_mmr_root_digest::<Block>(&header);
-		assert_eq!(extracted, Some(mmr_root_hash));
-	}
-
 	#[test]
 	fn keystore_vs_validator_set() {
 		let keys = &[Keyring::Alice];
@@ -1363,7 +1327,7 @@ pub(crate) mod tests {
 
 		let create_finality_proof = |block_num: NumberFor<Block>| {
 			let commitment = Commitment {
-				payload: Payload::new(known_payload_ids::MMR_ROOT_ID, vec![]),
+				payload: Payload::from_single_entry(known_payloads::MMR_ROOT_ID, vec![]),
 				block_number: block_num,
 				validator_set_id: validator_set.id(),
 			};
@@ -1482,7 +1446,7 @@ pub(crate) mod tests {
 			block_number: NumberFor<Block>,
 		) -> VoteMessage<NumberFor<Block>, AuthorityId, Signature> {
 			let commitment = Commitment {
-				payload: Payload::new(*b"BF", vec![]),
+				payload: Payload::from_single_entry(*b"BF", vec![]),
 				block_number,
 				validator_set_id: 0,
 			};
@@ -1574,7 +1538,7 @@ pub(crate) mod tests {
 
 			// import/append BEEFY justification for session boundary block 10
 			let commitment = Commitment {
-				payload: Payload::new(known_payload_ids::MMR_ROOT_ID, vec![]),
+				payload: Payload::from_single_entry(known_payloads::MMR_ROOT_ID, vec![]),
 				block_number: 10,
 				validator_set_id: validator_set.id(),
 			};
@@ -1608,7 +1572,7 @@ pub(crate) mod tests {
 
 			// import/append BEEFY justification for block 12
 			let commitment = Commitment {
-				payload: Payload::new(known_payload_ids::MMR_ROOT_ID, vec![]),
+				payload: Payload::from_single_entry(known_payloads::MMR_ROOT_ID, vec![]),
 				block_number: 12,
 				validator_set_id: validator_set.id(),
 			};

substrate/primitives/beefy/Cargo.toml

@@ -18,6 +18,7 @@ scale-info = { version = "2.1.1", default-features = false, features = ["derive"
 sp-api = { version = "4.0.0-dev", default-features = false, path = "../api" }
 sp-application-crypto = { version = "6.0.0", default-features = false, path = "../application-crypto" }
 sp-core = { version = "6.0.0", default-features = false, path = "../core" }
+sp-mmr-primitives = { version = "4.0.0-dev", default-features = false, path = "../merkle-mountain-range" }
 sp-runtime = { version = "6.0.0", default-features = false, path = "../runtime" }
 sp-std = { version = "4.0.0", default-features = false, path = "../std" }
 
@@ -33,6 +34,7 @@ std = [
 	"sp-api/std",
 	"sp-application-crypto/std",
 	"sp-core/std",
+	"sp-mmr-primitives/std",
 	"sp-runtime/std",
 	"sp-std/std",
 ]

substrate/primitives/beefy/src/commitment.rs

@@ -19,61 +19,7 @@ use codec::{Decode, Encode, Error, Input};
 use scale_info::TypeInfo;
 use sp_std::{cmp, prelude::*};
 
-use crate::ValidatorSetId;
-
-/// Id of different payloads in the [`Commitment`] data
-pub type BeefyPayloadId = [u8; 2];
-
-/// Registry of all known [`BeefyPayloadId`].
-pub mod known_payload_ids {
-	use crate::BeefyPayloadId;
-
-	/// A [`Payload`](super::Payload) identifier for Merkle Mountain Range root hash.
-	///
-	/// Encoded value should contain a [`crate::MmrRootHash`] type (i.e. 32-bytes hash).
-	pub const MMR_ROOT_ID: BeefyPayloadId = *b"mh";
-}
-
-/// A BEEFY payload type allowing for future extensibility of adding additional kinds of payloads.
-///
-/// The idea is to store a vector of SCALE-encoded values with an extra identifier.
-/// Identifiers MUST be sorted by the [`BeefyPayloadId`] to allow efficient lookup of expected
-/// value. Duplicated identifiers are disallowed. It's okay for different implementations to only
-/// support a subset of possible values.
-#[derive(Decode, Encode, Debug, PartialEq, Eq, Clone, Ord, PartialOrd, Hash, TypeInfo)]
-pub struct Payload(Vec<(BeefyPayloadId, Vec<u8>)>);
-
-impl Payload {
-	/// Construct a new payload given an initial vallue
-	pub fn new(id: BeefyPayloadId, value: Vec<u8>) -> Self {
-		Self(vec![(id, value)])
-	}
-
-	/// Returns a raw payload under given `id`.
-	///
-	/// If the [`BeefyPayloadId`] is not found in the payload `None` is returned.
-	pub fn get_raw(&self, id: &BeefyPayloadId) -> Option<&Vec<u8>> {
-		let index = self.0.binary_search_by(|probe| probe.0.cmp(id)).ok()?;
-		Some(&self.0[index].1)
-	}
-
-	/// Returns a decoded payload value under given `id`.
-	///
-	/// In case the value is not there or it cannot be decoded does not match `None` is returned.
-	pub fn get_decoded<T: Decode>(&self, id: &BeefyPayloadId) -> Option<T> {
-		self.get_raw(id).and_then(|raw| T::decode(&mut &raw[..]).ok())
-	}
-
-	/// Push a `Vec<u8>` with a given id into the payload vec.
-	/// This method will internally sort the payload vec after every push.
-	///
-	/// Returns self to allow for daisy chaining.
-	pub fn push_raw(mut self, id: BeefyPayloadId, value: Vec<u8>) -> Self {
-		self.0.push((id, value));
-		self.0.sort_by_key(|(id, _)| *id);
-		self
-	}
-}
+use crate::{Payload, ValidatorSetId};
 
 /// A commitment signed by GRANDPA validators as part of BEEFY protocol.
 ///
@@ -302,14 +248,12 @@ impl<N, S> From<SignedCommitment<N, S>> for VersionedFinalityProof<N, S> {
 
 #[cfg(test)]
 mod tests {
+	use super::*;
+	use crate::{crypto, known_payloads, KEY_TYPE};
+	use codec::Decode;
 	use sp_core::{keccak_256, Pair};
 	use sp_keystore::{testing::KeyStore, SyncCryptoStore, SyncCryptoStorePtr};
 
-	use super::*;
-	use codec::Decode;
-
-	use crate::{crypto, KEY_TYPE};
-
 	type TestCommitment = Commitment<u128>;
 	type TestSignedCommitment = SignedCommitment<u128, crypto::Signature>;
 	type TestVersionedFinalityProof = VersionedFinalityProof<u128, crypto::Signature>;
@@ -341,7 +285,8 @@ mod tests {
 	#[test]
 	fn commitment_encode_decode() {
 		// given
-		let payload = Payload::new(known_payload_ids::MMR_ROOT_ID, "Hello World!".encode());
+		let payload =
+			Payload::from_single_entry(known_payloads::MMR_ROOT_ID, "Hello World!".encode());
 		let commitment: TestCommitment =
 			Commitment { payload, block_number: 5, validator_set_id: 0 };
 
@@ -362,7 +307,8 @@ mod tests {
 	#[test]
 	fn signed_commitment_encode_decode() {
 		// given
-		let payload = Payload::new(known_payload_ids::MMR_ROOT_ID, "Hello World!".encode());
+		let payload =
+			Payload::from_single_entry(known_payloads::MMR_ROOT_ID, "Hello World!".encode());
 		let commitment: TestCommitment =
 			Commitment { payload, block_number: 5, validator_set_id: 0 };
 
@@ -396,7 +342,8 @@ mod tests {
 	#[test]
 	fn signed_commitment_count_signatures() {
 		// given
-		let payload = Payload::new(known_payload_ids::MMR_ROOT_ID, "Hello World!".encode());
+		let payload =
+			Payload::from_single_entry(known_payloads::MMR_ROOT_ID, "Hello World!".encode());
 		let commitment: TestCommitment =
 			Commitment { payload, block_number: 5, validator_set_id: 0 };
 
@@ -421,7 +368,8 @@ mod tests {
 			block_number: u128,
 			validator_set_id: crate::ValidatorSetId,
 		) -> TestCommitment {
-			let payload = Payload::new(known_payload_ids::MMR_ROOT_ID, "Hello World!".encode());
+			let payload =
+				Payload::from_single_entry(known_payloads::MMR_ROOT_ID, "Hello World!".encode());
 			Commitment { payload, block_number, validator_set_id }
 		}
 
@@ -441,7 +389,8 @@ mod tests {
 
 	#[test]
 	fn versioned_commitment_encode_decode() {
-		let payload = Payload::new(known_payload_ids::MMR_ROOT_ID, "Hello World!".encode());
+		let payload =
+			Payload::from_single_entry(known_payloads::MMR_ROOT_ID, "Hello World!".encode());
 		let commitment: TestCommitment =
 			Commitment { payload, block_number: 5, validator_set_id: 0 };
 
@@ -467,7 +416,8 @@ mod tests {
 	#[test]
 	fn large_signed_commitment_encode_decode() {
 		// given
-		let payload = Payload::new(known_payload_ids::MMR_ROOT_ID, "Hello World!".encode());
+		let payload =
+			Payload::from_single_entry(known_payloads::MMR_ROOT_ID, "Hello World!".encode());
 		let commitment: TestCommitment =
 			Commitment { payload, block_number: 5, validator_set_id: 0 };
 

substrate/primitives/beefy/src/lib.rs

@@ -33,12 +33,11 @@
 
 mod commitment;
 pub mod mmr;
+mod payload;
 pub mod witness;
 
-pub use commitment::{
-	known_payload_ids, BeefyPayloadId, Commitment, Payload, SignedCommitment,
-	VersionedFinalityProof,
-};
+pub use commitment::{Commitment, SignedCommitment, VersionedFinalityProof};
+pub use payload::{known_payloads, BeefyPayloadId, Payload, PayloadProvider};
 
 use codec::{Codec, Decode, Encode};
 use scale_info::TypeInfo;

substrate/primitives/beefy/src/mmr.rs

@@ -17,8 +17,8 @@
 
 //! BEEFY + MMR utilties.
 //!
-//! While BEEFY can be used completely indepentently as an additional consensus gadget,
-//! it is designed around a main use case of making bridging standalone networks together.
+//! While BEEFY can be used completely independently as an additional consensus gadget,
+//! it is designed around a main use case of bridging standalone networks together.
 //! For that use case it's common to use some aggregated data structure (like MMR) to be
 //! used in conjunction with BEEFY, to be able to efficiently prove any past blockchain data.
 //!
@@ -26,9 +26,13 @@
 //! but we imagine they will be useful for other chains that either want to bridge with Polkadot
 //! or are completely standalone, but heavily inspired by Polkadot.
 
-use crate::Vec;
+use crate::{crypto::AuthorityId, ConsensusLog, MmrRootHash, Vec, BEEFY_ENGINE_ID};
 use codec::{Decode, Encode, MaxEncodedLen};
 use scale_info::TypeInfo;
+use sp_runtime::{
+	generic::OpaqueDigestItemId,
+	traits::{Block, Header},
+};
 
 /// A provider for extra data that gets added to the Mmr leaf
 pub trait BeefyDataProvider<ExtraData> {
@@ -121,9 +125,78 @@ pub struct BeefyAuthoritySet<MerkleRoot> {
 /// Details of the next BEEFY authority set.
 pub type BeefyNextAuthoritySet<MerkleRoot> = BeefyAuthoritySet<MerkleRoot>;
 
+/// Extract the MMR root hash from a digest in the given header, if it exists.
+pub fn find_mmr_root_digest<B: Block>(header: &B::Header) -> Option<MmrRootHash> {
+	let id = OpaqueDigestItemId::Consensus(&BEEFY_ENGINE_ID);
+
+	let filter = |log: ConsensusLog<AuthorityId>| match log {
+		ConsensusLog::MmrRoot(root) => Some(root),
+		_ => None,
+	};
+	header.digest().convert_first(|l| l.try_to(id).and_then(filter))
+}
+
+#[cfg(feature = "std")]
+pub use mmr_root_provider::MmrRootProvider;
+#[cfg(feature = "std")]
+mod mmr_root_provider {
+	use super::*;
+	use crate::{known_payloads, payload::PayloadProvider, Payload};
+	use sp_api::ProvideRuntimeApi;
+	use sp_mmr_primitives::MmrApi;
+	use sp_runtime::generic::BlockId;
+	use sp_std::{marker::PhantomData, sync::Arc};
+
+	/// A [`crate::Payload`] provider where payload is Merkle Mountain Range root hash.
+	///
+	/// Encoded payload contains a [`crate::MmrRootHash`] type (i.e. 32-bytes hash).
+	pub struct MmrRootProvider<B, R> {
+		runtime: Arc<R>,
+		_phantom: PhantomData<B>,
+	}
+
+	impl<B, R> MmrRootProvider<B, R>
+	where
+		B: Block,
+		R: ProvideRuntimeApi<B>,
+		R::Api: MmrApi<B, MmrRootHash>,
+	{
+		/// Create new BEEFY Payload provider with MMR Root as payload.
+		pub fn new(runtime: Arc<R>) -> Self {
+			Self { runtime, _phantom: PhantomData }
+		}
+
+		/// Simple wrapper that gets MMR root from header digests or from client state.
+		fn mmr_root_from_digest_or_runtime(&self, header: &B::Header) -> Option<MmrRootHash> {
+			find_mmr_root_digest::<B>(header).or_else(|| {
+				self.runtime
+					.runtime_api()
+					.mmr_root(&BlockId::hash(header.hash()))
+					.ok()
+					.and_then(|r| r.ok())
+			})
+		}
+	}
+
+	impl<B: Block, R> PayloadProvider<B> for MmrRootProvider<B, R>
+	where
+		B: Block,
+		R: ProvideRuntimeApi<B>,
+		R::Api: MmrApi<B, MmrRootHash>,
+	{
+		fn payload(&self, header: &B::Header) -> Option<Payload> {
+			self.mmr_root_from_digest_or_runtime(header).map(|mmr_root| {
+				Payload::from_single_entry(known_payloads::MMR_ROOT_ID, mmr_root.encode())
+			})
+		}
+	}
+}
+
 #[cfg(test)]
 mod tests {
 	use super::*;
+	use crate::H256;
+	use sp_runtime::{traits::BlakeTwo256, Digest, DigestItem, OpaqueExtrinsic};
 
 	#[test]
 	fn should_construct_version_correctly() {
@@ -147,4 +220,30 @@ mod tests {
 	fn should_panic_if_minor_too_large() {
 		MmrLeafVersion::new(0, 32);
 	}
+
+	#[test]
+	fn extract_mmr_root_digest() {
+		type Header = sp_runtime::generic::Header<u64, BlakeTwo256>;
+		type Block = sp_runtime::generic::Block<Header, OpaqueExtrinsic>;
+		let mut header = Header::new(
+			1u64,
+			Default::default(),
+			Default::default(),
+			Default::default(),
+			Digest::default(),
+		);
+
+		// verify empty digest shows nothing
+		assert!(find_mmr_root_digest::<Block>(&header).is_none());
+
+		let mmr_root_hash = H256::random();
+		header.digest_mut().push(DigestItem::Consensus(
+			BEEFY_ENGINE_ID,
+			ConsensusLog::<AuthorityId>::MmrRoot(mmr_root_hash).encode(),
+		));
+
+		// verify validator set is correctly extracted from digest
+		let extracted = find_mmr_root_digest::<Block>(&header);
+		assert_eq!(extracted, Some(mmr_root_hash));
+	}
 }

substrate/primitives/beefy/src/payload.rs

@@ -0,0 +1,105 @@
+// This file is part of Substrate.
+
+// Copyright (C) 2022 Parity Technologies (UK) Ltd.
+// SPDX-License-Identifier: Apache-2.0
+
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// 	http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use codec::{Decode, Encode};
+use scale_info::TypeInfo;
+use sp_runtime::traits::Block;
+use sp_std::prelude::*;
+
+/// Id of different payloads in the [`crate::Commitment`] data.
+pub type BeefyPayloadId = [u8; 2];
+
+/// Registry of all known [`BeefyPayloadId`].
+pub mod known_payloads {
+	use crate::BeefyPayloadId;
+
+	/// A [`Payload`](super::Payload) identifier for Merkle Mountain Range root hash.
+	///
+	/// Encoded value should contain a [`crate::MmrRootHash`] type (i.e. 32-bytes hash).
+	pub const MMR_ROOT_ID: BeefyPayloadId = *b"mh";
+}
+
+/// A BEEFY payload type allowing for future extensibility of adding additional kinds of payloads.
+///
+/// The idea is to store a vector of SCALE-encoded values with an extra identifier.
+/// Identifiers MUST be sorted by the [`BeefyPayloadId`] to allow efficient lookup of expected
+/// value. Duplicated identifiers are disallowed. It's okay for different implementations to only
+/// support a subset of possible values.
+#[derive(Decode, Encode, Debug, PartialEq, Eq, Clone, Ord, PartialOrd, Hash, TypeInfo)]
+pub struct Payload(Vec<(BeefyPayloadId, Vec<u8>)>);
+
+impl Payload {
+	/// Construct a new payload given an initial vallue
+	pub fn from_single_entry(id: BeefyPayloadId, value: Vec<u8>) -> Self {
+		Self(vec![(id, value)])
+	}
+
+	/// Returns a raw payload under given `id`.
+	///
+	/// If the [`BeefyPayloadId`] is not found in the payload `None` is returned.
+	pub fn get_raw(&self, id: &BeefyPayloadId) -> Option<&Vec<u8>> {
+		let index = self.0.binary_search_by(|probe| probe.0.cmp(id)).ok()?;
+		Some(&self.0[index].1)
+	}
+
+	/// Returns a decoded payload value under given `id`.
+	///
+	/// In case the value is not there or it cannot be decoded does not match `None` is returned.
+	pub fn get_decoded<T: Decode>(&self, id: &BeefyPayloadId) -> Option<T> {
+		self.get_raw(id).and_then(|raw| T::decode(&mut &raw[..]).ok())
+	}
+
+	/// Push a `Vec<u8>` with a given id into the payload vec.
+	/// This method will internally sort the payload vec after every push.
+	///
+	/// Returns self to allow for daisy chaining.
+	pub fn push_raw(mut self, id: BeefyPayloadId, value: Vec<u8>) -> Self {
+		self.0.push((id, value));
+		self.0.sort_by_key(|(id, _)| *id);
+		self
+	}
+}
+
+/// Trait for custom BEEFY payload providers.
+pub trait PayloadProvider<B: Block> {
+	/// Provide BEEFY payload if available for `header`.
+	fn payload(&self, header: &B::Header) -> Option<Payload>;
+}
+
+#[cfg(test)]
+mod tests {
+	use super::*;
+
+	#[test]
+	fn payload_methods_work_as_expected() {
+		let id1: BeefyPayloadId = *b"hw";
+		let msg1: String = "1. Hello World!".to_string();
+		let id2: BeefyPayloadId = *b"yb";
+		let msg2: String = "2. Yellow Board!".to_string();
+		let id3: BeefyPayloadId = *b"cs";
+		let msg3: String = "3. Cello Cord!".to_string();
+
+		let payload = Payload::from_single_entry(id1, msg1.encode())
+			.push_raw(id2, msg2.encode())
+			.push_raw(id3, msg3.encode());
+
+		assert_eq!(payload.get_decoded(&id1), Some(msg1));
+		assert_eq!(payload.get_decoded(&id2), Some(msg2));
+		assert_eq!(payload.get_raw(&id3), Some(&msg3.encode()));
+		assert_eq!(payload.get_raw(&known_payloads::MMR_ROOT_ID), None);
+	}
+}

substrate/primitives/beefy/src/witness.rs

@@ -81,7 +81,7 @@ mod tests {
 	use super::*;
 	use codec::Decode;
 
-	use crate::{crypto, known_payload_ids, Payload, KEY_TYPE};
+	use crate::{crypto, known_payloads, Payload, KEY_TYPE};
 
 	type TestCommitment = Commitment<u128>;
 	type TestSignedCommitment = SignedCommitment<u128, crypto::Signature>;
@@ -111,8 +111,10 @@ mod tests {
 	}
 
 	fn signed_commitment() -> TestSignedCommitment {
-		let payload =
-			Payload::new(known_payload_ids::MMR_ROOT_ID, "Hello World!".as_bytes().to_vec());
+		let payload = Payload::from_single_entry(
+			known_payloads::MMR_ROOT_ID,
+			"Hello World!".as_bytes().to_vec(),
+		);
 		let commitment: TestCommitment =
 			Commitment { payload, block_number: 5, validator_set_id: 0 };