pallet-beefy-mmr: add API for BEEFY Authority Sets (#11406)

* pallet-beefy: add Config::OnNewValidatorSet type

Add a hook to pallet-beefy for doing specific work when
BEEFY validator set changes.

For example, this can be used by pallet-beefy-mmr to cache
a lightweight MMR root over validators and make it available
to light clients.

* pallet-beefy-mmr: implement OnNewValidatorSet

Implement pallet-beefy::OnNewValidatorSet to be notified of BEEFY
validator set changes. Use the notifications to compute and cache
a light weight 'BEEFY authority set' which is an MMR root over
BEEFY validator set plus some extra info.

Previously, pallet-beefy-mmr was interogating pallet-beefy about
validator set id on every block to find out when it needs to recompute
the authority set.
By using the event-driven approach in this commit, we also save one
extra state interogation per block.

* pallet-beefy-mmr: add new authority_set() API

Expose current and next BEEFY authority sets through runtime API.
These can be directly used by light clients to avoid having them
compute them themselves based on BEEFY validator sets.

Signed-off-by: acatangiu <adrian@parity.io>

* rename BeefyMmr exposed runtime api

substrate/Cargo.lock

@@ -511,10 +511,12 @@ dependencies = [
 name = "beefy-merkle-tree"
 version = "4.0.0-dev"
 dependencies = [
+ "beefy-primitives",
  "env_logger 0.9.0",
  "hex",
  "hex-literal",
  "log",
+ "sp-api",
  "tiny-keccak",
 ]
 
@@ -10646,6 +10648,7 @@ dependencies = [
 name = "substrate-test-runtime"
 version = "2.0.0"
 dependencies = [
+ "beefy-merkle-tree",
  "beefy-primitives",
  "cfg-if 1.0.0",
  "frame-support",

substrate/frame/beefy-mmr/primitives/Cargo.toml

@@ -13,6 +13,9 @@ hex = { version = "0.4", default-features = false, optional = true }
 log = { version = "0.4", default-features = false, optional = true }
 tiny-keccak = { version = "2.0.2", features = ["keccak"], optional = true }
 
+beefy-primitives = { version = "4.0.0-dev", default-features = false, path = "../../../primitives/beefy" }
+sp-api = { version = "4.0.0-dev", default-features = false, path = "../../../primitives/api" }
+
 [dev-dependencies]
 env_logger = "0.9"
 hex = "0.4"
@@ -22,4 +25,7 @@ hex-literal = "0.3"
 debug = ["hex", "hex/std", "log"]
 default = ["debug", "keccak", "std"]
 keccak = ["tiny-keccak"]
-std = []
+std = [
+	"beefy-primitives/std",
+	"sp-api/std"
+]

substrate/frame/beefy-mmr/primitives/src/lib.rs

@@ -36,6 +36,8 @@ extern crate alloc;
 #[cfg(not(feature = "std"))]
 use alloc::vec::Vec;
 
+use beefy_primitives::mmr::{BeefyAuthoritySet, BeefyNextAuthoritySet};
+
 /// Supported hashing output size.
 ///
 /// The size is restricted to 32 bytes to allow for a more optimised implementation.
@@ -375,6 +377,21 @@ where
 	}
 }
 
+sp_api::decl_runtime_apis! {
+	/// API useful for BEEFY light clients.
+	pub trait BeefyMmrApi<H>
+	where
+		H: From<Hash> + Into<Hash>,
+		BeefyAuthoritySet<H>: sp_api::Decode,
+	{
+		/// Return the currently active BEEFY authority set proof.
+		fn authority_set_proof() -> BeefyAuthoritySet<H>;
+
+		/// Return the next/queued BEEFY authority set proof.
+		fn next_authority_set_proof() -> BeefyNextAuthoritySet<H>;
+	}
+}
+
 #[cfg(test)]
 mod tests {
 	use super::*;

substrate/frame/beefy-mmr/src/lib.rs

@@ -36,7 +36,10 @@
 use sp_runtime::traits::{Convert, Hash, Member};
 use sp_std::prelude::*;
 
-use beefy_primitives::mmr::{BeefyDataProvider, BeefyNextAuthoritySet, MmrLeaf, MmrLeafVersion};
+use beefy_primitives::{
+	mmr::{BeefyAuthoritySet, BeefyDataProvider, BeefyNextAuthoritySet, MmrLeaf, MmrLeafVersion},
+	ValidatorSet as BeefyValidatorSet,
+};
 use pallet_mmr::{LeafDataProvider, ParentNumberAndHash};
 
 use frame_support::{crypto::ecdsa::ECDSAExt, traits::Get};
@@ -124,6 +127,12 @@ pub mod pallet {
 		type BeefyDataProvider: BeefyDataProvider<Self::LeafExtra>;
 	}
 
+	/// Details of current BEEFY authority set.
+	#[pallet::storage]
+	#[pallet::getter(fn beefy_authorities)]
+	pub type BeefyAuthorities<T: Config> =
+		StorageValue<_, BeefyAuthoritySet<MerkleRootOf<T>>, ValueQuery>;
+
 	/// Details of next BEEFY authority set.
 	///
 	/// This storage entry is used as cache for calls to `update_beefy_next_authority_set`.
@@ -149,7 +158,7 @@ where
 			version: T::LeafVersion::get(),
 			parent_number_and_hash: ParentNumberAndHash::<T>::leaf_data(),
 			leaf_extra: T::BeefyDataProvider::extra_data(),
-			beefy_next_authority_set: Pallet::<T>::update_beefy_next_authority_set(),
+			beefy_next_authority_set: Pallet::<T>::beefy_next_authorities(),
 		}
 	}
 }
@@ -163,35 +172,55 @@ where
 	}
 }
 
+impl<T> beefy_primitives::OnNewValidatorSet<<T as pallet_beefy::Config>::BeefyId> for Pallet<T>
+where
+	T: pallet::Config,
+	MerkleRootOf<T>: From<beefy_merkle_tree::Hash> + Into<beefy_merkle_tree::Hash>,
+{
+	/// Compute and cache BEEFY authority sets based on updated BEEFY validator sets.
+	fn on_new_validator_set(
+		current_set: &BeefyValidatorSet<<T as pallet_beefy::Config>::BeefyId>,
+		next_set: &BeefyValidatorSet<<T as pallet_beefy::Config>::BeefyId>,
+	) {
+		let current = Pallet::<T>::compute_authority_set(current_set);
+		let next = Pallet::<T>::compute_authority_set(next_set);
+		// cache the result
+		BeefyAuthorities::<T>::put(&current);
+		BeefyNextAuthorities::<T>::put(&next);
+	}
+}
+
 impl<T: Config> Pallet<T>
 where
 	MerkleRootOf<T>: From<beefy_merkle_tree::Hash> + Into<beefy_merkle_tree::Hash>,
 {
-	/// Returns details of the next BEEFY authority set.
+	/// Return the currently active BEEFY authority set proof.
+	pub fn authority_set_proof() -> BeefyAuthoritySet<MerkleRootOf<T>> {
+		Pallet::<T>::beefy_authorities()
+	}
+
+	/// Return the next/queued BEEFY authority set proof.
+	pub fn next_authority_set_proof() -> BeefyNextAuthoritySet<MerkleRootOf<T>> {
+		Pallet::<T>::beefy_next_authorities()
+	}
+
+	/// Returns details of a BEEFY authority set.
 	///
 	/// Details contain authority set id, authority set length and a merkle root,
 	/// constructed from uncompressed secp256k1 public keys converted to Ethereum addresses
 	/// of the next BEEFY authority set.
-	///
-	/// This function will use a storage-cached entry in case the set didn't change, or compute and
-	/// cache new one in case it did.
-	fn update_beefy_next_authority_set() -> BeefyNextAuthoritySet<MerkleRootOf<T>> {
-		let id = pallet_beefy::Pallet::<T>::validator_set_id() + 1;
-		let current_next = Self::beefy_next_authorities();
-		// avoid computing the merkle tree if validator set id didn't change.
-		if id == current_next.id {
-			return current_next
-		}
-
-		let beefy_addresses = pallet_beefy::Pallet::<T>::next_authorities()
+	fn compute_authority_set(
+		validator_set: &BeefyValidatorSet<<T as pallet_beefy::Config>::BeefyId>,
+	) -> BeefyAuthoritySet<MerkleRootOf<T>> {
+		let id = validator_set.id();
+		let beefy_addresses = validator_set
+			.validators()
 			.into_iter()
+			.cloned()
 			.map(T::BeefyAuthorityToMerkleLeaf::convert)
 			.collect::<Vec<_>>();
 		let len = beefy_addresses.len() as u32;
 		let root = beefy_merkle_tree::merkle_root::<Self, _, _>(beefy_addresses).into();
-		let next_set = BeefyNextAuthoritySet { id, len, root };
-		// cache the result
-		BeefyNextAuthorities::<T>::put(&next_set);
-		next_set
+		BeefyAuthoritySet { id, len, root }
 	}
 }

substrate/frame/beefy-mmr/src/mock.rs

@@ -125,6 +125,7 @@ impl pallet_mmr::Config for Test {
 impl pallet_beefy::Config for Test {
 	type BeefyId = BeefyId;
 	type MaxAuthorities = ConstU32<100>;
+	type OnNewValidatorSet = BeefyMmr;
 }
 
 parameter_types! {

substrate/frame/beefy-mmr/src/tests.rs

@@ -149,3 +149,53 @@ fn should_contain_valid_leaf_data() {
 		}
 	);
 }
+
+#[test]
+fn should_update_authorities() {
+	new_test_ext(vec![1, 2, 3, 4]).execute_with(|| {
+		let auth_set = BeefyMmr::authority_set_proof();
+		let next_auth_set = BeefyMmr::next_authority_set_proof();
+
+		// check current authority set
+		assert_eq!(0, auth_set.id);
+		assert_eq!(2, auth_set.len);
+		let want: H256 =
+			hex!("176e73f1bf656478b728e28dd1a7733c98621b8acf830bff585949763dca7a96").into();
+		assert_eq!(want, auth_set.root);
+
+		// next authority set should have same validators but different id
+		assert_eq!(1, next_auth_set.id);
+		assert_eq!(auth_set.len, next_auth_set.len);
+		assert_eq!(auth_set.root, next_auth_set.root);
+
+		let announced_set = next_auth_set;
+		init_block(1);
+		let auth_set = BeefyMmr::authority_set_proof();
+		let next_auth_set = BeefyMmr::next_authority_set_proof();
+
+		// check new auth are expected ones
+		assert_eq!(announced_set, auth_set);
+		assert_eq!(1, auth_set.id);
+		// check next auth set
+		assert_eq!(2, next_auth_set.id);
+		let want: H256 =
+			hex!("9c6b2c1b0d0b25a008e6c882cc7b415f309965c72ad2b944ac0931048ca31cd5").into();
+		assert_eq!(2, next_auth_set.len);
+		assert_eq!(want, next_auth_set.root);
+
+		let announced_set = next_auth_set;
+		init_block(2);
+		let auth_set = BeefyMmr::authority_set_proof();
+		let next_auth_set = BeefyMmr::next_authority_set_proof();
+
+		// check new auth are expected ones
+		assert_eq!(announced_set, auth_set);
+		assert_eq!(2, auth_set.id);
+		// check next auth set
+		assert_eq!(3, next_auth_set.id);
+		let want: H256 =
+			hex!("9c6b2c1b0d0b25a008e6c882cc7b415f309965c72ad2b944ac0931048ca31cd5").into();
+		assert_eq!(2, next_auth_set.len);
+		assert_eq!(want, next_auth_set.root);
+	});
+}

substrate/frame/beefy/src/lib.rs

@@ -32,7 +32,9 @@ use sp_runtime::{
 };
 use sp_std::prelude::*;
 
-use beefy_primitives::{AuthorityIndex, ConsensusLog, ValidatorSet, BEEFY_ENGINE_ID};
+use beefy_primitives::{
+	AuthorityIndex, ConsensusLog, OnNewValidatorSet, ValidatorSet, BEEFY_ENGINE_ID,
+};
 
 #[cfg(test)]
 mod mock;
@@ -58,6 +60,13 @@ pub mod pallet {
 
 		/// The maximum number of authorities that can be added.
 		type MaxAuthorities: Get<u32>;
+
+		/// A hook to act on the new BEEFY validator set.
+		///
+		/// For some applications it might be beneficial to make the BEEFY validator set available
+		/// externally apart from having it in the storage. For instance you might cache a light
+		/// weight MMR root over validators and make it available for Light Clients.
+		type OnNewValidatorSet: OnNewValidatorSet<<Self as Config>::BeefyId>;
 	}
 
 	#[pallet::pallet]
@@ -118,20 +127,29 @@ impl<T: Config> Pallet<T> {
 	) {
 		<Authorities<T>>::put(&new);
 
-		let next_id = Self::validator_set_id() + 1u64;
-		<ValidatorSetId<T>>::put(next_id);
-		if let Some(validator_set) = ValidatorSet::<T::BeefyId>::new(new, next_id) {
-			let log = DigestItem::Consensus(
-				BEEFY_ENGINE_ID,
-				ConsensusLog::AuthoritiesChange(validator_set).encode(),
-			);
-			<frame_system::Pallet<T>>::deposit_log(log);
-		}
+		let new_id = Self::validator_set_id() + 1u64;
+		<ValidatorSetId<T>>::put(new_id);
 
 		<NextAuthorities<T>>::put(&queued);
+
+		if let Some(validator_set) = ValidatorSet::<T::BeefyId>::new(new, new_id) {
+			let log = DigestItem::Consensus(
+				BEEFY_ENGINE_ID,
+				ConsensusLog::AuthoritiesChange(validator_set.clone()).encode(),
+			);
+			<frame_system::Pallet<T>>::deposit_log(log);
+
+			let next_id = new_id + 1;
+			if let Some(next_validator_set) = ValidatorSet::<T::BeefyId>::new(queued, next_id) {
+				<T::OnNewValidatorSet as OnNewValidatorSet<_>>::on_new_validator_set(
+					&validator_set,
+					&next_validator_set,
+				);
+			}
+		}
 	}
 
-	fn initialize_authorities(authorities: &[T::BeefyId]) -> Result<(), ()> {
+	fn initialize_authorities(authorities: &Vec<T::BeefyId>) -> Result<(), ()> {
 		if authorities.is_empty() {
 			return Ok(())
 		}
@@ -141,12 +159,25 @@ impl<T: Config> Pallet<T> {
 		}
 
 		let bounded_authorities =
-			BoundedSlice::<T::BeefyId, T::MaxAuthorities>::try_from(authorities)?;
+			BoundedSlice::<T::BeefyId, T::MaxAuthorities>::try_from(authorities.as_slice())?;
 
+		let id = 0;
 		<Authorities<T>>::put(bounded_authorities);
-		<ValidatorSetId<T>>::put(0);
+		<ValidatorSetId<T>>::put(id);
 		// Like `pallet_session`, initialize the next validator set as well.
 		<NextAuthorities<T>>::put(bounded_authorities);
+
+		if let Some(validator_set) = ValidatorSet::<T::BeefyId>::new(authorities.clone(), id) {
+			let next_id = id + 1;
+			if let Some(next_validator_set) =
+				ValidatorSet::<T::BeefyId>::new(authorities.clone(), next_id)
+			{
+				<T::OnNewValidatorSet as OnNewValidatorSet<_>>::on_new_validator_set(
+					&validator_set,
+					&next_validator_set,
+				);
+			}
+		}
 		Ok(())
 	}
 }

substrate/frame/beefy/src/mock.rs

@@ -87,6 +87,7 @@ impl frame_system::Config for Test {
 impl pallet_beefy::Config for Test {
 	type BeefyId = BeefyId;
 	type MaxAuthorities = ConstU32<100>;
+	type OnNewValidatorSet = ();
 }
 
 parameter_types! {

substrate/primitives/beefy/src/lib.rs

@@ -154,6 +154,20 @@ pub struct VoteMessage<Number, Id, Signature> {
 	pub signature: Signature,
 }
 
+/// New BEEFY validator set notification hook.
+pub trait OnNewValidatorSet<AuthorityId> {
+	/// Function called by the pallet when BEEFY validator set changes.
+	fn on_new_validator_set(
+		validator_set: &ValidatorSet<AuthorityId>,
+		next_validator_set: &ValidatorSet<AuthorityId>,
+	);
+}
+
+/// No-op implementation of [OnNewValidatorSet].
+impl<AuthorityId> OnNewValidatorSet<AuthorityId> for () {
+	fn on_new_validator_set(_: &ValidatorSet<AuthorityId>, _: &ValidatorSet<AuthorityId>) {}
+}
+
 sp_api::decl_runtime_apis! {
 	/// API necessary for BEEFY voters.
 	pub trait BeefyApi

substrate/primitives/beefy/src/mmr.rs

@@ -95,10 +95,10 @@ impl MmrLeafVersion {
 	}
 }
 
-/// Details of the next BEEFY authority set.
+/// Details of a BEEFY authority set.
 #[derive(Debug, Default, PartialEq, Eq, Clone, Encode, Decode, TypeInfo, MaxEncodedLen)]
-pub struct BeefyNextAuthoritySet<MerkleRoot> {
-	/// Id of the next set.
+pub struct BeefyAuthoritySet<MerkleRoot> {
+	/// Id of the set.
 	///
 	/// Id is required to correlate BEEFY signed commitments with the validator set.
 	/// Light Client can easily verify that the commitment witness it is getting is
@@ -106,11 +106,11 @@ pub struct BeefyNextAuthoritySet<MerkleRoot> {
 	pub id: crate::ValidatorSetId,
 	/// Number of validators in the set.
 	///
-	/// Some BEEFY Light Clients may use an interactive protocol to verify only subset
+	/// Some BEEFY Light Clients may use an interactive protocol to verify only a subset
 	/// of signatures. We put set length here, so that these clients can verify the minimal
 	/// number of required signatures.
 	pub len: u32,
-	/// Merkle Root Hash build from BEEFY AuthorityIds.
+	/// Merkle Root Hash built from BEEFY AuthorityIds.
 	///
 	/// This is used by Light Clients to confirm that the commitments are signed by the correct
 	/// validator set. Light Clients using interactive protocol, might verify only subset of
@@ -118,6 +118,9 @@ pub struct BeefyNextAuthoritySet<MerkleRoot> {
 	pub root: MerkleRoot,
 }
 
+/// Details of the next BEEFY authority set.
+pub type BeefyNextAuthoritySet<MerkleRoot> = BeefyAuthoritySet<MerkleRoot>;
+
 #[cfg(test)]
 mod tests {
 	use super::*;

substrate/test-utils/runtime/Cargo.toml

@@ -14,6 +14,7 @@ targets = ["x86_64-unknown-linux-gnu"]
 
 [dependencies]
 beefy-primitives = { version = "4.0.0-dev", default-features = false, path = "../../primitives/beefy" }
+beefy-merkle-tree = { version = "4.0.0-dev", default-features = false, path = "../../frame/beefy-mmr/primitives" }
 sp-application-crypto = { version = "6.0.0", default-features = false, path = "../../primitives/application-crypto" }
 sp-consensus-aura = { version = "0.10.0-dev", default-features = false, path = "../../primitives/consensus/aura" }
 sp-consensus-babe = { version = "0.10.0-dev", default-features = false, path = "../../primitives/consensus/babe" }
@@ -67,6 +68,7 @@ default = [
 ]
 std = [
 	"beefy-primitives/std",
+	"beefy-merkle-tree/std",
 	"sp-application-crypto/std",
 	"sp-consensus-aura/std",
 	"sp-consensus-babe/std",

substrate/test-utils/runtime/src/lib.rs

@@ -947,6 +947,16 @@ cfg_if! {
 				}
 			}
 
+			impl beefy_merkle_tree::BeefyMmrApi<Block, beefy_primitives::MmrRootHash> for RuntimeApi {
+				fn authority_set_proof() -> beefy_primitives::mmr::BeefyAuthoritySet<beefy_primitives::MmrRootHash> {
+					Default::default()
+				}
+
+				fn next_authority_set_proof() -> beefy_primitives::mmr::BeefyNextAuthoritySet<beefy_primitives::MmrRootHash> {
+					Default::default()
+				}
+			}
+
 			impl frame_system_rpc_runtime_api::AccountNonceApi<Block, AccountId, Index> for Runtime {
 				fn account_nonce(_account: AccountId) -> Index {
 					0