Update BEEFY+MMR integration. (#3480)

* Update MMR leaf.

* Revert to older substrate.

* Add version docs.

* Fix spellcheck.

polkadot/Cargo.lock

@@ -539,6 +539,11 @@ dependencies = [
  "sp-runtime",
 ]
 
+[[package]]
+name = "beefy-merkle-tree"
+version = "0.1.0"
+source = "git+https://github.com/paritytech/grandpa-bridge-gadget?branch=master#2e450ac733d55b2f5e42a304afa287be6abcc53b"
+
 [[package]]
 name = "beefy-primitives"
 version = "0.1.0"
@@ -4725,6 +4730,30 @@ dependencies = [
  "sp-std",
 ]
 
+[[package]]
+name = "pallet-beefy-mmr"
+version = "0.1.0"
+source = "git+https://github.com/paritytech/grandpa-bridge-gadget?branch=master#2e450ac733d55b2f5e42a304afa287be6abcc53b"
+dependencies = [
+ "beefy-merkle-tree",
+ "beefy-primitives",
+ "frame-support",
+ "frame-system",
+ "hex",
+ "libsecp256k1",
+ "log",
+ "pallet-beefy",
+ "pallet-mmr",
+ "pallet-mmr-primitives",
+ "pallet-session",
+ "parity-scale-codec",
+ "serde",
+ "sp-core",
+ "sp-io",
+ "sp-runtime",
+ "sp-std",
+]
+
 [[package]]
 name = "pallet-bounties"
 version = "4.0.0-dev"
@@ -6778,7 +6807,6 @@ dependencies = [
 name = "polkadot-runtime-common"
 version = "0.9.8"
 dependencies = [
- "beefy-primitives",
  "bitvec",
  "frame-benchmarking",
  "frame-support",
@@ -6791,9 +6819,8 @@ dependencies = [
  "pallet-authorship",
  "pallet-babe",
  "pallet-balances",
- "pallet-beefy",
+ "pallet-beefy-mmr",
  "pallet-election-provider-multi-phase",
- "pallet-mmr",
  "pallet-offences",
  "pallet-session",
  "pallet-staking",
@@ -7909,6 +7936,7 @@ dependencies = [
  "pallet-babe",
  "pallet-balances",
  "pallet-beefy",
+ "pallet-beefy-mmr",
  "pallet-bridge-dispatch",
  "pallet-bridge-grandpa",
  "pallet-bridge-messages",

polkadot/runtime/common/Cargo.toml

@@ -14,7 +14,6 @@ serde = { version = "1.0.123", default-features = false }
 serde_derive = { version = "1.0.117", optional = true }
 static_assertions = "1.1.0"
 
-beefy-primitives = { git = "https://github.com/paritytech/grandpa-bridge-gadget", branch = "master", default-features = false }
 sp-api = { git = "https://github.com/paritytech/substrate", branch = "master", default-features = false }
 inherents = { package = "sp-inherents", git = "https://github.com/paritytech/substrate", branch = "master", default-features = false }
 sp-std = { package = "sp-std", git = "https://github.com/paritytech/substrate", branch = "master", default-features = false }
@@ -36,8 +35,7 @@ pallet-offences = { git = "https://github.com/paritytech/substrate", branch = "m
 pallet-transaction-payment = { git = "https://github.com/paritytech/substrate", branch = "master", default-features = false }
 pallet-treasury = { git = "https://github.com/paritytech/substrate", branch = "master", default-features = false }
 pallet-election-provider-multi-phase = { git = "https://github.com/paritytech/substrate", branch = "master", default-features = false }
-pallet-beefy = { git = "https://github.com/paritytech/grandpa-bridge-gadget", branch = "master", default-features = false }
+pallet-beefy-mmr = { git = "https://github.com/paritytech/grandpa-bridge-gadget", branch = "master", default-features = false }
-pallet-mmr = { git = "https://github.com/paritytech/substrate", branch = "master", default-features = false }
 
 frame-benchmarking = { git = "https://github.com/paritytech/substrate", branch = "master", default-features = false, optional = true }
 pallet-babe = { git = "https://github.com/paritytech/substrate", branch = "master", default-features=false, optional = true }
@@ -67,7 +65,6 @@ libsecp256k1 = "0.3.5"
 default = ["std"]
 no_std = []
 std = [
-	"beefy-primitives/std",
 	"bitvec/std",
 	"parity-scale-codec/std",
 	"log/std",
@@ -83,8 +80,7 @@ std = [
 	"frame-support/std",
 	"pallet-authorship/std",
 	"pallet-balances/std",
-	"pallet-beefy/std",
+	"pallet-beefy-mmr/std",
-	"pallet-mmr/std",
 	"pallet-session/std",
 	"pallet-staking/std",
 	"pallet-timestamp/std",

polkadot/runtime/common/src/lib.rs

@@ -24,7 +24,6 @@ pub mod auctions;
 pub mod crowdloan;
 pub mod purchase;
 pub mod impls;
-pub mod mmr;
 pub mod paras_sudo_wrapper;
 pub mod paras_registrar;
 pub mod slot_range;
@@ -37,8 +36,7 @@ mod mock;
 #[cfg(test)]
 mod integration_tests;
 
-use beefy_primitives::crypto::AuthorityId as BeefyId;
+use primitives::v1::{AssignmentId, BlockNumber, ValidatorId};
-use primitives::v1::{AccountId, AssignmentId, BlockNumber, ValidatorId};
 use sp_runtime::{Perquintill, Perbill, FixedPointNumber};
 use frame_system::limits;
 use frame_support::{
@@ -181,20 +179,6 @@ impl<T: pallet_session::Config> OneSessionHandler<T::AccountId> for AssignmentSe
 	fn on_disabled(_: usize) { }
 }
 
-/// Generates a `BeefyId` from the given `AccountId`. The resulting `BeefyId` is
-/// a dummy value and this is a utility function meant to be used when migration
-/// session keys.
-pub fn dummy_beefy_id_from_account_id(a: AccountId) -> BeefyId {
-	let mut id = BeefyId::default();
-	let id_raw: &mut [u8] = id.as_mut();
-
-	// NOTE: AccountId is 32 bytes, whereas BeefyId is 33 bytes.
-	id_raw[1..].copy_from_slice(a.as_ref());
-	id_raw[0..4].copy_from_slice(b"beef");
-
-	id
-}
-
 #[cfg(test)]
 mod multiplier_tests {
 	use super::*;
@@ -297,15 +281,4 @@ mod multiplier_tests {
 			println!("block = {} multiplier {:?}", blocks, multiplier);
 		}
 	}
-
-	#[test]
-	fn generate_dummy_unique_beefy_id_from_account_id() {
-		let acc1 = AccountId::new([0; 32]);
-		let acc2 = AccountId::new([1; 32]);
-
-		let beefy_id1 = dummy_beefy_id_from_account_id(acc1);
-		let beefy_id2 = dummy_beefy_id_from_account_id(acc2);
-
-		assert_ne!(beefy_id1, beefy_id2);
-	}
 }

polkadot/runtime/common/src/mmr.rs

@@ -1,226 +0,0 @@
-// Copyright 2020 Parity Technologies (UK) Ltd.
-// This file is part of Polkadot.
-
-// Polkadot is free software: you can redistribute it and/or modify
-// it under the terms of the GNU General Public License as published by
-// the Free Software Foundation, either version 3 of the License, or
-// (at your option) any later version.
-
-// Polkadot is distributed in the hope that it will be useful,
-// but WITHOUT ANY WARRANTY; without even the implied warranty of
-// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-// GNU General Public License for more details.
-
-// You should have received a copy of the GNU General Public License
-// along with Polkadot.  If not, see <http://www.gnu.org/licenses/>.
-
-//! A pallet responsible for creating Merkle Mountain Range (MMR) leaf for current block.
-
-use beefy_primitives::ValidatorSetId;
-use sp_core::H256;
-use sp_runtime::traits::Convert;
-use sp_std::prelude::*;
-use frame_support::RuntimeDebug;
-use pallet_mmr::primitives::LeafDataProvider;
-use parity_scale_codec::{Encode, Decode};
-use runtime_parachains::paras;
-pub use pallet::*;
-
-/// A BEEFY consensus digest item with MMR root hash.
-pub struct DepositBeefyDigest<T>(sp_std::marker::PhantomData<T>);
-
-impl<T> pallet_mmr::primitives::OnNewRoot<beefy_primitives::MmrRootHash> for DepositBeefyDigest<T> where
-	T: pallet_mmr::Config<Hash = beefy_primitives::MmrRootHash>,
-	T: pallet_beefy::Config,
-{
-	fn on_new_root(root: &<T as pallet_mmr::Config>::Hash) {
-		let digest = sp_runtime::generic::DigestItem::Consensus(
-			beefy_primitives::BEEFY_ENGINE_ID,
-			parity_scale_codec::Encode::encode(
-				&beefy_primitives::ConsensusLog::<<T as pallet_beefy::Config>::BeefyId>::MmrRoot(*root)
-			),
-		);
-		<frame_system::Pallet<T>>::deposit_log(digest);
-	}
-}
-
-/// Convert BEEFY `secp256k1` public keys into uncompressed form
-pub struct UncompressBeefyEcdsaKeys;
-impl Convert<beefy_primitives::crypto::AuthorityId, Vec<u8>> for UncompressBeefyEcdsaKeys {
-	fn convert(a: beefy_primitives::crypto::AuthorityId) -> Vec<u8> {
-		use sp_core::crypto::Public;
-		let compressed_key = a.as_slice();
-		// TODO [ToDr] Temporary workaround until we have a better way to get uncompressed keys.
-		secp256k1::PublicKey::parse_slice(compressed_key, Some(secp256k1::PublicKeyFormat::Compressed))
-			.map(|pub_key| pub_key.serialize().to_vec())
-			.map_err(|_| {
-				log::error!(target: "runtime::beefy", "Invalid BEEFY PublicKey format!");
-			})
-			.unwrap_or_default()
-	}
-}
-
-/// A leaf that gets added every block to the MMR constructed by `[pallet_mmr]`.
-#[derive(RuntimeDebug, PartialEq, Eq, Clone, Encode, Decode)]
-pub struct MmrLeaf<BlockNumber, Hash, MerkleRoot> {
-	/// Current block parent number and hash.
-	pub parent_number_and_hash: (BlockNumber, Hash),
-	/// A merkle root of all registered parachain heads.
-	pub parachain_heads: MerkleRoot,
-	/// A merkle root of the next BEEFY authority set.
-	pub beefy_next_authority_set: BeefyNextAuthoritySet<MerkleRoot>,
-}
-
-/// Details of the next BEEFY authority set.
-#[derive(RuntimeDebug, Default, PartialEq, Eq, Clone, Encode, Decode)]
-pub struct BeefyNextAuthoritySet<MerkleRoot> {
-	/// Id of the next set.
-	///
-	/// Id is required to correlate BEEFY signed commitments with the validator set.
-	/// Light Client can easily verify that the commitment witness it is getting is
-	/// produced by the latest validator set.
-	pub id: ValidatorSetId,
-	/// Number of validators in the set.
-	///
-	/// Some BEEFY Light Clients may use an interactive protocol to verify only subset
-	/// of signatures. We put set length here, so that these clients can verify the minimal
-	/// number of required signatures.
-	pub len: u32,
-	/// Merkle Root Hash build from BEEFY `AuthorityIds`.
-	///
-	/// This is used by Light Clients to confirm that the commitments are signed by the correct
-	/// validator set. Light Clients using interactive protocol, might verify only subset of
-	/// signatures, hence don't require the full list here (will receive inclusion proofs).
-	pub root: MerkleRoot,
-}
-
-type MerkleRootOf<T> = <T as pallet_mmr::Config>::Hash;
-
-/// A type that is able to return current list of parachain heads that end up in the MMR leaf.
-pub trait ParachainHeadsProvider {
-	/// Return a list of encoded parachain heads.
-	fn encoded_heads() -> Vec<Vec<u8>>;
-}
-
-/// A default implementation for runtimes without parachains.
-impl ParachainHeadsProvider for () {
-	fn encoded_heads() -> Vec<Vec<u8>> {
-		Default::default()
-	}
-}
-
-impl<T: Config + paras::Config> ParachainHeadsProvider for paras::Pallet<T> {
-	fn encoded_heads() -> Vec<Vec<u8>> {
-		paras::Pallet::<T>::parachains()
-			.into_iter()
-			.map(paras::Pallet::<T>::para_head)
-			.map(|maybe_para_head| maybe_para_head.encode())
-			.collect()
-	}
-}
-
-#[frame_support::pallet]
-pub mod pallet {
-	use frame_support::pallet_prelude::*;
-	use super::*;
-
-	#[pallet::pallet]
-	#[pallet::generate_store(pub(super) trait Store)]
-	pub struct Pallet<T>(_);
-
-	/// The module's configuration trait.
-	#[pallet::config]
-	#[pallet::disable_frame_system_supertrait_check]
-	pub trait Config: pallet_mmr::Config + pallet_beefy::Config {
-		/// Convert BEEFY `AuthorityId` to a form that would end up in the Merkle Tree.
-		///
-		/// For instance for ECDSA (`secp256k1`) we want to store uncompressed public keys (65 bytes)
-		/// to simplify using them on Ethereum chain, but the rest of the Substrate codebase
-		/// is storing them compressed (33 bytes) for efficiency reasons.
-		type BeefyAuthorityToMerkleLeaf: Convert<<Self as pallet_beefy::Config>::BeefyId, Vec<u8>>;
-
-		/// Retrieve a list of current parachain heads.
-		///
-		/// The trait is implemented for `paras` module, but since not all chains might have parachains,
-		/// and we want to keep the MMR leaf structure uniform, it's possible to use `()` as well to
-		/// simply put dummy data to the leaf.
-		type ParachainHeads: ParachainHeadsProvider;
-	}
-
-	/// Details of next BEEFY authority set.
-	///
-	/// This storage entry is used as cache for calls to [`update_beefy_next_authority_set`].
-	#[pallet::storage]
-	#[pallet::getter(fn beefy_next_authorities)]
-	pub type BeefyNextAuthorities<T: Config> = StorageValue<
-		_,
-		BeefyNextAuthoritySet<MerkleRootOf<T>>,
-		ValueQuery,
-	>;
-}
-
-impl<T: Config> LeafDataProvider for Pallet<T> where
-	MerkleRootOf<T>: From<H256>,
-{
-	type LeafData = MmrLeaf<
-		<T as frame_system::Config>::BlockNumber,
-		<T as frame_system::Config>::Hash,
-		MerkleRootOf<T>,
-	>;
-
-	fn leaf_data() -> Self::LeafData {
-		MmrLeaf {
-			parent_number_and_hash: frame_system::Pallet::<T>::leaf_data(),
-			parachain_heads: Pallet::<T>::parachain_heads_merkle_root(),
-			beefy_next_authority_set: Pallet::<T>::update_beefy_next_authority_set(),
-		}
-	}
-}
-
-impl<T: Config> Pallet<T> where
-	MerkleRootOf<T>: From<H256>,
-	<T as pallet_beefy::Config>::BeefyId:
-{
-	/// Returns latest root hash of a merkle tree constructed from all registered parachain headers.
-	///
-	/// NOTE this does not include parathreads - only parachains are part of the merkle tree.
-	///
-	/// NOTE This is an initial and inefficient implementation, which re-constructs
-	/// the merkle tree every block. Instead we should update the merkle root in `[Self::on_initialize]`
-	/// call of this pallet and update the merkle tree efficiently (use on-chain storage to persist inner nodes).
-	fn parachain_heads_merkle_root() -> MerkleRootOf<T> {
-		let para_heads = T::ParachainHeads::encoded_heads();
-		sp_io::trie::keccak_256_ordered_root(para_heads).into()
-	}
-
-	/// Returns details of the next BEEFY authority set.
-	///
-	/// Details contain authority set id, authority set length and a merkle root,
-	/// constructed from uncompressed `secp256k1` public keys of the next BEEFY authority set.
-	///
-	/// This function will use a storage-cached entry in case the set didn't change, or compute and cache
-	/// new one in case it did.
-	fn update_beefy_next_authority_set() -> BeefyNextAuthoritySet<MerkleRootOf<T>> {
-		let id = pallet_beefy::Pallet::<T>::validator_set_id() + 1;
-		let current_next = Self::beefy_next_authorities();
-		// avoid computing the merkle tree if validator set id didn't change.
-		if id == current_next.id {
-			return current_next;
-		}
-
-		let beefy_public_keys = pallet_beefy::Pallet::<T>::next_authorities()
-			.into_iter()
-			.map(T::BeefyAuthorityToMerkleLeaf::convert)
-			.collect::<Vec<_>>();
-		let len = beefy_public_keys.len() as u32;
-		let root: MerkleRootOf<T> = sp_io::trie::keccak_256_ordered_root(beefy_public_keys).into();
-		let next_set = BeefyNextAuthoritySet {
-			id,
-			len,
-			root,
-		};
-		// cache the result
-		BeefyNextAuthorities::<T>::put(&next_set);
-		next_set
-	}
-}

polkadot/runtime/rococo/Cargo.toml

@@ -35,6 +35,7 @@ pallet-authority-discovery = { git = "https://github.com/paritytech/substrate",
 pallet-authorship = { git = "https://github.com/paritytech/substrate", branch = "master", default-features = false }
 pallet-babe = { git = "https://github.com/paritytech/substrate", branch = "master", default-features = false }
 pallet-beefy = { git = "https://github.com/paritytech/grandpa-bridge-gadget", branch = "master", default-features = false }
+pallet-beefy-mmr = { git = "https://github.com/paritytech/grandpa-bridge-gadget", branch = "master", default-features = false }
 pallet-balances = { git = "https://github.com/paritytech/substrate", branch = "master", default-features = false }
 pallet-collective = { git = "https://github.com/paritytech/substrate", branch = "master", default-features = false }
 pallet-grandpa = { git = "https://github.com/paritytech/substrate", branch = "master", default-features = false }
@@ -104,6 +105,7 @@ std = [
 	"pallet-bridge-messages/std",
 	"pallet-collective/std",
 	"pallet-beefy/std",
+	"pallet-beefy-mmr/std",
 	"pallet-grandpa/std",
 	"pallet-sudo/std",
 	"pallet-membership/std",

polkadot/runtime/rococo/src/lib.rs

@@ -32,7 +32,6 @@ use primitives::v1::{
 	SessionInfo as SessionInfoData,
 };
 use runtime_common::{
-	mmr as mmr_common,
 	SlowAdjustingFeeUpdate, impls::ToAuthor, BlockHashCount, BlockWeights, BlockLength, RocksDbWeight,
 };
 use runtime_parachains::{
@@ -65,6 +64,7 @@ use sp_core::{OpaqueMetadata, RuntimeDebug};
 use sp_staking::SessionIndex;
 use pallet_session::historical as session_historical;
 use beefy_primitives::crypto::AuthorityId as BeefyId;
+use beefy_primitives::mmr::MmrLeafVersion;
 use pallet_mmr_primitives as mmr;
 use frame_system::EnsureRoot;
 use runtime_common::{paras_sudo_wrapper, paras_registrar, xcm_sender, auctions, crowdloan, slots};
@@ -237,7 +237,7 @@ construct_runtime! {
 		// Bridges support.
 		Mmr: pallet_mmr::{Pallet, Storage},
 		Beefy: pallet_beefy::{Pallet, Config<T>, Storage},
-		MmrLeaf: mmr_common::{Pallet, Storage},
+		MmrLeaf: pallet_beefy_mmr::{Pallet, Storage},
 
 		// It might seem strange that we add both sides of the bridge to the same runtime. We do this because this
 		// runtime as shared by both the Rococo and Wococo chains. When running as Rococo we only use
@@ -825,27 +825,57 @@ impl pallet_mmr::Config for Runtime {
 	const INDEXING_PREFIX: &'static [u8] = b"mmr";
 	type Hashing = Keccak256;
 	type Hash = <Keccak256 as traits::Hash>::Output;
-	type OnNewRoot = mmr_common::DepositBeefyDigest<Runtime>;
+	type OnNewRoot = pallet_beefy_mmr::DepositBeefyDigest<Runtime>;
 	type WeightInfo = ();
-	type LeafData = mmr_common::Pallet<Runtime>;
+	type LeafData = pallet_beefy_mmr::Pallet<Runtime>;
 }
 
-impl mmr_common::Config for Runtime {
+pub struct ParasProvider;
-	type BeefyAuthorityToMerkleLeaf = mmr_common::UncompressBeefyEcdsaKeys;
+impl pallet_beefy_mmr::ParachainHeadsProvider for ParasProvider {
-	type ParachainHeads = Paras;
+	fn parachain_heads() -> Vec<(u32, Vec<u8>)> {
+		Paras::parachains()
+			.into_iter()
+			.filter_map(|id| {
+				Paras::para_head(&id).map(|head| (id.into(), head.0))
+			})
+			.collect()
+	}
 }
 
 parameter_types! {
-	// This is a pretty unscientific cap.
+	/// Version of the produced MMR leaf.
-	//
+	///
-	// Note that once this is hit the pallet will essentially throttle incoming requests down to one
+	/// The version consists of two parts;
-	// call per block.
+	/// - `major` (3 bits)
+	/// - `minor` (5 bits)
+	///
+	/// `major` should be updated only if decoding the previous MMR Leaf format from the payload
+	/// is not possible (i.e. backward incompatible change).
+	/// `minor` should be updated if fields are added to the previous MMR Leaf, which given SCALE
+	/// encoding does not prevent old leafs from being decoded.
+	///
+	/// Hence we expect `major` to be changed really rarely (think never).
+	/// See [`MmrLeafVersion`] type documentation for more details.
+	pub LeafVersion: MmrLeafVersion = MmrLeafVersion::new(0, 0);
+}
+
+impl pallet_beefy_mmr::Config for Runtime {
+	type LeafVersion = LeafVersion;
+	type BeefyAuthorityToMerkleLeaf = pallet_beefy_mmr::BeefyEcdsaToEthereum;
+	type ParachainHeads = ParasProvider;
+}
+
+parameter_types! {
+	/// This is a pretty unscientific cap.
+	///
+	/// Note that once this is hit the pallet will essentially throttle incoming requests down to one
+	/// call per block.
 	pub const MaxRequests: u32 = 4 * HOURS as u32;
 
-	// Number of headers to keep.
+	/// Number of headers to keep.
-	//
+	///
-	// Assuming the worst case of every header being finalized, we will keep headers at least for a
+	/// Assuming the worst case of every header being finalized, we will keep headers at least for a
-	// week.
+	/// week.
 	pub const HeadersToKeep: u32 = 7 * DAYS as u32;
 }