Merkle Mountain Range & BEEFY integration (#2101)

* Switch branch.

* Implement basic MMR leaf.

* Revert "Switch branch."

This reverts commit 7f4d41c67f27ca560c53fc63fd3bd06ac182403c.

* Bump substrate.

* Integrate BEEFY.

Bump all.

Fix missing imports.

* Use beefy pallet to get authorities.

* Bump BEEFY repo.

* Use next authority set instead of the current one.

* Start BEEFY service.

* Fix BEEFY start up.

* Cache BEEFY authority set.

* Add BEEFY ValidatorSetId to MMR

* Fix code.

* Apply suggestions from code review

Co-authored-by: André Silva <123550+andresilva@users.noreply.github.com>
Co-authored-by: Hernando Castano <HCastano@users.noreply.github.com>

* Review grumbles.

* Update beefy repo.

* Work-around missing protocol.

* Revert "Work-around missing protocol."

This reverts commit 0a6257a8bccc1c67e966898cdedc408c6469ffd6.

* Add beefy peers set config.

* Expose storage of BEEFY.

* Uncompress BEEFY keys for merkle tree.

* Update ordering.

* Switch to branch.

* Bump deps.

* Switch to custom beefy.

* Add MMR RuntimeApi and custom rpc.

* Add set length details.

* Fix compilation.

* Expose MmrLeaf storage.

* Expose MmrLeaf storage.

* Don't use session handler, and rather compute & cache beefy details on call.

* Don't use session handler, and rather compute & cache beefy details on call.

* Fixes.

* Update Cargo.lock.

* Switch back to master.

* Update lockfile.

* Fix xcm print issue.

* Cargo.lock.

* Use master branch.

* Remove extra dep.

* Fix tests.

* Update Cargo.lock

* Add BEEFY & MMR to westend.

* Implement session keys migration.

* Update testnet script.

* start BEEFY for all node types

* Update Cargo.lock

* fix Cargo.toml

* resolve another merge conflict

* add Westend BEEFY keys

* Apply suggestions from code review

Co-authored-by: Hernando Castano <HCastano@users.noreply.github.com>

* Update BEEFY.

* Add Rococo BEEFY keys

* resolve merge issue

* fix pallet indices

* fix Westend OldSessionKey

* remove unused imports in Westend runtime

* Fix compilation for Westend.

* address review

* start BEEFY gadget conditionally

* address review again

* fix typo

* remove duplicate

* remove another duplicate

* well

* add missing stuff

* cleanup Cargo.toml files

- revert unnecessary changes
- add missing /std dependencies
- remove unused dependencies

* runtime: remove unused structs from rococo runtime

* node: cleanup service

Co-authored-by: André Silva <123550+andresilva@users.noreply.github.com>
Co-authored-by: Hernando Castano <HCastano@users.noreply.github.com>
Co-authored-by: adoerr <0xad@gmx.net>
Co-authored-by: André Silva <andrerfosilva@gmail.com>

polkadot/runtime/westend/src/lib.rs

@@ -31,6 +31,7 @@ use primitives::v1::{
 	InboundDownwardMessage, InboundHrmpMessage, SessionInfo,
 };
 use runtime_common::{
+	mmr as mmr_common,
 	SlowAdjustingFeeUpdate, CurrencyToVote,
 	impls::ToAuthor,
 	BlockHashCount, BlockWeights, BlockLength, RocksDbWeight, OffchainSolutionWeightLimit,
@@ -41,7 +42,7 @@ use sp_runtime::{
 	ApplyExtrinsicResult, KeyTypeId, Perbill, curve::PiecewiseLinear,
 	transaction_validity::{TransactionValidity, TransactionSource, TransactionPriority},
 	traits::{
-		BlakeTwo256, Block as BlockT, OpaqueKeys, ConvertInto, AccountIdLookup,
+		Keccak256, BlakeTwo256, Block as BlockT, OpaqueKeys, ConvertInto, AccountIdLookup,
 		Extrinsic as ExtrinsicT, SaturatedConversion, Verify,
 	},
 };
@@ -63,6 +64,8 @@ use authority_discovery_primitives::AuthorityId as AuthorityDiscoveryId;
 use pallet_transaction_payment::{FeeDetails, RuntimeDispatchInfo};
 use pallet_session::historical as session_historical;
 use frame_system::{EnsureRoot};
+use beefy_primitives::ecdsa::AuthorityId as BeefyId;
+use pallet_mmr_primitives as mmr;
 
 #[cfg(feature = "std")]
 pub use pallet_staking::StakerStatus;
@@ -263,6 +266,18 @@ parameter_types! {
 	pub const Offset: BlockNumber = 0;
 }
 
+// TODO [ToDr] Remove while BEEFY runtime upgrade is done.
+impl_opaque_keys! {
+	pub struct OldSessionKeys {
+		pub grandpa: Grandpa,
+		pub babe: Babe,
+		pub im_online: ImOnline,
+		pub para_validator: ParachainSessionKeyPlaceholder<Runtime>,
+		pub para_assignment: AssignmentSessionKeyPlaceholder<Runtime>,
+		pub authority_discovery: AuthorityDiscovery,
+	}
+}
+
 impl_opaque_keys! {
 	pub struct SessionKeys {
 		pub grandpa: Grandpa,
@@ -271,6 +286,35 @@ impl_opaque_keys! {
 		pub para_validator: ParachainSessionKeyPlaceholder<Runtime>,
 		pub para_assignment: AssignmentSessionKeyPlaceholder<Runtime>,
 		pub authority_discovery: AuthorityDiscovery,
+		pub beefy: Beefy,
+	}
+}
+
+fn transform_session_keys(v: AccountId, old: OldSessionKeys) -> SessionKeys {
+	SessionKeys {
+		grandpa: old.grandpa,
+		babe: old.babe,
+		im_online: old.im_online,
+		para_validator: old.para_validator,
+		para_assignment: old.para_assignment,
+		authority_discovery: old.authority_discovery,
+		beefy: {
+			// We need to produce a dummy value that's unique for the validator.
+			let mut id = BeefyId::default();
+			let id_raw: &mut [u8] = id.as_mut();
+			id_raw.copy_from_slice(v.as_ref());
+			id_raw[0..4].copy_from_slice(b"beef");
+			id
+		},
+	}
+}
+
+// When this is removed, should also remove `OldSessionKeys`.
+pub struct UpgradeSessionKeys;
+impl frame_support::traits::OnRuntimeUpgrade for UpgradeSessionKeys {
+	fn on_runtime_upgrade() -> frame_support::weights::Weight {
+		Session::upgrade_keys::<OldSessionKeys, _>(transform_session_keys);
+		Perbill::from_percent(50) * BlockWeights::get().max_block
 	}
 }
 
@@ -581,6 +625,24 @@ impl pallet_sudo::Config for Runtime {
 	type Call = Call;
 }
 
+impl pallet_beefy::Config for Runtime {
+	type AuthorityId = BeefyId;
+}
+
+impl pallet_mmr::Config for Runtime {
+	const INDEXING_PREFIX: &'static [u8] = b"mmr";
+	type Hashing = Keccak256;
+	type Hash = <Keccak256 as sp_runtime::traits::Hash>::Output;
+	type OnNewRoot = mmr_common::DepositBeefyDigest<Runtime>;
+	type WeightInfo = ();
+	type LeafData = mmr_common::Pallet<Runtime>;
+}
+
+impl mmr_common::Config for Runtime {
+	type BeefyAuthorityToMerkleLeaf = mmr_common::UncompressBeefyEcdsaKeys;
+	type ParachainHeads = ();
+}
+
 parameter_types! {
 	// One storage item; key size 32, value size 8; .
 	pub const ProxyDepositBase: Balance = deposit(1, 8);
@@ -739,6 +801,11 @@ construct_runtime! {
 
 		// Election pallet. Only works with staking, but placed here to maintain indices.
 		ElectionProviderMultiPhase: pallet_election_provider_multi_phase::{Pallet, Call, Storage, Event<T>, ValidateUnsigned} = 24,
+
+		// Bridges support.
+		Mmr: pallet_mmr::{Pallet, Call, Storage} = 28,
+		Beefy: pallet_beefy::{Pallet, Config<T>, Storage} = 29,
+		MmrLeaf: mmr_common::{Pallet, Storage} = 30,
 	}
 }
 
@@ -788,7 +855,7 @@ pub type Executive = frame_executive::Executive<
 	frame_system::ChainContext<Runtime>,
 	Runtime,
 	AllPallets,
-	BabeEpochConfigMigrations
+	(BabeEpochConfigMigrations, UpgradeSessionKeys),
 >;
 /// The payload being signed in transactions.
 pub type SignedPayload = generic::SignedPayload<Call, SignedExtra>;
@@ -917,6 +984,45 @@ sp_api::impl_runtime_apis! {
 		}
 	}
 
+	impl beefy_primitives::BeefyApi<Block, BeefyId> for Runtime {
+		fn validator_set() -> beefy_primitives::ValidatorSet<BeefyId> {
+			Beefy::validator_set()
+		}
+	}
+
+	impl pallet_mmr_primitives::MmrApi<Block, Hash> for Runtime {
+		fn generate_proof(leaf_index: u64)
+			-> Result<(mmr::EncodableOpaqueLeaf, mmr::Proof<Hash>), mmr::Error>
+		{
+			Mmr::generate_proof(leaf_index)
+				.map(|(leaf, proof)| (mmr::EncodableOpaqueLeaf::from_leaf(&leaf), proof))
+		}
+
+		fn verify_proof(leaf: mmr::EncodableOpaqueLeaf, proof: mmr::Proof<Hash>)
+			-> Result<(), mmr::Error>
+		{
+			pub type Leaf = <
+				<Runtime as pallet_mmr::Config>::LeafData as mmr::LeafDataProvider
+			>::LeafData;
+
+			let leaf: Leaf = leaf
+				.into_opaque_leaf()
+				.try_decode()
+				.ok_or(mmr::Error::Verify)?;
+			Mmr::verify_leaf(leaf, proof)
+		}
+
+		fn verify_proof_stateless(
+			root: Hash,
+			leaf: mmr::EncodableOpaqueLeaf,
+			proof: mmr::Proof<Hash>
+		) -> Result<(), mmr::Error> {
+			type MmrHashing = <Runtime as pallet_mmr::Config>::Hashing;
+			let node = mmr::DataOrHash::Data(leaf.into_opaque_leaf());
+			pallet_mmr::verify_leaf_proof::<MmrHashing, _>(root, node, proof)
+		}
+	}
+
 	impl fg_primitives::GrandpaApi<Block> for Runtime {
 		fn grandpa_authorities() -> Vec<(GrandpaId, u64)> {
 			Grandpa::grandpa_authorities()