Merkle Mountain Range & BEEFY integration (#2101)

* Switch branch.

* Implement basic MMR leaf.

* Revert "Switch branch."

This reverts commit 7f4d41c67f27ca560c53fc63fd3bd06ac182403c.

* Bump substrate.

* Integrate BEEFY.

Bump all.

Fix missing imports.

* Use beefy pallet to get authorities.

* Bump BEEFY repo.

* Use next authority set instead of the current one.

* Start BEEFY service.

* Fix BEEFY start up.

* Cache BEEFY authority set.

* Add BEEFY ValidatorSetId to MMR

* Fix code.

* Apply suggestions from code review

Co-authored-by: André Silva <123550+andresilva@users.noreply.github.com>
Co-authored-by: Hernando Castano <HCastano@users.noreply.github.com>

* Review grumbles.

* Update beefy repo.

* Work-around missing protocol.

* Revert "Work-around missing protocol."

This reverts commit 0a6257a8bccc1c67e966898cdedc408c6469ffd6.

* Add beefy peers set config.

* Expose storage of BEEFY.

* Uncompress BEEFY keys for merkle tree.

* Update ordering.

* Switch to branch.

* Bump deps.

* Switch to custom beefy.

* Add MMR RuntimeApi and custom rpc.

* Add set length details.

* Fix compilation.

* Expose MmrLeaf storage.

* Expose MmrLeaf storage.

* Don't use session handler, and rather compute & cache beefy details on call.

* Don't use session handler, and rather compute & cache beefy details on call.

* Fixes.

* Update Cargo.lock.

* Switch back to master.

* Update lockfile.

* Fix xcm print issue.

* Cargo.lock.

* Use master branch.

* Remove extra dep.

* Fix tests.

* Update Cargo.lock

* Add BEEFY & MMR to westend.

* Implement session keys migration.

* Update testnet script.

* start BEEFY for all node types

* Update Cargo.lock

* fix Cargo.toml

* resolve another merge conflict

* add Westend BEEFY keys

* Apply suggestions from code review

Co-authored-by: Hernando Castano <HCastano@users.noreply.github.com>

* Update BEEFY.

* Add Rococo BEEFY keys

* resolve merge issue

* fix pallet indices

* fix Westend OldSessionKey

* remove unused imports in Westend runtime

* Fix compilation for Westend.

* address review

* start BEEFY gadget conditionally

* address review again

* fix typo

* remove duplicate

* remove another duplicate

* well

* add missing stuff

* cleanup Cargo.toml files

- revert unnecessary changes
- add missing /std dependencies
- remove unused dependencies

* runtime: remove unused structs from rococo runtime

* node: cleanup service

Co-authored-by: André Silva <123550+andresilva@users.noreply.github.com>
Co-authored-by: Hernando Castano <HCastano@users.noreply.github.com>
Co-authored-by: adoerr <0xad@gmx.net>
Co-authored-by: André Silva <andrerfosilva@gmail.com>

polkadot/rpc/src/lib.rs

@@ -74,8 +74,16 @@ pub struct GrandpaDeps<B> {
 	pub finality_provider: Arc<FinalityProofProvider<B, Block>>,
 }
 
+/// Dependencies for BEEFY
+pub struct BeefyDeps<BeefySignature> {
+	/// Receives notifications about signed commitment events from BEEFY.
+	pub beefy_commitment_stream: beefy_gadget::notification::BeefySignedCommitmentStream<Block, BeefySignature>,
+	/// Executor to drive the subscription manager in the BEEFY RPC handler.
+	pub subscription_executor: sc_rpc::SubscriptionTaskExecutor,
+}
+
 /// Full client dependencies
-pub struct FullDeps<C, P, SC, B> {
+pub struct FullDeps<C, P, SC, B, BS> {
 	/// The client instance to use.
 	pub client: Arc<C>,
 	/// Transaction pool instance.
@@ -90,13 +98,16 @@ pub struct FullDeps<C, P, SC, B> {
 	pub babe: BabeDeps,
 	/// GRANDPA specific dependencies.
 	pub grandpa: GrandpaDeps<B>,
+	/// BEEFY specific dependencies.
+	pub beefy: BeefyDeps<BS>,
 }
 
 /// Instantiate all RPC extensions.
-pub fn create_full<C, P, SC, B>(deps: FullDeps<C, P, SC, B>) -> RpcExtension where
+pub fn create_full<C, P, SC, B, BS>(deps: FullDeps<C, P, SC, B, BS>) -> RpcExtension where
 	C: ProvideRuntimeApi<Block> + HeaderBackend<Block> + AuxStore +
 		HeaderMetadata<Block, Error=BlockChainError> + Send + Sync + 'static,
 	C::Api: frame_rpc_system::AccountNonceApi<Block, AccountId, Nonce>,
+	C::Api: pallet_mmr_rpc::MmrRuntimeApi<Block, <Block as sp_runtime::traits::Block>::Hash>,
 	C::Api: pallet_transaction_payment_rpc::TransactionPaymentRuntimeApi<Block, Balance>,
 	C::Api: BabeApi<Block>,
 	C::Api: BlockBuilder<Block>,
@@ -104,11 +115,13 @@ pub fn create_full<C, P, SC, B>(deps: FullDeps<C, P, SC, B>) -> RpcExtension whe
 	SC: SelectChain<Block> + 'static,
 	B: sc_client_api::Backend<Block> + Send + Sync + 'static,
 	B::State: sc_client_api::StateBackend<sp_runtime::traits::HashFor<Block>>,
+	BS: Clone + Send + parity_scale_codec::Encode + 'static,
 {
 	use frame_rpc_system::{FullSystem, SystemApi};
+	use pallet_mmr_rpc::{MmrApi, Mmr};
 	use pallet_transaction_payment_rpc::{TransactionPayment, TransactionPaymentApi};
-	use sc_finality_grandpa_rpc::{GrandpaApi, GrandpaRpcHandler};
 	use sc_consensus_babe_rpc::BabeRpcHandler;
+	use sc_finality_grandpa_rpc::{GrandpaApi, GrandpaRpcHandler};
 
 	let mut io = jsonrpc_core::IoHandler::default();
 	let FullDeps {
@@ -119,6 +132,7 @@ pub fn create_full<C, P, SC, B>(deps: FullDeps<C, P, SC, B>) -> RpcExtension whe
 		deny_unsafe,
 		babe,
 		grandpa,
+		beefy,
 	} = deps;
 	let BabeDeps {
 		keystore,
@@ -139,6 +153,9 @@ pub fn create_full<C, P, SC, B>(deps: FullDeps<C, P, SC, B>) -> RpcExtension whe
 	io.extend_with(
 		TransactionPaymentApi::to_delegate(TransactionPayment::new(client.clone()))
 	);
+	io.extend_with(
+		MmrApi::to_delegate(Mmr::new(client.clone()))
+	);
 	io.extend_with(
 		sc_consensus_babe_rpc::BabeApi::to_delegate(
 			BabeRpcHandler::new(
@@ -169,6 +186,14 @@ pub fn create_full<C, P, SC, B>(deps: FullDeps<C, P, SC, B>) -> RpcExtension whe
 			deny_unsafe,
 		))
 	);
+
+	io.extend_with(beefy_gadget_rpc::BeefyApi::to_delegate(
+		beefy_gadget_rpc::BeefyRpcHandler::new(
+			beefy.beefy_commitment_stream,
+			beefy.subscription_executor,
+		),
+	));
+
 	io
 }