update Substrate/Polkadot/Cumulus refs (#1562)

* update Substrate/Polkadot/Cumulus refs

* finality-grandpa 0.16

* fix miillau-runtime compilation

* fix rialto runtime compilation

* fixed rialto-parachain runtime compilation

* backport GRANDPA test fixes

* helper instead of removed record_all_keys

* substrate-relay is compiling

* millau-bridge-node at least compiles

* rialto-bridge-node at least compiles

* rialto-parachain-collator compiles

* fixings tests (wip)

* fmt

* fixed BEEFY alert

* clippy

* removed unused dep

* -extra var

* move Leaf to mod mmr

* fix benchmarks

bridges/bin/millau/node/Cargo.toml

@@ -10,7 +10,7 @@ license = "GPL-3.0-or-later WITH Classpath-exception-2.0"
 
 [dependencies]
 clap = { version = "3.1", features = ["derive"] }
-jsonrpc-core = "18.0"
+jsonrpsee = { version = "0.15.1", features = ["server"] }
 serde_json = "1.0.79"
 
 # Bridge dependencies

bridges/bin/millau/node/src/service.rs

@@ -16,19 +16,7 @@
 
 //! Service and ServiceFactory implementation. Specialized wrapper over substrate service.
 
-// =====================================================================================
-// =====================================================================================
-// =====================================================================================
-// UPDATE GUIDE:
-// 1) replace everything with node-template/src/service.rs contents (found in main Substrate repo);
-// 2) from old code keep `rpc_extensions_builder` - we use our own custom RPCs;
-// 3) from old code keep the Beefy gadget;
-// 4) fix compilation errors;
-// 5) test :)
-// =====================================================================================
-// =====================================================================================
-// =====================================================================================
-
+use jsonrpsee::RpcModule;
 use millau_runtime::{self, opaque::Block, RuntimeApi};
 use sc_client_api::{BlockBackend, ExecutorProvider};
 use sc_consensus_aura::{ImportQueueParams, SlotProportion, StartAuraParams};
@@ -83,6 +71,8 @@ pub fn new_partial(
 				FullSelectChain,
 			>,
 			sc_finality_grandpa::LinkHalf<Block, FullClient, FullSelectChain>,
+			beefy_gadget::BeefyVoterLinks<Block>,
+			beefy_gadget::BeefyRPCLinks<Block>,
 			Option<Telemetry>,
 		),
 	>,
@@ -140,11 +130,18 @@ pub fn new_partial(
 		telemetry.as_ref().map(|x| x.handle()),
 	)?;
 
+	let (beefy_block_import, beefy_voter_links, beefy_rpc_links) =
+		beefy_gadget::beefy_block_import_and_links(
+			grandpa_block_import.clone(),
+			backend.clone(),
+			client.clone(),
+		);
+
 	let slot_duration = sc_consensus_aura::slot_duration(&*client)?;
 
 	let import_queue =
 		sc_consensus_aura::import_queue::<AuraPair, _, _, _, _, _, _>(ImportQueueParams {
-			block_import: grandpa_block_import.clone(),
+			block_import: beefy_block_import,
 			justification_import: Some(Box::new(grandpa_block_import.clone())),
 			client: client.clone(),
 			create_inherent_data_providers: move |_, ()| async move {
@@ -175,7 +172,7 @@ pub fn new_partial(
 		keystore_container,
 		select_chain,
 		transaction_pool,
-		other: (grandpa_block_import, grandpa_link, telemetry),
+		other: (grandpa_block_import, grandpa_link, beefy_voter_links, beefy_rpc_links, telemetry),
 	})
 }
 
@@ -196,7 +193,7 @@ pub fn new_full(mut config: Configuration) -> Result<TaskManager, ServiceError>
 		mut keystore_container,
 		select_chain,
 		transaction_pool,
-		other: (block_import, grandpa_link, mut telemetry),
+		other: (block_import, grandpa_link, beefy_voter_links, beefy_rpc_links, mut telemetry),
 	} = new_partial(&config)?;
 
 	if let Some(url) = &config.keystore_remote {
@@ -264,18 +261,16 @@ pub fn new_full(mut config: Configuration) -> Result<TaskManager, ServiceError>
 	let enable_grandpa = !config.disable_grandpa;
 	let prometheus_registry = config.prometheus_registry().cloned();
 	let shared_voter_state = SharedVoterState::empty();
-	let (beefy_commitment_link, beefy_commitment_stream) =
-		beefy_gadget::notification::BeefySignedCommitmentStream::<Block>::channel();
-	let (beefy_best_block_link, beefy_best_block_stream) =
-		beefy_gadget::notification::BeefyBestBlockStream::<Block>::channel();
 
 	let rpc_extensions_builder = {
 		use sc_finality_grandpa::FinalityProofProvider as GrandpaFinalityProofProvider;
 
-		use pallet_transaction_payment_rpc::{TransactionPayment, TransactionPaymentApi};
-		use sc_finality_grandpa_rpc::{GrandpaApi, GrandpaRpcHandler};
+		use beefy_gadget_rpc::{Beefy, BeefyApiServer};
+		use pallet_mmr_rpc::{Mmr, MmrApiServer};
+		use pallet_transaction_payment_rpc::{TransactionPayment, TransactionPaymentApiServer};
+		use sc_finality_grandpa_rpc::{Grandpa, GrandpaApiServer};
 		use sc_rpc::DenyUnsafe;
-		use substrate_frame_rpc_system::{FullSystem, SystemApi};
+		use substrate_frame_rpc_system::{System, SystemApiServer};
 
 		let backend = backend.clone();
 		let client = client.clone();
@@ -291,33 +286,33 @@ pub fn new_full(mut config: Configuration) -> Result<TaskManager, ServiceError>
 		);
 
 		Box::new(move |_, subscription_executor: sc_rpc::SubscriptionTaskExecutor| {
-			let mut io = jsonrpc_core::IoHandler::default();
-			io.extend_with(SystemApi::to_delegate(FullSystem::new(
-				client.clone(),
-				pool.clone(),
-				DenyUnsafe::No,
-			)));
-			io.extend_with(TransactionPaymentApi::to_delegate(TransactionPayment::new(
-				client.clone(),
-			)));
-			io.extend_with(GrandpaApi::to_delegate(GrandpaRpcHandler::new(
-				shared_authority_set.clone(),
-				shared_voter_state.clone(),
-				justification_stream.clone(),
-				subscription_executor.clone(),
-				finality_proof_provider.clone(),
-			)));
-			io.extend_with(beefy_gadget_rpc::BeefyApi::to_delegate(
-				beefy_gadget_rpc::BeefyRpcHandler::<Block>::new(
-					beefy_commitment_stream.clone(),
-					beefy_best_block_stream.clone(),
+			let mut io = RpcModule::new(());
+			let map_err = |e| sc_service::Error::Other(format!("{}", e));
+			io.merge(System::new(client.clone(), pool.clone(), DenyUnsafe::No).into_rpc())
+				.map_err(map_err)?;
+			io.merge(TransactionPayment::new(client.clone()).into_rpc()).map_err(map_err)?;
+			io.merge(
+				Grandpa::new(
+					subscription_executor.clone(),
+					shared_authority_set.clone(),
+					shared_voter_state.clone(),
+					justification_stream.clone(),
+					finality_proof_provider.clone(),
+				)
+				.into_rpc(),
+			)
+			.map_err(map_err)?;
+			io.merge(
+				Beefy::<Block>::new(
+					beefy_rpc_links.from_voter_justif_stream.clone(),
+					beefy_rpc_links.from_voter_best_beefy_stream.clone(),
 					subscription_executor,
 				)
-				.map_err(|e| sc_service::Error::Other(format!("{}", e)))?,
-			));
-			io.extend_with(pallet_mmr_rpc::MmrApi::to_delegate(pallet_mmr_rpc::Mmr::new(
-				client.clone(),
-			)));
+				.map_err(|e| sc_service::Error::Other(format!("{}", e)))?
+				.into_rpc(),
+			)
+			.map_err(map_err)?;
+			io.merge(Mmr::new(client.clone()).into_rpc()).map_err(map_err)?;
 			Ok(io)
 		})
 	};
@@ -328,7 +323,7 @@ pub fn new_full(mut config: Configuration) -> Result<TaskManager, ServiceError>
 		keystore: keystore_container.sync_keystore(),
 		task_manager: &mut task_manager,
 		transaction_pool: transaction_pool.clone(),
-		rpc_extensions_builder,
+		rpc_builder: rpc_extensions_builder.clone(),
 		backend: backend.clone(),
 		system_rpc_tx,
 		config,
@@ -397,11 +392,10 @@ pub fn new_full(mut config: Configuration) -> Result<TaskManager, ServiceError>
 		runtime: client,
 		key_store: keystore.clone(),
 		network: network.clone(),
-		signed_commitment_sender: beefy_commitment_link,
-		beefy_best_block_sender: beefy_best_block_link,
 		min_block_delta: 2,
 		prometheus_registry: prometheus_registry.clone(),
 		protocol_name: beefy_protocol_name,
+		links: beefy_voter_links,
 	};
 
 	// Start the BEEFY bridge gadget.

bridges/bin/millau/runtime/src/lib.rs

@@ -52,9 +52,7 @@ use pallet_transaction_payment::{FeeDetails, Multiplier, RuntimeDispatchInfo};
 use sp_api::impl_runtime_apis;
 use sp_consensus_aura::sr25519::AuthorityId as AuraId;
 use sp_core::{crypto::KeyTypeId, OpaqueMetadata};
-use sp_mmr_primitives::{
-	DataOrHash, EncodableOpaqueLeaf, Error as MmrError, LeafDataProvider, Proof as MmrProof,
-};
+use sp_mmr_primitives::{DataOrHash, EncodableOpaqueLeaf, Error as MmrError, Proof as MmrProof};
 use sp_runtime::{
 	create_runtime_str, generic, impl_opaque_keys,
 	traits::{Block as BlockT, IdentityLookup, Keccak256, NumberFor, OpaqueKeys},
@@ -238,6 +236,8 @@ impl pallet_aura::Config for Runtime {
 
 impl pallet_beefy::Config for Runtime {
 	type BeefyId = BeefyId;
+	type MaxAuthorities = MaxAuthorities;
+	type OnNewValidatorSet = MmrLeaf;
 }
 
 impl pallet_grandpa::Config for Runtime {
@@ -256,12 +256,21 @@ impl pallet_grandpa::Config for Runtime {
 	type MaxAuthorities = MaxAuthorities;
 }
 
-type MmrHash = <Keccak256 as sp_runtime::traits::Hash>::Output;
+/// MMR helper types.
+mod mmr {
+	use super::Runtime;
+	pub use pallet_mmr::primitives::*;
+	use sp_runtime::traits::Keccak256;
+
+	pub type Leaf = <<Runtime as pallet_mmr::Config>::LeafData as LeafDataProvider>::LeafData;
+	pub type Hash = <Keccak256 as sp_runtime::traits::Hash>::Output;
+	pub type Hashing = <Runtime as pallet_mmr::Config>::Hashing;
+}
 
 impl pallet_mmr::Config for Runtime {
 	const INDEXING_PREFIX: &'static [u8] = b"mmr";
 	type Hashing = Keccak256;
-	type Hash = MmrHash;
+	type Hash = mmr::Hash;
 	type OnNewRoot = pallet_beefy_mmr::DepositBeefyDigest<Runtime>;
 	type WeightInfo = ();
 	type LeafData = pallet_beefy_mmr::Pallet<Runtime>;
@@ -355,6 +364,7 @@ impl pallet_transaction_payment::Config for Runtime {
 		AdjustmentVariable,
 		MinimumMultiplier,
 	>;
+	type Event = Event;
 }
 
 impl pallet_sudo::Config for Runtime {
@@ -562,7 +572,7 @@ construct_runtime!(
 
 		Timestamp: pallet_timestamp::{Pallet, Call, Storage, Inherent},
 		Balances: pallet_balances::{Pallet, Call, Storage, Config<T>, Event<T>},
-		TransactionPayment: pallet_transaction_payment::{Pallet, Storage},
+		TransactionPayment: pallet_transaction_payment::{Pallet, Storage, Event<T>},
 
 		// Consensus support.
 		Session: pallet_session::{Pallet, Call, Storage, Event, Config<T>},
@@ -744,41 +754,69 @@ impl_runtime_apis! {
 		}
 	}
 
-	impl sp_mmr_primitives::MmrApi<Block, MmrHash> for Runtime {
+	impl sp_mmr_primitives::MmrApi<Block, mmr::Hash> for Runtime {
 		fn generate_proof(leaf_index: u64)
-			-> Result<(EncodableOpaqueLeaf, MmrProof<MmrHash>), MmrError>
+			-> Result<(EncodableOpaqueLeaf, MmrProof<mmr::Hash>), MmrError>
 		{
-			Mmr::generate_proof(leaf_index)
-				.map(|(leaf, proof)| (EncodableOpaqueLeaf::from_leaf(&leaf), proof))
+			Mmr::generate_batch_proof(vec![leaf_index])
+				.and_then(|(leaves, proof)| Ok((
+					mmr::EncodableOpaqueLeaf::from_leaf(&leaves[0]),
+					mmr::BatchProof::into_single_leaf_proof(proof)?
+				)))
 		}
 
-		fn verify_proof(leaf: EncodableOpaqueLeaf, proof: MmrProof<MmrHash>)
+		fn verify_proof(leaf: EncodableOpaqueLeaf, proof: MmrProof<mmr::Hash>)
 			-> Result<(), MmrError>
 		{
-			pub type Leaf = <
-				<Runtime as pallet_mmr::Config>::LeafData as LeafDataProvider
-			>::LeafData;
-
-			let leaf: Leaf = leaf
+			let leaf: mmr::Leaf = leaf
 				.into_opaque_leaf()
 				.try_decode()
 				.ok_or(MmrError::Verify)?;
-			Mmr::verify_leaf(leaf, proof)
+			Mmr::verify_leaves(vec![leaf], mmr::Proof::into_batch_proof(proof))
 		}
 
 		fn verify_proof_stateless(
-			root: MmrHash,
+			root: mmr::Hash,
 			leaf: EncodableOpaqueLeaf,
-			proof: MmrProof<MmrHash>
+			proof: MmrProof<mmr::Hash>
 		) -> Result<(), MmrError> {
-			type MmrHashing = <Runtime as pallet_mmr::Config>::Hashing;
 			let node = DataOrHash::Data(leaf.into_opaque_leaf());
-			pallet_mmr::verify_leaf_proof::<MmrHashing, _>(root, node, proof)
+			pallet_mmr::verify_leaves_proof::<mmr::Hashing, _>(
+				root,
+				vec![node],
+				pallet_mmr::primitives::Proof::into_batch_proof(proof),
+			)
 		}
 
-		fn mmr_root() -> Result<MmrHash, MmrError> {
+		fn mmr_root() -> Result<mmr::Hash, MmrError> {
 			Ok(Mmr::mmr_root())
 		}
+
+		fn generate_batch_proof(leaf_indices: Vec<pallet_mmr::primitives::LeafIndex>)
+			-> Result<(Vec<mmr::EncodableOpaqueLeaf>, mmr::BatchProof<mmr::Hash>), mmr::Error>
+		{
+			Mmr::generate_batch_proof(leaf_indices)
+				.map(|(leaves, proof)| (leaves.into_iter().map(|leaf| mmr::EncodableOpaqueLeaf::from_leaf(&leaf)).collect(), proof))
+		}
+
+		fn verify_batch_proof(leaves: Vec<mmr::EncodableOpaqueLeaf>, proof: mmr::BatchProof<mmr::Hash>)
+			-> Result<(), mmr::Error>
+		{
+			let leaves = leaves.into_iter().map(|leaf|
+				leaf.into_opaque_leaf()
+				.try_decode()
+				.ok_or(mmr::Error::Verify)).collect::<Result<Vec<mmr::Leaf>, mmr::Error>>()?;
+			Mmr::verify_leaves(leaves, proof)
+		}
+
+		fn verify_batch_proof_stateless(
+			root: mmr::Hash,
+			leaves: Vec<mmr::EncodableOpaqueLeaf>,
+			proof: mmr::BatchProof<mmr::Hash>
+		) -> Result<(), mmr::Error> {
+			let nodes = leaves.into_iter().map(|leaf|mmr::DataOrHash::Data(leaf.into_opaque_leaf())).collect();
+			pallet_mmr::verify_leaves_proof::<mmr::Hashing, _>(root, nodes, proof)
+		}
 	}
 
 	impl fg_primitives::GrandpaApi<Block> for Runtime {

bridges/bin/millau/runtime/src/xcm_config.rs

@@ -156,6 +156,7 @@ impl xcm_executor::Config for XcmConfig {
 	type FeeManager = ();
 	type MessageExporter = ();
 	type UniversalAliases = Nothing;
+	type CallDispatcher = Call;
 }
 
 /// Type to convert an `Origin` type value into a `MultiLocation` value which represents an interior
@@ -270,7 +271,7 @@ mod tests {
 	fn xcm_messages_are_sent_using_bridge_router() {
 		new_test_ext().execute_with(|| {
 			let xcm: Xcm<()> = vec![Instruction::Trap(42)].into();
-			let expected_fee = MultiAssets::from((Here, 4_345_002_552_u64));
+			let expected_fee = MultiAssets::from((Here, 4_259_858_152_u64));
 			let expected_hash =
 				([0u8, 0u8, 0u8, 0u8], 1u64).using_encoded(sp_io::hashing::blake2_256);