Expose MMR root through runtime API - use it in BEEFY client (#11183)

* beefy-gadget: allow custom runtime api provider

* beefy-gadget: use mock runtime api in tests

* pallet-mmr: expose mmr root from state through runtime API

* beefy-gadget: get mmr root from runtime state

* pallet-beefy-mmr: remove MmrRoot from header digests

* frame/mmr: move mmr primitives out of frame

* frame/mmr: completely move primitives out of frame

* address review comments

* beefy-mmr: bring back mmr root from header digest

* clippy fixes for rustc 1.60

* address review comments

substrate/Cargo.lock

@@ -502,6 +502,7 @@ dependencies = [
  "sp-finality-grandpa",
  "sp-keyring",
  "sp-keystore",
+ "sp-mmr-primitives",
  "sp-runtime",
  "sp-tracing",
  "strum",
@@ -5652,7 +5653,6 @@ dependencies = [
  "log 0.4.14",
  "pallet-beefy",
  "pallet-mmr",
- "pallet-mmr-primitives",
  "pallet-session",
  "parity-scale-codec",
  "scale-info",
@@ -6081,27 +6081,11 @@ dependencies = [
  "frame-support",
  "frame-system",
  "hex-literal",
- "pallet-mmr-primitives",
  "parity-scale-codec",
  "scale-info",
  "sp-core",
  "sp-io",
- "sp-runtime",
+ "sp-mmr-primitives",
- "sp-std",
-]
-
-[[package]]
-name = "pallet-mmr-primitives"
-version = "4.0.0-dev"
-dependencies = [
- "frame-support",
- "frame-system",
- "hex-literal",
- "log 0.4.14",
- "parity-scale-codec",
- "serde",
- "sp-api",
- "sp-core",
  "sp-runtime",
  "sp-std",
 ]
@@ -6113,13 +6097,13 @@ dependencies = [
  "jsonrpc-core",
  "jsonrpc-core-client",
  "jsonrpc-derive",
- "pallet-mmr-primitives",
  "parity-scale-codec",
  "serde",
  "serde_json",
  "sp-api",
  "sp-blockchain",
  "sp-core",
+ "sp-mmr-primitives",
  "sp-runtime",
 ]
 
@@ -10140,6 +10124,21 @@ dependencies = [
  "zstd",
 ]
 
+[[package]]
+name = "sp-mmr-primitives"
+version = "4.0.0-dev"
+dependencies = [
+ "hex-literal",
+ "log 0.4.14",
+ "parity-scale-codec",
+ "serde",
+ "sp-api",
+ "sp-core",
+ "sp-debug-derive",
+ "sp-runtime",
+ "sp-std",
+]
+
 [[package]]
 name = "sp-npos-elections"
 version = "4.0.0-dev"

substrate/Cargo.toml

@@ -101,7 +101,6 @@ members = [
 	"frame/lottery",
 	"frame/membership",
 	"frame/merkle-mountain-range",
-	"frame/merkle-mountain-range/primitives",
 	"frame/merkle-mountain-range/rpc",
 	"frame/multisig",
 	"frame/nicks",
@@ -172,6 +171,7 @@ members = [
 	"primitives/keyring",
 	"primitives/keystore",
 	"primitives/maybe-compressed-blob",
+	"primitives/merkle-mountain-range",
 	"primitives/npos-elections",
 	"primitives/npos-elections/fuzzer",
 	"primitives/offchain",

substrate/bin/node/runtime/src/lib.rs

@@ -1261,7 +1261,7 @@ impl pallet_mmr::Config for Runtime {
 	const INDEXING_PREFIX: &'static [u8] = b"mmr";
 	type Hashing = <Runtime as frame_system::Config>::Hashing;
 	type Hash = <Runtime as frame_system::Config>::Hash;
-	type LeafData = frame_system::Pallet<Self>;
+	type LeafData = pallet_mmr::ParentNumberAndHash<Self>;
 	type OnNewRoot = ();
 	type WeightInfo = ();
 }
@@ -1804,6 +1804,10 @@ impl_runtime_apis! {
 			let node = mmr::DataOrHash::Data(leaf.into_opaque_leaf());
 			pallet_mmr::verify_leaf_proof::<mmr::Hashing, _>(root, node, proof)
 		}
+
+		fn mmr_root() -> Result<mmr::Hash, mmr::Error> {
+			Ok(Mmr::mmr_root())
+		}
 	}
 
 	impl sp_session::SessionKeys<Block> for Runtime {

substrate/client/beefy/Cargo.toml

@@ -27,6 +27,7 @@ sp-blockchain = { version = "4.0.0-dev", path = "../../primitives/blockchain" }
 sp-consensus = { version = "0.10.0-dev", path = "../../primitives/consensus/common" }
 sp-core = { version = "6.0.0", path = "../../primitives/core" }
 sp-keystore = { version = "0.12.0", path = "../../primitives/keystore" }
+sp-mmr-primitives = { version = "4.0.0-dev", path = "../../primitives/merkle-mountain-range" }
 sp-runtime = { version = "6.0.0", path = "../../primitives/runtime" }
 
 sc-chain-spec = { version = "4.0.0-dev", path = "../../client/chain-spec" }

substrate/client/beefy/src/lib.rs

@@ -27,9 +27,10 @@ use sp_api::ProvideRuntimeApi;
 use sp_blockchain::HeaderBackend;
 use sp_consensus::SyncOracle;
 use sp_keystore::SyncCryptoStorePtr;
+use sp_mmr_primitives::MmrApi;
 use sp_runtime::traits::Block;
 
-use beefy_primitives::BeefyApi;
+use beefy_primitives::{BeefyApi, MmrRootHash};
 
 use crate::notification::{BeefyBestBlockSender, BeefySignedCommitmentSender};
 
@@ -87,7 +88,7 @@ pub fn beefy_peers_set_config(
 /// of today, Rust does not allow a type alias to be used as a trait bound. Tracking
 /// issue is <https://github.com/rust-lang/rust/issues/41517>.
 pub trait Client<B, BE>:
-	BlockchainEvents<B> + HeaderBackend<B> + Finalizer<B, BE> + ProvideRuntimeApi<B> + Send + Sync
+	BlockchainEvents<B> + HeaderBackend<B> + Finalizer<B, BE> + Send + Sync
 where
 	B: Block,
 	BE: Backend<B>,
@@ -110,18 +111,21 @@ where
 }
 
 /// BEEFY gadget initialization parameters.
-pub struct BeefyParams<B, BE, C, N>
+pub struct BeefyParams<B, BE, C, N, R>
 where
 	B: Block,
 	BE: Backend<B>,
 	C: Client<B, BE>,
-	C::Api: BeefyApi<B>,
+	R: ProvideRuntimeApi<B>,
+	R::Api: BeefyApi<B> + MmrApi<B, MmrRootHash>,
 	N: GossipNetwork<B> + Clone + SyncOracle + Send + Sync + 'static,
 {
 	/// BEEFY client
 	pub client: Arc<C>,
 	/// Client Backend
 	pub backend: Arc<BE>,
+	/// Runtime Api Provider
+	pub runtime: Arc<R>,
 	/// Local key store
 	pub key_store: Option<SyncCryptoStorePtr>,
 	/// Gossip network
@@ -138,21 +142,22 @@ where
 	pub protocol_name: std::borrow::Cow<'static, str>,
 }
 
-#[cfg(not(test))]
 /// Start the BEEFY gadget.
 ///
 /// This is a thin shim around running and awaiting a BEEFY worker.
-pub async fn start_beefy_gadget<B, BE, C, N>(beefy_params: BeefyParams<B, BE, C, N>)
+pub async fn start_beefy_gadget<B, BE, C, N, R>(beefy_params: BeefyParams<B, BE, C, N, R>)
 where
 	B: Block,
 	BE: Backend<B>,
 	C: Client<B, BE>,
-	C::Api: BeefyApi<B>,
+	R: ProvideRuntimeApi<B>,
+	R::Api: BeefyApi<B> + MmrApi<B, MmrRootHash>,
 	N: GossipNetwork<B> + Clone + SyncOracle + Send + Sync + 'static,
 {
 	let BeefyParams {
 		client,
 		backend,
+		runtime,
 		key_store,
 		network,
 		signed_commitment_sender,
@@ -188,6 +193,7 @@ where
 	let worker_params = worker::WorkerParams {
 		client,
 		backend,
+		runtime,
 		key_store: key_store.into(),
 		signed_commitment_sender,
 		beefy_best_block_sender,
@@ -198,7 +204,7 @@ where
 		sync_oracle,
 	};
 
-	let worker = worker::BeefyWorker::<_, _, _, _>::new(worker_params);
+	let worker = worker::BeefyWorker::<_, _, _, _, _>::new(worker_params);
 
 	worker.run().await
 }

substrate/client/beefy/src/metrics.rs

@@ -18,9 +18,7 @@
 
 //! BEEFY Prometheus metrics definition
 
-#[cfg(not(test))]
+use prometheus::{register, Counter, Gauge, PrometheusError, Registry, U64};
-use prometheus::{register, PrometheusError, Registry};
-use prometheus::{Counter, Gauge, U64};
 
 /// BEEFY metrics exposed through Prometheus
 pub(crate) struct Metrics {
@@ -39,7 +37,6 @@ pub(crate) struct Metrics {
 }
 
 impl Metrics {
-	#[cfg(not(test))]
 	pub(crate) fn register(registry: &Registry) -> Result<Self, PrometheusError> {
 		Ok(Self {
 			beefy_validator_set_id: register(

substrate/client/beefy/src/tests.rs

@@ -28,18 +28,20 @@ use sc_chain_spec::{ChainSpec, GenericChainSpec};
 use sc_client_api::HeaderBackend;
 use sc_consensus::BoxJustificationImport;
 use sc_keystore::LocalKeystore;
-use sc_network::{config::ProtocolConfig, NetworkService};
+use sc_network::config::ProtocolConfig;
-use sc_network_gossip::GossipEngine;
 use sc_network_test::{
 	Block, BlockImportAdapter, FullPeerConfig, PassThroughVerifier, Peer, PeersClient,
-	PeersFullClient, TestNetFactory,
+	TestNetFactory,
 };
 use sc_utils::notification::NotificationReceiver;
 
 use beefy_primitives::{
-	crypto::AuthorityId, ConsensusLog, MmrRootHash, ValidatorSet, BEEFY_ENGINE_ID,
+	crypto::AuthorityId, BeefyApi, ConsensusLog, MmrRootHash, ValidatorSet, BEEFY_ENGINE_ID,
 	KEY_TYPE as BeefyKeyType,
 };
+use sp_mmr_primitives::{EncodableOpaqueLeaf, Error as MmrError, LeafIndex, MmrApi, Proof};
+
+use sp_api::{ApiRef, ProvideRuntimeApi};
 use sp_consensus::BlockOrigin;
 use sp_core::H256;
 use sp_keystore::{SyncCryptoStore, SyncCryptoStorePtr};
@@ -47,19 +49,16 @@ use sp_runtime::{
 	codec::Encode, generic::BlockId, traits::Header as HeaderT, BuildStorage, DigestItem, Storage,
 };
 
-use substrate_test_runtime_client::{runtime::Header, Backend, ClientExt};
+use substrate_test_runtime_client::{runtime::Header, ClientExt};
 
-use crate::{
+use crate::{beefy_protocol_name, keystore::tests::Keyring as BeefyKeyring, notification::*};
-	beefy_protocol_name,
-	keystore::tests::Keyring as BeefyKeyring,
-	notification::*,
-	worker::{tests::TestModifiers, BeefyWorker},
-};
 
-const BEEFY_PROTOCOL_NAME: &'static str = "/beefy/1";
+pub(crate) const BEEFY_PROTOCOL_NAME: &'static str = "/beefy/1";
+const GOOD_MMR_ROOT: MmrRootHash = MmrRootHash::repeat_byte(0xbf);
+const BAD_MMR_ROOT: MmrRootHash = MmrRootHash::repeat_byte(0x42);
 
-type BeefyValidatorSet = ValidatorSet<AuthorityId>;
+pub(crate) type BeefyValidatorSet = ValidatorSet<AuthorityId>;
-type BeefyPeer = Peer<PeerData, PeersClient>;
+pub(crate) type BeefyPeer = Peer<PeerData, PeersClient>;
 
 #[derive(Debug, Serialize, Deserialize)]
 struct Genesis(std::collections::BTreeMap<String, String>);
@@ -101,27 +100,13 @@ fn beefy_protocol_name() {
 #[allow(dead_code)]
 #[derive(Clone)]
 pub(crate) struct BeefyLinkHalf {
-	signed_commitment_stream: BeefySignedCommitmentStream<Block>,
+	pub signed_commitment_stream: BeefySignedCommitmentStream<Block>,
-	beefy_best_block_stream: BeefyBestBlockStream<Block>,
+	pub beefy_best_block_stream: BeefyBestBlockStream<Block>,
 }
 
 #[derive(Default)]
 pub(crate) struct PeerData {
 	pub(crate) beefy_link_half: Mutex<Option<BeefyLinkHalf>>,
-	pub(crate) test_modifiers: Option<TestModifiers>,
-}
-
-impl PeerData {
-	pub(crate) fn use_validator_set(&mut self, validator_set: &ValidatorSet<AuthorityId>) {
-		if let Some(tm) = self.test_modifiers.as_mut() {
-			tm.active_validators = validator_set.clone();
-		} else {
-			self.test_modifiers = Some(TestModifiers {
-				active_validators: validator_set.clone(),
-				corrupt_mmr_roots: false,
-			});
-		}
-	}
 }
 
 pub(crate) struct BeefyTestNet {
@@ -153,17 +138,19 @@ impl BeefyTestNet {
 		count: usize,
 		session_length: u64,
 		validator_set: &BeefyValidatorSet,
+		include_mmr_digest: bool,
 	) {
 		self.peer(0).generate_blocks(count, BlockOrigin::File, |builder| {
 			let mut block = builder.build().unwrap().block;
 
-			let block_num = *block.header.number();
+			if include_mmr_digest {
-			let num_byte = block_num.to_le_bytes().into_iter().next().unwrap();
+				let block_num = *block.header.number();
-			let mmr_root = MmrRootHash::repeat_byte(num_byte);
+				let num_byte = block_num.to_le_bytes().into_iter().next().unwrap();
+				let mmr_root = MmrRootHash::repeat_byte(num_byte);
+				add_mmr_digest(&mut block.header, mmr_root);
+			}
 
-			add_mmr_digest(&mut block.header, mmr_root);
+			if *block.header.number() % session_length == 0 {
-
-			if block_num % session_length == 0 {
 				add_auth_change_digest(&mut block.header, validator_set.clone());
 			}
 
@@ -223,6 +210,79 @@ impl TestNetFactory for BeefyTestNet {
 	}
 }
 
+macro_rules! create_test_api {
+    ( $api_name:ident, mmr_root: $mmr_root:expr, $($inits:expr),+ ) => {
+		pub(crate) mod $api_name {
+			use super::*;
+
+			#[derive(Clone, Default)]
+			pub(crate) struct TestApi {}
+
+			// compiler gets confused and warns us about unused inner
+			#[allow(dead_code)]
+			pub(crate) struct RuntimeApi {
+				inner: TestApi,
+			}
+
+			impl ProvideRuntimeApi<Block> for TestApi {
+				type Api = RuntimeApi;
+				fn runtime_api<'a>(&'a self) -> ApiRef<'a, Self::Api> {
+					RuntimeApi { inner: self.clone() }.into()
+				}
+			}
+			sp_api::mock_impl_runtime_apis! {
+				impl BeefyApi<Block> for RuntimeApi {
+					fn validator_set() -> Option<BeefyValidatorSet> {
+						BeefyValidatorSet::new(make_beefy_ids(&[$($inits),+]), 0)
+					}
+				}
+
+				impl MmrApi<Block, MmrRootHash> for RuntimeApi {
+					fn generate_proof(_leaf_index: LeafIndex)
+						-> Result<(EncodableOpaqueLeaf, Proof<MmrRootHash>), MmrError> {
+						unimplemented!()
+					}
+
+					fn verify_proof(_leaf: EncodableOpaqueLeaf, _proof: Proof<MmrRootHash>)
+						-> Result<(), MmrError> {
+						unimplemented!()
+					}
+
+					fn verify_proof_stateless(
+						_root: MmrRootHash,
+						_leaf: EncodableOpaqueLeaf,
+						_proof: Proof<MmrRootHash>
+					) -> Result<(), MmrError> {
+						unimplemented!()
+					}
+
+					fn mmr_root() -> Result<MmrRootHash, MmrError> {
+						Ok($mmr_root)
+					}
+				}
+			}
+		}
+	};
+}
+
+create_test_api!(two_validators, mmr_root: GOOD_MMR_ROOT, BeefyKeyring::Alice, BeefyKeyring::Bob);
+create_test_api!(
+	four_validators,
+	mmr_root: GOOD_MMR_ROOT,
+	BeefyKeyring::Alice,
+	BeefyKeyring::Bob,
+	BeefyKeyring::Charlie,
+	BeefyKeyring::Dave
+);
+create_test_api!(
+	bad_four_validators,
+	mmr_root: BAD_MMR_ROOT,
+	BeefyKeyring::Alice,
+	BeefyKeyring::Bob,
+	BeefyKeyring::Charlie,
+	BeefyKeyring::Dave
+);
+
 fn add_mmr_digest(header: &mut Header, mmr_hash: MmrRootHash) {
 	header.digest_mut().push(DigestItem::Consensus(
 		BEEFY_ENGINE_ID,
@@ -248,54 +308,43 @@ pub(crate) fn create_beefy_keystore(authority: BeefyKeyring) -> SyncCryptoStoreP
 	keystore
 }
 
-pub(crate) fn create_beefy_worker(
-	peer: &BeefyPeer,
-	key: &BeefyKeyring,
-	min_block_delta: u32,
-) -> BeefyWorker<Block, PeersFullClient, Backend, Arc<NetworkService<Block, H256>>> {
-	let keystore = create_beefy_keystore(*key);
-
-	let (signed_commitment_sender, signed_commitment_stream) =
-		BeefySignedCommitmentStream::<Block>::channel();
-	let (beefy_best_block_sender, beefy_best_block_stream) =
-		BeefyBestBlockStream::<Block>::channel();
-
-	let beefy_link_half = BeefyLinkHalf { signed_commitment_stream, beefy_best_block_stream };
-	*peer.data.beefy_link_half.lock() = Some(beefy_link_half);
-	let test_modifiers = peer.data.test_modifiers.clone().unwrap();
-
-	let network = peer.network_service().clone();
-	let sync_oracle = network.clone();
-	let gossip_validator = Arc::new(crate::gossip::GossipValidator::new());
-	let gossip_engine =
-		GossipEngine::new(network, BEEFY_PROTOCOL_NAME, gossip_validator.clone(), None);
-	let worker_params = crate::worker::WorkerParams {
-		client: peer.client().as_client(),
-		backend: peer.client().as_backend(),
-		key_store: Some(keystore).into(),
-		signed_commitment_sender,
-		beefy_best_block_sender,
-		gossip_engine,
-		gossip_validator,
-		min_block_delta,
-		metrics: None,
-		sync_oracle,
-	};
-
-	BeefyWorker::<_, _, _, _>::new(worker_params, test_modifiers)
-}
-
 // Spawns beefy voters. Returns a future to spawn on the runtime.
-fn initialize_beefy(
+fn initialize_beefy<API>(
 	net: &mut BeefyTestNet,
-	peers: &[BeefyKeyring],
+	peers: Vec<(usize, &BeefyKeyring, Arc<API>)>,
 	min_block_delta: u32,
-) -> impl Future<Output = ()> {
+) -> impl Future<Output = ()>
+where
+	API: ProvideRuntimeApi<Block> + Default + Sync + Send,
+	API::Api: BeefyApi<Block> + MmrApi<Block, MmrRootHash>,
+{
 	let voters = FuturesUnordered::new();
 
-	for (peer_id, key) in peers.iter().enumerate() {
+	for (peer_id, key, api) in peers.into_iter() {
-		let worker = create_beefy_worker(&net.peers[peer_id], key, min_block_delta);
+		let peer = &net.peers[peer_id];
-		let gadget = worker.run();
+
+		let keystore = create_beefy_keystore(*key);
+
+		let (signed_commitment_sender, signed_commitment_stream) =
+			BeefySignedCommitmentStream::<Block>::channel();
+		let (beefy_best_block_sender, beefy_best_block_stream) =
+			BeefyBestBlockStream::<Block>::channel();
+		let beefy_link_half = BeefyLinkHalf { signed_commitment_stream, beefy_best_block_stream };
+		*peer.data.beefy_link_half.lock() = Some(beefy_link_half);
+
+		let beefy_params = crate::BeefyParams {
+			client: peer.client().as_client(),
+			backend: peer.client().as_backend(),
+			runtime: api.clone(),
+			key_store: Some(keystore),
+			network: peer.network_service().clone(),
+			signed_commitment_sender,
+			beefy_best_block_sender,
+			min_block_delta,
+			prometheus_registry: None,
+			protocol_name: BEEFY_PROTOCOL_NAME.into(),
+		};
+		let gadget = crate::start_beefy_gadget::<_, _, _, _, _>(beefy_params);
 
 		fn assert_send<T: Send>(_: &T) {}
 		assert_send(&gadget);
@@ -443,13 +492,12 @@ fn beefy_finalizing_blocks() {
 
 	let mut net = BeefyTestNet::new(2, 0);
 
-	for i in 0..peers.len() {
+	let api = Arc::new(two_validators::TestApi {});
-		net.peer(i).data.use_validator_set(&validator_set);
+	let beefy_peers = peers.iter().enumerate().map(|(id, key)| (id, key, api.clone())).collect();
-	}
+	runtime.spawn(initialize_beefy(&mut net, beefy_peers, min_block_delta));
-	runtime.spawn(initialize_beefy(&mut net, peers, min_block_delta));
 
 	// push 42 blocks including `AuthorityChange` digests every 10 blocks.
-	net.generate_blocks(42, session_len, &validator_set);
+	net.generate_blocks(42, session_len, &validator_set, true);
 	net.block_until_sync();
 
 	let net = Arc::new(Mutex::new(net));
@@ -477,19 +525,18 @@ fn lagging_validators() {
 	sp_tracing::try_init_simple();
 
 	let mut runtime = Runtime::new().unwrap();
-	let peers = &[BeefyKeyring::Charlie, BeefyKeyring::Dave];
+	let peers = &[BeefyKeyring::Alice, BeefyKeyring::Bob];
 	let validator_set = ValidatorSet::new(make_beefy_ids(peers), 0).unwrap();
 	let session_len = 30;
 	let min_block_delta = 1;
 
 	let mut net = BeefyTestNet::new(2, 0);
-	for i in 0..peers.len() {
+	let api = Arc::new(two_validators::TestApi {});
-		net.peer(i).data.use_validator_set(&validator_set);
+	let beefy_peers = peers.iter().enumerate().map(|(id, key)| (id, key, api.clone())).collect();
-	}
+	runtime.spawn(initialize_beefy(&mut net, beefy_peers, min_block_delta));
-	runtime.spawn(initialize_beefy(&mut net, peers, min_block_delta));
 
 	// push 42 blocks including `AuthorityChange` digests every 30 blocks.
-	net.generate_blocks(42, session_len, &validator_set);
+	net.generate_blocks(42, session_len, &validator_set, true);
 	net.block_until_sync();
 
 	let net = Arc::new(Mutex::new(net));
@@ -498,7 +545,7 @@ fn lagging_validators() {
 	// diff-power-of-two rule.
 	finalize_block_and_wait_for_beefy(&net, peers, &mut runtime, &[15], &[1, 9, 13, 14, 15]);
 
-	// Charlie finalizes #25, Dave lags behind
+	// Alice finalizes #25, Bob lags behind
 	let finalize = BlockId::number(25);
 	let (best_blocks, signed_commitments) = get_beefy_streams(&mut *net.lock(), peers);
 	net.lock().peer(0).client().as_client().finalize_block(finalize, None).unwrap();
@@ -507,7 +554,7 @@ fn lagging_validators() {
 	streams_empty_after_timeout(best_blocks, &net, &mut runtime, timeout);
 	streams_empty_after_timeout(signed_commitments, &net, &mut runtime, None);
 
-	// Dave catches up and also finalizes #25
+	// Bob catches up and also finalizes #25
 	let (best_blocks, signed_commitments) = get_beefy_streams(&mut *net.lock(), peers);
 	net.lock().peer(1).client().as_client().finalize_block(finalize, None).unwrap();
 	// expected beefy finalizes block #17 from diff-power-of-two
@@ -530,16 +577,23 @@ fn correct_beefy_payload() {
 	let min_block_delta = 2;
 
 	let mut net = BeefyTestNet::new(4, 0);
-	for i in 0..peers.len() {
+
-		net.peer(i).data.use_validator_set(&validator_set);
+	// Alice, Bob, Charlie will vote on good payloads
-	}
+	let good_api = Arc::new(four_validators::TestApi {});
+	let good_peers = [BeefyKeyring::Alice, BeefyKeyring::Bob, BeefyKeyring::Charlie]
+		.iter()
+		.enumerate()
+		.map(|(id, key)| (id, key, good_api.clone()))
+		.collect();
+	runtime.spawn(initialize_beefy(&mut net, good_peers, min_block_delta));
 
 	// Dave will vote on bad mmr roots
-	net.peer(3).data.test_modifiers.as_mut().map(|tm| tm.corrupt_mmr_roots = true);
+	let bad_api = Arc::new(bad_four_validators::TestApi {});
-	runtime.spawn(initialize_beefy(&mut net, peers, min_block_delta));
+	let bad_peers = vec![(3, &BeefyKeyring::Dave, bad_api)];
+	runtime.spawn(initialize_beefy(&mut net, bad_peers, min_block_delta));
 
 	// push 10 blocks
-	net.generate_blocks(12, session_len, &validator_set);
+	net.generate_blocks(12, session_len, &validator_set, false);
 	net.block_until_sync();
 
 	let net = Arc::new(Mutex::new(net));

substrate/client/beefy/src/worker.rs

@@ -26,9 +26,10 @@ use parking_lot::Mutex;
 use sc_client_api::{Backend, FinalityNotification, FinalityNotifications};
 use sc_network_gossip::GossipEngine;
 
-use sp_api::BlockId;
+use sp_api::{BlockId, ProvideRuntimeApi};
 use sp_arithmetic::traits::AtLeast32Bit;
 use sp_consensus::SyncOracle;
+use sp_mmr_primitives::MmrApi;
 use sp_runtime::{
 	generic::OpaqueDigestItemId,
 	traits::{Block, Header, NumberFor},
@@ -52,12 +53,10 @@ use crate::{
 	Client,
 };
 
-pub(crate) struct WorkerParams<B, BE, C, SO>
+pub(crate) struct WorkerParams<B: Block, BE, C, R, SO> {
-where
-	B: Block,
-{
 	pub client: Arc<C>,
 	pub backend: Arc<BE>,
+	pub runtime: Arc<R>,
 	pub key_store: BeefyKeystore,
 	pub signed_commitment_sender: BeefySignedCommitmentSender<B>,
 	pub beefy_best_block_sender: BeefyBestBlockSender<B>,
@@ -69,15 +68,10 @@ where
 }
 
 /// A BEEFY worker plays the BEEFY protocol
-pub(crate) struct BeefyWorker<B, C, BE, SO>
+pub(crate) struct BeefyWorker<B: Block, BE, C, R, SO> {
-where
-	B: Block,
-	BE: Backend<B>,
-	C: Client<B, BE>,
-	SO: SyncOracle + Send + Sync + Clone + 'static,
-{
 	client: Arc<C>,
 	backend: Arc<BE>,
+	runtime: Arc<R>,
 	key_store: BeefyKeystore,
 	signed_commitment_sender: BeefySignedCommitmentSender<B>,
 	gossip_engine: Arc<Mutex<GossipEngine<B>>>,
@@ -99,17 +93,15 @@ where
 	sync_oracle: SO,
 	// keep rustc happy
 	_backend: PhantomData<BE>,
-	#[cfg(test)]
-	// behavior modifiers used in tests
-	test_res: tests::TestModifiers,
 }
 
-impl<B, C, BE, SO> BeefyWorker<B, C, BE, SO>
+impl<B, BE, C, R, SO> BeefyWorker<B, BE, C, R, SO>
 where
 	B: Block + Codec,
 	BE: Backend<B>,
 	C: Client<B, BE>,
-	C::Api: BeefyApi<B>,
+	R: ProvideRuntimeApi<B>,
+	R::Api: BeefyApi<B> + MmrApi<B, MmrRootHash>,
 	SO: SyncOracle + Send + Sync + Clone + 'static,
 {
 	/// Return a new BEEFY worker instance.
@@ -118,15 +110,11 @@ where
 	/// BEEFY pallet has been deployed on-chain.
 	///
 	/// The BEEFY pallet is needed in order to keep track of the BEEFY authority set.
-	pub(crate) fn new(
+	pub(crate) fn new(worker_params: WorkerParams<B, BE, C, R, SO>) -> Self {
-		worker_params: WorkerParams<B, BE, C, SO>,
-		#[cfg(test)]
-		// behavior modifiers used in tests
-		test_res: tests::TestModifiers,
-	) -> Self {
 		let WorkerParams {
 			client,
 			backend,
+			runtime,
 			key_store,
 			signed_commitment_sender,
 			beefy_best_block_sender,
@@ -144,6 +132,7 @@ where
 		BeefyWorker {
 			client: client.clone(),
 			backend,
+			runtime,
 			key_store,
 			signed_commitment_sender,
 			gossip_engine: Arc::new(Mutex::new(gossip_engine)),
@@ -159,20 +148,9 @@ where
 			beefy_best_block_sender,
 			sync_oracle,
 			_backend: PhantomData,
-			#[cfg(test)]
-			test_res,
 		}
 	}
-}
 
-impl<B, C, BE, SO> BeefyWorker<B, C, BE, SO>
-where
-	B: Block,
-	BE: Backend<B>,
-	C: Client<B, BE>,
-	C::Api: BeefyApi<B>,
-	SO: SyncOracle + Send + Sync + Clone + 'static,
-{
 	/// Return `Some(number)` if we should be voting on block `number` now,
 	/// return `None` if there is no block we should vote on now.
 	fn current_vote_target(&self) -> Option<NumberFor<B>> {
@@ -396,7 +374,7 @@ where
 		};
 		let target_hash = target_header.hash();
 
-		let mmr_root = if let Some(hash) = self.extract_mmr_root_digest(&target_header) {
+		let mmr_root = if let Some(hash) = self.get_mmr_root_digest(&target_header) {
 			hash
 		} else {
 			warn!(target: "beefy", "🥩 No MMR root digest found for: {:?}", target_hash);
@@ -458,13 +436,12 @@ where
 	}
 
 	/// Wait for BEEFY runtime pallet to be available.
-	#[cfg(not(test))]
 	async fn wait_for_runtime_pallet(&mut self) {
 		self.client
 			.finality_notification_stream()
 			.take_while(|notif| {
 				let at = BlockId::hash(notif.header.hash());
-				if let Some(active) = self.client.runtime_api().validator_set(&at).ok().flatten() {
+				if let Some(active) = self.runtime.runtime_api().validator_set(&at).ok().flatten() {
 					if active.id() == GENESIS_AUTHORITY_SET_ID {
 						// When starting from genesis, there is no session boundary digest.
 						// Just initialize `rounds` to Block #1 as BEEFY mandatory block.
@@ -488,13 +465,6 @@ where
 		self.finality_notifications = self.client.finality_notification_stream();
 	}
 
-	/// For tests don't use runtime pallet. Start rounds from block #1.
-	#[cfg(test)]
-	async fn wait_for_runtime_pallet(&mut self) {
-		let active = self.test_res.active_validators.clone();
-		self.init_session_at(active, 1u32.into());
-	}
-
 	/// Main loop for BEEFY worker.
 	///
 	/// Wait for BEEFY runtime pallet to be available, then start the main async loop
@@ -550,20 +520,15 @@ where
 		}
 	}
 
-	/// Simple wrapper over mmr root extraction.
+	/// Simple wrapper that gets MMR root from header digests or from client state.
-	#[cfg(not(test))]
+	fn get_mmr_root_digest(&self, header: &B::Header) -> Option<MmrRootHash> {
-	fn extract_mmr_root_digest(&self, header: &B::Header) -> Option<MmrRootHash> {
+		find_mmr_root_digest::<B>(header).or_else(|| {
-		find_mmr_root_digest::<B>(header)
+			self.runtime
-	}
+				.runtime_api()
-
+				.mmr_root(&BlockId::hash(header.hash()))
-	/// For tests, have the option to modify mmr root.
+				.ok()
-	#[cfg(test)]
+				.and_then(|r| r.ok())
-	fn extract_mmr_root_digest(&self, header: &B::Header) -> Option<MmrRootHash> {
+		})
-		let mut mmr_root = find_mmr_root_digest::<B>(header);
-		if self.test_res.corrupt_mmr_roots {
-			mmr_root.as_mut().map(|hash| *hash ^= MmrRootHash::random());
-		}
-		mmr_root
 	}
 }
 
@@ -658,11 +623,16 @@ pub(crate) mod tests {
 	use super::*;
 	use crate::{
 		keystore::tests::Keyring,
-		tests::{create_beefy_worker, get_beefy_streams, make_beefy_ids, BeefyTestNet},
+		notification::{BeefyBestBlockStream, BeefySignedCommitmentStream},
+		tests::{
+			create_beefy_keystore, get_beefy_streams, make_beefy_ids, two_validators::TestApi,
+			BeefyPeer, BeefyTestNet, BEEFY_PROTOCOL_NAME,
+		},
 	};
 
 	use futures::{executor::block_on, future::poll_fn, task::Poll};
 
+	use crate::tests::BeefyLinkHalf;
 	use sc_client_api::HeaderBackend;
 	use sc_network::NetworkService;
 	use sc_network_test::{PeersFullClient, TestNetFactory};
@@ -672,10 +642,40 @@ pub(crate) mod tests {
 		Backend,
 	};
 
-	#[derive(Clone)]
+	fn create_beefy_worker(
-	pub struct TestModifiers {
+		peer: &BeefyPeer,
-		pub active_validators: ValidatorSet<AuthorityId>,
+		key: &Keyring,
-		pub corrupt_mmr_roots: bool,
+		min_block_delta: u32,
+	) -> BeefyWorker<Block, Backend, PeersFullClient, TestApi, Arc<NetworkService<Block, H256>>> {
+		let keystore = create_beefy_keystore(*key);
+
+		let (signed_commitment_sender, signed_commitment_stream) =
+			BeefySignedCommitmentStream::<Block>::channel();
+		let (beefy_best_block_sender, beefy_best_block_stream) =
+			BeefyBestBlockStream::<Block>::channel();
+		let beefy_link_half = BeefyLinkHalf { signed_commitment_stream, beefy_best_block_stream };
+		*peer.data.beefy_link_half.lock() = Some(beefy_link_half);
+
+		let api = Arc::new(TestApi {});
+		let network = peer.network_service().clone();
+		let sync_oracle = network.clone();
+		let gossip_validator = Arc::new(crate::gossip::GossipValidator::new());
+		let gossip_engine =
+			GossipEngine::new(network, BEEFY_PROTOCOL_NAME, gossip_validator.clone(), None);
+		let worker_params = crate::worker::WorkerParams {
+			client: peer.client().as_client(),
+			backend: peer.client().as_backend(),
+			runtime: api,
+			key_store: Some(keystore).into(),
+			signed_commitment_sender,
+			beefy_best_block_sender,
+			gossip_engine,
+			gossip_validator,
+			min_block_delta,
+			metrics: None,
+			sync_oracle,
+		};
+		BeefyWorker::<_, _, _, _, _>::new(worker_params)
 	}
 
 	#[test]
@@ -825,7 +825,6 @@ pub(crate) mod tests {
 		let keys = &[Keyring::Alice];
 		let validator_set = ValidatorSet::new(make_beefy_ids(keys), 0).unwrap();
 		let mut net = BeefyTestNet::new(1, 0);
-		net.peer(0).data.use_validator_set(&validator_set);
 		let mut worker = create_beefy_worker(&net.peer(0), &keys[0], 1);
 
 		// rounds not initialized -> should vote: `None`
@@ -833,8 +832,9 @@ pub(crate) mod tests {
 
 		let set_up = |worker: &mut BeefyWorker<
 			Block,
-			PeersFullClient,
 			Backend,
+			PeersFullClient,
+			TestApi,
 			Arc<NetworkService<Block, H256>>,
 		>,
 		              best_grandpa: u64,
@@ -889,7 +889,6 @@ pub(crate) mod tests {
 		let keys = &[Keyring::Alice];
 		let validator_set = ValidatorSet::new(make_beefy_ids(keys), 0).unwrap();
 		let mut net = BeefyTestNet::new(1, 0);
-		net.peer(0).data.use_validator_set(&validator_set);
 		let mut worker = create_beefy_worker(&net.peer(0), &keys[0], 1);
 
 		// keystore doesn't contain other keys than validators'
@@ -913,7 +912,6 @@ pub(crate) mod tests {
 		let keys = &[Keyring::Alice];
 		let validator_set = ValidatorSet::new(make_beefy_ids(keys), 0).unwrap();
 		let mut net = BeefyTestNet::new(1, 0);
-		net.peer(0).data.use_validator_set(&validator_set);
 		let mut worker = create_beefy_worker(&net.peer(0), &keys[0], 1);
 
 		let (mut best_block_streams, _) = get_beefy_streams(&mut net, keys);
@@ -939,7 +937,7 @@ pub(crate) mod tests {
 		// generate 2 blocks, try again expect success
 		let (mut best_block_streams, _) = get_beefy_streams(&mut net, keys);
 		let mut best_block_stream = best_block_streams.drain(..).next().unwrap();
-		net.generate_blocks(2, 10, &validator_set);
+		net.generate_blocks(2, 10, &validator_set, false);
 
 		worker.set_best_beefy_block(2);
 		assert_eq!(worker.best_beefy_block, Some(2));
@@ -961,7 +959,6 @@ pub(crate) mod tests {
 		let keys = &[Keyring::Alice];
 		let validator_set = ValidatorSet::new(make_beefy_ids(keys), 0).unwrap();
 		let mut net = BeefyTestNet::new(1, 0);
-		net.peer(0).data.use_validator_set(&validator_set);
 		let mut worker = create_beefy_worker(&net.peer(0), &keys[0], 1);
 
 		assert!(worker.rounds.is_none());

substrate/client/db/src/lib.rs

@@ -106,6 +106,7 @@ const DEFAULT_CHILD_RATIO: (usize, usize) = (1, 10);
 pub type DbState<B> =
 	sp_state_machine::TrieBackend<Arc<dyn sp_state_machine::Storage<HashFor<B>>>, HashFor<B>>;
 
+#[cfg(feature = "with-parity-db")]
 /// Length of a [`DbHash`].
 const DB_HASH_LEN: usize = 32;
 

substrate/frame/beefy-mmr/Cargo.toml

@@ -18,7 +18,6 @@ serde = { version = "1.0.136", optional = true }
 frame-support = { version = "4.0.0-dev", path = "../support", default-features = false }
 frame-system = { version = "4.0.0-dev", path = "../system", default-features = false }
 pallet-mmr = { version = "4.0.0-dev", path = "../merkle-mountain-range", default-features = false }
-pallet-mmr-primitives = { version = "4.0.0-dev", path = "../merkle-mountain-range/primitives", default-features = false }
 pallet-session = { version = "4.0.0-dev", path = "../session", default-features = false }
 
 sp-core = { version = "6.0.0", path = "../../primitives/core", default-features = false }
@@ -46,7 +45,6 @@ std = [
     "k256/std",
     "log/std",
     "pallet-beefy/std",
-    "pallet-mmr-primitives/std",
     "pallet-mmr/std",
     "pallet-session/std",
     "serde",

substrate/frame/beefy-mmr/src/lib.rs

@@ -37,7 +37,7 @@ use sp_runtime::traits::{Convert, Hash, Member};
 use sp_std::prelude::*;
 
 use beefy_primitives::mmr::{BeefyDataProvider, BeefyNextAuthoritySet, MmrLeaf, MmrLeafVersion};
-use pallet_mmr::primitives::LeafDataProvider;
+use pallet_mmr::{LeafDataProvider, ParentNumberAndHash};
 
 use frame_support::traits::Get;
 
@@ -150,7 +150,7 @@ where
 	fn leaf_data() -> Self::LeafData {
 		MmrLeaf {
 			version: T::LeafVersion::get(),
-			parent_number_and_hash: frame_system::Pallet::<T>::leaf_data(),
+			parent_number_and_hash: ParentNumberAndHash::<T>::leaf_data(),
 			leaf_extra: T::BeefyDataProvider::extra_data(),
 			beefy_next_authority_set: Pallet::<T>::update_beefy_next_authority_set(),
 		}

substrate/frame/beefy-mmr/src/tests.rs

@@ -49,7 +49,7 @@ fn offchain_key(pos: usize) -> Vec<u8> {
 }
 
 fn read_mmr_leaf(ext: &mut TestExternalities, index: usize) -> MmrLeaf {
-	type Node = pallet_mmr_primitives::DataOrHash<Keccak256, MmrLeaf>;
+	type Node = pallet_mmr::primitives::DataOrHash<Keccak256, MmrLeaf>;
 	ext.persist_offchain_overlay();
 	let offchain_db = ext.offchain_db();
 	offchain_db

substrate/frame/merkle-mountain-range/Cargo.toml

@@ -18,6 +18,7 @@ mmr-lib = { package = "ckb-merkle-mountain-range", default-features = false, ver
 
 sp-core = { version = "6.0.0", default-features = false, path = "../../primitives/core" }
 sp-io = { version = "6.0.0", default-features = false, path = "../../primitives/io" }
+sp-mmr-primitives = { version = "4.0.0-dev", default-features = false, path = "../../primitives/merkle-mountain-range" }
 sp-runtime = { version = "6.0.0", default-features = false, path = "../../primitives/runtime" }
 sp-std = { version = "4.0.0", default-features = false, path = "../../primitives/std" }
 
@@ -25,8 +26,6 @@ frame-benchmarking = { version = "4.0.0-dev", default-features = false, path = "
 frame-support = { version = "4.0.0-dev", default-features = false, path = "../support" }
 frame-system = { version = "4.0.0-dev", default-features = false, path = "../system" }
 
-pallet-mmr-primitives = { version = "4.0.0-dev", default-features = false, path = "./primitives" }
-
 [dev-dependencies]
 env_logger = "0.9"
 hex-literal = "0.3"
@@ -39,12 +38,12 @@ std = [
 	"mmr-lib/std",
 	"sp-core/std",
 	"sp-io/std",
+	"sp-mmr-primitives/std",
 	"sp-runtime/std",
 	"sp-std/std",
 	"frame-benchmarking/std",
 	"frame-support/std",
 	"frame-system/std",
-	"pallet-mmr-primitives/std",
 ]
 runtime-benchmarks = ["frame-benchmarking/runtime-benchmarks"]
 try-runtime = ["frame-support/try-runtime"]

substrate/frame/merkle-mountain-range/rpc/Cargo.toml

@@ -22,9 +22,8 @@ serde = { version = "1.0.136", features = ["derive"] }
 sp-api = { version = "4.0.0-dev", path = "../../../primitives/api" }
 sp-blockchain = { version = "4.0.0-dev", path = "../../../primitives/blockchain" }
 sp-core = { version = "6.0.0", path = "../../../primitives/core" }
+sp-mmr-primitives = { version = "4.0.0-dev", default-features = false, path = "../../../primitives/merkle-mountain-range" }
 sp-runtime = { version = "6.0.0", path = "../../../primitives/runtime" }
 
-pallet-mmr-primitives = { version = "4.0.0-dev", path = "../primitives" }
-
 [dev-dependencies]
 serde_json = "1.0.79"

substrate/frame/merkle-mountain-range/rpc/src/lib.rs

@@ -26,13 +26,13 @@ use jsonrpc_core::{Error, ErrorCode, Result};
 use jsonrpc_derive::rpc;
 use serde::{Deserialize, Serialize};
 
-use pallet_mmr_primitives::{Error as MmrError, Proof};
 use sp_api::ProvideRuntimeApi;
 use sp_blockchain::HeaderBackend;
 use sp_core::Bytes;
+use sp_mmr_primitives::{Error as MmrError, LeafIndex, Proof};
 use sp_runtime::{generic::BlockId, traits::Block as BlockT};
 
-pub use pallet_mmr_primitives::{LeafIndex, MmrApi as MmrRuntimeApi};
+pub use sp_mmr_primitives::MmrApi as MmrRuntimeApi;
 
 /// Retrieved MMR leaf and its proof.
 #[derive(Serialize, Deserialize, Debug, Clone, PartialEq, Eq)]
@@ -42,7 +42,7 @@ pub struct LeafProof<BlockHash> {
 	pub block_hash: BlockHash,
 	/// SCALE-encoded leaf data.
 	pub leaf: Bytes,
-	/// SCALE-encoded proof data. See [pallet_mmr_primitives::Proof].
+	/// SCALE-encoded proof data. See [sp_mmr_primitives::Proof].
 	pub proof: Bytes,
 }
 

substrate/frame/merkle-mountain-range/src/lib.rs

@@ -58,7 +58,7 @@
 
 use codec::Encode;
 use frame_support::weights::Weight;
-use sp_runtime::traits;
+use sp_runtime::traits::{self, One, Saturating};
 
 #[cfg(any(feature = "runtime-benchmarks", test))]
 mod benchmarking;
@@ -70,7 +70,30 @@ mod mock;
 mod tests;
 
 pub use pallet::*;
-pub use pallet_mmr_primitives::{self as primitives, NodeIndex};
+pub use sp_mmr_primitives::{self as primitives, Error, LeafDataProvider, LeafIndex, NodeIndex};
+
+/// The most common use case for MMRs is to store historical block hashes,
+/// so that any point in time in the future we can receive a proof about some past
+/// blocks without using excessive on-chain storage.
+///
+/// Hence we implement the [LeafDataProvider] for [ParentNumberAndHash] which is a
+/// crate-local wrapper over [frame_system::Pallet]. Since the current block hash
+/// is not available (since the block is not finished yet),
+/// we use the `parent_hash` here along with parent block number.
+pub struct ParentNumberAndHash<T: frame_system::Config> {
+	_phanthom: sp_std::marker::PhantomData<T>,
+}
+
+impl<T: frame_system::Config> LeafDataProvider for ParentNumberAndHash<T> {
+	type LeafData = (<T as frame_system::Config>::BlockNumber, <T as frame_system::Config>::Hash);
+
+	fn leaf_data() -> Self::LeafData {
+		(
+			frame_system::Pallet::<T>::block_number().saturating_sub(One::one()),
+			frame_system::Pallet::<T>::parent_hash(),
+		)
+	}
+}
 
 pub trait WeightInfo {
 	fn on_initialize(peaks: NodeIndex) -> Weight;
@@ -161,7 +184,7 @@ pub mod pallet {
 	/// Current size of the MMR (number of leaves).
 	#[pallet::storage]
 	#[pallet::getter(fn mmr_leaves)]
-	pub type NumberOfLeaves<T, I = ()> = StorageValue<_, NodeIndex, ValueQuery>;
+	pub type NumberOfLeaves<T, I = ()> = StorageValue<_, LeafIndex, ValueQuery>;
 
 	/// Hashes of the nodes in the MMR.
 	///
@@ -240,7 +263,7 @@ impl<T: Config<I>, I: 'static> Pallet<T, I> {
 	/// all the leaves to be present.
 	/// It may return an error or panic if used incorrectly.
 	pub fn generate_proof(
-		leaf_index: NodeIndex,
+		leaf_index: LeafIndex,
 	) -> Result<(LeafOf<T, I>, primitives::Proof<<T as Config<I>>::Hash>), primitives::Error> {
 		let mmr: ModuleMmr<mmr::storage::OffchainStorage, T, I> = mmr::Mmr::new(Self::mmr_leaves());
 		mmr.generate_proof(leaf_index)
@@ -272,4 +295,9 @@ impl<T: Config<I>, I: 'static> Pallet<T, I> {
 			Err(primitives::Error::Verify.log_debug("The proof is incorrect."))
 		}
 	}
+
+	/// Return the on-chain MMR root hash.
+	pub fn mmr_root() -> <T as Config<I>>::Hash {
+		Self::mmr_root_hash()
+	}
 }

substrate/frame/merkle-mountain-range/src/mmr/mod.rs

@@ -19,7 +19,7 @@ mod mmr;
 pub mod storage;
 pub mod utils;
 
-use crate::primitives::FullLeaf;
+use sp_mmr_primitives::{DataOrHash, FullLeaf};
 use sp_runtime::traits;
 
 pub use self::mmr::{verify_leaf_proof, Mmr};
@@ -28,7 +28,7 @@ pub use self::mmr::{verify_leaf_proof, Mmr};
 pub type NodeOf<T, I, L> = Node<<T as crate::Config<I>>::Hashing, L>;
 
 /// A node stored in the MMR.
-pub type Node<H, L> = crate::primitives::DataOrHash<H, L>;
+pub type Node<H, L> = DataOrHash<H, L>;
 
 /// Default Merging & Hashing behavior for MMR.
 pub struct Hasher<H, L>(sp_std::marker::PhantomData<(H, L)>);

substrate/frame/merkle-mountain-range/src/mock.rs

@@ -20,8 +20,8 @@ use crate::*;
 
 use codec::{Decode, Encode};
 use frame_support::traits::{ConstU32, ConstU64};
-use pallet_mmr_primitives::{Compact, LeafDataProvider};
 use sp_core::H256;
+use sp_mmr_primitives::{Compact, LeafDataProvider};
 use sp_runtime::{
 	testing::Header,
 	traits::{BlakeTwo256, IdentityLookup, Keccak256},
@@ -74,7 +74,7 @@ impl Config for Test {
 
 	type Hashing = Keccak256;
 	type Hash = H256;
-	type LeafData = Compact<Keccak256, (frame_system::Pallet<Test>, LeafData)>;
+	type LeafData = Compact<Keccak256, (ParentNumberAndHash<Test>, LeafData)>;
 	type OnNewRoot = ();
 	type WeightInfo = ();
 }

substrate/frame/merkle-mountain-range/src/tests.rs

@@ -19,11 +19,11 @@ use crate::{mmr::utils, mock::*, *};
 
 use frame_support::traits::OnInitialize;
 use mmr_lib::helper;
-use pallet_mmr_primitives::{Compact, Proof};
 use sp_core::{
 	offchain::{testing::TestOffchainExt, OffchainDbExt, OffchainWorkerExt},
 	H256,
 };
+use sp_mmr_primitives::{Compact, Proof};
 
 pub(crate) fn new_test_ext() -> sp_io::TestExternalities {
 	frame_system::GenesisConfig::default().build_storage::<Test>().unwrap().into()

substrate/primitives/merkle-mountain-range/Cargo.toml

@@ -1,12 +1,12 @@
 [package]
-name = "pallet-mmr-primitives"
+name = "sp-mmr-primitives"
 version = "4.0.0-dev"
 authors = ["Parity Technologies <admin@parity.io>"]
 edition = "2021"
 license = "Apache-2.0"
 homepage = "https://substrate.io"
 repository = "https://github.com/paritytech/substrate/"
-description = "FRAME Merkle Mountain Range primitives."
+description = "Merkle Mountain Range primitives."
 
 [package.metadata.docs.rs]
 targets = ["x86_64-unknown-linux-gnu"]
@@ -16,13 +16,11 @@ codec = { package = "parity-scale-codec", version = "3.0.0", default-features =
 log = { version = "0.4.14", default-features = false }
 serde = { version = "1.0.136", optional = true, features = ["derive"] }
 
-sp-api = { version = "4.0.0-dev", default-features = false, path = "../../../primitives/api" }
+sp-api = { version = "4.0.0-dev", default-features = false, path = "../api" }
-sp-core = { version = "6.0.0", default-features = false, path = "../../../primitives/core" }
+sp-core = { version = "6.0.0", default-features = false, path = "../core" }
-sp-runtime = { version = "6.0.0", default-features = false, path = "../../../primitives/runtime" }
+sp-debug-derive = { version = "4.0.0", default-features = false, path = "../debug-derive" }
-sp-std = { version = "4.0.0", default-features = false, path = "../../../primitives/std" }
+sp-runtime = { version = "6.0.0", default-features = false, path = "../runtime" }
-
+sp-std = { version = "4.0.0", default-features = false, path = "../std" }
-frame-support = { version = "4.0.0-dev", default-features = false, path = "../../support" }
-frame-system = { version = "4.0.0-dev", default-features = false, path = "../../system" }
 
 [dev-dependencies]
 hex-literal = "0.3"
@@ -35,8 +33,7 @@ std = [
 	"serde",
 	"sp-api/std",
 	"sp-core/std",
+	"sp-debug-derive/std",
 	"sp-runtime/std",
 	"sp-std/std",
-	"frame-support/std",
-	"frame-system/std",
 ]

substrate/primitives/merkle-mountain-range/src/lib.rs

@@ -20,8 +20,8 @@
 #![cfg_attr(not(feature = "std"), no_std)]
 #![warn(missing_docs)]
 
-use frame_support::RuntimeDebug;
+use sp_debug_derive::RuntimeDebug;
-use sp_runtime::traits::{self, One, Saturating};
+use sp_runtime::traits;
 use sp_std::fmt;
 #[cfg(not(feature = "std"))]
 use sp_std::prelude::Vec;
@@ -57,21 +57,6 @@ impl LeafDataProvider for () {
 	}
 }
 
-/// The most common use case for MMRs is to store historical block hashes,
-/// so that any point in time in the future we can receive a proof about some past
-/// blocks without using excessive on-chain storage.
-///
-/// Hence we implement the [LeafDataProvider] for [frame_system::Pallet]. Since the
-/// current block hash is not available (since the block is not finished yet),
-/// we use the `parent_hash` here along with parent block number.
-impl<T: frame_system::Config> LeafDataProvider for frame_system::Pallet<T> {
-	type LeafData = (<T as frame_system::Config>::BlockNumber, <T as frame_system::Config>::Hash);
-
-	fn leaf_data() -> Self::LeafData {
-		(Self::block_number().saturating_sub(One::one()), Self::parent_hash())
-	}
-}
-
 /// New MMR root notification hook.
 pub trait OnNewRoot<Hash> {
 	/// Function called by the pallet in case new MMR root has been computed.
@@ -83,6 +68,17 @@ impl<Hash> OnNewRoot<Hash> for () {
 	fn on_new_root(_root: &Hash) {}
 }
 
+/// A MMR proof data for one of the leaves.
+#[derive(codec::Encode, codec::Decode, RuntimeDebug, Clone, PartialEq, Eq)]
+pub struct Proof<Hash> {
+	/// The index of the leaf the proof is for.
+	pub leaf_index: LeafIndex,
+	/// Number of leaves in MMR, when the proof was generated.
+	pub leaf_count: NodeIndex,
+	/// Proof elements (hashes of siblings of inner nodes on the path to the leaf).
+	pub items: Vec<Hash>,
+}
+
 /// A full leaf content stored in the offchain-db.
 pub trait FullLeaf: Clone + PartialEq + fmt::Debug {
 	/// Encode the leaf either in it's full or compact form.
@@ -97,6 +93,80 @@ impl<T: codec::Encode + codec::Decode + Clone + PartialEq + fmt::Debug> FullLeaf
 	}
 }
 
+/// A helper type to allow using arbitrary SCALE-encoded leaf data in the RuntimeApi.
+///
+/// The point is to be able to verify MMR proofs from external MMRs, where we don't
+/// know the exact leaf type, but it's enough for us to have it SCALE-encoded.
+///
+/// Note the leaf type should be encoded in its compact form when passed through this type.
+/// See [FullLeaf] documentation for details.
+///
+/// This type does not implement SCALE encoding/decoding on purpose to avoid confusion,
+/// it would have to be SCALE-compatible with the concrete leaf type, but due to SCALE limitations
+/// it's not possible to know how many bytes the encoding of concrete leaf type uses.
+#[cfg_attr(feature = "std", derive(serde::Serialize, serde::Deserialize))]
+#[derive(RuntimeDebug, Clone, PartialEq)]
+pub struct OpaqueLeaf(
+	/// Raw bytes of the leaf type encoded in its compact form.
+	///
+	/// NOTE it DOES NOT include length prefix (like `Vec<u8>` encoding would).
+	#[cfg_attr(feature = "std", serde(with = "sp_core::bytes"))]
+	pub Vec<u8>,
+);
+
+impl OpaqueLeaf {
+	/// Convert a concrete MMR leaf into an opaque type.
+	pub fn from_leaf<T: FullLeaf>(leaf: &T) -> Self {
+		let encoded_leaf = leaf.using_encoded(|d| d.to_vec(), true);
+		OpaqueLeaf::from_encoded_leaf(encoded_leaf)
+	}
+
+	/// Create a `OpaqueLeaf` given raw bytes of compact-encoded leaf.
+	pub fn from_encoded_leaf(encoded_leaf: Vec<u8>) -> Self {
+		OpaqueLeaf(encoded_leaf)
+	}
+
+	/// Attempt to decode the leaf into expected concrete type.
+	pub fn try_decode<T: codec::Decode>(&self) -> Option<T> {
+		codec::Decode::decode(&mut &*self.0).ok()
+	}
+}
+
+impl FullLeaf for OpaqueLeaf {
+	fn using_encoded<R, F: FnOnce(&[u8]) -> R>(&self, f: F, _compact: bool) -> R {
+		f(&self.0)
+	}
+}
+
+/// A type-safe wrapper for the concrete leaf type.
+///
+/// This structure serves merely to avoid passing raw `Vec<u8>` around.
+/// It must be `Vec<u8>`-encoding compatible.
+///
+/// It is different from [`OpaqueLeaf`], because it does implement `Codec`
+/// and the encoding has to match raw `Vec<u8>` encoding.
+#[derive(codec::Encode, codec::Decode, RuntimeDebug, PartialEq, Eq)]
+pub struct EncodableOpaqueLeaf(pub Vec<u8>);
+
+impl EncodableOpaqueLeaf {
+	/// Convert a concrete leaf into encodable opaque version.
+	pub fn from_leaf<T: FullLeaf>(leaf: &T) -> Self {
+		let opaque = OpaqueLeaf::from_leaf(leaf);
+		Self::from_opaque_leaf(opaque)
+	}
+
+	/// Given an opaque leaf, make it encodable.
+	pub fn from_opaque_leaf(opaque: OpaqueLeaf) -> Self {
+		Self(opaque.0)
+	}
+
+	/// Try to convert into a [OpaqueLeaf].
+	pub fn into_opaque_leaf(self) -> OpaqueLeaf {
+		// wrap into `OpaqueLeaf` type
+		OpaqueLeaf::from_encoded_leaf(self.0)
+	}
+}
+
 /// An element representing either full data or it's hash.
 ///
 /// See [Compact] to see how it may be used in practice to reduce the size
@@ -281,17 +351,6 @@ impl_leaf_data_for_tuple!(A:0, B:1, C:2);
 impl_leaf_data_for_tuple!(A:0, B:1, C:2, D:3);
 impl_leaf_data_for_tuple!(A:0, B:1, C:2, D:3, E:4);
 
-/// A MMR proof data for one of the leaves.
-#[derive(codec::Encode, codec::Decode, RuntimeDebug, Clone, PartialEq, Eq)]
-pub struct Proof<Hash> {
-	/// The index of the leaf the proof is for.
-	pub leaf_index: LeafIndex,
-	/// Number of leaves in MMR, when the proof was generated.
-	pub leaf_count: NodeIndex,
-	/// Proof elements (hashes of siblings of inner nodes on the path to the leaf).
-	pub items: Vec<Hash>,
-}
-
 /// Merkle Mountain Range operation error.
 #[derive(RuntimeDebug, codec::Encode, codec::Decode, PartialEq, Eq)]
 pub enum Error {
@@ -334,80 +393,6 @@ impl Error {
 	}
 }
 
-/// A helper type to allow using arbitrary SCALE-encoded leaf data in the RuntimeApi.
-///
-/// The point is to be able to verify MMR proofs from external MMRs, where we don't
-/// know the exact leaf type, but it's enough for us to have it SCALE-encoded.
-///
-/// Note the leaf type should be encoded in its compact form when passed through this type.
-/// See [FullLeaf] documentation for details.
-///
-/// This type does not implement SCALE encoding/decoding on purpose to avoid confusion,
-/// it would have to be SCALE-compatible with the concrete leaf type, but due to SCALE limitations
-/// it's not possible to know how many bytes the encoding of concrete leaf type uses.
-#[cfg_attr(feature = "std", derive(serde::Serialize, serde::Deserialize))]
-#[derive(RuntimeDebug, Clone, PartialEq)]
-pub struct OpaqueLeaf(
-	/// Raw bytes of the leaf type encoded in its compact form.
-	///
-	/// NOTE it DOES NOT include length prefix (like `Vec<u8>` encoding would).
-	#[cfg_attr(feature = "std", serde(with = "sp_core::bytes"))]
-	pub Vec<u8>,
-);
-
-impl OpaqueLeaf {
-	/// Convert a concrete MMR leaf into an opaque type.
-	pub fn from_leaf<T: FullLeaf>(leaf: &T) -> Self {
-		let encoded_leaf = leaf.using_encoded(|d| d.to_vec(), true);
-		OpaqueLeaf::from_encoded_leaf(encoded_leaf)
-	}
-
-	/// Create a `OpaqueLeaf` given raw bytes of compact-encoded leaf.
-	pub fn from_encoded_leaf(encoded_leaf: Vec<u8>) -> Self {
-		OpaqueLeaf(encoded_leaf)
-	}
-
-	/// Attempt to decode the leaf into expected concrete type.
-	pub fn try_decode<T: codec::Decode>(&self) -> Option<T> {
-		codec::Decode::decode(&mut &*self.0).ok()
-	}
-}
-
-impl FullLeaf for OpaqueLeaf {
-	fn using_encoded<R, F: FnOnce(&[u8]) -> R>(&self, f: F, _compact: bool) -> R {
-		f(&self.0)
-	}
-}
-
-/// A type-safe wrapper for the concrete leaf type.
-///
-/// This structure serves merely to avoid passing raw `Vec<u8>` around.
-/// It must be `Vec<u8>`-encoding compatible.
-///
-/// It is different from [`OpaqueLeaf`], because it does implement `Codec`
-/// and the encoding has to match raw `Vec<u8>` encoding.
-#[derive(codec::Encode, codec::Decode, RuntimeDebug, PartialEq, Eq)]
-pub struct EncodableOpaqueLeaf(pub Vec<u8>);
-
-impl EncodableOpaqueLeaf {
-	/// Convert a concrete leaf into encodable opaque version.
-	pub fn from_leaf<T: FullLeaf>(leaf: &T) -> Self {
-		let opaque = OpaqueLeaf::from_leaf(leaf);
-		Self::from_opaque_leaf(opaque)
-	}
-
-	/// Given an opaque leaf, make it encodable.
-	pub fn from_opaque_leaf(opaque: OpaqueLeaf) -> Self {
-		Self(opaque.0)
-	}
-
-	/// Try to convert into a [OpaqueLeaf].
-	pub fn into_opaque_leaf(self) -> OpaqueLeaf {
-		// wrap into `OpaqueLeaf` type
-		OpaqueLeaf::from_encoded_leaf(self.0)
-	}
-}
-
 sp_api::decl_runtime_apis! {
 	/// API to interact with MMR pallet.
 	pub trait MmrApi<Hash: codec::Codec> {
@@ -429,6 +414,9 @@ sp_api::decl_runtime_apis! {
 		/// The leaf data is expected to be encoded in it's compact form.
 		fn verify_proof_stateless(root: Hash, leaf: EncodableOpaqueLeaf, proof: Proof<Hash>)
 			-> Result<(), Error>;
+
+		/// Return the on-chain MMR root hash.
+		fn mmr_root() -> Result<Hash, Error>;
 	}
 }