feat: Rebrand Polkadot/Substrate references to PezkuwiChain

This commit systematically rebrands various references from Parity Technologies'
Polkadot/Substrate ecosystem to PezkuwiChain within the kurdistan-sdk.

Key changes include:
- Updated external repository URLs (zombienet-sdk, parity-db, parity-scale-codec, wasm-instrument) to point to pezkuwichain forks.
- Modified internal documentation and code comments to reflect PezkuwiChain naming and structure.
- Replaced direct references to  with  or specific paths within the  for XCM, Pezkuwi, and other modules.
- Cleaned up deprecated  issue and PR references in various  and  files, particularly in  and  modules.
- Adjusted image and logo URLs in documentation to point to PezkuwiChain assets.
- Removed or rephrased comments related to external Polkadot/Substrate PRs and issues.

This is a significant step towards fully customizing the SDK for the PezkuwiChain ecosystem.

bizinikiwi/client/merkle-mountain-range/Cargo.toml

@@ -0,0 +1,49 @@
+[package]
+name = "mmr-gadget"
+version = "29.0.0"
+authors.workspace = true
+edition.workspace = true
+license = "GPL-3.0-or-later WITH Classpath-exception-2.0"
+repository.workspace = true
+description = "MMR Client gadget for bizinikiwi"
+homepage.workspace = true
+
+[lints]
+workspace = true
+
+# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
+
+[dependencies]
+codec = { workspace = true, default-features = true }
+futures = { workspace = true }
+log = { workspace = true, default-features = true }
+pezsc-client-api = { workspace = true, default-features = true }
+pezsc-offchain = { workspace = true, default-features = true }
+pezsp-api = { workspace = true, default-features = true }
+pezsp-blockchain = { workspace = true, default-features = true }
+pezsp-consensus = { workspace = true, default-features = true }
+pezsp-consensus-beefy = { workspace = true, default-features = true }
+pezsp-core = { workspace = true, default-features = true }
+pezsp-mmr-primitives = { workspace = true, default-features = true }
+pezsp-runtime = { workspace = true, default-features = true }
+
+[dev-dependencies]
+parking_lot = { workspace = true, default-features = true }
+pezsc-block-builder = { workspace = true, default-features = true }
+pezsp-tracing = { workspace = true, default-features = true }
+bizinikiwi-test-runtime-client = { workspace = true }
+tokio = { workspace = true, default-features = true }
+
+[features]
+runtime-benchmarks = [
+	"pezsc-block-builder/runtime-benchmarks",
+	"pezsc-client-api/runtime-benchmarks",
+	"pezsc-offchain/runtime-benchmarks",
+	"pezsp-api/runtime-benchmarks",
+	"pezsp-blockchain/runtime-benchmarks",
+	"pezsp-consensus-beefy/runtime-benchmarks",
+	"pezsp-consensus/runtime-benchmarks",
+	"pezsp-mmr-primitives/runtime-benchmarks",
+	"pezsp-runtime/runtime-benchmarks",
+	"bizinikiwi-test-runtime-client/runtime-benchmarks",
+]

bizinikiwi/client/merkle-mountain-range/rpc/Cargo.toml

@@ -0,0 +1,40 @@
+[package]
+name = "mmr-rpc"
+version = "28.0.0"
+authors.workspace = true
+edition.workspace = true
+license = "Apache-2.0"
+homepage.workspace = true
+repository.workspace = true
+description = "Node-specific RPC methods for interaction with Merkle Mountain Range pallet."
+
+[lints]
+workspace = true
+
+[package.metadata.docs.rs]
+targets = ["x86_64-unknown-linux-gnu"]
+
+[dependencies]
+codec = { workspace = true, default-features = true }
+jsonrpsee = { features = [
+	"client-core",
+	"macros",
+	"server-core",
+], workspace = true }
+serde = { features = ["derive"], workspace = true, default-features = true }
+pezsp-api = { workspace = true, default-features = true }
+pezsp-blockchain = { workspace = true, default-features = true }
+pezsp-core = { workspace = true, default-features = true }
+pezsp-mmr-primitives = { workspace = true, default-features = true }
+pezsp-runtime = { workspace = true, default-features = true }
+
+[dev-dependencies]
+serde_json = { workspace = true, default-features = true }
+
+[features]
+runtime-benchmarks = [
+	"pezsp-api/runtime-benchmarks",
+	"pezsp-blockchain/runtime-benchmarks",
+	"pezsp-mmr-primitives/runtime-benchmarks",
+	"pezsp-runtime/runtime-benchmarks",
+]

bizinikiwi/client/merkle-mountain-range/rpc/src/lib.rs

@@ -0,0 +1,402 @@
+// This file is part of Bizinikiwi.
+
+// Copyright (C) Parity Technologies (UK) Ltd.
+// SPDX-License-Identifier: Apache-2.0
+
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// 	http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#![warn(missing_docs)]
+#![warn(unused_crate_dependencies)]
+
+//! Node-specific RPC methods for interaction with Merkle Mountain Range pallet.
+
+use std::{marker::PhantomData, sync::Arc};
+
+use codec::{Codec, Decode, Encode};
+use jsonrpsee::{
+	core::{async_trait, RpcResult},
+	proc_macros::rpc,
+	types::{error::ErrorObject, ErrorObjectOwned},
+};
+use serde::{Deserialize, Serialize};
+
+use pezsp_api::{ApiExt, ProvideRuntimeApi};
+use pezsp_blockchain::HeaderBackend;
+use pezsp_core::{
+	offchain::{storage::OffchainDb, OffchainDbExt, OffchainStorage},
+	Bytes,
+};
+use pezsp_mmr_primitives::{AncestryProof as MmrAncestryProof, Error as MmrError, LeafProof};
+use pezsp_runtime::traits::{Block as BlockT, NumberFor};
+
+pub use pezsp_mmr_primitives::MmrApi as MmrRuntimeApi;
+
+const RUNTIME_ERROR: i32 = 8000;
+const MMR_ERROR: i32 = 8010;
+
+/// Retrieved MMR leaves and their proof.
+#[derive(Serialize, Deserialize, Debug, Clone, PartialEq, Eq)]
+#[serde(rename_all = "camelCase")]
+pub struct LeavesProof<BlockHash> {
+	/// Block hash the proof was generated for.
+	pub block_hash: BlockHash,
+	/// SCALE-encoded vector of `LeafData`.
+	pub leaves: Bytes,
+	/// SCALE-encoded proof data. See [pezsp_mmr_primitives::LeafProof].
+	pub proof: Bytes,
+}
+
+impl<BlockHash> LeavesProof<BlockHash> {
+	/// Create new `LeavesProof` from a given vector of `Leaf` and a
+	/// [pezsp_mmr_primitives::LeafProof].
+	pub fn new<Leaf, MmrHash>(
+		block_hash: BlockHash,
+		leaves: Vec<Leaf>,
+		proof: LeafProof<MmrHash>,
+	) -> Self
+	where
+		Leaf: Encode,
+		MmrHash: Encode,
+	{
+		Self { block_hash, leaves: Bytes(leaves.encode()), proof: Bytes(proof.encode()) }
+	}
+}
+
+/// MMR RPC methods.
+#[rpc(client, server)]
+pub trait MmrApi<BlockHash, BlockNumber, MmrHash> {
+	/// Get the MMR root hash for the current best block.
+	#[method(name = "mmr_root")]
+	fn mmr_root(&self, at: Option<BlockHash>) -> RpcResult<MmrHash>;
+
+	/// Generate an MMR proof for the given `block_numbers`.
+	///
+	/// This method calls into a runtime with MMR pallet included and attempts to generate
+	/// an MMR proof for the set of blocks that have the given `block_numbers` with the MMR root at
+	/// `best_known_block_number`. `best_known_block_number` must be larger than all the
+	/// `block_numbers` for the function to succeed.
+	///
+	/// Optionally via `at`, a block hash at which the runtime should be queried can be specified.
+	/// Optionally via `best_known_block_number`, the proof can be generated using the MMR's state
+	/// at a specific best block. Note that if `best_known_block_number` is provided, then also
+	/// specifying the block hash via `at` isn't super-useful here, unless you're generating proof
+	/// using non-finalized blocks where there are several competing forks. That's because MMR state
+	/// will be fixed to the state with `best_known_block_number`, which already points to
+	/// some historical block.
+	///
+	/// Returns the (full) leaves and a proof for these leaves (compact encoding, i.e. hash of
+	/// the leaves). Both parameters are SCALE-encoded.
+	/// The order of entries in the `leaves` field of the returned struct
+	/// is the same as the order of the entries in `block_numbers` supplied
+	#[method(name = "mmr_generateProof")]
+	fn generate_proof(
+		&self,
+		block_numbers: Vec<BlockNumber>,
+		best_known_block_number: Option<BlockNumber>,
+		at: Option<BlockHash>,
+	) -> RpcResult<LeavesProof<BlockHash>>;
+
+	/// Generate an MMR ancestry proof for the given `prev_block_number`.
+	///
+	/// This method calls into a runtime with MMR pallet included and attempts to generate
+	/// an MMR ancestry proof for the MMR root at the prior block with number `prev_block_number`,
+	/// with the reference MMR root at `best_known_block_number`. `best_known_block_number` must be
+	/// larger than the `prev_block_number` for the function to succeed.
+	///
+	/// Optionally via `at`, a block hash at which the runtime should be queried can be specified.
+	/// Optionally via `best_known_block_number`, the proof can be generated using the MMR's state
+	/// at a specific best block. Note that if `best_known_block_number` is provided, then also
+	/// specifying the block hash via `at` isn't super-useful here, unless you're generating proof
+	/// using non-finalized blocks where there are several competing forks. That's because MMR state
+	/// will be fixed to the state with `best_known_block_number`, which already points to
+	/// some historical block.
+	///
+	/// Returns the SCALE-encoded ancestry proof for the prior block's MMR root against the MMR root
+	/// of the best block specified. The order of entries in the `leaves` field of the returned
+	/// struct is the same as the order of the entries in `block_numbers` supplied
+	#[method(name = "mmr_generateAncestryProof")]
+	fn generate_ancestry_proof(
+		&self,
+		prev_block_number: BlockNumber,
+		best_known_block_number: Option<BlockNumber>,
+		at: Option<BlockHash>,
+	) -> RpcResult<MmrAncestryProof<MmrHash>>;
+
+	/// Verify an MMR `proof`.
+	///
+	/// This method calls into a runtime with MMR pallet included and attempts to verify
+	/// an MMR proof.
+	///
+	/// Returns `true` if the proof is valid, else returns the verification error.
+	#[method(name = "mmr_verifyProof")]
+	fn verify_proof(&self, proof: LeavesProof<BlockHash>) -> RpcResult<bool>;
+
+	/// Verify an MMR `proof` statelessly given an `mmr_root`.
+	///
+	/// This method calls into a runtime with MMR pallet included and attempts to verify
+	/// an MMR proof against a provided MMR root.
+	///
+	/// Returns `true` if the proof is valid, else returns the verification error.
+	#[method(name = "mmr_verifyProofStateless")]
+	fn verify_proof_stateless(
+		&self,
+		mmr_root: MmrHash,
+		proof: LeavesProof<BlockHash>,
+	) -> RpcResult<bool>;
+}
+
+/// MMR RPC methods.
+pub struct Mmr<Client, Block, S> {
+	client: Arc<Client>,
+	offchain_db: OffchainDb<S>,
+	_marker: PhantomData<Block>,
+}
+
+impl<C, B, S> Mmr<C, B, S> {
+	/// Create new `Mmr` with the given reference to the client.
+	pub fn new(client: Arc<C>, offchain_storage: S) -> Self {
+		Self { client, _marker: Default::default(), offchain_db: OffchainDb::new(offchain_storage) }
+	}
+}
+
+#[async_trait]
+impl<Client, Block, MmrHash, S> MmrApiServer<<Block as BlockT>::Hash, NumberFor<Block>, MmrHash>
+	for Mmr<Client, (Block, MmrHash), S>
+where
+	Block: BlockT,
+	Client: Send + Sync + 'static + ProvideRuntimeApi<Block> + HeaderBackend<Block>,
+	Client::Api: MmrRuntimeApi<Block, MmrHash, NumberFor<Block>>,
+	MmrHash: Codec + Send + Sync + 'static,
+	S: OffchainStorage + 'static,
+{
+	fn mmr_root(&self, at: Option<<Block as BlockT>::Hash>) -> RpcResult<MmrHash> {
+		let block_hash = at.unwrap_or_else(||
+			// If the block hash is not supplied assume the best block.
+			self.client.info().best_hash);
+		let api = self.client.runtime_api();
+		let mmr_root = api
+			.mmr_root(block_hash)
+			.map_err(runtime_error_into_rpc_error)?
+			.map_err(mmr_error_into_rpc_error)?;
+		Ok(mmr_root)
+	}
+
+	fn generate_proof(
+		&self,
+		block_numbers: Vec<NumberFor<Block>>,
+		best_known_block_number: Option<NumberFor<Block>>,
+		at: Option<<Block as BlockT>::Hash>,
+	) -> RpcResult<LeavesProof<<Block as BlockT>::Hash>> {
+		let mut api = self.client.runtime_api();
+		let block_hash = at.unwrap_or_else(||
+			// If the block hash is not supplied assume the best block.
+			self.client.info().best_hash);
+
+		api.register_extension(OffchainDbExt::new(self.offchain_db.clone()));
+
+		let (leaves, proof) = api
+			.generate_proof(block_hash, block_numbers, best_known_block_number)
+			.map_err(runtime_error_into_rpc_error)?
+			.map_err(mmr_error_into_rpc_error)?;
+
+		Ok(LeavesProof::new(block_hash, leaves, proof))
+	}
+
+	fn generate_ancestry_proof(
+		&self,
+		prev_block_number: NumberFor<Block>,
+		best_known_block_number: Option<NumberFor<Block>>,
+		at: Option<<Block as BlockT>::Hash>,
+	) -> RpcResult<MmrAncestryProof<MmrHash>> {
+		let mut api = self.client.runtime_api();
+		let block_hash = at.unwrap_or_else(||
+			// If the block hash is not supplied assume the best block.
+			self.client.info().best_hash);
+
+		api.register_extension(OffchainDbExt::new(self.offchain_db.clone()));
+
+		let proof = api
+			.generate_ancestry_proof(block_hash, prev_block_number, best_known_block_number)
+			.map_err(runtime_error_into_rpc_error)?
+			.map_err(mmr_error_into_rpc_error)?;
+
+		Ok(proof)
+	}
+
+	fn verify_proof(&self, proof: LeavesProof<<Block as BlockT>::Hash>) -> RpcResult<bool> {
+		let mut api = self.client.runtime_api();
+
+		let leaves = Decode::decode(&mut &proof.leaves.0[..]).map_err(invalid_params)?;
+
+		let decoded_proof = Decode::decode(&mut &proof.proof.0[..]).map_err(invalid_params)?;
+
+		api.register_extension(OffchainDbExt::new(self.offchain_db.clone()));
+
+		api.verify_proof(proof.block_hash, leaves, decoded_proof)
+			.map_err(runtime_error_into_rpc_error)?
+			.map_err(mmr_error_into_rpc_error)?;
+
+		Ok(true)
+	}
+
+	fn verify_proof_stateless(
+		&self,
+		mmr_root: MmrHash,
+		proof: LeavesProof<<Block as BlockT>::Hash>,
+	) -> RpcResult<bool> {
+		let api = self.client.runtime_api();
+
+		let leaves = Decode::decode(&mut &proof.leaves.0[..]).map_err(invalid_params)?;
+
+		let decoded_proof = Decode::decode(&mut &proof.proof.0[..]).map_err(invalid_params)?;
+
+		api.verify_proof_stateless(proof.block_hash, mmr_root, leaves, decoded_proof)
+			.map_err(runtime_error_into_rpc_error)?
+			.map_err(mmr_error_into_rpc_error)?;
+
+		Ok(true)
+	}
+}
+
+/// Converts an mmr-specific error into a [`CallError`].
+fn mmr_error_into_rpc_error(err: MmrError) -> ErrorObjectOwned {
+	let error_code = MMR_ERROR +
+		match err {
+			MmrError::LeafNotFound => 1,
+			MmrError::GenerateProof => 2,
+			MmrError::Verify => 3,
+			MmrError::InvalidNumericOp => 4,
+			MmrError::InvalidBestKnownBlock => 5,
+			_ => 0,
+		};
+
+	ErrorObject::owned(error_code, err.to_string(), Some(format!("{:?}", err)))
+}
+
+/// Converts a runtime trap into a [`CallError`].
+fn runtime_error_into_rpc_error(err: impl std::fmt::Debug) -> ErrorObjectOwned {
+	ErrorObject::owned(RUNTIME_ERROR, "Runtime trapped", Some(format!("{:?}", err)))
+}
+
+fn invalid_params(e: impl std::error::Error) -> ErrorObjectOwned {
+	ErrorObject::owned(
+		jsonrpsee::types::error::ErrorCode::InvalidParams.code(),
+		e.to_string(),
+		None::<()>,
+	)
+}
+
+#[cfg(test)]
+mod tests {
+	use super::*;
+	use pezsp_core::H256;
+
+	#[test]
+	fn should_serialize_leaf_proof() {
+		// given
+		let leaf = vec![1_u8, 2, 3, 4];
+		let proof = LeafProof {
+			leaf_indices: vec![1],
+			leaf_count: 9,
+			items: vec![H256::repeat_byte(1), H256::repeat_byte(2)],
+		};
+
+		let leaf_proof = LeavesProof::new(H256::repeat_byte(0), vec![leaf], proof);
+
+		// when
+		let actual = serde_json::to_string(&leaf_proof).unwrap();
+
+		// then
+		assert_eq!(
+			actual,
+			r#"{"blockHash":"0x0000000000000000000000000000000000000000000000000000000000000000","leaves":"0x041001020304","proof":"0x04010000000000000009000000000000000801010101010101010101010101010101010101010101010101010101010101010202020202020202020202020202020202020202020202020202020202020202"}"#
+		);
+	}
+
+	#[test]
+	fn should_serialize_leaves_proof() {
+		// given
+		let leaf_a = vec![1_u8, 2, 3, 4];
+		let leaf_b = vec![2_u8, 2, 3, 4];
+		let proof = LeafProof {
+			leaf_indices: vec![1, 2],
+			leaf_count: 9,
+			items: vec![H256::repeat_byte(1), H256::repeat_byte(2)],
+		};
+
+		let leaf_proof = LeavesProof::new(H256::repeat_byte(0), vec![leaf_a, leaf_b], proof);
+
+		// when
+		let actual = serde_json::to_string(&leaf_proof).unwrap();
+
+		// then
+		assert_eq!(
+			actual,
+			r#"{"blockHash":"0x0000000000000000000000000000000000000000000000000000000000000000","leaves":"0x0810010203041002020304","proof":"0x080100000000000000020000000000000009000000000000000801010101010101010101010101010101010101010101010101010101010101010202020202020202020202020202020202020202020202020202020202020202"}"#
+		);
+	}
+
+	#[test]
+	fn should_deserialize_leaf_proof() {
+		// given
+		let expected = LeavesProof {
+			block_hash: H256::repeat_byte(0),
+			leaves: Bytes(vec![vec![1_u8, 2, 3, 4]].encode()),
+			proof: Bytes(
+				LeafProof {
+					leaf_indices: vec![1],
+					leaf_count: 9,
+					items: vec![H256::repeat_byte(1), H256::repeat_byte(2)],
+				}
+				.encode(),
+			),
+		};
+
+		// when
+		let actual: LeavesProof<H256> = serde_json::from_str(r#"{
+			"blockHash":"0x0000000000000000000000000000000000000000000000000000000000000000",
+			"leaves":"0x041001020304",
+			"proof":"0x04010000000000000009000000000000000801010101010101010101010101010101010101010101010101010101010101010202020202020202020202020202020202020202020202020202020202020202"
+		}"#).unwrap();
+
+		// then
+		assert_eq!(actual, expected);
+	}
+
+	#[test]
+	fn should_deserialize_leaves_proof() {
+		// given
+		let expected = LeavesProof {
+			block_hash: H256::repeat_byte(0),
+			leaves: Bytes(vec![vec![1_u8, 2, 3, 4], vec![2_u8, 2, 3, 4]].encode()),
+			proof: Bytes(
+				LeafProof {
+					leaf_indices: vec![1, 2],
+					leaf_count: 9,
+					items: vec![H256::repeat_byte(1), H256::repeat_byte(2)],
+				}
+				.encode(),
+			),
+		};
+
+		// when
+		let actual: LeavesProof<H256> = serde_json::from_str(r#"{
+			"blockHash":"0x0000000000000000000000000000000000000000000000000000000000000000",
+			"leaves":"0x0810010203041002020304",
+			"proof":"0x080100000000000000020000000000000009000000000000000801010101010101010101010101010101010101010101010101010101010101010202020202020202020202020202020202020202020202020202020202020202"
+		}"#).unwrap();
+
+		// then
+		assert_eq!(actual, expected);
+	}
+}

bizinikiwi/client/merkle-mountain-range/src/aux_schema.rs

@@ -0,0 +1,238 @@
+// This file is part of Bizinikiwi.
+
+// Copyright (C) Parity Technologies (UK) Ltd.
+// SPDX-License-Identifier: GPL-3.0-or-later WITH Classpath-exception-2.0
+
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with this program. If not, see <https://www.gnu.org/licenses/>.
+
+//! Schema for MMR-gadget state persisted in the aux-db.
+
+use crate::LOG_TARGET;
+use codec::{Decode, Encode};
+use log::{info, trace};
+use pezsc_client_api::backend::AuxStore;
+use pezsp_blockchain::{Error as ClientError, Result as ClientResult};
+use pezsp_runtime::traits::{Block, NumberFor};
+
+const VERSION_KEY: &[u8] = b"mmr_auxschema_version";
+const GADGET_STATE: &[u8] = b"mmr_gadget_state";
+
+const CURRENT_VERSION: u32 = 1;
+pub(crate) type PersistedState<B> = NumberFor<B>;
+
+pub(crate) fn write_current_version<B: AuxStore>(backend: &B) -> ClientResult<()> {
+	info!(target: LOG_TARGET, "write aux schema version {:?}", CURRENT_VERSION);
+	AuxStore::insert_aux(backend, &[(VERSION_KEY, CURRENT_VERSION.encode().as_slice())], &[])
+}
+
+/// Write gadget state.
+pub(crate) fn write_gadget_state<B: Block, BE: AuxStore>(
+	backend: &BE,
+	state: &PersistedState<B>,
+) -> ClientResult<()> {
+	trace!(target: LOG_TARGET, "persisting {:?}", state);
+	backend.insert_aux(&[(GADGET_STATE, state.encode().as_slice())], &[])
+}
+
+fn load_decode<B: AuxStore, T: Decode>(backend: &B, key: &[u8]) -> ClientResult<Option<T>> {
+	match backend.get_aux(key)? {
+		None => Ok(None),
+		Some(t) => T::decode(&mut &t[..])
+			.map_err(|e| ClientError::Backend(format!("MMR aux DB is corrupted: {}", e)))
+			.map(Some),
+	}
+}
+
+/// Load persistent data from backend.
+pub(crate) fn load_state<B, BE>(backend: &BE) -> ClientResult<Option<PersistedState<B>>>
+where
+	B: Block,
+	BE: AuxStore,
+{
+	let version: Option<u32> = load_decode(backend, VERSION_KEY)?;
+
+	match version {
+		None => (),
+		Some(1) => return load_decode::<_, PersistedState<B>>(backend, GADGET_STATE),
+		other =>
+			return Err(ClientError::Backend(format!("Unsupported MMR aux DB version: {:?}", other))),
+	}
+
+	// No persistent state found in DB.
+	Ok(None)
+}
+
+/// Load or initialize persistent data from backend.
+pub(crate) fn load_or_init_state<B, BE>(
+	backend: &BE,
+	default: NumberFor<B>,
+) -> pezsp_blockchain::Result<NumberFor<B>>
+where
+	B: Block,
+	BE: AuxStore,
+{
+	// Initialize gadget best_canon from AUX DB or from pallet genesis.
+	if let Some(best) = load_state::<B, BE>(backend)? {
+		info!(target: LOG_TARGET, "Loading MMR best canonicalized state from db: {:?}.", best);
+		Ok(best)
+	} else {
+		info!(
+			target: LOG_TARGET,
+			"Loading MMR from pallet genesis on what appears to be the first startup: {:?}.",
+			default
+		);
+		write_current_version(backend)?;
+		write_gadget_state::<B, BE>(backend, &default)?;
+		Ok(default)
+	}
+}
+
+#[cfg(test)]
+pub(crate) mod tests {
+	use super::*;
+	use crate::test_utils::{run_test_with_mmr_gadget_pre_post_using_client, MmrBlock, MockClient};
+	use parking_lot::Mutex;
+	use pezsp_runtime::generic::BlockId;
+	use std::{sync::Arc, time::Duration};
+	use bizinikiwi_test_runtime_client::{runtime::Block, Backend};
+
+	#[test]
+	fn should_load_persistent_sanity_checks() {
+		let client = MockClient::new();
+		let backend = &*client.backend;
+
+		// version not available in db -> None
+		assert_eq!(load_state::<Block, Backend>(backend).unwrap(), None);
+
+		// populate version in db
+		write_current_version(backend).unwrap();
+		// verify correct version is retrieved
+		assert_eq!(load_decode(backend, VERSION_KEY).unwrap(), Some(CURRENT_VERSION));
+
+		// version is available in db but state isn't -> None
+		assert_eq!(load_state::<Block, Backend>(backend).unwrap(), None);
+	}
+
+	#[test]
+	fn should_persist_progress_across_runs() {
+		pezsp_tracing::try_init_simple();
+
+		let client = Arc::new(MockClient::new());
+		let backend = client.backend.clone();
+
+		// version not available in db -> None
+		assert_eq!(load_decode::<Backend, Option<u32>>(&*backend, VERSION_KEY).unwrap(), None);
+		// state not available in db -> None
+		assert_eq!(load_state::<Block, Backend>(&*backend).unwrap(), None);
+		// run the gadget while importing and finalizing 3 blocks
+		run_test_with_mmr_gadget_pre_post_using_client(
+			client.clone(),
+			|_| async {},
+			|client| async move {
+				let a1 = client.import_block(&BlockId::Number(0), b"a1", Some(0)).await;
+				let a2 = client.import_block(&BlockId::Number(1), b"a2", Some(1)).await;
+				let a3 = client.import_block(&BlockId::Number(2), b"a3", Some(2)).await;
+				client.finalize_block(a3.hash(), Some(3));
+				tokio::time::sleep(Duration::from_millis(200)).await;
+				// a1, a2, a3 were canonicalized
+				client.assert_canonicalized(&[&a1, &a2, &a3]);
+			},
+		);
+
+		// verify previous progress was persisted and run the gadget again
+		run_test_with_mmr_gadget_pre_post_using_client(
+			client.clone(),
+			|client| async move {
+				let backend = &*client.backend;
+				// check there is both version and best canon available in db before running gadget
+				assert_eq!(load_decode(backend, VERSION_KEY).unwrap(), Some(CURRENT_VERSION));
+				assert_eq!(load_state::<Block, Backend>(backend).unwrap(), Some(3));
+			},
+			|client| async move {
+				let a4 = client.import_block(&BlockId::Number(3), b"a4", Some(3)).await;
+				let a5 = client.import_block(&BlockId::Number(4), b"a5", Some(4)).await;
+				let a6 = client.import_block(&BlockId::Number(5), b"a6", Some(5)).await;
+				client.finalize_block(a6.hash(), Some(6));
+				tokio::time::sleep(Duration::from_millis(200)).await;
+
+				// a4, a5, a6 were canonicalized
+				client.assert_canonicalized(&[&a4, &a5, &a6]);
+				// check persisted best canon was updated
+				assert_eq!(load_state::<Block, Backend>(&*client.backend).unwrap(), Some(6));
+			},
+		);
+	}
+
+	#[test]
+	fn should_resume_from_persisted_state() {
+		pezsp_tracing::try_init_simple();
+
+		let client = Arc::new(MockClient::new());
+		let blocks = Arc::new(Mutex::new(Vec::<MmrBlock>::new()));
+		let blocks_clone = blocks.clone();
+
+		// run the gadget while importing and finalizing 3 blocks
+		run_test_with_mmr_gadget_pre_post_using_client(
+			client.clone(),
+			|_| async {},
+			|client| async move {
+				let mut blocks = blocks_clone.lock();
+				blocks.push(client.import_block(&BlockId::Number(0), b"a1", Some(0)).await);
+				blocks.push(client.import_block(&BlockId::Number(1), b"a2", Some(1)).await);
+				blocks.push(client.import_block(&BlockId::Number(2), b"a3", Some(2)).await);
+				client.finalize_block(blocks.last().unwrap().hash(), Some(3));
+				tokio::time::sleep(Duration::from_millis(200)).await;
+				// a1, a2, a3 were canonicalized
+				let slice: Vec<&MmrBlock> = blocks.iter().collect();
+				client.assert_canonicalized(&slice);
+
+				// now manually move them back to non-canon/temp location
+				for mmr_block in slice {
+					client.undo_block_canonicalization(mmr_block)
+				}
+			},
+		);
+
+		let blocks_clone = blocks.clone();
+		// verify new gadget continues from block 4 and ignores 1, 2, 3 based on persisted state
+		run_test_with_mmr_gadget_pre_post_using_client(
+			client.clone(),
+			|client| async move {
+				let blocks = blocks_clone.lock();
+				let slice: Vec<&MmrBlock> = blocks.iter().collect();
+
+				// verify persisted state says a1, a2, a3 were canonicalized,
+				assert_eq!(load_state::<Block, Backend>(&*client.backend).unwrap(), Some(3));
+				// but actually they are NOT canon (we manually reverted them earlier).
+				client.assert_not_canonicalized(&slice);
+			},
+			|client| async move {
+				let a4 = client.import_block(&BlockId::Number(3), b"a4", Some(3)).await;
+				let a5 = client.import_block(&BlockId::Number(4), b"a5", Some(4)).await;
+				let a6 = client.import_block(&BlockId::Number(5), b"a6", Some(5)).await;
+				client.finalize_block(a6.hash(), Some(6));
+				tokio::time::sleep(Duration::from_millis(200)).await;
+
+				let block_1_to_3 = blocks.lock();
+				let slice: Vec<&MmrBlock> = block_1_to_3.iter().collect();
+				// verify a1, a2, a3 are still NOT canon (skipped by gadget based on data in aux db)
+				client.assert_not_canonicalized(&slice);
+				// but a4, a5, a6 were canonicalized
+				client.assert_canonicalized(&[&a4, &a5, &a6]);
+				// check persisted best canon was updated
+				assert_eq!(load_state::<Block, Backend>(&*client.backend).unwrap(), Some(6));
+			},
+		);
+	}
+}

bizinikiwi/client/merkle-mountain-range/src/lib.rs

@@ -0,0 +1,283 @@
+// This file is part of Bizinikiwi.
+
+// Copyright (C) Parity Technologies (UK) Ltd.
+// SPDX-License-Identifier: GPL-3.0-or-later WITH Classpath-exception-2.0
+
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with this program. If not, see <https://www.gnu.org/licenses/>.
+
+//! # MMR offchain gadget
+//!
+//! The MMR offchain gadget is run alongside `pezpallet-mmr` to assist it with offchain
+//! canonicalization of finalized MMR leaves and nodes.
+//! The gadget should only be run on nodes that have Indexing API enabled (otherwise
+//! `pezpallet-mmr` cannot write to offchain and this gadget has nothing to do).
+//!
+//! The runtime `pezpallet-mmr` creates one new MMR leaf per block and all inner MMR parent nodes
+//! generated by the MMR when adding said leaf. MMR nodes are stored both in:
+//! - on-chain storage - hashes only; not full leaf content;
+//! - off-chain storage - via Indexing API, full leaf content (and all internal nodes as well) is
+//!   saved to the Off-chain DB using a key derived from `parent_hash` and node index in MMR. The
+//!   `parent_hash` is also used within the key to avoid conflicts and overwrites on forks (leaf
+//!   data is only allowed to reference data coming from parent block).
+//!
+//! This gadget is driven by block finality and in responsible for pruning stale forks from
+//! offchain db, and moving finalized forks under a "canonical" key based solely on node `pos`
+//! in the MMR.
+
+#![warn(missing_docs)]
+
+mod aux_schema;
+mod offchain_mmr;
+#[cfg(test)]
+pub mod test_utils;
+
+use crate::offchain_mmr::OffchainMmr;
+use futures::StreamExt;
+use log::{debug, error, trace, warn};
+use pezsc_client_api::{Backend, BlockchainEvents, FinalityNotification, FinalityNotifications};
+use pezsc_offchain::OffchainDb;
+use pezsp_api::ProvideRuntimeApi;
+use pezsp_blockchain::{HeaderBackend, HeaderMetadata};
+use pezsp_consensus_beefy::MmrRootHash;
+use pezsp_mmr_primitives::{utils, LeafIndex, MmrApi};
+use pezsp_runtime::traits::{Block, Header, NumberFor};
+use std::{marker::PhantomData, sync::Arc};
+
+/// Logging target for the mmr gadget.
+pub const LOG_TARGET: &str = "mmr";
+
+/// A convenience MMR client trait that defines all the type bounds a MMR client
+/// has to satisfy and defines some helper methods.
+pub trait MmrClient<B, BE>:
+	BlockchainEvents<B> + HeaderBackend<B> + HeaderMetadata<B> + ProvideRuntimeApi<B>
+where
+	B: Block,
+	BE: Backend<B>,
+	Self::Api: MmrApi<B, MmrRootHash, NumberFor<B>>,
+{
+	/// Get the block number where the mmr pallet was added to the runtime.
+	fn first_mmr_block_num(&self, notification: &FinalityNotification<B>) -> Option<NumberFor<B>> {
+		let best_block_hash = notification.header.hash();
+		let best_block_number = *notification.header.number();
+		match self.runtime_api().mmr_leaf_count(best_block_hash) {
+			Ok(Ok(mmr_leaf_count)) => {
+				match utils::first_mmr_block_num::<B::Header>(best_block_number, mmr_leaf_count) {
+					Ok(first_mmr_block) => {
+						debug!(
+							target: LOG_TARGET,
+							"pezpallet-mmr detected at block {:?} with genesis at block {:?}",
+							best_block_number,
+							first_mmr_block
+						);
+						Some(first_mmr_block)
+					},
+					Err(e) => {
+						error!(
+							target: LOG_TARGET,
+							"Error calculating the first mmr block: {:?}", e
+						);
+						None
+					},
+				}
+			},
+			_ => {
+				trace!(
+					target: LOG_TARGET,
+					"pezpallet-mmr not detected at block {:?} ... (best finalized {:?})",
+					best_block_number,
+					notification.header.number()
+				);
+				None
+			},
+		}
+	}
+}
+
+impl<B, BE, T> MmrClient<B, BE> for T
+where
+	B: Block,
+	BE: Backend<B>,
+	T: BlockchainEvents<B> + HeaderBackend<B> + HeaderMetadata<B> + ProvideRuntimeApi<B>,
+	T::Api: MmrApi<B, MmrRootHash, NumberFor<B>>,
+{
+	// empty
+}
+
+struct OffchainMmrBuilder<B: Block, BE: Backend<B>, C> {
+	backend: Arc<BE>,
+	client: Arc<C>,
+	offchain_db: OffchainDb<BE::OffchainStorage>,
+	indexing_prefix: Vec<u8>,
+
+	_phantom: PhantomData<B>,
+}
+
+impl<B, BE, C> OffchainMmrBuilder<B, BE, C>
+where
+	B: Block,
+	BE: Backend<B>,
+	C: MmrClient<B, BE>,
+	C::Api: MmrApi<B, MmrRootHash, NumberFor<B>>,
+{
+	async fn try_build(
+		self,
+		finality_notifications: &mut FinalityNotifications<B>,
+	) -> Option<OffchainMmr<B, BE, C>> {
+		while let Some(notification) = finality_notifications.next().await {
+			if let Some(first_mmr_block_num) = self.client.first_mmr_block_num(&notification) {
+				let mut offchain_mmr = OffchainMmr::new(
+					self.backend,
+					self.client,
+					self.offchain_db,
+					self.indexing_prefix,
+					first_mmr_block_num,
+				)?;
+				// We need to make sure all blocks leading up to current notification
+				// have also been canonicalized.
+				offchain_mmr.canonicalize_catch_up(&notification);
+				// We have to canonicalize and prune the blocks in the finality
+				// notification that lead to building the offchain-mmr as well.
+				offchain_mmr.canonicalize_and_prune(notification);
+				return Some(offchain_mmr);
+			}
+		}
+
+		error!(
+			target: LOG_TARGET,
+			"Finality notifications stream closed unexpectedly. \
+			Couldn't build the canonicalization engine",
+		);
+		None
+	}
+}
+
+/// A MMR Gadget.
+pub struct MmrGadget<B: Block, BE: Backend<B>, C> {
+	finality_notifications: FinalityNotifications<B>,
+
+	_phantom: PhantomData<(B, BE, C)>,
+}
+
+impl<B, BE, C> MmrGadget<B, BE, C>
+where
+	B: Block,
+	<B::Header as Header>::Number: Into<LeafIndex>,
+	BE: Backend<B>,
+	C: MmrClient<B, BE>,
+	C::Api: MmrApi<B, MmrRootHash, NumberFor<B>>,
+{
+	async fn run(mut self, builder: OffchainMmrBuilder<B, BE, C>) {
+		let mut offchain_mmr = match builder.try_build(&mut self.finality_notifications).await {
+			Some(offchain_mmr) => offchain_mmr,
+			None => return,
+		};
+
+		while let Some(notification) = self.finality_notifications.next().await {
+			offchain_mmr.canonicalize_and_prune(notification);
+		}
+	}
+
+	/// Create and run the MMR gadget.
+	pub async fn start(client: Arc<C>, backend: Arc<BE>, indexing_prefix: Vec<u8>) {
+		let offchain_db = match backend.offchain_storage() {
+			Some(offchain_storage) => OffchainDb::new(offchain_storage),
+			None => {
+				warn!(
+					target: LOG_TARGET,
+					"Can't spawn a MmrGadget for a node without offchain storage."
+				);
+				return;
+			},
+		};
+
+		let mmr_gadget = MmrGadget::<B, BE, C> {
+			finality_notifications: client.finality_notification_stream(),
+
+			_phantom: Default::default(),
+		};
+		mmr_gadget
+			.run(OffchainMmrBuilder {
+				backend,
+				client,
+				offchain_db,
+				indexing_prefix,
+				_phantom: Default::default(),
+			})
+			.await
+	}
+}
+
+#[cfg(test)]
+mod tests {
+	use crate::test_utils::run_test_with_mmr_gadget;
+	use pezsp_runtime::generic::BlockId;
+	use std::time::Duration;
+
+	#[test]
+	fn mmr_first_block_is_computed_correctly() {
+		// Check the case where the first block is also the first block with MMR.
+		run_test_with_mmr_gadget(|client| async move {
+			// G -> A1 -> A2
+			//      |
+			//      | -> first mmr block
+
+			let a1 = client.import_block(&BlockId::Number(0), b"a1", Some(0)).await;
+			let a2 = client.import_block(&BlockId::Hash(a1.hash()), b"a2", Some(1)).await;
+
+			client.finalize_block(a1.hash(), Some(1));
+			tokio::time::sleep(Duration::from_millis(200)).await;
+			// expected finalized heads: a1
+			client.assert_canonicalized(&[&a1]);
+			client.assert_not_pruned(&[&a2]);
+		});
+
+		// Check the case where the first block with MMR comes later.
+		run_test_with_mmr_gadget(|client| async move {
+			// G -> A1 -> A2 -> A3 -> A4 -> A5 -> A6
+			//                        |
+			//                        | -> first mmr block
+
+			let a1 = client.import_block(&BlockId::Number(0), b"a1", None).await;
+			let a2 = client.import_block(&BlockId::Hash(a1.hash()), b"a2", None).await;
+			let a3 = client.import_block(&BlockId::Hash(a2.hash()), b"a3", None).await;
+			let a4 = client.import_block(&BlockId::Hash(a3.hash()), b"a4", Some(0)).await;
+			let a5 = client.import_block(&BlockId::Hash(a4.hash()), b"a5", Some(1)).await;
+			let a6 = client.import_block(&BlockId::Hash(a5.hash()), b"a6", Some(2)).await;
+
+			client.finalize_block(a5.hash(), Some(2));
+			tokio::time::sleep(Duration::from_millis(200)).await;
+			// expected finalized heads: a4, a5
+			client.assert_canonicalized(&[&a4, &a5]);
+			client.assert_not_pruned(&[&a6]);
+		});
+	}
+
+	#[test]
+	fn does_not_panic_on_invalid_num_mmr_blocks() {
+		run_test_with_mmr_gadget(|client| async move {
+			// G -> A1
+			//      |
+			//      | -> first mmr block
+
+			let a1 = client.import_block(&BlockId::Number(0), b"a1", Some(0)).await;
+
+			// Simulate the case where the runtime says that there are 2 mmr_blocks when in fact
+			// there is only 1.
+			client.finalize_block(a1.hash(), Some(2));
+			tokio::time::sleep(Duration::from_millis(200)).await;
+			// expected finalized heads: -
+			client.assert_not_canonicalized(&[&a1]);
+		});
+	}
+}

bizinikiwi/client/merkle-mountain-range/src/offchain_mmr.rs

@@ -0,0 +1,450 @@
+// This file is part of Bizinikiwi.
+
+// Copyright (C) Parity Technologies (UK) Ltd.
+// SPDX-License-Identifier: GPL-3.0-or-later WITH Classpath-exception-2.0
+
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with this program. If not, see <https://www.gnu.org/licenses/>.
+
+//! Logic for canonicalizing MMR offchain entries for finalized forks,
+//! and for pruning MMR offchain entries for stale forks.
+
+#![warn(missing_docs)]
+
+use crate::{aux_schema, MmrClient, LOG_TARGET};
+use log::{debug, error, info, warn};
+use pezsc_client_api::{Backend, FinalityNotification};
+use pezsc_offchain::OffchainDb;
+use pezsp_blockchain::CachedHeaderMetadata;
+use pezsp_consensus_beefy::MmrRootHash;
+use pezsp_core::offchain::{DbExternalities, StorageKind};
+use pezsp_mmr_primitives::{utils, utils::NodesUtils, MmrApi, NodeIndex};
+use pezsp_runtime::{
+	traits::{Block, Header, NumberFor, One},
+	Saturating,
+};
+use std::{collections::VecDeque, sync::Arc};
+
+/// `OffchainMMR` exposes MMR offchain canonicalization and pruning logic.
+pub struct OffchainMmr<B: Block, BE: Backend<B>, C> {
+	backend: Arc<BE>,
+	client: Arc<C>,
+	offchain_db: OffchainDb<BE::OffchainStorage>,
+	indexing_prefix: Vec<u8>,
+	first_mmr_block: NumberFor<B>,
+	best_canonicalized: NumberFor<B>,
+}
+
+impl<B, BE, C> OffchainMmr<B, BE, C>
+where
+	BE: Backend<B>,
+	B: Block,
+	C: MmrClient<B, BE>,
+	C::Api: MmrApi<B, MmrRootHash, NumberFor<B>>,
+{
+	pub fn new(
+		backend: Arc<BE>,
+		client: Arc<C>,
+		offchain_db: OffchainDb<BE::OffchainStorage>,
+		indexing_prefix: Vec<u8>,
+		first_mmr_block: NumberFor<B>,
+	) -> Option<Self> {
+		let mut best_canonicalized = first_mmr_block.saturating_sub(One::one());
+		best_canonicalized = aux_schema::load_or_init_state::<B, BE>(&*backend, best_canonicalized)
+			.map_err(|e| error!(target: LOG_TARGET, "Error loading state from aux db: {:?}", e))
+			.ok()?;
+
+		Some(Self {
+			backend,
+			client,
+			offchain_db,
+			indexing_prefix,
+			first_mmr_block,
+			best_canonicalized,
+		})
+	}
+
+	fn node_temp_offchain_key(&self, pos: NodeIndex, parent_hash: B::Hash) -> Vec<u8> {
+		NodesUtils::node_temp_offchain_key::<B::Header>(&self.indexing_prefix, pos, parent_hash)
+	}
+
+	fn node_canon_offchain_key(&self, pos: NodeIndex) -> Vec<u8> {
+		NodesUtils::node_canon_offchain_key(&self.indexing_prefix, pos)
+	}
+
+	fn write_gadget_state_or_log(&self) {
+		if let Err(e) =
+			aux_schema::write_gadget_state::<B, BE>(&*self.backend, &self.best_canonicalized)
+		{
+			debug!(target: LOG_TARGET, "error saving state: {:?}", e);
+		}
+	}
+
+	fn header_metadata_or_log(
+		&self,
+		hash: B::Hash,
+		action: &str,
+	) -> Option<CachedHeaderMetadata<B>> {
+		match self.client.header_metadata(hash) {
+			Ok(header) => Some(header),
+			_ => {
+				debug!(
+					target: LOG_TARGET,
+					"Block {} not found. Couldn't {} associated branch.", hash, action
+				);
+				None
+			},
+		}
+	}
+
+	fn right_branch_ending_in_block_or_log(
+		&self,
+		block_num: NumberFor<B>,
+		action: &str,
+	) -> Option<Vec<NodeIndex>> {
+		match utils::block_num_to_leaf_index::<B::Header>(block_num, self.first_mmr_block) {
+			Ok(leaf_idx) => {
+				let branch = NodesUtils::right_branch_ending_in_leaf(leaf_idx);
+				debug!(
+					target: LOG_TARGET,
+					"Nodes to {} for block {}: {:?}", action, block_num, branch
+				);
+				Some(branch)
+			},
+			Err(e) => {
+				error!(
+					target: LOG_TARGET,
+					"Error converting block number {} to leaf index: {:?}. \
+					Couldn't {} associated branch.",
+					block_num,
+					e,
+					action
+				);
+				None
+			},
+		}
+	}
+
+	fn prune_branch(&mut self, block_hash: &B::Hash) {
+		let action = "prune";
+		let header = match self.header_metadata_or_log(*block_hash, action) {
+			Some(header) => header,
+			_ => return,
+		};
+
+		// We prune the leaf associated with the provided block and all the nodes added by that
+		// leaf.
+		let stale_nodes = match self.right_branch_ending_in_block_or_log(header.number, action) {
+			Some(nodes) => nodes,
+			None => {
+				// If we can't convert the block number to a leaf index, the chain state is probably
+				// corrupted. We only log the error, hoping that the chain state will be fixed.
+				return;
+			},
+		};
+
+		for pos in stale_nodes {
+			let temp_key = self.node_temp_offchain_key(pos, header.parent);
+			self.offchain_db.local_storage_clear(StorageKind::PERSISTENT, &temp_key);
+			debug!(target: LOG_TARGET, "Pruned elem at pos {} with temp key {:?}", pos, temp_key);
+		}
+	}
+
+	fn canonicalize_branch(&mut self, block_hash: B::Hash) {
+		let action = "canonicalize";
+		let header = match self.header_metadata_or_log(block_hash, action) {
+			Some(header) => header,
+			_ => return,
+		};
+
+		// Don't canonicalize branches corresponding to blocks for which the MMR pallet
+		// wasn't yet initialized.
+		if header.number < self.first_mmr_block {
+			return;
+		}
+
+		// We "canonicalize" the leaf associated with the provided block
+		// and all the nodes added by that leaf.
+		let to_canon_nodes = match self.right_branch_ending_in_block_or_log(header.number, action) {
+			Some(nodes) => nodes,
+			None => {
+				// If we can't convert the block number to a leaf index, the chain state is probably
+				// corrupted. We only log the error, hoping that the chain state will be fixed.
+				self.best_canonicalized = header.number;
+				return;
+			},
+		};
+
+		for pos in to_canon_nodes {
+			let temp_key = self.node_temp_offchain_key(pos, header.parent);
+			if let Some(elem) =
+				self.offchain_db.local_storage_get(StorageKind::PERSISTENT, &temp_key)
+			{
+				let canon_key = self.node_canon_offchain_key(pos);
+				self.offchain_db.local_storage_set(StorageKind::PERSISTENT, &canon_key, &elem);
+				self.offchain_db.local_storage_clear(StorageKind::PERSISTENT, &temp_key);
+				debug!(
+					target: LOG_TARGET,
+					"Moved elem at pos {} from temp key {:?} to canon key {:?}",
+					pos,
+					temp_key,
+					canon_key
+				);
+			} else {
+				debug!(
+					target: LOG_TARGET,
+					"Couldn't canonicalize elem at pos {} using temp key {:?}", pos, temp_key
+				);
+			}
+		}
+		if self.best_canonicalized != header.number.saturating_sub(One::one()) {
+			warn!(
+				target: LOG_TARGET,
+				"Detected canonicalization skip: best {:?} current {:?}.",
+				self.best_canonicalized,
+				header.number,
+			);
+		}
+		self.best_canonicalized = header.number;
+	}
+
+	/// In case of missed finality notifications (node restarts for example),
+	/// make sure to also canon everything leading up to `notification.tree_route`.
+	pub fn canonicalize_catch_up(&mut self, notification: &FinalityNotification<B>) {
+		let first = notification.tree_route.first().unwrap_or(&notification.hash);
+		if let Some(mut header) = self.header_metadata_or_log(*first, "canonicalize") {
+			let mut to_canon = VecDeque::<<B as Block>::Hash>::new();
+			// Walk up the chain adding all blocks newer than `self.best_canonicalized`.
+			loop {
+				header = match self.header_metadata_or_log(header.parent, "canonicalize") {
+					Some(header) => header,
+					_ => break,
+				};
+				if header.number <= self.best_canonicalized {
+					break;
+				}
+				to_canon.push_front(header.hash);
+			}
+			// Canonicalize all blocks leading up to current finality notification.
+			for hash in to_canon.drain(..) {
+				self.canonicalize_branch(hash);
+			}
+			self.write_gadget_state_or_log();
+		}
+	}
+
+	fn handle_potential_pallet_reset(&mut self, notification: &FinalityNotification<B>) {
+		if let Some(first_mmr_block_num) = self.client.first_mmr_block_num(&notification) {
+			if first_mmr_block_num != self.first_mmr_block {
+				info!(
+					target: LOG_TARGET,
+					"pezpallet-mmr reset detected at block {:?} with new genesis at block {:?}",
+					notification.header.number(),
+					first_mmr_block_num
+				);
+				self.first_mmr_block = first_mmr_block_num;
+				self.best_canonicalized = first_mmr_block_num.saturating_sub(One::one());
+				self.write_gadget_state_or_log();
+			}
+		}
+	}
+
+	/// Move leafs and nodes added by finalized blocks in offchain db from _fork-aware key_ to
+	/// _canonical key_.
+	/// Prune leafs and nodes added by stale blocks in offchain db from _fork-aware key_.
+	pub fn canonicalize_and_prune(&mut self, notification: FinalityNotification<B>) {
+		// Update the first MMR block in case of a pallet reset.
+		self.handle_potential_pallet_reset(&notification);
+
+		// Move offchain MMR nodes for finalized blocks to canonical keys.
+		for hash in notification.tree_route.iter().chain(std::iter::once(&notification.hash)) {
+			self.canonicalize_branch(*hash);
+		}
+		self.write_gadget_state_or_log();
+
+		// Remove offchain MMR nodes for stale forks.
+		notification.stale_blocks.iter().for_each(|s| self.prune_branch(&s.hash));
+	}
+}
+
+#[cfg(test)]
+mod tests {
+	use crate::test_utils::{run_test_with_mmr_gadget, run_test_with_mmr_gadget_pre_post};
+	use parking_lot::Mutex;
+	use pezsp_runtime::generic::BlockId;
+	use std::{sync::Arc, time::Duration};
+
+	#[test]
+	fn canonicalize_and_prune_works_correctly() {
+		run_test_with_mmr_gadget(|client| async move {
+			//                     -> D4 -> D5
+			// G -> A1 -> A2 -> A3 -> A4
+			//   -> B1 -> B2 -> B3
+			//   -> C1
+
+			let a1 = client.import_block(&BlockId::Number(0), b"a1", Some(0)).await;
+			let a2 = client.import_block(&BlockId::Hash(a1.hash()), b"a2", Some(1)).await;
+			let a3 = client.import_block(&BlockId::Hash(a2.hash()), b"a3", Some(2)).await;
+			let a4 = client.import_block(&BlockId::Hash(a3.hash()), b"a4", Some(3)).await;
+
+			let b1 = client.import_block(&BlockId::Number(0), b"b1", Some(0)).await;
+			let b2 = client.import_block(&BlockId::Hash(b1.hash()), b"b2", Some(1)).await;
+			let b3 = client.import_block(&BlockId::Hash(b2.hash()), b"b3", Some(2)).await;
+
+			let c1 = client.import_block(&BlockId::Number(0), b"c1", Some(0)).await;
+
+			let d4 = client.import_block(&BlockId::Hash(a3.hash()), b"d4", Some(3)).await;
+			let d5 = client.import_block(&BlockId::Hash(d4.hash()), b"d5", Some(4)).await;
+
+			client.finalize_block(a3.hash(), Some(3));
+			tokio::time::sleep(Duration::from_millis(200)).await;
+			// expected finalized heads: a1, a2, a3
+			client.assert_canonicalized(&[&a1, &a2, &a3]);
+			// expected stale heads: c1
+			// expected pruned heads because of temp key collision: b1
+			client.assert_pruned(&[&c1, &b1]);
+
+			client.finalize_block(d5.hash(), Some(5));
+			tokio::time::sleep(Duration::from_millis(200)).await;
+			// expected finalized heads: d4, d5,
+			client.assert_canonicalized(&[&d4, &d5]);
+			// expected stale heads: b1, b2, b3, a4
+			client.assert_pruned(&[&b1, &b2, &b3, &a4]);
+		})
+	}
+
+	#[test]
+	fn canonicalize_and_prune_handles_pallet_reset() {
+		run_test_with_mmr_gadget(|client| async move {
+			// G -> A1 -> A2 -> A3 -> A4 -> A5
+			//      |           |
+			//      |           | -> pallet reset
+			//      |
+			//      | -> first finality notification
+
+			let a1 = client.import_block(&BlockId::Number(0), b"a1", Some(0)).await;
+			let a2 = client.import_block(&BlockId::Hash(a1.hash()), b"a2", Some(1)).await;
+			let a3 = client.import_block(&BlockId::Hash(a2.hash()), b"a3", Some(0)).await;
+			let a4 = client.import_block(&BlockId::Hash(a3.hash()), b"a4", Some(1)).await;
+			let a5 = client.import_block(&BlockId::Hash(a4.hash()), b"a5", Some(2)).await;
+
+			client.finalize_block(a1.hash(), Some(1));
+			tokio::time::sleep(Duration::from_millis(200)).await;
+			// expected finalized heads: a1
+			client.assert_canonicalized(&[&a1]);
+			// a2 shouldn't be either canonicalized or pruned. It should be handled as part of the
+			// reset process.
+			client.assert_not_canonicalized(&[&a2]);
+
+			client.finalize_block(a5.hash(), Some(3));
+			tokio::time::sleep(Duration::from_millis(200)).await;
+			//expected finalized heads: a3, a4, a5,
+			client.assert_canonicalized(&[&a3, &a4, &a5]);
+		})
+	}
+
+	#[test]
+	fn canonicalize_catchup_works_correctly() {
+		let mmr_blocks = Arc::new(Mutex::new(vec![]));
+		let mmr_blocks_ref = mmr_blocks.clone();
+		run_test_with_mmr_gadget_pre_post(
+			|client| async move {
+				// G -> A1 -> A2
+				//      |     |
+				//      |     | -> finalized without gadget (missed notification)
+				//      |
+				//      | -> first mmr block
+
+				let a1 = client.import_block(&BlockId::Number(0), b"a1", Some(0)).await;
+				let a2 = client.import_block(&BlockId::Hash(a1.hash()), b"a2", Some(1)).await;
+
+				client.finalize_block(a2.hash(), Some(2));
+
+				let mut mmr_blocks = mmr_blocks_ref.lock();
+				mmr_blocks.push(a1);
+				mmr_blocks.push(a2);
+			},
+			|client| async move {
+				// G -> A1 -> A2 -> A3 -> A4
+				//      |     |     |     |
+				//      |     |     |     | -> finalized after starting gadget
+				//      |     |     |
+				//      |     |     | -> gadget start
+				//      |     |
+				//      |     | -> finalized before starting gadget (missed notification)
+				//      |
+				//      | -> first mmr block
+				let blocks = mmr_blocks.lock();
+				let a1 = blocks[0].clone();
+				let a2 = blocks[1].clone();
+				let a3 = client.import_block(&BlockId::Hash(a2.hash()), b"a3", Some(2)).await;
+				let a4 = client.import_block(&BlockId::Hash(a3.hash()), b"a4", Some(3)).await;
+
+				client.finalize_block(a4.hash(), Some(4));
+				tokio::time::sleep(Duration::from_millis(200)).await;
+				// expected finalized heads: a1, a2 _and_ a3, a4.
+				client.assert_canonicalized(&[&a1, &a2, &a3, &a4]);
+			},
+		)
+	}
+
+	#[test]
+	fn canonicalize_catchup_works_correctly_with_pallet_reset() {
+		let mmr_blocks = Arc::new(Mutex::new(vec![]));
+		let mmr_blocks_ref = mmr_blocks.clone();
+		run_test_with_mmr_gadget_pre_post(
+			|client| async move {
+				// G -> A1 -> A2
+				//      |     |
+				//      |     | -> finalized without gadget (missed notification)
+				//      |
+				//      | -> first mmr block
+
+				let a1 = client.import_block(&BlockId::Number(0), b"a1", Some(0)).await;
+				let a2 = client.import_block(&BlockId::Hash(a1.hash()), b"a2", Some(0)).await;
+
+				client.finalize_block(a2.hash(), Some(1));
+
+				let mut mmr_blocks = mmr_blocks_ref.lock();
+				mmr_blocks.push(a1);
+				mmr_blocks.push(a2);
+			},
+			|client| async move {
+				// G -> A1 -> A2 -> A3 -> A4
+				//      |     |     |     |
+				//      |     |     |     | -> finalized after starting gadget
+				//      |     |     |
+				//      |     |     | -> gadget start
+				//      |     |
+				//      |     | -> finalized before gadget start (missed notification)
+				//      |     |    + pallet reset
+				//      |
+				//      | -> first mmr block
+				let blocks = mmr_blocks.lock();
+				let a1 = blocks[0].clone();
+				let a2 = blocks[1].clone();
+				let a3 = client.import_block(&BlockId::Hash(a2.hash()), b"a3", Some(1)).await;
+				let a4 = client.import_block(&BlockId::Hash(a3.hash()), b"a4", Some(2)).await;
+
+				client.finalize_block(a4.hash(), Some(3));
+				tokio::time::sleep(Duration::from_millis(200)).await;
+				// a1 shouldn't be either canonicalized or pruned. It should be handled as part of
+				// the reset process. Checking only that it wasn't pruned. Because of temp key
+				// collision with a2 we can't check that it wasn't canonicalized.
+				client.assert_not_pruned(&[&a1]);
+				// expected finalized heads: a4, a5.
+				client.assert_canonicalized(&[&a2, &a3, &a4]);
+			},
+		)
+	}
+}

bizinikiwi/client/merkle-mountain-range/src/test_utils.rs

@@ -0,0 +1,381 @@
+// This file is part of Bizinikiwi.
+
+// Copyright (C) Parity Technologies (UK) Ltd.
+// SPDX-License-Identifier: GPL-3.0-or-later WITH Classpath-exception-2.0
+
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with this program. If not, see <https://www.gnu.org/licenses/>.
+
+//! Test utilities.
+
+use crate::MmrGadget;
+use parking_lot::Mutex;
+use pezsc_block_builder::BlockBuilderBuilder;
+use pezsc_client_api::{
+	Backend as BackendT, BlockchainEvents, FinalityNotifications, ImportNotifications,
+	StorageEventStream, StorageKey,
+};
+use pezsc_offchain::OffchainDb;
+use pezsp_api::{ApiRef, ProvideRuntimeApi};
+use pezsp_blockchain::{BlockStatus, CachedHeaderMetadata, HeaderBackend, HeaderMetadata, Info};
+use pezsp_consensus::BlockOrigin;
+use pezsp_core::{
+	offchain::{DbExternalities, StorageKind},
+	H256,
+};
+use pezsp_mmr_primitives as mmr;
+use pezsp_mmr_primitives::{utils::NodesUtils, LeafIndex, NodeIndex};
+use pezsp_runtime::{
+	generic::BlockId,
+	traits::{Block as BlockT, Header as HeaderT},
+};
+use std::{future::Future, sync::Arc, time::Duration};
+use bizinikiwi_test_runtime_client::{
+	runtime::{Block, BlockNumber, Hash, Header},
+	Backend, BlockBuilderExt, Client, ClientBlockImportExt, ClientExt, DefaultTestClientBuilderExt,
+	TestClientBuilder, TestClientBuilderExt,
+};
+use tokio::runtime::Runtime;
+
+type MmrHash = H256;
+
+pub(crate) struct MockRuntimeApiData {
+	pub(crate) num_blocks: BlockNumber,
+}
+
+#[derive(Clone)]
+pub(crate) struct MockRuntimeApi {
+	pub(crate) data: Arc<Mutex<MockRuntimeApiData>>,
+}
+
+impl MockRuntimeApi {
+	pub(crate) const INDEXING_PREFIX: &'static [u8] = b"mmr_test";
+}
+
+#[derive(Clone, Debug)]
+pub(crate) struct MmrBlock {
+	pub(crate) block: Block,
+	pub(crate) leaf_idx: Option<LeafIndex>,
+	pub(crate) leaf_data: Vec<u8>,
+}
+
+/// Which kind of key type to use.
+#[derive(Clone, Copy)]
+pub enum OffchainKeyType {
+	/// Temporary key.
+	Temp,
+	/// Cononical key.
+	Canon,
+}
+
+impl MmrBlock {
+	pub fn hash(&self) -> Hash {
+		self.block.hash()
+	}
+
+	pub fn parent_hash(&self) -> Hash {
+		*self.block.header.parent_hash()
+	}
+
+	pub fn get_offchain_key(&self, node: NodeIndex, key_type: OffchainKeyType) -> Vec<u8> {
+		match key_type {
+			OffchainKeyType::Temp => NodesUtils::node_temp_offchain_key::<Header>(
+				MockRuntimeApi::INDEXING_PREFIX,
+				node,
+				self.parent_hash(),
+			),
+			OffchainKeyType::Canon =>
+				NodesUtils::node_canon_offchain_key(MockRuntimeApi::INDEXING_PREFIX, node),
+		}
+	}
+}
+
+pub(crate) struct MockClient {
+	pub(crate) client: Mutex<Client<Backend>>,
+	pub(crate) backend: Arc<Backend>,
+	pub(crate) runtime_api_params: Arc<Mutex<MockRuntimeApiData>>,
+}
+
+impl MockClient {
+	pub(crate) fn new() -> Self {
+		let client_builder = TestClientBuilder::new().enable_offchain_indexing_api();
+		let (client, backend) = client_builder.build_with_backend();
+		MockClient {
+			client: Mutex::new(client),
+			backend,
+			runtime_api_params: Arc::new(Mutex::new(MockRuntimeApiData { num_blocks: 0 })),
+		}
+	}
+
+	pub(crate) fn offchain_db(&self) -> OffchainDb<<Backend as BackendT<Block>>::OffchainStorage> {
+		OffchainDb::new(self.backend.offchain_storage().unwrap())
+	}
+
+	pub async fn import_block(
+		&self,
+		at: &BlockId<Block>,
+		name: &[u8],
+		maybe_leaf_idx: Option<LeafIndex>,
+	) -> MmrBlock {
+		let client = self.client.lock();
+
+		let hash = client.expect_block_hash_from_id(&at).unwrap();
+		let mut block_builder = BlockBuilderBuilder::new(&*client)
+			.on_parent_block(hash)
+			.fetch_parent_block_number(&*client)
+			.unwrap()
+			.build()
+			.unwrap();
+		// Make sure the block has a different hash than its siblings
+		block_builder
+			.push_storage_change(b"name".to_vec(), Some(name.to_vec()))
+			.unwrap();
+		let block = block_builder.build().unwrap().block;
+		client.import(BlockOrigin::Own, block.clone()).await.unwrap();
+
+		let parent_hash = *block.header.parent_hash();
+		// Simulate writing MMR nodes in offchain storage
+		if let Some(leaf_idx) = maybe_leaf_idx {
+			let mut offchain_db = self.offchain_db();
+			for node in NodesUtils::right_branch_ending_in_leaf(leaf_idx) {
+				let temp_key = NodesUtils::node_temp_offchain_key::<Header>(
+					MockRuntimeApi::INDEXING_PREFIX,
+					node,
+					parent_hash,
+				);
+				offchain_db.local_storage_set(
+					StorageKind::PERSISTENT,
+					&temp_key,
+					parent_hash.as_ref(),
+				)
+			}
+		}
+
+		MmrBlock { block, leaf_idx: maybe_leaf_idx, leaf_data: parent_hash.as_ref().to_vec() }
+	}
+
+	pub fn finalize_block(&self, hash: Hash, maybe_num_mmr_blocks: Option<BlockNumber>) {
+		let client = self.client.lock();
+		if let Some(num_mmr_blocks) = maybe_num_mmr_blocks {
+			self.runtime_api_params.lock().num_blocks = num_mmr_blocks;
+		}
+
+		client.finalize_block(hash, None).unwrap();
+	}
+
+	pub fn undo_block_canonicalization(&self, mmr_block: &MmrBlock) {
+		let mut offchain_db = self.offchain_db();
+		for node in NodesUtils::right_branch_ending_in_leaf(mmr_block.leaf_idx.unwrap()) {
+			let canon_key = mmr_block.get_offchain_key(node, OffchainKeyType::Canon);
+			let val = offchain_db.local_storage_get(StorageKind::PERSISTENT, &canon_key).unwrap();
+			offchain_db.local_storage_clear(StorageKind::PERSISTENT, &canon_key);
+
+			let temp_key = mmr_block.get_offchain_key(node, OffchainKeyType::Temp);
+			offchain_db.local_storage_set(StorageKind::PERSISTENT, &temp_key, &val);
+		}
+	}
+
+	pub fn check_offchain_storage<F>(
+		&self,
+		key_type: OffchainKeyType,
+		blocks: &[&MmrBlock],
+		mut f: F,
+	) where
+		F: FnMut(Option<Vec<u8>>, &MmrBlock),
+	{
+		let mut offchain_db = self.offchain_db();
+		for mmr_block in blocks {
+			for node in NodesUtils::right_branch_ending_in_leaf(mmr_block.leaf_idx.unwrap()) {
+				let temp_key = mmr_block.get_offchain_key(node, key_type);
+				let val = offchain_db.local_storage_get(StorageKind::PERSISTENT, &temp_key);
+				f(val, mmr_block);
+			}
+		}
+	}
+
+	pub fn assert_pruned(&self, blocks: &[&MmrBlock]) {
+		self.check_offchain_storage(OffchainKeyType::Temp, blocks, |val, _block| {
+			assert!(val.is_none());
+		})
+	}
+
+	pub fn assert_not_pruned(&self, blocks: &[&MmrBlock]) {
+		self.check_offchain_storage(OffchainKeyType::Temp, blocks, |val, block| {
+			assert_eq!(val.as_ref(), Some(&block.leaf_data));
+		})
+	}
+
+	pub fn assert_canonicalized(&self, blocks: &[&MmrBlock]) {
+		self.check_offchain_storage(OffchainKeyType::Canon, blocks, |val, block| {
+			assert_eq!(val.as_ref(), Some(&block.leaf_data));
+		});
+
+		self.assert_pruned(blocks);
+	}
+
+	pub fn assert_not_canonicalized(&self, blocks: &[&MmrBlock]) {
+		self.check_offchain_storage(OffchainKeyType::Canon, blocks, |val, _block| {
+			assert!(val.is_none());
+		});
+
+		self.assert_not_pruned(blocks);
+	}
+}
+
+impl HeaderMetadata<Block> for MockClient {
+	type Error = <Client<Backend> as HeaderMetadata<Block>>::Error;
+
+	fn header_metadata(&self, hash: Hash) -> Result<CachedHeaderMetadata<Block>, Self::Error> {
+		self.client.lock().header_metadata(hash)
+	}
+
+	fn insert_header_metadata(&self, _hash: Hash, _header_metadata: CachedHeaderMetadata<Block>) {
+		todo!()
+	}
+
+	fn remove_header_metadata(&self, _hash: Hash) {
+		todo!()
+	}
+}
+
+impl HeaderBackend<Block> for MockClient {
+	fn header(&self, hash: Hash) -> pezsc_client_api::blockchain::Result<Option<Header>> {
+		self.client.lock().header(hash)
+	}
+
+	fn info(&self) -> Info<Block> {
+		self.client.lock().info()
+	}
+
+	fn status(&self, hash: Hash) -> pezsc_client_api::blockchain::Result<BlockStatus> {
+		self.client.lock().status(hash)
+	}
+
+	fn number(&self, hash: Hash) -> pezsc_client_api::blockchain::Result<Option<BlockNumber>> {
+		self.client.lock().number(hash)
+	}
+
+	fn hash(&self, number: BlockNumber) -> pezsc_client_api::blockchain::Result<Option<Hash>> {
+		self.client.lock().hash(number)
+	}
+}
+
+impl BlockchainEvents<Block> for MockClient {
+	fn import_notification_stream(&self) -> ImportNotifications<Block> {
+		unimplemented!()
+	}
+
+	fn every_import_notification_stream(&self) -> ImportNotifications<Block> {
+		unimplemented!()
+	}
+
+	fn finality_notification_stream(&self) -> FinalityNotifications<Block> {
+		self.client.lock().finality_notification_stream()
+	}
+
+	fn storage_changes_notification_stream(
+		&self,
+		_filter_keys: Option<&[StorageKey]>,
+		_child_filter_keys: Option<&[(StorageKey, Option<Vec<StorageKey>>)]>,
+	) -> pezsc_client_api::blockchain::Result<StorageEventStream<Hash>> {
+		unimplemented!()
+	}
+}
+
+impl ProvideRuntimeApi<Block> for MockClient {
+	type Api = MockRuntimeApi;
+
+	fn runtime_api(&self) -> ApiRef<'_, Self::Api> {
+		MockRuntimeApi { data: self.runtime_api_params.clone() }.into()
+	}
+}
+
+pezsp_api::mock_impl_runtime_apis! {
+	impl mmr::MmrApi<Block, MmrHash, BlockNumber> for MockRuntimeApi {
+		fn mmr_root() -> Result<MmrHash, mmr::Error> {
+			Err(mmr::Error::PalletNotIncluded)
+		}
+
+		fn mmr_leaf_count(&self) -> Result<LeafIndex, mmr::Error> {
+			Ok(self.data.lock().num_blocks)
+		}
+
+		fn generate_proof(
+			&self,
+			_block_numbers: Vec<u64>,
+			_best_known_block_number: Option<u64>,
+		) -> Result<(Vec<mmr::EncodableOpaqueLeaf>, mmr::LeafProof<MmrHash>), mmr::Error> {
+			Err(mmr::Error::PalletNotIncluded)
+		}
+
+		fn verify_proof(_leaves: Vec<mmr::EncodableOpaqueLeaf>, _proof: mmr::LeafProof<MmrHash>)
+			-> Result<(), mmr::Error>
+		{
+			Err(mmr::Error::PalletNotIncluded)
+		}
+
+		fn verify_proof_stateless(
+			_root: MmrHash,
+			_leaves: Vec<mmr::EncodableOpaqueLeaf>,
+			_proof: mmr::LeafProof<MmrHash>
+		) -> Result<(), mmr::Error> {
+			Err(mmr::Error::PalletNotIncluded)
+		}
+	}
+}
+
+pub(crate) fn run_test_with_mmr_gadget<F, Fut>(post_gadget: F)
+where
+	F: FnOnce(Arc<MockClient>) -> Fut + 'static,
+	Fut: Future<Output = ()>,
+{
+	run_test_with_mmr_gadget_pre_post(|_| async {}, post_gadget);
+}
+
+pub(crate) fn run_test_with_mmr_gadget_pre_post<F, G, RetF, RetG>(pre_gadget: F, post_gadget: G)
+where
+	F: FnOnce(Arc<MockClient>) -> RetF + 'static,
+	G: FnOnce(Arc<MockClient>) -> RetG + 'static,
+	RetF: Future<Output = ()>,
+	RetG: Future<Output = ()>,
+{
+	let client = Arc::new(MockClient::new());
+	run_test_with_mmr_gadget_pre_post_using_client(client, pre_gadget, post_gadget)
+}
+
+pub(crate) fn run_test_with_mmr_gadget_pre_post_using_client<F, G, RetF, RetG>(
+	client: Arc<MockClient>,
+	pre_gadget: F,
+	post_gadget: G,
+) where
+	F: FnOnce(Arc<MockClient>) -> RetF + 'static,
+	G: FnOnce(Arc<MockClient>) -> RetG + 'static,
+	RetF: Future<Output = ()>,
+	RetG: Future<Output = ()>,
+{
+	let client_clone = client.clone();
+	let runtime = Runtime::new().unwrap();
+	runtime.block_on(async move { pre_gadget(client_clone).await });
+
+	let client_clone = client.clone();
+	runtime.spawn(async move {
+		let backend = client_clone.backend.clone();
+		MmrGadget::start(client_clone, backend, MockRuntimeApi::INDEXING_PREFIX.to_vec()).await
+	});
+
+	runtime.block_on(async move {
+		tokio::time::sleep(Duration::from_millis(200)).await;
+
+		post_gadget(client).await
+	});
+}