Move beefy-merkle-tree to utils/binary-merkle-tree and make it generic (#13076)

* move BeefyMmrApi to pallet-beefy-mmr

* fix test_utils use pallet-beefy-mmr BeefyMmrApi

* Move beefy-merkle-tree to utils and Rename to Merkle-tree

* fix fmt and test

* Update merkle-tree to binary-merkle-tree and Remove Keccak256 mod from merkle-tree

* change merkle-tree name to binary-merkle-tree

* mirr fix

substrate/Cargo.lock

@@ -543,14 +543,14 @@ dependencies = [
 ]
 
 [[package]]
-name = "beefy-merkle-tree"
+name = "binary-merkle-tree"
 version = "4.0.0-dev"
 dependencies = [
  "array-bytes",
  "env_logger",
+ "hash-db",
  "log",
- "sp-api",
- "sp-beefy",
+ "sp-core",
  "sp-runtime",
 ]
 
@@ -5499,7 +5499,7 @@ name = "pallet-beefy-mmr"
 version = "4.0.0-dev"
 dependencies = [
  "array-bytes",
- "beefy-merkle-tree",
+ "binary-merkle-tree",
  "frame-support",
  "frame-system",
  "log",
@@ -5509,6 +5509,7 @@ dependencies = [
  "parity-scale-codec",
  "scale-info",
  "serde",
+ "sp-api",
  "sp-beefy",
  "sp-core",
  "sp-io",
@@ -10699,7 +10700,6 @@ dependencies = [
 name = "substrate-test-runtime"
 version = "2.0.0"
 dependencies = [
- "beefy-merkle-tree",
  "cfg-if",
  "frame-support",
  "frame-system",
@@ -10708,6 +10708,7 @@ dependencies = [
  "log",
  "memory-db",
  "pallet-babe",
+ "pallet-beefy-mmr",
  "pallet-timestamp",
  "parity-scale-codec",
  "sc-block-builder",

substrate/Cargo.toml

@@ -85,7 +85,6 @@ members = [
 	"frame/balances",
 	"frame/beefy",
 	"frame/beefy-mmr",
-	"frame/beefy-mmr/primitives",
 	"frame/benchmarking",
 	"frame/benchmarking/pov",
 	"frame/bounties",
@@ -246,6 +245,7 @@ members = [
 	"utils/frame/rpc/client",
 	"utils/prometheus",
 	"utils/wasm-builder",
+	"utils/binary-merkle-tree",
 ]
 
 # The list of dependencies below (which can be both direct and indirect dependencies) are crates

substrate/frame/beefy-mmr/Cargo.toml

@@ -14,7 +14,7 @@ codec = { package = "parity-scale-codec", version = "3.2.2", default-features =
 log = { version = "0.4.17", default-features = false }
 scale-info = { version = "2.1.1", default-features = false, features = ["derive"] }
 serde = { version = "1.0.136", optional = true }
-beefy-merkle-tree = { version = "4.0.0-dev", default-features = false, path = "./primitives" }
+binary-merkle-tree = { version = "4.0.0-dev", default-features = false, path = "../../utils/binary-merkle-tree" }
 beefy-primitives = { version = "4.0.0-dev", default-features = false, path = "../../primitives/beefy", package = "sp-beefy" }
 frame-support = { version = "4.0.0-dev", default-features = false, path = "../support" }
 frame-system = { version = "4.0.0-dev", default-features = false, path = "../system" }
@@ -25,6 +25,7 @@ sp-core = { version = "7.0.0", default-features = false, path = "../../primitive
 sp-io = { version = "7.0.0", default-features = false, path = "../../primitives/io" }
 sp-runtime = { version = "7.0.0", default-features = false, path = "../../primitives/runtime" }
 sp-std = { version = "5.0.0", default-features = false, path = "../../primitives/std" }
+sp-api = { version = "4.0.0-dev", default-features = false, path = "../../primitives/api" }
 
 [dev-dependencies]
 array-bytes = "4.1"
@@ -34,7 +35,7 @@ sp-staking = { version = "4.0.0-dev", path = "../../primitives/staking" }
 default = ["std"]
 std = [
 	"array-bytes",
-	"beefy-merkle-tree/std",
+	"binary-merkle-tree/std",
 	"beefy-primitives/std",
 	"codec/std",
 	"frame-support/std",
@@ -49,5 +50,6 @@ std = [
 	"sp-io/std",
 	"sp-runtime/std",
 	"sp-std/std",
+	"sp-api/std",
 ]
 try-runtime = ["frame-support/try-runtime"]

substrate/frame/beefy-mmr/src/lib.rs

@@ -200,10 +200,24 @@ impl<T: Config> Pallet<T> {
 			.map(T::BeefyAuthorityToMerkleLeaf::convert)
 			.collect::<Vec<_>>();
 		let len = beefy_addresses.len() as u32;
-		let root = beefy_merkle_tree::merkle_root::<<T as pallet_mmr::Config>::Hashing, _>(
+		let root = binary_merkle_tree::merkle_root::<<T as pallet_mmr::Config>::Hashing, _>(
 			beefy_addresses,
 		)
 		.into();
 		BeefyAuthoritySet { id, len, root }
 	}
 }
+
+sp_api::decl_runtime_apis! {
+	/// API useful for BEEFY light clients.
+	pub trait BeefyMmrApi<H>
+	where
+		BeefyAuthoritySet<H>: sp_api::Decode,
+	{
+		/// Return the currently active BEEFY authority set proof.
+		fn authority_set_proof() -> BeefyAuthoritySet<H>;
+
+		/// Return the next/queued BEEFY authority set proof.
+		fn next_authority_set_proof() -> BeefyNextAuthoritySet<H>;
+	}
+}

substrate/frame/beefy-mmr/src/mock.rs

@@ -147,7 +147,7 @@ impl BeefyDataProvider<Vec<u8>> for DummyDataProvider {
 	fn extra_data() -> Vec<u8> {
 		let mut col = vec![(15, vec![1, 2, 3]), (5, vec![4, 5, 6])];
 		col.sort();
-		beefy_merkle_tree::merkle_root::<<Test as pallet_mmr::Config>::Hashing, _>(
+		binary_merkle_tree::merkle_root::<<Test as pallet_mmr::Config>::Hashing, _>(
 			col.into_iter().map(|pair| pair.encode()),
 		)
 		.as_ref()

substrate/test-utils/runtime/Cargo.toml

@@ -14,7 +14,7 @@ targets = ["x86_64-unknown-linux-gnu"]
 
 [dependencies]
 beefy-primitives = { version = "4.0.0-dev", default-features = false, path = "../../primitives/beefy", package = "sp-beefy" }
-beefy-merkle-tree = { version = "4.0.0-dev", default-features = false, path = "../../frame/beefy-mmr/primitives" }
+pallet-beefy-mmr = { version = "4.0.0-dev", default-features = false, path = "../../frame/beefy-mmr" }
 sp-application-crypto = { version = "7.0.0", default-features = false, path = "../../primitives/application-crypto" }
 sp-consensus-aura = { version = "0.10.0-dev", default-features = false, path = "../../primitives/consensus/aura" }
 sp-consensus-babe = { version = "0.10.0-dev", default-features = false, path = "../../primitives/consensus/babe" }
@@ -67,7 +67,7 @@ default = [
 ]
 std = [
 	"beefy-primitives/std",
-	"beefy-merkle-tree/std",
+	"pallet-beefy-mmr/std",
 	"sp-application-crypto/std",
 	"sp-consensus-aura/std",
 	"sp-consensus-babe/std",

substrate/test-utils/runtime/src/lib.rs

@@ -978,7 +978,7 @@ cfg_if! {
 				}
 			}
 
-			impl beefy_merkle_tree::BeefyMmrApi<Block, beefy_primitives::MmrRootHash> for Runtime {
+			impl pallet_beefy_mmr::BeefyMmrApi<Block, beefy_primitives::MmrRootHash> for Runtime {
 				fn authority_set_proof() -> beefy_primitives::mmr::BeefyAuthoritySet<beefy_primitives::MmrRootHash> {
 					Default::default()
 				}

substrate/utils/binary-merkle-tree/Cargo.toml

@@ -1,5 +1,5 @@
 [package]
-name = "beefy-merkle-tree"
+name = "binary-merkle-tree"
 version = "4.0.0-dev"
 authors = ["Parity Technologies <admin@parity.io>"]
 edition = "2021"
@@ -11,20 +11,18 @@ homepage = "https://substrate.io"
 [dependencies]
 array-bytes = { version = "4.1", optional = true }
 log = { version = "0.4", default-features = false, optional = true }
-
-beefy-primitives = { version = "4.0.0-dev", default-features = false, path = "../../../primitives/beefy", package = "sp-beefy" }
-sp-api = { version = "4.0.0-dev", default-features = false, path = "../../../primitives/api" }
-sp-runtime = { version = "7.0.0", default-features = false, path = "../../../primitives/runtime" }
+hash-db = { version = "0.15.2", default-features = false }
 
 [dev-dependencies]
 array-bytes = "4.1"
 env_logger = "0.9"
+sp-core = { version = "7.0.0", path = "../../primitives/core" }
+sp-runtime = { version = "7.0.0", path = "../../primitives/runtime" }
 
 [features]
 debug = ["array-bytes", "log"]
 default = ["debug", "std"]
 std = [
-	"beefy-primitives/std",
-	"sp-api/std",
-	"sp-runtime/std"
+    "log/std",
+    "hash-db/std"
 ]

substrate/utils/binary-merkle-tree/src/lib.rs

@@ -31,40 +31,42 @@
 //! efficient by removing the need to track which side each intermediate hash is concatenated on.
 //!
 //! If the number of leaves is not even, last leaf (hash of) is promoted to the upper layer.
+#[cfg(not(feature = "std"))]
+extern crate alloc;
+#[cfg(not(feature = "std"))]
+use alloc::vec;
+#[cfg(not(feature = "std"))]
+use alloc::vec::Vec;
 
-pub use sp_runtime::traits::Keccak256;
-use sp_runtime::{app_crypto::sp_core, sp_std, traits::Hash as HashT};
-use sp_std::{vec, vec::Vec};
-
-use beefy_primitives::mmr::{BeefyAuthoritySet, BeefyNextAuthoritySet};
+use hash_db::Hasher;
 
 /// Construct a root hash of a Binary Merkle Tree created from given leaves.
 ///
 /// See crate-level docs for details about Merkle Tree construction.
 ///
 /// In case an empty list of leaves is passed the function returns a 0-filled hash.
-pub fn merkle_root<H, I>(leaves: I) -> H::Output
+pub fn merkle_root<H, I>(leaves: I) -> H::Out
 where
-	H: HashT,
-	H::Output: Default + AsRef<[u8]> + PartialOrd,
+	H: Hasher,
+	H::Out: Default + AsRef<[u8]> + PartialOrd,
 	I: IntoIterator,
 	I::Item: AsRef<[u8]>,
 {
-	let iter = leaves.into_iter().map(|l| <H as HashT>::hash(l.as_ref()));
+	let iter = leaves.into_iter().map(|l| <H as Hasher>::hash(l.as_ref()));
 	merkelize::<H, _, _>(iter, &mut ()).into()
 }
 
-fn merkelize<H, V, I>(leaves: I, visitor: &mut V) -> H::Output
+fn merkelize<H, V, I>(leaves: I, visitor: &mut V) -> H::Out
 where
-	H: HashT,
-	H::Output: Default + AsRef<[u8]> + PartialOrd,
-	V: Visitor<H::Output>,
-	I: Iterator<Item = H::Output>,
+	H: Hasher,
+	H::Out: Default + AsRef<[u8]> + PartialOrd,
+	V: Visitor<H::Out>,
+	I: Iterator<Item = H::Out>,
 {
 	let upper = Vec::with_capacity((leaves.size_hint().1.unwrap_or(0).saturating_add(1)) / 2);
 	let mut next = match merkelize_row::<H, _, _>(leaves, upper, visitor) {
 		Ok(root) => return root,
-		Err(next) if next.is_empty() => return H::Output::default(),
+		Err(next) if next.is_empty() => return H::Out::default(),
 		Err(next) => next,
 	};
 
@@ -139,17 +141,17 @@ impl<T> Visitor<T> for () {
 /// # Panic
 ///
 /// The function will panic if given `leaf_index` is greater than the number of leaves.
-pub fn merkle_proof<H, I, T>(leaves: I, leaf_index: usize) -> MerkleProof<H::Output, T>
+pub fn merkle_proof<H, I, T>(leaves: I, leaf_index: usize) -> MerkleProof<H::Out, T>
 where
-	H: HashT,
-	H::Output: Default + Copy + AsRef<[u8]> + PartialOrd,
+	H: Hasher,
+	H::Out: Default + Copy + AsRef<[u8]> + PartialOrd,
 	I: IntoIterator<Item = T>,
 	I::IntoIter: ExactSizeIterator,
 	T: AsRef<[u8]>,
 {
 	let mut leaf = None;
 	let iter = leaves.into_iter().enumerate().map(|(idx, l)| {
-		let hash = <H as HashT>::hash(l.as_ref());
+		let hash = <H as Hasher>::hash(l.as_ref());
 		if idx == leaf_index {
 			leaf = Some(l);
 		}
@@ -234,28 +236,28 @@ impl<'a, H, T: AsRef<[u8]>> From<&'a T> for Leaf<'a, H> {
 ///
 /// The proof must not contain the root hash.
 pub fn verify_proof<'a, H, P, L>(
-	root: &'a H::Output,
+	root: &'a H::Out,
 	proof: P,
 	number_of_leaves: usize,
 	leaf_index: usize,
 	leaf: L,
 ) -> bool
 where
-	H: HashT,
-	H::Output: PartialEq + AsRef<[u8]> + PartialOrd,
-	P: IntoIterator<Item = H::Output>,
-	L: Into<Leaf<'a, H::Output>>,
+	H: Hasher,
+	H::Out: PartialEq + AsRef<[u8]> + PartialOrd,
+	P: IntoIterator<Item = H::Out>,
+	L: Into<Leaf<'a, H::Out>>,
 {
 	if leaf_index >= number_of_leaves {
 		return false
 	}
 
 	let leaf_hash = match leaf.into() {
-		Leaf::Value(content) => <H as HashT>::hash(content),
+		Leaf::Value(content) => <H as Hasher>::hash(content),
 		Leaf::Hash(hash) => hash,
 	};
 
-	let hash_len = <H as sp_core::Hasher>::LENGTH;
+	let hash_len = <H as Hasher>::LENGTH;
 	let mut combined = vec![0_u8; hash_len * 2];
 	let computed = proof.into_iter().fold(leaf_hash, |a, b| {
 		if a < b {
@@ -265,7 +267,7 @@ where
 			combined[..hash_len].copy_from_slice(&b.as_ref());
 			combined[hash_len..].copy_from_slice(&a.as_ref());
 		}
-		let hash = <H as HashT>::hash(&combined);
+		let hash = <H as Hasher>::hash(&combined);
 		#[cfg(feature = "debug")]
 		log::debug!(
 			"[verify_proof]: (a, b) {:?}, {:?} => {:?} ({:?}) hash",
@@ -287,20 +289,20 @@ where
 /// empty iterator) an `Err` with the inner nodes of upper layer is returned.
 fn merkelize_row<H, V, I>(
 	mut iter: I,
-	mut next: Vec<H::Output>,
+	mut next: Vec<H::Out>,
 	visitor: &mut V,
-) -> Result<H::Output, Vec<H::Output>>
+) -> Result<H::Out, Vec<H::Out>>
 where
-	H: HashT,
-	H::Output: AsRef<[u8]> + PartialOrd,
-	V: Visitor<H::Output>,
-	I: Iterator<Item = H::Output>,
+	H: Hasher,
+	H::Out: AsRef<[u8]> + PartialOrd,
+	V: Visitor<H::Out>,
+	I: Iterator<Item = H::Out>,
 {
 	#[cfg(feature = "debug")]
 	log::debug!("[merkelize_row]");
 	next.clear();
 
-	let hash_len = <H as sp_core::Hasher>::LENGTH;
+	let hash_len = <H as Hasher>::LENGTH;
 	let mut index = 0;
 	let mut combined = vec![0_u8; hash_len * 2];
 	loop {
@@ -326,7 +328,7 @@ where
 					combined[hash_len..].copy_from_slice(a.as_ref());
 				}
 
-				next.push(<H as HashT>::hash(&combined));
+				next.push(<H as Hasher>::hash(&combined));
 			},
 			// Odd number of items. Promote the item to the upper layer.
 			(Some(a), None) if !next.is_empty() => {
@@ -347,24 +349,11 @@ where
 	}
 }
 
-sp_api::decl_runtime_apis! {
-	/// API useful for BEEFY light clients.
-	pub trait BeefyMmrApi<H>
-	where
-		BeefyAuthoritySet<H>: sp_api::Decode,
-	{
-		/// Return the currently active BEEFY authority set proof.
-		fn authority_set_proof() -> BeefyAuthoritySet<H>;
-
-		/// Return the next/queued BEEFY authority set proof.
-		fn next_authority_set_proof() -> BeefyNextAuthoritySet<H>;
-	}
-}
-
 #[cfg(test)]
 mod tests {
 	use super::*;
-	use crate::sp_core::H256;
+	use sp_core::H256;
+	use sp_runtime::traits::Keccak256;
 
 	#[test]
 	fn should_generate_empty_root() {