pezkuwichain/pezkuwi-subxt
1
0
Fork 0
mirror of https://github.com/pezkuwichain/pezkuwi-subxt.git synced 2026-06-14 09:51:10 +00:00
Code Issues Packages Projects Releases Wiki Activity

pallet-mmr: move offchain logic to client-side gadget (#12753)

Browse Source 
* Move MMR utils methods from pallet to primitives

Signed-off-by: Serban Iorga <serban@parity.io>

* Add method to MmrApi

* Move forks expanding logic from babe to primitives

* Implement MMR gadget

* Remove prunning logic from the MMR pallet

* Code review changes: 1st iteration

* Replace MaybeCanonEngine with CanonEngineBuilder

* fix mmr_leaves_count() for kitchen sink demo

* Update client/merkle-mountain-range/src/canon_engine.rs

Co-authored-by: Adrian Catangiu <adrian@parity.io>

* Code review changes: 2nd iteration

* fix INDEXING_PREFIX

* impl review comments

* add documentation and minor rename

Signed-off-by: Serban Iorga <serban@parity.io>
Co-authored-by: Adrian Catangiu <adrian@parity.io>
This commit is contained in:
Serban Iorga
2022-11-29 16:39:52 +02:00
committed by GitHub
parent d56214c21f
commit ff439ee335
21 changed files with 1161 additions and 561 deletions
Download Patch File Download Diff File Expand all files Collapse all files
substrate/frame/merkle-mountain-range/Cargo.toml
-2
View File
@@ -13,7 +13,6 @@ targets = ["x86_64-unknown-linux-gnu"]
[dependencies]
codec = { package = "parity-scale-codec", version = "3.0.0", default-features = false }
mmr-lib = { package = "ckb-merkle-mountain-range", version = "0.5.2", default-features = false }
scale-info = { version = "2.1.1", default-features = false, features = ["derive"] }
frame-benchmarking = { version = "4.0.0-dev", default-features = false, optional = true, path = "../benchmarking" }
frame-support = { version = "4.0.0-dev", default-features = false, path = "../support" }
@@ -36,7 +35,6 @@ std = [
"frame-benchmarking?/std",
"frame-support/std",
"frame-system/std",
"mmr-lib/std",
"scale-info/std",
"sp-core/std",
"sp-io/std",
substrate/frame/merkle-mountain-range/rpc/src/lib.rs
+1 -1
View File
@@ -238,7 +238,7 @@ fn mmr_error_into_rpc_error(err: MmrError) -> CallError {
MmrError::LeafNotFound => 1,
MmrError::GenerateProof => 2,
MmrError::Verify => 3,
MmrError::BlockNumToLeafIndex => 4,
MmrError::InvalidNumericOp => 4,
MmrError::InvalidBestKnownBlock => 5,
_ => 0,
};
substrate/frame/merkle-mountain-range/src/lib.rs
+23 -77
View File
@@ -24,7 +24,7 @@
//!
//! The MMR pallet constructs an MMR from leaf data obtained on every block from
//! `LeafDataProvider`. MMR nodes are stored both in:
//! - on-chain storage - hashes only; not full leaf content)
//! - on-chain storage - hashes only; not full leaf content;
//! - off-chain storage - via Indexing API we push full leaf content (and all internal nodes as
//! well) to the Off-chain DB, so that the data is available for Off-chain workers.
//! Hashing used for MMR is configurable independently from the rest of the runtime (i.e. not using
@@ -56,10 +56,9 @@
//! NOTE This pallet is experimental and not proven to work in production.
#![cfg_attr(not(feature = "std"), no_std)]
use codec::Encode;
use frame_support::{log, traits::Get, weights::Weight};
use frame_support::{log, weights::Weight};
use sp_runtime::{
traits::{self, CheckedSub, One, Saturating, UniqueSaturatedInto},
traits::{self, One, Saturating},
SaturatedConversion,
};
@@ -73,7 +72,10 @@ mod mock;
mod tests;
pub use pallet::*;
pub use sp_mmr_primitives::{self as primitives, Error, LeafDataProvider, LeafIndex, NodeIndex};
use sp_mmr_primitives::utils;
pub use sp_mmr_primitives::{
self as primitives, utils::NodesUtils, Error, LeafDataProvider, LeafIndex, NodeIndex,
};
use sp_std::prelude::*;
/// The most common use case for MMRs is to store historical block hashes,
@@ -219,7 +221,7 @@ pub mod pallet {
fn on_initialize(_n: T::BlockNumber) -> Weight {
use primitives::LeafDataProvider;
let leaves = Self::mmr_leaves();
let peaks_before = mmr::utils::NodesUtils::new(leaves).number_of_peaks();
let peaks_before = sp_mmr_primitives::utils::NodesUtils::new(leaves).number_of_peaks();
let data = T::LeafData::leaf_data();
// append new leaf to MMR
@@ -242,46 +244,10 @@ pub mod pallet {
<NumberOfLeaves<T, I>>::put(leaves);
<RootHash<T, I>>::put(root);
let peaks_after = mmr::utils::NodesUtils::new(leaves).number_of_peaks();
let peaks_after = sp_mmr_primitives::utils::NodesUtils::new(leaves).number_of_peaks();
T::WeightInfo::on_initialize(peaks_before.max(peaks_after))
}
fn offchain_worker(n: T::BlockNumber) {
use mmr::storage::{OffchainStorage, Storage};
// The MMR nodes can be found in offchain db under either:
// - fork-unique keys `(prefix, pos, parent_hash)`, or,
// - "canonical" keys `(prefix, pos)`,
// depending on how many blocks in the past the node at position `pos` was
// added to the MMR.
//
// For the fork-unique keys, the MMR pallet depends on
// `frame_system::block_hash(parent_num)` mappings to find the relevant parent block
// hashes, so it is limited by `frame_system::BlockHashCount` in terms of how many
// historical forks it can track. Nodes added to MMR by block `N` can be found in
// offchain db at:
// - fork-unique keys `(prefix, pos, parent_hash)` when (`N` >= `latest_block` -
// `frame_system::BlockHashCount`);
// - "canonical" keys `(prefix, pos)` when (`N` < `latest_block` -
// `frame_system::BlockHashCount`);
//
// The offchain worker is responsible for maintaining the nodes' positions in
// offchain db as the chain progresses by moving a rolling window of the same size as
// `frame_system::block_hash` map, where nodes/leaves added by blocks that are just
// about to exit the window are "canonicalized" so that their offchain key no longer
// depends on `parent_hash`.
//
// This approach works to eliminate fork-induced leaf collisions in offchain db,
// under the assumption that no fork will be deeper than `frame_system::BlockHashCount`
// blocks:
// entries pertaining to block `N` where `N < current-BlockHashCount` are moved to a
// key based solely on block number. The only way to have collisions is if two
// competing forks are deeper than `frame_system::BlockHashCount` blocks and they
// both "canonicalize" their view of block `N`
// Once a block is canonicalized, all MMR entries pertaining to sibling blocks from
// other forks are pruned from offchain db.
Storage::<OffchainStorage, T, I, LeafOf<T, I>>::canonicalize_and_prune(n);
}
}
}
@@ -313,11 +279,15 @@ impl<T: Config<I>, I: 'static> Pallet<T, I> {
/// Build offchain key from `parent_hash` of block that originally added node `pos` to MMR.
///
/// This combination makes the offchain (key,value) entry resilient to chain forks.
fn node_offchain_key(
fn node_temp_offchain_key(
pos: NodeIndex,
parent_hash: <T as frame_system::Config>::Hash,
) -> sp_std::prelude::Vec<u8> {
(T::INDEXING_PREFIX, pos, parent_hash).encode()
NodesUtils::node_temp_offchain_key::<<T as frame_system::Config>::Header>(
&T::INDEXING_PREFIX,
pos,
parent_hash,
)
}
/// Build canonical offchain key for node `pos` in MMR.
@@ -326,18 +296,7 @@ impl<T: Config<I>, I: 'static> Pallet<T, I> {
/// Never read keys using `node_canon_offchain_key` unless you sure that
/// there's no `node_offchain_key` key in the storage.
fn node_canon_offchain_key(pos: NodeIndex) -> sp_std::prelude::Vec<u8> {
(T::INDEXING_PREFIX, pos).encode()
}
/// Return size of rolling window of leaves saved in offchain under fork-unique keys.
///
/// Leaves outside this window are canonicalized.
/// Window size is `frame_system::BlockHashCount - 1` to make sure fork-unique keys
/// can be built using `frame_system::block_hash` map.
fn offchain_canonicalization_window() -> LeafIndex {
let window_size: LeafIndex =
<T as frame_system::Config>::BlockHashCount::get().unique_saturated_into();
window_size.saturating_sub(1)
NodesUtils::node_canon_offchain_key(&T::INDEXING_PREFIX, pos)
}
/// Provide the parent number for the block that added `leaf_index` to the MMR.
@@ -355,30 +314,17 @@ impl<T: Config<I>, I: 'static> Pallet<T, I> {
.saturating_add(leaf_index.saturated_into())
}
/// Convert a `block_num` into a leaf index.
fn block_num_to_leaf_index(block_num: T::BlockNumber) -> Result<LeafIndex, primitives::Error>
/// Convert a block number into a leaf index.
fn block_num_to_leaf_index(block_num: T::BlockNumber) -> Result<LeafIndex, Error>
where
T: frame_system::Config,
{
// leaf_idx = (leaves_count - 1) - (current_block_num - block_num);
let best_block_num = <frame_system::Pallet<T>>::block_number();
let blocks_diff = best_block_num.checked_sub(&block_num).ok_or_else(|| {
primitives::Error::BlockNumToLeafIndex
.log_debug("The provided block_number is greater than the best block number.")
})?;
let blocks_diff_as_leaf_idx = blocks_diff.try_into().map_err(|_| {
primitives::Error::BlockNumToLeafIndex
.log_debug("The `blocks_diff` couldn't be converted to `LeafIndex`.")
})?;
let first_mmr_block = utils::first_mmr_block_num::<T::Header>(
<frame_system::Pallet<T>>::block_number(),
Self::mmr_leaves(),
)?;
let leaf_idx = Self::mmr_leaves()
.checked_sub(1)
.and_then(|last_leaf_idx| last_leaf_idx.checked_sub(blocks_diff_as_leaf_idx))
.ok_or_else(|| {
primitives::Error::BlockNumToLeafIndex
.log_debug("There aren't enough leaves in the chain.")
})?;
Ok(leaf_idx)
utils::block_num_to_leaf_index::<T::Header>(block_num, first_mmr_block)
}
/// Generate an MMR proof for the given `block_numbers`.
substrate/frame/merkle-mountain-range/src/mmr/mmr.rs
+1 -1
View File
@@ -18,12 +18,12 @@
use crate::{
mmr::{
storage::{OffchainStorage, RuntimeStorage, Storage},
utils::NodesUtils,
Hasher, Node, NodeOf,
},
primitives::{self, Error, NodeIndex},
Config, HashingOf,
};
use sp_mmr_primitives::{mmr_lib, utils::NodesUtils};
use sp_std::prelude::*;
/// Stateless verification of the proof for a batch of leaves.
substrate/frame/merkle-mountain-range/src/mmr/mod.rs
+1 -2
View File
@@ -17,9 +17,8 @@
mod mmr;
pub mod storage;
pub mod utils;
use sp_mmr_primitives::{DataOrHash, FullLeaf};
use sp_mmr_primitives::{mmr_lib, DataOrHash, FullLeaf};
use sp_runtime::traits;
pub use self::mmr::{verify_leaves_proof, Mmr};
substrate/frame/merkle-mountain-range/src/mmr/storage.rs
+26 -183
View File
@@ -18,16 +18,16 @@
//! An MMR storage implementation.
use codec::Encode;
use frame_support::log::{debug, error, trace};
use mmr_lib::helper;
use frame_support::log::{debug, trace};
use sp_core::offchain::StorageKind;
use sp_io::{offchain, offchain_index};
use sp_io::offchain_index;
use sp_mmr_primitives::{mmr_lib, mmr_lib::helper, utils::NodesUtils};
use sp_std::iter::Peekable;
#[cfg(not(feature = "std"))]
use sp_std::prelude::*;
use crate::{
mmr::{utils::NodesUtils, Node, NodeOf},
mmr::{Node, NodeOf},
primitives::{self, NodeIndex},
Config, Nodes, NumberOfLeaves, Pallet,
};
@@ -48,51 +48,6 @@ pub struct RuntimeStorage;
/// DOES NOT support adding new items to the MMR.
pub struct OffchainStorage;
/// Suffix of key for the 'pruning_map'.
///
/// Nodes and leaves are initially saved under fork-specific keys in offchain db,
/// eventually they are "canonicalized" and this map is used to prune non-canon entries.
const OFFCHAIN_PRUNING_MAP_KEY_SUFFIX: &str = "pruning_map";
/// Used to store offchain mappings of `BlockNumber -> Vec[Hash]` to track all forks.
/// Size of this offchain map is at most `frame_system::BlockHashCount`, its entries are pruned
/// as part of the mechanism that prunes the forks this map tracks.
pub(crate) struct PruningMap<T, I>(sp_std::marker::PhantomData<(T, I)>);
impl<T, I> PruningMap<T, I>
where
T: Config<I>,
I: 'static,
{
pub(crate) fn pruning_map_offchain_key(block: T::BlockNumber) -> sp_std::prelude::Vec<u8> {
(T::INDEXING_PREFIX, block, OFFCHAIN_PRUNING_MAP_KEY_SUFFIX).encode()
}
/// Append `hash` to the list of parent hashes for `block` in offchain db.
pub fn append(block: T::BlockNumber, hash: <T as frame_system::Config>::Hash) {
let map_key = Self::pruning_map_offchain_key(block);
offchain::local_storage_get(StorageKind::PERSISTENT, &map_key)
.and_then(|v| codec::Decode::decode(&mut &*v).ok())
.or_else(|| Some(Vec::<<T as frame_system::Config>::Hash>::new()))
.map(|mut parents| {
parents.push(hash);
offchain::local_storage_set(
StorageKind::PERSISTENT,
&map_key,
&Encode::encode(&parents),
);
});
}
/// Remove list of parent hashes for `block` from offchain db and return it.
pub fn remove(block: T::BlockNumber) -> Option<Vec<<T as frame_system::Config>::Hash>> {
let map_key = Self::pruning_map_offchain_key(block);
offchain::local_storage_get(StorageKind::PERSISTENT, &map_key).and_then(|v| {
offchain::local_storage_clear(StorageKind::PERSISTENT, &map_key);
codec::Decode::decode(&mut &*v).ok()
})
}
}
/// A storage layer for MMR.
///
/// There are two different implementations depending on the use case.
@@ -111,100 +66,6 @@ where
I: 'static,
L: primitives::FullLeaf,
{
/// Move nodes and leaves added by block `N` in offchain db from _fork-aware key_ to
/// _canonical key_,
/// where `N` is `frame_system::BlockHashCount` blocks behind current block number.
///
/// This "canonicalization" process is required because the _fork-aware key_ value depends
/// on `frame_system::block_hash(block_num)` map which only holds the last
/// `frame_system::BlockHashCount` blocks.
///
/// For the canonicalized block, prune all nodes pertaining to other forks from offchain db.
///
/// Should only be called from offchain context, because it requires both read and write
/// access to offchain db.
pub(crate) fn canonicalize_and_prune(block: T::BlockNumber) {
// Add "block_num -> hash" mapping to offchain db,
// with all forks pushing hashes to same entry (same block number).
let parent_hash = <frame_system::Pallet<T>>::parent_hash();
PruningMap::<T, I>::append(block, parent_hash);
// Effectively move a rolling window of fork-unique leaves. Once out of the window, leaves
// are "canonicalized" in offchain by moving them under `Pallet::node_canon_offchain_key`.
let leaves = NumberOfLeaves::<T, I>::get();
let window_size = Pallet::<T, I>::offchain_canonicalization_window();
if leaves >= window_size {
// Move the rolling window towards the end of `block_num->hash` mappings available
// in the runtime: we "canonicalize" the leaf at the end,
let to_canon_leaf = leaves.saturating_sub(window_size);
// and all the nodes added by that leaf.
let to_canon_nodes = NodesUtils::right_branch_ending_in_leaf(to_canon_leaf);
debug!(
target: "runtime::mmr::offchain", "Nodes to canon for leaf {}: {:?}",
to_canon_leaf, to_canon_nodes
);
// For this block number there may be node entries saved from multiple forks.
let to_canon_block_num =
Pallet::<T, I>::leaf_index_to_parent_block_num(to_canon_leaf, leaves);
// Only entries under this hash (retrieved from state on current canon fork) are to be
// persisted. All entries added by same block number on other forks will be cleared.
let to_canon_hash = <frame_system::Pallet<T>>::block_hash(to_canon_block_num);
Self::canonicalize_nodes_for_hash(&to_canon_nodes, to_canon_hash);
// Get all the forks to prune, also remove them from the offchain pruning_map.
PruningMap::<T, I>::remove(to_canon_block_num)
.map(|forks| {
Self::prune_nodes_for_forks(&to_canon_nodes, forks);
})
.unwrap_or_else(|| {
error!(
target: "runtime::mmr::offchain",
"Offchain: could not prune: no entry in pruning map for block {:?}",
to_canon_block_num
);
})
}
}
fn prune_nodes_for_forks(nodes: &[NodeIndex], forks: Vec<<T as frame_system::Config>::Hash>) {
for hash in forks {
for pos in nodes {
let key = Pallet::<T, I>::node_offchain_key(*pos, hash);
debug!(
target: "runtime::mmr::offchain",
"Clear elem at pos {} with key {:?}",
pos, key
);
offchain::local_storage_clear(StorageKind::PERSISTENT, &key);
}
}
}
fn canonicalize_nodes_for_hash(
to_canon_nodes: &[NodeIndex],
to_canon_hash: <T as frame_system::Config>::Hash,
) {
for pos in to_canon_nodes {
let key = Pallet::<T, I>::node_offchain_key(*pos, to_canon_hash);
// Retrieve the element from Off-chain DB under fork-aware key.
if let Some(elem) = offchain::local_storage_get(StorageKind::PERSISTENT, &key) {
let canon_key = Pallet::<T, I>::node_canon_offchain_key(*pos);
// Add under new canon key.
offchain::local_storage_set(StorageKind::PERSISTENT, &canon_key, &elem);
debug!(
target: "runtime::mmr::offchain",
"Moved elem at pos {} from key {:?} to canon key {:?}",
pos, key, canon_key
);
} else {
error!(
target: "runtime::mmr::offchain",
"Could not canonicalize elem at pos {} using key {:?}",
pos, key
);
}
}
}
}
impl<T, I, L> mmr_lib::MMRStore<NodeOf<T, I, L>> for Storage<OffchainStorage, T, I, L>
@@ -218,42 +79,31 @@ where
// Find out which leaf added node `pos` in the MMR.
let ancestor_leaf_idx = NodesUtils::leaf_index_that_added_node(pos);
let window_size = Pallet::<T, I>::offchain_canonicalization_window();
// Leaves older than this window should have been canonicalized.
if leaves.saturating_sub(ancestor_leaf_idx) > window_size {
let key = Pallet::<T, I>::node_canon_offchain_key(pos);
debug!(
target: "runtime::mmr::offchain", "offchain db get {}: leaf idx {:?}, key {:?}",
pos, ancestor_leaf_idx, key
);
// Just for safety, to easily handle runtime upgrades where any of the window params
// change and maybe we mess up storage migration,
// return _if and only if_ node is found (in normal conditions it's always found),
if let Some(elem) = sp_io::offchain::local_storage_get(StorageKind::PERSISTENT, &key) {
return Ok(codec::Decode::decode(&mut &*elem).ok())
}
// BUT if we DID MESS UP, fall through to searching node using fork-specific key.
// We should only get here when trying to generate proofs. The client requests
// for proofs for finalized blocks, which should usually be already canonicalized,
// unless the MMR client gadget has a delay.
let key = Pallet::<T, I>::node_canon_offchain_key(pos);
debug!(
target: "runtime::mmr::offchain", "offchain db get {}: leaf idx {:?}, canon key {:?}",
pos, ancestor_leaf_idx, key
);
// Try to retrieve the element from Off-chain DB.
if let Some(elem) = sp_io::offchain::local_storage_get(StorageKind::PERSISTENT, &key) {
return Ok(codec::Decode::decode(&mut &*elem).ok())
}
// Leaves still within the window will be found in offchain db under fork-aware keys.
// Fall through to searching node using fork-specific key.
let ancestor_parent_block_num =
Pallet::<T, I>::leaf_index_to_parent_block_num(ancestor_leaf_idx, leaves);
let ancestor_parent_hash = <frame_system::Pallet<T>>::block_hash(ancestor_parent_block_num);
let key = Pallet::<T, I>::node_offchain_key(pos, ancestor_parent_hash);
let temp_key = Pallet::<T, I>::node_temp_offchain_key(pos, ancestor_parent_hash);
debug!(
target: "runtime::mmr::offchain", "offchain db get {}: leaf idx {:?}, hash {:?}, key {:?}",
pos, ancestor_leaf_idx, ancestor_parent_hash, key
target: "runtime::mmr::offchain",
"offchain db get {}: leaf idx {:?}, hash {:?}, temp key {:?}",
pos, ancestor_leaf_idx, ancestor_parent_hash, temp_key
);
// Retrieve the element from Off-chain DB.
Ok(sp_io::offchain::local_storage_get(StorageKind::PERSISTENT, &key)
.or_else(|| {
// Again, this is just us being extra paranoid.
// We get here only if we mess up a storage migration for a runtime upgrades where
// say the window is increased, and for a little while following the upgrade there's
// leaves inside new 'window' that had been already canonicalized before upgrade.
let key = Pallet::<T, I>::node_canon_offchain_key(pos);
sp_io::offchain::local_storage_get(StorageKind::PERSISTENT, &key)
})
Ok(sp_io::offchain::local_storage_get(StorageKind::PERSISTENT, &temp_key)
.and_then(|v| codec::Decode::decode(&mut &*v).ok()))
}
@@ -342,22 +192,15 @@ where
) {
let encoded_node = node.encode();
// We store this leaf offchain keyed by `(parent_hash, node_index)` to make it
// fork-resistant. Offchain worker task will "canonicalize" it
// `frame_system::BlockHashCount` blocks later, when we are not worried about forks anymore
// (multi-era-deep forks should not happen).
let key = Pallet::<T, I>::node_offchain_key(pos, parent_hash);
// fork-resistant. The MMR client gadget task will "canonicalize" it on the first
// finality notification that follows, when we are not worried about forks anymore.
let temp_key = Pallet::<T, I>::node_temp_offchain_key(pos, parent_hash);
debug!(
target: "runtime::mmr::offchain", "offchain db set: pos {} parent_hash {:?} key {:?}",
pos, parent_hash, key
pos, parent_hash, temp_key
);
// Indexing API is used to store the full node content.
offchain_index::set(&key, &encoded_node);
// We also directly save the full node under the "canonical" key.
// This is superfluous for the normal case - this entry will possibly be overwritten
// by forks, and will also be overwritten by "offchain_worker canonicalization".
// But it is required for blocks imported during initial sync where none of the above apply
// (`offchain_worker` doesn't run for initial sync blocks).
offchain_index::set(&Pallet::<T, I>::node_canon_offchain_key(pos), &encoded_node);
offchain_index::set(&temp_key, &encoded_node);
}
}
substrate/frame/merkle-mountain-range/src/mmr/utils.rs
-176
View File
@@ -1,176 +0,0 @@
// This file is part of Substrate.
// Copyright (C) 2020-2022 Parity Technologies (UK) Ltd.
// SPDX-License-Identifier: Apache-2.0
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//! Merkle Mountain Range utilities.
use crate::primitives::{LeafIndex, NodeIndex};
use mmr_lib::helper;
/// MMR nodes & size -related utilities.
pub struct NodesUtils {
no_of_leaves: LeafIndex,
}
impl NodesUtils {
/// Create new instance of MMR nodes utilities for given number of leaves.
pub fn new(no_of_leaves: LeafIndex) -> Self {
Self { no_of_leaves }
}
/// Calculate number of peaks in the MMR.
pub fn number_of_peaks(&self) -> NodeIndex {
self.number_of_leaves().count_ones() as NodeIndex
}
/// Return the number of leaves in the MMR.
pub fn number_of_leaves(&self) -> LeafIndex {
self.no_of_leaves
}
/// Calculate the total size of MMR (number of nodes).
pub fn size(&self) -> NodeIndex {
2 * self.no_of_leaves - self.number_of_peaks()
}
/// Calculate `LeafIndex` for the leaf that added `node_index` to the MMR.
pub fn leaf_index_that_added_node(node_index: NodeIndex) -> LeafIndex {
let rightmost_leaf_pos = Self::rightmost_leaf_node_index_from_pos(node_index);
Self::leaf_node_index_to_leaf_index(rightmost_leaf_pos)
}
// Translate a _leaf_ `NodeIndex` to its `LeafIndex`.
fn leaf_node_index_to_leaf_index(pos: NodeIndex) -> LeafIndex {
if pos == 0 {
return 0
}
let peaks = helper::get_peaks(pos);
(pos + peaks.len() as u64) >> 1
}
// Starting from any node position get position of rightmost leaf; this is the leaf
// responsible for the addition of node `pos`.
fn rightmost_leaf_node_index_from_pos(pos: NodeIndex) -> NodeIndex {
pos - (helper::pos_height_in_tree(pos) as u64)
}
/// Starting from any leaf index, get the sequence of positions of the nodes added
/// to the mmr when this leaf was added (inclusive of the leaf's position itself).
/// That is, all of these nodes are right children of their respective parents.
pub fn right_branch_ending_in_leaf(leaf_index: LeafIndex) -> crate::Vec<NodeIndex> {
let pos = helper::leaf_index_to_pos(leaf_index);
let num_parents = leaf_index.trailing_ones() as u64;
return (pos..=pos + num_parents).collect()
}
}
#[cfg(test)]
mod tests {
use super::*;
use mmr_lib::helper::leaf_index_to_pos;
#[test]
fn should_calculate_node_index_from_leaf_index() {
for index in 0..100000 {
let pos = leaf_index_to_pos(index);
assert_eq!(NodesUtils::leaf_node_index_to_leaf_index(pos), index);
}
}
#[test]
fn should_calculate_right_branch_correctly() {
fn left_jump_sequence(leaf_index: LeafIndex) -> Vec<u64> {
let pos = leaf_index_to_pos(leaf_index);
let mut right_branch_ending_in_leaf = vec![pos];
let mut next_pos = pos + 1;
while mmr_lib::helper::pos_height_in_tree(next_pos) > 0 {
right_branch_ending_in_leaf.push(next_pos);
next_pos += 1;
}
right_branch_ending_in_leaf
}
for leaf_index in 0..100000 {
let pos = mmr_lib::helper::leaf_index_to_pos(leaf_index);
assert_eq!(NodesUtils::right_branch_ending_in_leaf(pos), left_jump_sequence(pos));
}
}
#[test]
fn should_calculate_rightmost_leaf_node_index_from_pos() {
for pos in 0..100000 {
let leaf_pos = NodesUtils::rightmost_leaf_node_index_from_pos(pos);
let leaf_index = NodesUtils::leaf_node_index_to_leaf_index(leaf_pos);
assert!(NodesUtils::right_branch_ending_in_leaf(leaf_index).contains(&pos));
}
}
#[test]
fn should_calculate_depth_correctly() {
assert_eq!(
vec![0, 1, 2, 3, 4, 9, 15, 21]
.into_iter()
.map(|n| NodesUtils::new(n).number_of_leaves())
.collect::<Vec<_>>(),
vec![0, 1, 2, 3, 4, 9, 15, 21]
);
}
#[test]
fn should_calculate_number_of_peaks_correctly() {
assert_eq!(
vec![0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 21]
.into_iter()
.map(|n| NodesUtils::new(n).number_of_peaks())
.collect::<Vec<_>>(),
vec![0, 1, 1, 2, 1, 2, 2, 3, 1, 2, 2, 3, 2, 3, 3, 4, 3]
);
}
#[test]
fn should_calculate_the_size_correctly() {
let _ = env_logger::try_init();
let leaves = vec![0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 21];
let sizes = vec![0, 1, 3, 4, 7, 8, 10, 11, 15, 16, 18, 19, 22, 23, 25, 26, 39];
assert_eq!(
leaves
.clone()
.into_iter()
.map(|n| NodesUtils::new(n).size())
.collect::<Vec<_>>(),
sizes.clone()
);
// size cross-check
let mut actual_sizes = vec![];
for s in &leaves[1..] {
crate::tests::new_test_ext().execute_with(|| {
let mut mmr = crate::mmr::Mmr::<
crate::mmr::storage::RuntimeStorage,
crate::mock::Test,
_,
_,
>::new(0);
for i in 0..*s {
mmr.push(i);
}
actual_sizes.push(mmr.size());
})
}
assert_eq!(sizes[1..], actual_sizes[..]);
}
}
substrate/frame/merkle-mountain-range/src/tests.rs
+39 -223
View File
@@ -15,19 +15,14 @@
// See the License for the specific language governing permissions and
// limitations under the License.
use crate::{
mmr::{storage::PruningMap, utils},
mock::*,
*,
};
use crate::{mock::*, *};
use frame_support::traits::{Get, OnInitialize};
use mmr_lib::helper;
use sp_core::{
offchain::{testing::TestOffchainExt, OffchainDbExt, OffchainWorkerExt},
H256,
};
use sp_mmr_primitives::{Compact, Proof};
use sp_mmr_primitives::{mmr_lib::helper, utils, Compact, Proof};
pub(crate) fn new_test_ext() -> sp_io::TestExternalities {
frame_system::GenesisConfig::default().build_storage::<Test>().unwrap().into()
@@ -171,21 +166,26 @@ fn should_append_to_mmr_when_on_initialize_is_called() {
let offchain_db = ext.offchain_db();
let expected = Some(mmr::Node::Data(((0, H256::repeat_byte(1)), LeafData::new(1))));
assert_eq!(offchain_db.get(&MMR::node_offchain_key(0, parent_b1)).map(decode_node), expected);
assert_eq!(offchain_db.get(&MMR::node_canon_offchain_key(0)).map(decode_node), expected);
assert_eq!(
offchain_db.get(&MMR::node_temp_offchain_key(0, parent_b1)).map(decode_node),
expected
);
let expected = Some(mmr::Node::Data(((1, H256::repeat_byte(2)), LeafData::new(2))));
assert_eq!(offchain_db.get(&MMR::node_offchain_key(1, parent_b2)).map(decode_node), expected);
assert_eq!(offchain_db.get(&MMR::node_canon_offchain_key(1)).map(decode_node), expected);
assert_eq!(
offchain_db.get(&MMR::node_temp_offchain_key(1, parent_b2)).map(decode_node),
expected
);
let expected = Some(mmr::Node::Hash(hex(
"672c04a9cd05a644789d769daa552d35d8de7c33129f8a7cbf49e595234c4854",
)));
assert_eq!(offchain_db.get(&MMR::node_offchain_key(2, parent_b2)).map(decode_node), expected);
assert_eq!(offchain_db.get(&MMR::node_canon_offchain_key(2)).map(decode_node), expected);
assert_eq!(
offchain_db.get(&MMR::node_temp_offchain_key(2, parent_b2)).map(decode_node),
expected
);
assert_eq!(offchain_db.get(&MMR::node_offchain_key(3, parent_b2)), None);
assert_eq!(offchain_db.get(&MMR::node_canon_offchain_key(3)), None);
assert_eq!(offchain_db.get(&MMR::node_temp_offchain_key(3, parent_b2)), None);
}
#[test]
@@ -219,6 +219,27 @@ fn should_construct_larger_mmr_correctly() {
});
}
#[test]
fn should_calculate_the_size_correctly() {
let _ = env_logger::try_init();
let leaves = vec![0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 21];
let sizes = vec![0, 1, 3, 4, 7, 8, 10, 11, 15, 16, 18, 19, 22, 23, 25, 26, 39];
// size cross-check
let mut actual_sizes = vec![];
for s in &leaves[1..] {
new_test_ext().execute_with(|| {
let mut mmr = mmr::Mmr::<mmr::storage::RuntimeStorage, crate::mock::Test, _, _>::new(0);
for i in 0..*s {
mmr.push(i);
}
actual_sizes.push(mmr.size());
})
}
assert_eq!(sizes[1..], actual_sizes[..]);
}
#[test]
fn should_generate_proofs_correctly() {
let _ = env_logger::try_init();
@@ -698,208 +719,6 @@ fn should_verify_on_the_next_block_since_there_is_no_pruning_yet() {
});
}
#[test]
fn should_verify_pruning_map() {
use sp_core::offchain::StorageKind;
use sp_io::offchain;
let _ = env_logger::try_init();
let mut ext = new_test_ext();
register_offchain_ext(&mut ext);
ext.execute_with(|| {
type TestPruningMap = PruningMap<Test, ()>;
fn offchain_decoded(key: Vec<u8>) -> Option<Vec<H256>> {
offchain::local_storage_get(StorageKind::PERSISTENT, &key)
.and_then(|v| codec::Decode::decode(&mut &*v).ok())
}
// test append
{
TestPruningMap::append(1, H256::repeat_byte(1));
TestPruningMap::append(2, H256::repeat_byte(21));
TestPruningMap::append(2, H256::repeat_byte(22));
TestPruningMap::append(3, H256::repeat_byte(31));
TestPruningMap::append(3, H256::repeat_byte(32));
TestPruningMap::append(3, H256::repeat_byte(33));
// `0` not present
let map_key = TestPruningMap::pruning_map_offchain_key(0);
assert_eq!(offchain::local_storage_get(StorageKind::PERSISTENT, &map_key), None);
// verify `1` entries
let map_key = TestPruningMap::pruning_map_offchain_key(1);
let expected = vec![H256::repeat_byte(1)];
assert_eq!(offchain_decoded(map_key), Some(expected));
// verify `2` entries
let map_key = TestPruningMap::pruning_map_offchain_key(2);
let expected = vec![H256::repeat_byte(21), H256::repeat_byte(22)];
assert_eq!(offchain_decoded(map_key), Some(expected));
// verify `3` entries
let map_key = TestPruningMap::pruning_map_offchain_key(3);
let expected =
vec![H256::repeat_byte(31), H256::repeat_byte(32), H256::repeat_byte(33)];
assert_eq!(offchain_decoded(map_key), Some(expected));
// `4` not present
let map_key = TestPruningMap::pruning_map_offchain_key(4);
assert_eq!(offchain::local_storage_get(StorageKind::PERSISTENT, &map_key), None);
}
// test remove
{
// `0` doesn't return anything
assert_eq!(TestPruningMap::remove(0), None);
// remove and verify `1` entries
let expected = vec![H256::repeat_byte(1)];
assert_eq!(TestPruningMap::remove(1), Some(expected));
// remove and verify `2` entries
let expected = vec![H256::repeat_byte(21), H256::repeat_byte(22)];
assert_eq!(TestPruningMap::remove(2), Some(expected));
// remove and verify `3` entries
let expected =
vec![H256::repeat_byte(31), H256::repeat_byte(32), H256::repeat_byte(33)];
assert_eq!(TestPruningMap::remove(3), Some(expected));
// `4` doesn't return anything
assert_eq!(TestPruningMap::remove(4), None);
// no entries left in offchain map
for block in 0..5 {
let map_key = TestPruningMap::pruning_map_offchain_key(block);
assert_eq!(offchain::local_storage_get(StorageKind::PERSISTENT, &map_key), None);
}
}
})
}
#[test]
fn should_canonicalize_offchain() {
use frame_support::traits::Hooks;
let _ = env_logger::try_init();
let mut ext = new_test_ext();
register_offchain_ext(&mut ext);
// adding 13 blocks that we'll later check have been canonicalized,
// (test assumes `13 < frame_system::BlockHashCount`).
let to_canon_count = 13u32;
// add 13 blocks and verify leaves and nodes for them have been added to
// offchain MMR using fork-proof keys.
for blocknum in 0..to_canon_count {
ext.execute_with(|| {
new_block();
<Pallet<Test> as Hooks<BlockNumber>>::offchain_worker(blocknum.into());
});
ext.persist_offchain_overlay();
}
let offchain_db = ext.offchain_db();
ext.execute_with(|| {
// verify leaves added by blocks 1..=13
for block_num in 1..=to_canon_count {
let parent_num: BlockNumber = (block_num - 1).into();
let leaf_index = u64::from(block_num - 1);
let pos = helper::leaf_index_to_pos(leaf_index.into());
let parent_hash = <frame_system::Pallet<Test>>::block_hash(parent_num);
// Available in offchain db under both fork-proof key and canon key.
// We'll later check it is pruned from fork-proof key.
let expected = Some(mmr::Node::Data((
(leaf_index, H256::repeat_byte(u8::try_from(block_num).unwrap())),
LeafData::new(block_num.into()),
)));
assert_eq!(
offchain_db.get(&MMR::node_canon_offchain_key(pos)).map(decode_node),
expected
);
assert_eq!(
offchain_db.get(&MMR::node_offchain_key(pos, parent_hash)).map(decode_node),
expected
);
}
// verify a couple of nodes and peaks:
// `pos` is node to verify,
// `leaf_index` is leaf that added node `pos`,
// `expected` is expected value of node at `pos`.
let verify = |pos: NodeIndex, leaf_index: LeafIndex, expected: H256| {
let parent_num: BlockNumber = leaf_index.try_into().unwrap();
let parent_hash = <frame_system::Pallet<Test>>::block_hash(parent_num);
// Available in offchain db under both fork-proof key and canon key.
// We'll later check it is pruned from fork-proof key.
let expected = Some(mmr::Node::Hash(expected));
assert_eq!(
offchain_db.get(&MMR::node_canon_offchain_key(pos)).map(decode_node),
expected
);
assert_eq!(
offchain_db.get(&MMR::node_offchain_key(pos, parent_hash)).map(decode_node),
expected
);
};
verify(2, 1, hex("672c04a9cd05a644789d769daa552d35d8de7c33129f8a7cbf49e595234c4854"));
verify(13, 7, hex("441bf63abc7cf9b9e82eb57b8111c883d50ae468d9fd7f301e12269fc0fa1e75"));
verify(21, 11, hex("f323ac1a7f56de5f40ed8df3e97af74eec0ee9d72883679e49122ffad2ffd03b"));
});
// add another `frame_system::BlockHashCount` blocks and verify all nodes and leaves
// added by our original `to_canon_count` blocks have now been canonicalized in offchain db.
let block_hash_size: u64 = <Test as frame_system::Config>::BlockHashCount::get();
let base = to_canon_count;
for blocknum in base..(base + u32::try_from(block_hash_size).unwrap()) {
ext.execute_with(|| {
new_block();
<Pallet<Test> as Hooks<BlockNumber>>::offchain_worker(blocknum.into());
});
ext.persist_offchain_overlay();
}
ext.execute_with(|| {
// verify leaves added by blocks 1..=13, should be in offchain under canon key.
for block_num in 1..=to_canon_count {
let leaf_index = u64::from(block_num - 1);
let pos = helper::leaf_index_to_pos(leaf_index.into());
let parent_num: BlockNumber = (block_num - 1).into();
let parent_hash = <frame_system::Pallet<Test>>::block_hash(parent_num);
// no longer available in fork-proof storage (was pruned),
assert_eq!(offchain_db.get(&MMR::node_offchain_key(pos, parent_hash)), None);
// but available using canon key.
assert_eq!(
offchain_db.get(&MMR::node_canon_offchain_key(pos)).map(decode_node),
Some(mmr::Node::Data((
(leaf_index, H256::repeat_byte(u8::try_from(block_num).unwrap())),
LeafData::new(block_num.into()),
)))
);
}
// also check some nodes and peaks:
// `pos` is node to verify,
// `leaf_index` is leaf that added node `pos`,
// `expected` is expected value of node at `pos`.
let verify = |pos: NodeIndex, leaf_index: LeafIndex, expected: H256| {
let parent_num: BlockNumber = leaf_index.try_into().unwrap();
let parent_hash = <frame_system::Pallet<Test>>::block_hash(parent_num);
// no longer available in fork-proof storage (was pruned),
assert_eq!(offchain_db.get(&MMR::node_offchain_key(pos, parent_hash)), None);
// but available using canon key.
assert_eq!(
offchain_db.get(&MMR::node_canon_offchain_key(pos)).map(decode_node),
Some(mmr::Node::Hash(expected))
);
};
verify(2, 1, hex("672c04a9cd05a644789d769daa552d35d8de7c33129f8a7cbf49e595234c4854"));
verify(13, 7, hex("441bf63abc7cf9b9e82eb57b8111c883d50ae468d9fd7f301e12269fc0fa1e75"));
verify(21, 11, hex("f323ac1a7f56de5f40ed8df3e97af74eec0ee9d72883679e49122ffad2ffd03b"));
});
}
#[test]
fn should_verify_canonicalized() {
use frame_support::traits::Hooks;
@@ -955,21 +774,18 @@ fn does_not_panic_when_generating_historical_proofs() {
register_offchain_ext(&mut ext);
ext.execute_with(|| {
// when leaf index is invalid
assert_eq!(
crate::Pallet::<Test>::generate_proof(vec![10], None),
Err(Error::BlockNumToLeafIndex),
);
assert_eq!(crate::Pallet::<Test>::generate_proof(vec![10], None), Err(Error::LeafNotFound),);
// when leaves count is invalid
assert_eq!(
crate::Pallet::<Test>::generate_proof(vec![3], Some(100)),
Err(Error::BlockNumToLeafIndex),
Err(Error::GenerateProof),
);
// when both leaf index and leaves count are invalid
assert_eq!(
crate::Pallet::<Test>::generate_proof(vec![10], Some(100)),
Err(Error::BlockNumToLeafIndex),
Err(Error::LeafNotFound),
);
});
}