pezkuwichain/pezkuwi-subxt
1
0
Fork 0
mirror of https://github.com/pezkuwichain/pezkuwi-subxt.git synced 2026-07-25 04:45:42 +00:00
Code Issues Packages Projects Releases Wiki Activity

pallet-mmr: fix offchain db for sync from zero (#12498)

Browse Source 
* pallet-mmr: cosmetic improvements

* pallet-mmr: fix offchain storage for initial sync

* address review comments

* pallet-mmr: change offchain fork-resistant key to `(prefix, pos, parent_hash)`

Do this so that both canon and fork-resitant keys have the same
`(prefix, pos).encode()` prefix. Might be useful in the future if we'd
be able to to "get" offchain db entries using key prefixes as well.

Signed-off-by: acatangiu <adrian@parity.io>

Signed-off-by: acatangiu <adrian@parity.io>
This commit is contained in:
Adrian Catangiu
2022-10-17 12:09:47 +03:00
committed by GitHub
parent 37538f0429
commit 4ae0d9a8e3
4 changed files with 163 additions and 108 deletions
Download Patch File Download Diff File Expand all files Collapse all files
substrate/frame/merkle-mountain-range/src/lib.rs
+66 -29
View File
@@ -57,9 +57,9 @@
#![cfg_attr(not(feature = "std"), no_std)]
use codec::Encode;
use frame_support::weights::Weight;
use frame_support::{log, traits::Get, weights::Weight};
use sp_runtime::{
traits::{self, CheckedSub, One, Saturating},
traits::{self, CheckedSub, One, Saturating, UniqueSaturatedInto},
SaturatedConversion,
};
@@ -103,6 +103,15 @@ pub trait WeightInfo {
fn on_initialize(peaks: NodeIndex) -> Weight;
}
/// A MMR specific to the pallet.
type ModuleMmr<StorageType, T, I> = mmr::Mmr<StorageType, T, I, LeafOf<T, I>>;
/// Leaf data.
type LeafOf<T, I> = <<T as Config<I>>::LeafData as primitives::LeafDataProvider>::LeafData;
/// Hashing used for the pallet.
pub(crate) type HashingOf<T, I> = <T as Config<I>>::Hashing;
#[frame_support::pallet]
pub mod pallet {
use super::*;
@@ -166,7 +175,7 @@ pub mod pallet {
/// Note that the leaf at each block MUST be unique. You may want to include a block hash or
/// block number as an easiest way to ensure that.
/// Also note that the leaf added by each block is expected to only reference data coming
/// from ancestor blocks (leaves are saved offchain using `(parent_hash, pos)` key to be
/// from ancestor blocks (leaves are saved offchain using `(pos, parent_hash)` key to be
/// fork-resistant, as such conflicts could only happen on 1-block deep forks, which means
/// two forks with identical line of ancestors compete to write the same offchain key, but
/// that's fine as long as leaves only contain data coming from ancestors - conflicting
@@ -212,12 +221,22 @@ pub mod pallet {
let leaves = Self::mmr_leaves();
let peaks_before = mmr::utils::NodesUtils::new(leaves).number_of_peaks();
let data = T::LeafData::leaf_data();
// append new leaf to MMR
let mut mmr: ModuleMmr<mmr::storage::RuntimeStorage, T, I> = mmr::Mmr::new(leaves);
mmr.push(data).expect("MMR push never fails.");
// update the size
let (leaves, root) = mmr.finalize().expect("MMR finalize never fails.");
// MMR push never fails, but better safe than sorry.
if mmr.push(data).is_none() {
log::error!(target: "runtime::mmr", "MMR push failed");
return T::WeightInfo::on_initialize(peaks_before)
}
// Update the size, `mmr.finalize()` should also never fail.
let (leaves, root) = match mmr.finalize() {
Ok((leaves, root)) => (leaves, root),
Err(e) => {
log::error!(target: "runtime::mmr", "MMR finalize failed: {:?}", e);
return T::WeightInfo::on_initialize(peaks_before)
},
};
<T::OnNewRoot as primitives::OnNewRoot<_>>::on_new_root(&root);
<NumberOfLeaves<T, I>>::put(leaves);
@@ -230,21 +249,35 @@ pub mod pallet {
fn offchain_worker(n: T::BlockNumber) {
use mmr::storage::{OffchainStorage, Storage};
// MMR pallet uses offchain storage to hold full MMR and leaves.
// The leaves are saved under fork-unique keys `(parent_hash, pos)`.
// MMR Runtime depends on `frame_system::block_hash(block_num)` mappings to find
// parent hashes for particular nodes or leaves.
// This MMR offchain worker function moves a rolling window of the same size
// as `frame_system::block_hash` map, where nodes/leaves added by blocks that are just
// The MMR nodes can be found in offchain db under either:
// - fork-unique keys `(prefix, pos, parent_hash)`, or,
// - "canonical" keys `(prefix, pos)`,
// depending on how many blocks in the past the node at position `pos` was
// added to the MMR.
//
// For the fork-unique keys, the MMR pallet depends on
// `frame_system::block_hash(parent_num)` mappings to find the relevant parent block
// hashes, so it is limited by `frame_system::BlockHashCount` in terms of how many
// historical forks it can track. Nodes added to MMR by block `N` can be found in
// offchain db at:
// - fork-unique keys `(prefix, pos, parent_hash)` when (`N` >= `latest_block` -
// `frame_system::BlockHashCount`);
// - "canonical" keys `(prefix, pos)` when (`N` < `latest_block` -
// `frame_system::BlockHashCount`);
//
// The offchain worker is responsible for maintaining the nodes' positions in
// offchain db as the chain progresses by moving a rolling window of the same size as
// `frame_system::block_hash` map, where nodes/leaves added by blocks that are just
// about to exit the window are "canonicalized" so that their offchain key no longer
// depends on `parent_hash` therefore on access to `frame_system::block_hash`.
// depends on `parent_hash`.
//
// This approach works to eliminate fork-induced leaf collisions in offchain db,
// under the assumption that no fork will be deeper than `frame_system::BlockHashCount`
// blocks (2400 blocks on Polkadot, Kusama, Rococo, etc):
// entries pertaining to block `N` where `N < current-2400` are moved to a key based
// solely on block number. The only way to have collisions is if two competing forks
// are deeper than 2400 blocks and they both "canonicalize" their view of block `N`.
// blocks:
// entries pertaining to block `N` where `N < current-BlockHashCount` are moved to a
// key based solely on block number. The only way to have collisions is if two
// competing forks are deeper than `frame_system::BlockHashCount` blocks and they
// both "canonicalize" their view of block `N`
// Once a block is canonicalized, all MMR entries pertaining to sibling blocks from
// other forks are pruned from offchain db.
Storage::<OffchainStorage, T, I, LeafOf<T, I>>::canonicalize_and_prune(n);
@@ -252,15 +285,6 @@ pub mod pallet {
}
}
/// A MMR specific to the pallet.
type ModuleMmr<StorageType, T, I> = mmr::Mmr<StorageType, T, I, LeafOf<T, I>>;
/// Leaf data.
type LeafOf<T, I> = <<T as Config<I>>::LeafData as primitives::LeafDataProvider>::LeafData;
/// Hashing used for the pallet.
pub(crate) type HashingOf<T, I> = <T as Config<I>>::Hashing;
/// Stateless MMR proof verification for batch of leaves.
///
/// This function can be used to verify received MMR [primitives::BatchProof] (`proof`)
@@ -290,19 +314,32 @@ impl<T: Config<I>, I: 'static> Pallet<T, I> {
///
/// This combination makes the offchain (key,value) entry resilient to chain forks.
fn node_offchain_key(
parent_hash: <T as frame_system::Config>::Hash,
pos: NodeIndex,
parent_hash: <T as frame_system::Config>::Hash,
) -> sp_std::prelude::Vec<u8> {
(T::INDEXING_PREFIX, parent_hash, pos).encode()
(T::INDEXING_PREFIX, pos, parent_hash).encode()
}
/// Build canonical offchain key for node `pos` in MMR.
///
/// Used for nodes added by now finalized blocks.
/// Never read keys using `node_canon_offchain_key` unless you sure that
/// there's no `node_offchain_key` key in the storage.
fn node_canon_offchain_key(pos: NodeIndex) -> sp_std::prelude::Vec<u8> {
(T::INDEXING_PREFIX, pos).encode()
}
/// Return size of rolling window of leaves saved in offchain under fork-unique keys.
///
/// Leaves outside this window are canonicalized.
/// Window size is `frame_system::BlockHashCount - 1` to make sure fork-unique keys
/// can be built using `frame_system::block_hash` map.
fn offchain_canonicalization_window() -> LeafIndex {
let window_size: LeafIndex =
<T as frame_system::Config>::BlockHashCount::get().unique_saturated_into();
window_size.saturating_sub(1)
}
/// Provide the parent number for the block that added `leaf_index` to the MMR.
fn leaf_index_to_parent_block_num(
leaf_index: LeafIndex,