feat: Rebrand Polkadot/Substrate references to PezkuwiChain
This commit systematically rebrands various references from Parity Technologies' Polkadot/Substrate ecosystem to PezkuwiChain within the kurdistan-sdk. Key changes include: - Updated external repository URLs (zombienet-sdk, parity-db, parity-scale-codec, wasm-instrument) to point to pezkuwichain forks. - Modified internal documentation and code comments to reflect PezkuwiChain naming and structure. - Replaced direct references to with or specific paths within the for XCM, Pezkuwi, and other modules. - Cleaned up deprecated issue and PR references in various and files, particularly in and modules. - Adjusted image and logo URLs in documentation to point to PezkuwiChain assets. - Removed or rephrased comments related to external Polkadot/Substrate PRs and issues. This is a significant step towards fully customizing the SDK for the PezkuwiChain ecosystem.
This commit is contained in:
@@ -0,0 +1,29 @@
|
||||
[package]
|
||||
name = "binary-merkle-tree"
|
||||
version = "13.0.0"
|
||||
authors.workspace = true
|
||||
edition.workspace = true
|
||||
license = "Apache-2.0"
|
||||
repository.workspace = true
|
||||
description = "A no-std/Bizinikiwi compatible library to construct binary merkle tree."
|
||||
homepage.workspace = true
|
||||
|
||||
[lints]
|
||||
workspace = true
|
||||
|
||||
[dependencies]
|
||||
array-bytes = { optional = true, workspace = true, default-features = true }
|
||||
codec = { workspace = true, features = ["derive"] }
|
||||
hash-db = { workspace = true }
|
||||
log = { optional = true, workspace = true }
|
||||
|
||||
[dev-dependencies]
|
||||
array-bytes = { workspace = true, default-features = true }
|
||||
pezsp-core = { workspace = true, default-features = true }
|
||||
pezsp-runtime = { workspace = true, default-features = true }
|
||||
|
||||
[features]
|
||||
debug = ["array-bytes", "log"]
|
||||
default = ["debug", "std"]
|
||||
runtime-benchmarks = ["pezsp-runtime/runtime-benchmarks"]
|
||||
std = ["codec/std", "hash-db/std", "log/std", "pezsp-core/std", "pezsp-runtime/std"]
|
||||
@@ -0,0 +1,866 @@
|
||||
// This file is part of Bizinikiwi.
|
||||
|
||||
// Copyright (C) Parity Technologies (UK) Ltd.
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
#![cfg_attr(not(feature = "std"), no_std)]
|
||||
#![warn(missing_docs)]
|
||||
|
||||
//! This crate implements a simple binary Merkle Tree utilities required for inter-op with Ethereum
|
||||
//! bridge & Solidity contract.
|
||||
//!
|
||||
//! The implementation is optimised for usage within Bizinikiwi Runtime and supports no-std
|
||||
//! compilation targets.
|
||||
//!
|
||||
//! Merkle Tree is constructed from arbitrary-length leaves, that are initially hashed using the
|
||||
//! same hasher as the inner nodes.
|
||||
//! Inner nodes are created by concatenating child hashes and hashing again. The implementation
|
||||
//! does not perform any sorting of the input data (leaves) nor when inner nodes are created.
|
||||
//!
|
||||
//! If the number of leaves is not even, last leaf (hash of) is promoted to the upper layer.
|
||||
#[cfg(not(feature = "std"))]
|
||||
extern crate alloc;
|
||||
#[cfg(not(feature = "std"))]
|
||||
use alloc::vec;
|
||||
#[cfg(not(feature = "std"))]
|
||||
use alloc::vec::Vec;
|
||||
|
||||
use codec::{Decode, Encode};
|
||||
use hash_db::Hasher;
|
||||
|
||||
/// Construct a root hash of a Binary Merkle Tree created from given leaves.
|
||||
///
|
||||
/// See crate-level docs for details about Merkle Tree construction.
|
||||
///
|
||||
/// In case an empty list of leaves is passed the function returns a 0-filled hash.
|
||||
pub fn merkle_root<H, I>(leaves: I) -> H::Out
|
||||
where
|
||||
H: Hasher,
|
||||
H::Out: Default + AsRef<[u8]>,
|
||||
I: IntoIterator,
|
||||
I::Item: AsRef<[u8]>,
|
||||
{
|
||||
let iter = leaves.into_iter().map(|l| <H as Hasher>::hash(l.as_ref()));
|
||||
merkelize::<H, _, _>(iter, &mut ()).into()
|
||||
}
|
||||
|
||||
/// Construct a root hash of a Binary Merkle Tree created from given leaves.
|
||||
///
|
||||
/// This is a raw version of the [`merkle_root`] function that expects the hashes of the leaves and
|
||||
/// not the leaves itself.
|
||||
///
|
||||
/// See crate-level docs for details about Merkle Tree construction.
|
||||
///
|
||||
/// In case an empty list of leaves is passed the function returns a 0-filled hash.
|
||||
pub fn merkle_root_raw<H, I>(leaves: I) -> H::Out
|
||||
where
|
||||
H: Hasher,
|
||||
H::Out: Default + AsRef<[u8]>,
|
||||
I: IntoIterator<Item = H::Out>,
|
||||
{
|
||||
merkelize::<H, _, _>(leaves.into_iter(), &mut ()).into()
|
||||
}
|
||||
|
||||
fn merkelize<H, V, I>(leaves: I, visitor: &mut V) -> H::Out
|
||||
where
|
||||
H: Hasher,
|
||||
H::Out: Default + AsRef<[u8]>,
|
||||
V: Visitor<H::Out>,
|
||||
I: Iterator<Item = H::Out>,
|
||||
{
|
||||
let upper = Vec::with_capacity((leaves.size_hint().1.unwrap_or(0).saturating_add(1)) / 2);
|
||||
let mut next = match merkelize_row::<H, _, _>(leaves, upper, visitor) {
|
||||
Ok(root) => return root,
|
||||
Err(next) if next.is_empty() => return H::Out::default(),
|
||||
Err(next) => next,
|
||||
};
|
||||
|
||||
let mut upper = Vec::with_capacity((next.len().saturating_add(1)) / 2);
|
||||
loop {
|
||||
visitor.move_up();
|
||||
|
||||
match merkelize_row::<H, _, _>(next.drain(..), upper, visitor) {
|
||||
Ok(root) => return root,
|
||||
Err(t) => {
|
||||
// swap collections to avoid allocations
|
||||
upper = next;
|
||||
next = t;
|
||||
},
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
/// A generated merkle proof.
|
||||
///
|
||||
/// The structure contains all necessary data to later on verify the proof and the leaf itself.
|
||||
#[derive(Debug, PartialEq, Eq, Encode, Decode)]
|
||||
pub struct MerkleProof<H, L> {
|
||||
/// Root hash of generated merkle tree.
|
||||
pub root: H,
|
||||
/// Proof items (does not contain the leaf hash, nor the root obviously).
|
||||
///
|
||||
/// This vec contains all inner node hashes necessary to reconstruct the root hash given the
|
||||
/// leaf hash.
|
||||
pub proof: Vec<H>,
|
||||
/// Number of leaves in the original tree.
|
||||
///
|
||||
/// This is needed to detect a case where we have an odd number of leaves that "get promoted"
|
||||
/// to upper layers.
|
||||
pub number_of_leaves: u32,
|
||||
/// Index of the leaf the proof is for (0-based).
|
||||
pub leaf_index: u32,
|
||||
/// Leaf content.
|
||||
pub leaf: L,
|
||||
}
|
||||
|
||||
/// A trait of object inspecting merkle root creation.
|
||||
///
|
||||
/// It can be passed to [`merkelize_row`] or [`merkelize`] functions and will be notified
|
||||
/// about tree traversal.
|
||||
trait Visitor<T> {
|
||||
/// We are moving one level up in the tree.
|
||||
fn move_up(&mut self);
|
||||
|
||||
/// We are creating an inner node from given `left` and `right` nodes.
|
||||
///
|
||||
/// Note that in case of last odd node in the row `right` might be empty.
|
||||
/// The method will also visit the `root` hash (level 0).
|
||||
///
|
||||
/// The `index` is an index of `left` item.
|
||||
fn visit(&mut self, index: u32, left: &Option<T>, right: &Option<T>);
|
||||
}
|
||||
|
||||
/// No-op implementation of the visitor.
|
||||
impl<T> Visitor<T> for () {
|
||||
fn move_up(&mut self) {}
|
||||
fn visit(&mut self, _index: u32, _left: &Option<T>, _right: &Option<T>) {}
|
||||
}
|
||||
|
||||
/// The struct collects a proof for single leaf.
|
||||
struct ProofCollection<T> {
|
||||
proof: Vec<T>,
|
||||
position: u32,
|
||||
}
|
||||
|
||||
impl<T> ProofCollection<T> {
|
||||
fn new(position: u32) -> Self {
|
||||
ProofCollection { proof: Default::default(), position }
|
||||
}
|
||||
}
|
||||
|
||||
impl<T: Copy> Visitor<T> for ProofCollection<T> {
|
||||
fn move_up(&mut self) {
|
||||
self.position /= 2;
|
||||
}
|
||||
|
||||
fn visit(&mut self, index: u32, left: &Option<T>, right: &Option<T>) {
|
||||
// we are at left branch - right goes to the proof.
|
||||
if self.position == index {
|
||||
if let Some(right) = right {
|
||||
self.proof.push(*right);
|
||||
}
|
||||
}
|
||||
// we are at right branch - left goes to the proof.
|
||||
if self.position == index + 1 {
|
||||
if let Some(left) = left {
|
||||
self.proof.push(*left);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Construct a Merkle Proof for leaves given by indices.
|
||||
///
|
||||
/// The function constructs a (partial) Merkle Tree first and stores all elements required
|
||||
/// to prove requested item (leaf) given the root hash.
|
||||
///
|
||||
/// Both the Proof and the Root Hash is returned.
|
||||
///
|
||||
/// # Panic
|
||||
///
|
||||
/// The function will panic if given `leaf_index` is greater than the number of leaves.
|
||||
pub fn merkle_proof<H, I, T>(leaves: I, leaf_index: u32) -> MerkleProof<H::Out, T>
|
||||
where
|
||||
H: Hasher,
|
||||
H::Out: Default + Copy + AsRef<[u8]>,
|
||||
I: IntoIterator<Item = T>,
|
||||
I::IntoIter: ExactSizeIterator,
|
||||
T: AsRef<[u8]>,
|
||||
{
|
||||
let mut leaf = None;
|
||||
let iter = leaves.into_iter().enumerate().map(|(idx, l)| {
|
||||
let hash = <H as Hasher>::hash(l.as_ref());
|
||||
if idx as u32 == leaf_index {
|
||||
leaf = Some(l);
|
||||
}
|
||||
hash
|
||||
});
|
||||
|
||||
let number_of_leaves = iter.len() as u32;
|
||||
let mut collect_proof = ProofCollection::new(leaf_index);
|
||||
|
||||
let root = merkelize::<H, _, _>(iter, &mut collect_proof);
|
||||
let leaf = leaf.expect("Requested `leaf_index` is greater than number of leaves.");
|
||||
|
||||
#[cfg(feature = "debug")]
|
||||
log::debug!(
|
||||
"[merkle_proof] Proof: {:?}",
|
||||
collect_proof
|
||||
.proof
|
||||
.iter()
|
||||
.map(|s| array_bytes::bytes2hex("", s))
|
||||
.collect::<Vec<_>>()
|
||||
);
|
||||
|
||||
MerkleProof { root, proof: collect_proof.proof, number_of_leaves, leaf_index, leaf }
|
||||
}
|
||||
|
||||
/// Construct a Merkle Proof for leaves given by indices.
|
||||
///
|
||||
/// This is a raw version of the [`merkle_proof`] function that expects the hashes of the leaves and
|
||||
/// not the leaves itself.
|
||||
///
|
||||
/// The function constructs a (partial) Merkle Tree first and stores all elements required
|
||||
/// to prove requested item (leaf) given the root hash.
|
||||
///
|
||||
/// Both the Proof and the Root Hash is returned.
|
||||
///
|
||||
/// # Panic
|
||||
///
|
||||
/// The function will panic if given `leaf_index` is greater than the number of leaves.
|
||||
pub fn merkle_proof_raw<H, I>(leaves: I, leaf_index: u32) -> MerkleProof<H::Out, H::Out>
|
||||
where
|
||||
H: Hasher,
|
||||
H::Out: Default + Copy + AsRef<[u8]>,
|
||||
I: IntoIterator<Item = H::Out>,
|
||||
I::IntoIter: ExactSizeIterator,
|
||||
{
|
||||
let mut leaf = None;
|
||||
let iter = leaves.into_iter().enumerate().map(|(idx, l)| {
|
||||
let hash = l;
|
||||
if idx as u32 == leaf_index {
|
||||
leaf = Some(l);
|
||||
}
|
||||
hash
|
||||
});
|
||||
|
||||
let number_of_leaves = iter.len() as u32;
|
||||
let mut collect_proof = ProofCollection::new(leaf_index);
|
||||
|
||||
let root = merkelize::<H, _, _>(iter, &mut collect_proof);
|
||||
let leaf = leaf.expect("Requested `leaf_index` is greater than number of leaves.");
|
||||
|
||||
#[cfg(feature = "debug")]
|
||||
log::debug!(
|
||||
"[merkle_proof] Proof: {:?}",
|
||||
collect_proof
|
||||
.proof
|
||||
.iter()
|
||||
.map(|s| array_bytes::bytes2hex("", s))
|
||||
.collect::<Vec<_>>()
|
||||
);
|
||||
|
||||
MerkleProof { root, proof: collect_proof.proof, number_of_leaves, leaf_index, leaf }
|
||||
}
|
||||
|
||||
/// Leaf node for proof verification.
|
||||
///
|
||||
/// Can be either a value that needs to be hashed first,
|
||||
/// or the hash itself.
|
||||
#[derive(Debug, PartialEq, Eq)]
|
||||
pub enum Leaf<'a, H> {
|
||||
/// Leaf content.
|
||||
Value(&'a [u8]),
|
||||
/// Hash of the leaf content.
|
||||
Hash(H),
|
||||
}
|
||||
|
||||
impl<'a, H, T: AsRef<[u8]>> From<&'a T> for Leaf<'a, H> {
|
||||
fn from(v: &'a T) -> Self {
|
||||
Leaf::Value(v.as_ref())
|
||||
}
|
||||
}
|
||||
|
||||
/// Verify Merkle Proof correctness versus given root hash.
|
||||
///
|
||||
/// The proof is NOT expected to contain leaf hash as the first
|
||||
/// element, but only all adjacent nodes required to eventually by process of
|
||||
/// concatenating and hashing end up with given root hash.
|
||||
///
|
||||
/// The proof must not contain the root hash.
|
||||
pub fn verify_proof<'a, H, P, L>(
|
||||
root: &'a H::Out,
|
||||
proof: P,
|
||||
number_of_leaves: u32,
|
||||
leaf_index: u32,
|
||||
leaf: L,
|
||||
) -> bool
|
||||
where
|
||||
H: Hasher,
|
||||
H::Out: PartialEq + AsRef<[u8]>,
|
||||
P: IntoIterator<Item = H::Out>,
|
||||
L: Into<Leaf<'a, H::Out>>,
|
||||
{
|
||||
if leaf_index >= number_of_leaves {
|
||||
return false;
|
||||
}
|
||||
|
||||
let leaf_hash = match leaf.into() {
|
||||
Leaf::Value(content) => <H as Hasher>::hash(content),
|
||||
Leaf::Hash(hash) => hash,
|
||||
};
|
||||
|
||||
let hash_len = <H as Hasher>::LENGTH;
|
||||
let mut combined = vec![0_u8; hash_len * 2];
|
||||
let mut position = leaf_index;
|
||||
let mut width = number_of_leaves;
|
||||
let computed = proof.into_iter().fold(leaf_hash, |a, b| {
|
||||
if position % 2 == 1 || position + 1 == width {
|
||||
combined[..hash_len].copy_from_slice(&b.as_ref());
|
||||
combined[hash_len..].copy_from_slice(&a.as_ref());
|
||||
} else {
|
||||
combined[..hash_len].copy_from_slice(&a.as_ref());
|
||||
combined[hash_len..].copy_from_slice(&b.as_ref());
|
||||
}
|
||||
let hash = <H as Hasher>::hash(&combined);
|
||||
#[cfg(feature = "debug")]
|
||||
log::debug!(
|
||||
"[verify_proof]: (a, b) {:?}, {:?} => {:?} ({:?}) hash",
|
||||
array_bytes::bytes2hex("", a),
|
||||
array_bytes::bytes2hex("", b),
|
||||
array_bytes::bytes2hex("", hash),
|
||||
array_bytes::bytes2hex("", &combined)
|
||||
);
|
||||
position /= 2;
|
||||
width = ((width - 1) / 2) + 1;
|
||||
hash
|
||||
});
|
||||
|
||||
root == &computed
|
||||
}
|
||||
|
||||
/// Processes a single row (layer) of a tree by taking pairs of elements,
|
||||
/// concatenating them, hashing and placing into resulting vector.
|
||||
///
|
||||
/// In case only one element is provided it is returned via `Ok` result, in any other case (also an
|
||||
/// empty iterator) an `Err` with the inner nodes of upper layer is returned.
|
||||
fn merkelize_row<H, V, I>(
|
||||
mut iter: I,
|
||||
mut next: Vec<H::Out>,
|
||||
visitor: &mut V,
|
||||
) -> Result<H::Out, Vec<H::Out>>
|
||||
where
|
||||
H: Hasher,
|
||||
H::Out: AsRef<[u8]>,
|
||||
V: Visitor<H::Out>,
|
||||
I: Iterator<Item = H::Out>,
|
||||
{
|
||||
#[cfg(feature = "debug")]
|
||||
log::debug!("[merkelize_row]");
|
||||
next.clear();
|
||||
|
||||
let hash_len = <H as Hasher>::LENGTH;
|
||||
let mut index = 0;
|
||||
let mut combined = vec![0_u8; hash_len * 2];
|
||||
loop {
|
||||
let a = iter.next();
|
||||
let b = iter.next();
|
||||
visitor.visit(index, &a, &b);
|
||||
|
||||
#[cfg(feature = "debug")]
|
||||
log::debug!(
|
||||
" {:?}\n {:?}",
|
||||
a.as_ref().map(|s| array_bytes::bytes2hex("", s)),
|
||||
b.as_ref().map(|s| array_bytes::bytes2hex("", s))
|
||||
);
|
||||
|
||||
index += 2;
|
||||
match (a, b) {
|
||||
(Some(a), Some(b)) => {
|
||||
combined[..hash_len].copy_from_slice(a.as_ref());
|
||||
combined[hash_len..].copy_from_slice(b.as_ref());
|
||||
|
||||
next.push(<H as Hasher>::hash(&combined));
|
||||
},
|
||||
// Odd number of items. Promote the item to the upper layer.
|
||||
(Some(a), None) if !next.is_empty() => {
|
||||
next.push(a);
|
||||
},
|
||||
// Last item = root.
|
||||
(Some(a), None) => return Ok(a),
|
||||
// Finish up, no more items.
|
||||
_ => {
|
||||
#[cfg(feature = "debug")]
|
||||
log::debug!(
|
||||
"[merkelize_row] Next: {:?}",
|
||||
next.iter().map(|s| array_bytes::bytes2hex("", s)).collect::<Vec<_>>()
|
||||
);
|
||||
return Err(next);
|
||||
},
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use pezsp_core::H256;
|
||||
use pezsp_runtime::traits::Keccak256;
|
||||
|
||||
#[test]
|
||||
fn should_generate_empty_root() {
|
||||
// given
|
||||
let data: Vec<[u8; 1]> = Default::default();
|
||||
|
||||
// when
|
||||
let out = merkle_root::<Keccak256, _>(data);
|
||||
|
||||
// then
|
||||
assert_eq!(
|
||||
array_bytes::bytes2hex("", out),
|
||||
"0000000000000000000000000000000000000000000000000000000000000000"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn should_generate_single_root() {
|
||||
// given
|
||||
let data = vec![array_bytes::hex2array_unchecked::<_, 20>(
|
||||
"E04CC55ebEE1cBCE552f250e85c57B70B2E2625b",
|
||||
)];
|
||||
|
||||
// when
|
||||
let out = merkle_root::<Keccak256, _>(data);
|
||||
|
||||
// then
|
||||
assert_eq!(
|
||||
array_bytes::bytes2hex("", out),
|
||||
"aeb47a269393297f4b0a3c9c9cfd00c7a4195255274cf39d83dabc2fcc9ff3d7"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn should_generate_root_pow_2() {
|
||||
// given
|
||||
let data = vec![
|
||||
array_bytes::hex2array_unchecked::<_, 20>("E04CC55ebEE1cBCE552f250e85c57B70B2E2625b"),
|
||||
array_bytes::hex2array_unchecked::<_, 20>("25451A4de12dcCc2D166922fA938E900fCc4ED24"),
|
||||
];
|
||||
|
||||
// when
|
||||
let out = merkle_root::<Keccak256, _>(data);
|
||||
|
||||
// then
|
||||
assert_eq!(
|
||||
array_bytes::bytes2hex("", out),
|
||||
"697ea2a8fe5b03468548a7a413424a6292ab44a82a6f5cc594c3fa7dda7ce402"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn should_generate_root_complex() {
|
||||
let test = |root, data| {
|
||||
assert_eq!(array_bytes::bytes2hex("", &merkle_root::<Keccak256, _>(data)), root);
|
||||
};
|
||||
|
||||
test(
|
||||
"aff1208e69c9e8be9b584b07ebac4e48a1ee9d15ce3afe20b77a4d29e4175aa3",
|
||||
vec!["a", "b", "c"],
|
||||
);
|
||||
|
||||
test(
|
||||
"b8912f7269068901f231a965adfefbc10f0eedcfa61852b103efd54dac7db3d7",
|
||||
vec!["a", "b", "a"],
|
||||
);
|
||||
|
||||
test(
|
||||
"dc8e73fe6903148ff5079baecc043983625c23b39f31537e322cd0deee09fa9c",
|
||||
vec!["a", "b", "a", "b"],
|
||||
);
|
||||
|
||||
test(
|
||||
"fb3b3be94be9e983ba5e094c9c51a7d96a4fa2e5d8e891df00ca89ba05bb1239",
|
||||
vec!["a", "b", "c", "d", "e", "f", "g", "h", "i", "j"],
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn should_generate_and_verify_proof_simple() {
|
||||
// given
|
||||
let data = vec!["a", "b", "c"];
|
||||
|
||||
// when
|
||||
let proof0 = merkle_proof::<Keccak256, _, _>(data.clone(), 0);
|
||||
assert!(verify_proof::<Keccak256, _, _>(
|
||||
&proof0.root,
|
||||
proof0.proof.clone(),
|
||||
data.len() as _,
|
||||
proof0.leaf_index,
|
||||
&proof0.leaf,
|
||||
));
|
||||
|
||||
let proof1 = merkle_proof::<Keccak256, _, _>(data.clone(), 1);
|
||||
assert!(verify_proof::<Keccak256, _, _>(
|
||||
&proof1.root,
|
||||
proof1.proof,
|
||||
data.len() as _,
|
||||
proof1.leaf_index,
|
||||
&proof1.leaf,
|
||||
));
|
||||
|
||||
let proof2 = merkle_proof::<Keccak256, _, _>(data.clone(), 2);
|
||||
assert!(verify_proof::<Keccak256, _, _>(
|
||||
&proof2.root,
|
||||
proof2.proof,
|
||||
data.len() as _,
|
||||
proof2.leaf_index,
|
||||
&proof2.leaf
|
||||
));
|
||||
|
||||
// then
|
||||
assert_eq!(
|
||||
array_bytes::bytes2hex("", &proof0.root),
|
||||
array_bytes::bytes2hex("", &proof1.root)
|
||||
);
|
||||
assert_eq!(
|
||||
array_bytes::bytes2hex("", &proof2.root),
|
||||
array_bytes::bytes2hex("", &proof1.root)
|
||||
);
|
||||
|
||||
assert!(!verify_proof::<Keccak256, _, _>(
|
||||
&array_bytes::hex2array_unchecked(
|
||||
"fb3b3be94be9e983ba5e094c9c51a7d96a4fa2e5d8e891df00ca89ba05bb1239"
|
||||
)
|
||||
.into(),
|
||||
proof0.proof,
|
||||
data.len() as _,
|
||||
proof0.leaf_index,
|
||||
&proof0.leaf
|
||||
));
|
||||
|
||||
assert!(!verify_proof::<Keccak256, _, _>(
|
||||
&proof0.root.into(),
|
||||
vec![],
|
||||
data.len() as _,
|
||||
proof0.leaf_index,
|
||||
&proof0.leaf
|
||||
));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn should_generate_and_verify_proof_complex() {
|
||||
// given
|
||||
let data = vec!["a", "b", "c", "d", "e", "f", "g", "h", "i", "j"];
|
||||
|
||||
for l in 0..data.len() as u32 {
|
||||
// when
|
||||
let proof = merkle_proof::<Keccak256, _, _>(data.clone(), l);
|
||||
// then
|
||||
assert!(verify_proof::<Keccak256, _, _>(
|
||||
&proof.root,
|
||||
proof.proof,
|
||||
data.len() as _,
|
||||
proof.leaf_index,
|
||||
&proof.leaf
|
||||
));
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn should_generate_and_verify_proof_large() {
|
||||
// given
|
||||
let mut data = vec![];
|
||||
for i in 1..16 {
|
||||
for c in 'a'..'z' {
|
||||
if c as usize % i != 0 {
|
||||
data.push(c.to_string());
|
||||
}
|
||||
}
|
||||
|
||||
for l in 0..data.len() as u32 {
|
||||
// when
|
||||
let proof = merkle_proof::<Keccak256, _, _>(data.clone(), l);
|
||||
// then
|
||||
assert!(verify_proof::<Keccak256, _, _>(
|
||||
&proof.root,
|
||||
proof.proof,
|
||||
data.len() as _,
|
||||
proof.leaf_index,
|
||||
&proof.leaf
|
||||
));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn should_generate_and_verify_proof_large_tree() {
|
||||
// given
|
||||
let mut data = vec![];
|
||||
for i in 0..6000 {
|
||||
data.push(format!("{}", i));
|
||||
}
|
||||
|
||||
for l in (0..data.len() as u32).step_by(13) {
|
||||
// when
|
||||
let proof = merkle_proof::<Keccak256, _, _>(data.clone(), l);
|
||||
// then
|
||||
assert!(verify_proof::<Keccak256, _, _>(
|
||||
&proof.root,
|
||||
proof.proof,
|
||||
data.len() as _,
|
||||
proof.leaf_index,
|
||||
&proof.leaf
|
||||
));
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
#[should_panic]
|
||||
fn should_panic_on_invalid_leaf_index() {
|
||||
merkle_proof::<Keccak256, _, _>(vec!["a"], 5);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn should_generate_and_verify_proof_on_test_data() {
|
||||
let addresses = vec![
|
||||
"0x9aF1Ca5941148eB6A3e9b9C741b69738292C533f",
|
||||
"0xDD6ca953fddA25c496165D9040F7F77f75B75002",
|
||||
"0x60e9C47B64Bc1C7C906E891255EaEC19123E7F42",
|
||||
"0xfa4859480Aa6D899858DE54334d2911E01C070df",
|
||||
"0x19B9b128470584F7209eEf65B69F3624549Abe6d",
|
||||
"0xC436aC1f261802C4494504A11fc2926C726cB83b",
|
||||
"0xc304C8C2c12522F78aD1E28dD86b9947D7744bd0",
|
||||
"0xDa0C2Cba6e832E55dE89cF4033affc90CC147352",
|
||||
"0xf850Fd22c96e3501Aad4CDCBf38E4AEC95622411",
|
||||
"0x684918D4387CEb5E7eda969042f036E226E50642",
|
||||
"0x963F0A1bFbb6813C0AC88FcDe6ceB96EA634A595",
|
||||
"0x39B38ad74b8bCc5CE564f7a27Ac19037A95B6099",
|
||||
"0xC2Dec7Fdd1fef3ee95aD88EC8F3Cd5bd4065f3C7",
|
||||
"0x9E311f05c2b6A43C2CCF16fB2209491BaBc2ec01",
|
||||
"0x927607C30eCE4Ef274e250d0bf414d4a210b16f0",
|
||||
"0x98882bcf85E1E2DFF780D0eB360678C1cf443266",
|
||||
"0xFBb50191cd0662049E7C4EE32830a4Cc9B353047",
|
||||
"0x963854fc2C358c48C3F9F0A598B9572c581B8DEF",
|
||||
"0xF9D7Bc222cF6e3e07bF66711e6f409E51aB75292",
|
||||
"0xF2E3fd32D063F8bBAcB9e6Ea8101C2edd899AFe6",
|
||||
"0x407a5b9047B76E8668570120A96d580589fd1325",
|
||||
"0xEAD9726FAFB900A07dAd24a43AE941d2eFDD6E97",
|
||||
"0x42f5C8D9384034A9030313B51125C32a526b6ee8",
|
||||
"0x158fD2529Bc4116570Eb7C80CC76FEf33ad5eD95",
|
||||
"0x0A436EE2E4dEF3383Cf4546d4278326Ccc82514E",
|
||||
"0x34229A215db8FeaC93Caf8B5B255e3c6eA51d855",
|
||||
"0xEb3B7CF8B1840242CB98A732BA464a17D00b5dDF",
|
||||
"0x2079692bf9ab2d6dc7D79BBDdEE71611E9aA3B72",
|
||||
"0x46e2A67e5d450e2Cf7317779f8274a2a630f3C9B",
|
||||
"0xA7Ece4A5390DAB18D08201aE18800375caD78aab",
|
||||
"0x15E1c0D24D62057Bf082Cb2253dA11Ef0d469570",
|
||||
"0xADDEF4C9b5687Eb1F7E55F2251916200A3598878",
|
||||
"0xe0B16Fb96F936035db2b5A68EB37D470fED2f013",
|
||||
"0x0c9A84993feaa779ae21E39F9793d09e6b69B62D",
|
||||
"0x3bc4D5148906F70F0A7D1e2756572655fd8b7B34",
|
||||
"0xFf4675C26903D5319795cbd3a44b109E7DDD9fDe",
|
||||
"0xCec4450569A8945C6D2Aba0045e4339030128a92",
|
||||
"0x85f0584B10950E421A32F471635b424063FD8405",
|
||||
"0xb38bEe7Bdc0bC43c096e206EFdFEad63869929E3",
|
||||
"0xc9609466274Fef19D0e58E1Ee3b321D5C141067E",
|
||||
"0xa08EA868cF75268E7401021E9f945BAe73872ecc",
|
||||
"0x67C9Cb1A29E964Fe87Ff669735cf7eb87f6868fE",
|
||||
"0x1B6BEF636aFcdd6085cD4455BbcC93796A12F6E2",
|
||||
"0x46B37b243E09540b55cF91C333188e7D5FD786dD",
|
||||
"0x8E719E272f62Fa97da93CF9C941F5e53AA09e44a",
|
||||
"0xa511B7E7DB9cb24AD5c89fBb6032C7a9c2EfA0a5",
|
||||
"0x4D11FDcAeD335d839132AD450B02af974A3A66f8",
|
||||
"0xB8cf790a5090E709B4619E1F335317114294E17E",
|
||||
"0x7f0f57eA064A83210Cafd3a536866ffD2C5eDCB3",
|
||||
"0xC03C848A4521356EF800e399D889e9c2A25D1f9E",
|
||||
"0xC6b03DF05cb686D933DD31fCa5A993bF823dc4FE",
|
||||
"0x58611696b6a8102cf95A32c25612E4cEF32b910F",
|
||||
"0x2ed4bC7197AEF13560F6771D930Bf907772DE3CE",
|
||||
"0x3C5E58f334306be029B0e47e119b8977B2639eb4",
|
||||
"0x288646a1a4FeeC560B349d210263c609aDF649a6",
|
||||
"0xb4F4981E0d027Dc2B3c86afA0D0fC03d317e83C0",
|
||||
"0xaAE4A87F8058feDA3971f9DEd639Ec9189aA2500",
|
||||
"0x355069DA35E598913d8736E5B8340527099960b8",
|
||||
"0x3cf5A0F274cd243C0A186d9fCBdADad089821B93",
|
||||
"0xca55155dCc4591538A8A0ca322a56EB0E4aD03C4",
|
||||
"0xE824D0268366ec5C4F23652b8eD70D552B1F2b8B",
|
||||
"0x84C3e9B25AE8a9b39FF5E331F9A597F2DCf27Ca9",
|
||||
"0xcA0018e278751De10d26539915d9c7E7503432FE",
|
||||
"0xf13077dE6191D6c1509ac7E088b8BE7Fe656c28b",
|
||||
"0x7a6bcA1ec9Db506e47ac6FD86D001c2aBc59C531",
|
||||
"0xeA7f9A2A9dd6Ba9bc93ca615C3Ddf26973146911",
|
||||
"0x8D0d8577e16F8731d4F8712BAbFa97aF4c453458",
|
||||
"0xB7a7855629dF104246997e9ACa0E6510df75d0ea",
|
||||
"0x5C1009BDC70b0C8Ab2e5a53931672ab448C17c89",
|
||||
"0x40B47D1AfefEF5eF41e0789F0285DE7b1C31631C",
|
||||
"0x5086933d549cEcEB20652CE00973703CF10Da373",
|
||||
"0xeb364f6FE356882F92ae9314fa96116Cf65F47d8",
|
||||
"0xdC4D31516A416cEf533C01a92D9a04bbdb85EE67",
|
||||
"0x9b36E086E5A274332AFd3D8509e12ca5F6af918d",
|
||||
"0xBC26394fF36e1673aE0608ce91A53B9768aD0D76",
|
||||
"0x81B5AB400be9e563fA476c100BE898C09966426c",
|
||||
"0x9d93C8ae5793054D28278A5DE6d4653EC79e90FE",
|
||||
"0x3B8E75804F71e121008991E3177fc942b6c28F50",
|
||||
"0xC6Eb5886eB43dD473f5BB4e21e56E08dA464D9B4",
|
||||
"0xfdf1277b71A73c813cD0e1a94B800f4B1Db66DBE",
|
||||
"0xc2ff2cCc98971556670e287Ff0CC39DA795231ad",
|
||||
"0x76b7E1473f0D0A87E9B4a14E2B179266802740f5",
|
||||
"0xA7Bc965660a6EF4687CCa4F69A97563163A3C2Ef",
|
||||
"0xB9C2b47888B9F8f7D03dC1de83F3F55E738CebD3",
|
||||
"0xEd400162E6Dd6bD2271728FFb04176bF770De94a",
|
||||
"0xE3E8331156700339142189B6E555DCb2c0962750",
|
||||
"0xbf62e342Bc7706a448EdD52AE871d9C4497A53b1",
|
||||
"0xb9d7A1A111eed75714a0AcD2dd467E872eE6B03D",
|
||||
"0x03942919DFD0383b8c574AB8A701d89fd4bfA69D",
|
||||
"0x0Ef4C92355D3c8c7050DFeb319790EFCcBE6fe9e",
|
||||
"0xA6895a3cf0C60212a73B3891948ACEcF1753f25E",
|
||||
"0x0Ed509239DB59ef3503ded3d31013C983d52803A",
|
||||
"0xc4CE8abD123BfAFc4deFf37c7D11DeCd5c350EE4",
|
||||
"0x4A4Bf59f7038eDcd8597004f35d7Ee24a7Bdd2d3",
|
||||
"0x5769E8e8A2656b5ed6b6e6fa2a2bFAeaf970BB87",
|
||||
"0xf9E15cCE181332F4F57386687c1776b66C377060",
|
||||
"0xc98f8d4843D56a46C21171900d3eE538Cc74dbb5",
|
||||
"0x3605965B47544Ce4302b988788B8195601AE4dEd",
|
||||
"0xe993BDfdcAac2e65018efeE0F69A12678031c71d",
|
||||
"0x274fDf8801385D3FAc954BCc1446Af45f5a8304c",
|
||||
"0xBFb3f476fcD6429F4a475bA23cEFdDdd85c6b964",
|
||||
"0x806cD16588Fe812ae740e931f95A289aFb4a4B50",
|
||||
"0xa89488CE3bD9C25C3aF797D1bbE6CA689De79d81",
|
||||
"0xd412f1AfAcf0Ebf3Cd324593A231Fc74CC488B12",
|
||||
"0xd1f715b2D7951d54bc31210BbD41852D9BF98Ed1",
|
||||
"0xf65aD707c344171F467b2ADba3d14f312219cE23",
|
||||
"0x2971a4b242e9566dEF7bcdB7347f5E484E11919B",
|
||||
"0x12b113D6827E07E7D426649fBd605f427da52314",
|
||||
"0x1c6CA45171CDb9856A6C9Dba9c5F1216913C1e97",
|
||||
"0x11cC6ee1d74963Db23294FCE1E3e0A0555779CeA",
|
||||
"0x8Aa1C721255CDC8F895E4E4c782D86726b068667",
|
||||
"0xA2cDC1f37510814485129aC6310b22dF04e9Bbf0",
|
||||
"0xCf531b71d388EB3f5889F1f78E0d77f6fb109767",
|
||||
"0xBe703e3545B2510979A0cb0C440C0Fba55c6dCB5",
|
||||
"0x30a35886F989db39c797D8C93880180Fdd71b0c8",
|
||||
"0x1071370D981F60c47A9Cd27ac0A61873a372cBB2",
|
||||
"0x3515d74A11e0Cb65F0F46cB70ecf91dD1712daaa",
|
||||
"0x50500a3c2b7b1229c6884505D00ac6Be29Aecd0C",
|
||||
"0x9A223c2a11D4FD3585103B21B161a2B771aDA3d1",
|
||||
"0xd7218df03AD0907e6c08E707B15d9BD14285e657",
|
||||
"0x76CfD72eF5f93D1a44aD1F80856797fBE060c70a",
|
||||
"0x44d093cB745944991EFF5cBa151AA6602d6f5420",
|
||||
"0x626516DfF43bf09A71eb6fd1510E124F96ED0Cde",
|
||||
"0x6530824632dfe099304E2DC5701cA99E6d031E08",
|
||||
"0x57e6c423d6a7607160d6379A0c335025A14DaFC0",
|
||||
"0x3966D4AD461Ef150E0B10163C81E79b9029E69c3",
|
||||
"0xF608aCfd0C286E23721a3c347b2b65039f6690F1",
|
||||
"0xbfB8FAac31A25646681936977837f7740fCd0072",
|
||||
"0xd80aa634a623a7ED1F069a1a3A28a173061705c7",
|
||||
"0x9122a77B36363e24e12E1E2D73F87b32926D3dF5",
|
||||
"0x62562f0d1cD31315bCCf176049B6279B2bfc39C2",
|
||||
"0x48aBF7A2a7119e5675059E27a7082ba7F38498b2",
|
||||
"0xb4596983AB9A9166b29517acD634415807569e5F",
|
||||
"0x52519D16E20BC8f5E96Da6d736963e85b2adA118",
|
||||
"0x7663893C3dC0850EfC5391f5E5887eD723e51B83",
|
||||
"0x5FF323a29bCC3B5b4B107e177EccEF4272959e61",
|
||||
"0xee6e499AdDf4364D75c05D50d9344e9daA5A9AdF",
|
||||
"0x1631b0BD31fF904aD67dD58994C6C2051CDe4E75",
|
||||
"0xbc208e9723D44B9811C428f6A55722a26204eEF2",
|
||||
"0xe76103a222Ee2C7Cf05B580858CEe625C4dc00E1",
|
||||
"0xC71Bb2DBC51760f4fc2D46D84464410760971B8a",
|
||||
"0xB4C18811e6BFe564D69E12c224FFc57351f7a7ff",
|
||||
"0xD11DB0F5b41061A887cB7eE9c8711438844C298A",
|
||||
"0xB931269934A3D4432c084bAAc3d0de8143199F4f",
|
||||
"0x070037cc85C761946ec43ea2b8A2d5729908A2a1",
|
||||
"0x2E34aa8C95Ffdbb37f14dCfBcA69291c55Ba48DE",
|
||||
"0x052D93e8d9220787c31d6D83f87eC7dB088E998f",
|
||||
"0x498dAC6C69b8b9ad645217050054840f1D91D029",
|
||||
"0xE4F7D60f9d84301e1fFFd01385a585F3A11F8E89",
|
||||
"0xEa637992f30eA06460732EDCBaCDa89355c2a107",
|
||||
"0x4960d8Da07c27CB6Be48a79B96dD70657c57a6bF",
|
||||
"0x7e471A003C8C9fdc8789Ded9C3dbe371d8aa0329",
|
||||
"0xd24265Cc10eecb9e8d355CCc0dE4b11C556E74D7",
|
||||
"0xDE59C8f7557Af779674f41CA2cA855d571018690",
|
||||
"0x2fA8A6b3b6226d8efC9d8f6EBDc73Ca33DDcA4d8",
|
||||
"0xe44102664c6c2024673Ff07DFe66E187Db77c65f",
|
||||
"0x94E3f4f90a5f7CBF2cc2623e66B8583248F01022",
|
||||
"0x0383EdBbc21D73DEd039E9C1Ff6bf56017b4CC40",
|
||||
"0x64C3E49898B88d1E0f0d02DA23E0c00A2Cd0cA99",
|
||||
"0xF4ccfB67b938d82B70bAb20975acFAe402E812E1",
|
||||
"0x4f9ee5829e9852E32E7BC154D02c91D8E203e074",
|
||||
"0xb006312eF9713463bB33D22De60444Ba95609f6B",
|
||||
"0x7Cbe76ef69B52110DDb2e3b441C04dDb11D63248",
|
||||
"0x70ADEEa65488F439392B869b1Df7241EF317e221",
|
||||
"0x64C0bf8AA36Ba590477585Bc0D2BDa7970769463",
|
||||
"0xA4cDc98593CE52d01Fe5Ca47CB3dA5320e0D7592",
|
||||
"0xc26B34D375533fFc4c5276282Fa5D660F3d8cbcB",
|
||||
];
|
||||
let root: H256 = array_bytes::hex2array_unchecked(
|
||||
"72b0acd7c302a84f1f6b6cefe0ba7194b7398afb440e1b44a9dbbe270394ca53",
|
||||
)
|
||||
.into();
|
||||
|
||||
let data = addresses
|
||||
.into_iter()
|
||||
.map(|address| array_bytes::hex2bytes_unchecked(&address))
|
||||
.collect::<Vec<_>>();
|
||||
|
||||
for l in 0..data.len() as u32 {
|
||||
// when
|
||||
let proof = merkle_proof::<Keccak256, _, _>(data.clone(), l);
|
||||
assert_eq!(array_bytes::bytes2hex("", &proof.root), array_bytes::bytes2hex("", &root));
|
||||
assert_eq!(proof.leaf_index, l);
|
||||
assert_eq!(&proof.leaf, &data[l as usize]);
|
||||
|
||||
// then
|
||||
assert!(verify_proof::<Keccak256, _, _>(
|
||||
&proof.root,
|
||||
proof.proof,
|
||||
data.len() as _,
|
||||
proof.leaf_index,
|
||||
&proof.leaf
|
||||
));
|
||||
}
|
||||
|
||||
let proof = merkle_proof::<Keccak256, _, _>(data.clone(), data.len() as u32 - 1);
|
||||
|
||||
assert_eq!(
|
||||
proof,
|
||||
MerkleProof {
|
||||
root,
|
||||
proof: vec![
|
||||
array_bytes::hex2array_unchecked(
|
||||
"340bcb1d49b2d82802ddbcf5b85043edb3427b65d09d7f758fbc76932ad2da2f"
|
||||
)
|
||||
.into(),
|
||||
array_bytes::hex2array_unchecked(
|
||||
"ba0580e5bd530bc93d61276df7969fb5b4ae8f1864b4a28c280249575198ff1f"
|
||||
)
|
||||
.into(),
|
||||
array_bytes::hex2array_unchecked(
|
||||
"d02609d2bbdb28aa25f58b85afec937d5a4c85d37925bce6d0cf802f9d76ba79"
|
||||
)
|
||||
.into(),
|
||||
array_bytes::hex2array_unchecked(
|
||||
"ae3f8991955ed884613b0a5f40295902eea0e0abe5858fc520b72959bc016d4e"
|
||||
)
|
||||
.into(),
|
||||
],
|
||||
number_of_leaves: data.len() as _,
|
||||
leaf_index: data.len() as u32 - 1,
|
||||
leaf: array_bytes::hex2array_unchecked::<_, 20>(
|
||||
"c26B34D375533fFc4c5276282Fa5D660F3d8cbcB"
|
||||
)
|
||||
.to_vec(),
|
||||
}
|
||||
);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,31 @@
|
||||
[package]
|
||||
name = "bizinikiwi-bip39"
|
||||
version = "0.4.7"
|
||||
license = "Apache-2.0"
|
||||
description = "Converting BIP39 entropy to valid Bizinikiwi (sr25519) SecretKeys"
|
||||
documentation = "https://docs.rs/bizinikiwi-bip39"
|
||||
authors.workspace = true
|
||||
edition.workspace = true
|
||||
repository.workspace = true
|
||||
|
||||
[dependencies]
|
||||
hmac = { workspace = true }
|
||||
pbkdf2 = { workspace = true }
|
||||
schnorrkel = { workspace = true }
|
||||
sha2 = { workspace = true }
|
||||
zeroize = { workspace = true }
|
||||
|
||||
[dev-dependencies]
|
||||
bip39 = { workspace = true, default-features = true }
|
||||
rustc-hex = { workspace = true, default-features = true }
|
||||
|
||||
[features]
|
||||
default = ["std"]
|
||||
std = [
|
||||
"hmac/std",
|
||||
"pbkdf2/std",
|
||||
"schnorrkel/std",
|
||||
"sha2/std",
|
||||
"zeroize/alloc",
|
||||
"zeroize/std",
|
||||
]
|
||||
@@ -0,0 +1,55 @@
|
||||
# Bizinikiwi BIP39
|
||||
|
||||
This is a crate for deriving secret keys for Ristretto compressed Ed25519 (should be compatible with Ed25519 at this
|
||||
time) from BIP39 phrases.
|
||||
|
||||
## Why?
|
||||
|
||||
The natural approach here would be to use the 64-byte seed generated from the BIP39 phrase, and use that to construct
|
||||
the key. This approach, while reasonable and fairly straight forward to implement, also means we would have to inherit
|
||||
all the characteristics of seed generation. Since we are breaking compatibility with both BIP32 and BIP44 anyway (which
|
||||
we are free to do as we are no longer using the Secp256k1 curve), there is also no reason why we should adhere to BIP39
|
||||
seed generation from the mnemonic.
|
||||
|
||||
BIP39 seed generation was designed to be compatible with user supplied brain wallet phrases as well as being extensible
|
||||
to wallets providing their own dictionaries and checksum mechanism. Issues with those two points:
|
||||
|
||||
1. Brain wallets are a horrible idea, simply because humans are bad entropy generators. It's next to impossible to
|
||||
educate users on how to use that feature in a secure manner. The 2048 rounds of PBKDF2 is a mere inconvenience that
|
||||
offers no real protection against dictionary attacks for anyone equipped with modern consumer hardware. Brain wallets
|
||||
have given users false sense of security. _People have lost money_ this way and wallet providers today tend to stick
|
||||
to CSPRNG supplied dictionary phrases.
|
||||
|
||||
2. Providing own dictionaries felt into the _you ain't gonna need it_ anti-pattern category on day 1. Wallet providers
|
||||
(be it hardware or software) typically want their products to be compatible with other wallets so that users can
|
||||
migrate to their product without having to migrate all their assets.
|
||||
|
||||
To achieve the above phrases have to be precisely encoded in _The One True Canonical Encoding_, for which UTF-8 NFKD was
|
||||
chosen. This is largely irrelevant (and even ignored) for English phrases, as they encode to basically just ASCII in
|
||||
virtually every character encoding known to mankind, but immediately becomes a problem for dictionaries that do use
|
||||
non-ASCII characters. Even if the right encoding is used and implemented correctly, there are still [other caveats
|
||||
present for some non-english dictionaries](https://github.com/bitcoin/bips/blob/master/bip-0039/bip-0039-wordlists.md),
|
||||
such as normalizing spaces to a canonical form, or making some latin based characters equivalent to their base in
|
||||
dictionary lookups (eg. Spanish `ñ` and `n` are meant to be interchangeable). Thinking about all of this gives me a
|
||||
headache, and opens doors for disagreements between buggy implementations, breaking compatibility.
|
||||
|
||||
BIP39 does already provide a form of the mnemonic that is free from all of these issues: the entropy byte array. Since
|
||||
verifying the checksum requires that we recover the entropy from which the phrase was generated, no extra work is
|
||||
actually needed here. Wallet implementors can encode the dictionaries in whatever encoding they find convenient (as
|
||||
long as they are the standard BIP39 dictionaries), no harm in using UTF-16 string primitives that Java and JavaScript
|
||||
provide. Since the dictionary is fixed and known, and the checksum is done on the entropy itself, the exact character
|
||||
encoding used becomes irrelevant, as are the precise codepoints and amount of whitespace around the words. It is thus
|
||||
much harder to create a buggy implementation.
|
||||
|
||||
PBKDF2 was kept in place, along with the password. Using 24 words (with its 256 bits entropy) makes the extra hashing
|
||||
redundant (if you could brute force 256 bit entropy, you can also just brute force secret keys), however some users
|
||||
might be still using 12 word phrases from other applications. There is no good reason to prohibit users from recovering
|
||||
their old wallets using 12 words that I can see, in which case the extra hashing does provide _some_ protection.
|
||||
Passwords are also a feature that some power users find useful - particularly for creating a decoy address with a small
|
||||
balance with empty password, while the funds proper are stored on an address that requires a password to be entered.
|
||||
|
||||
## Why not ditch BIP39 altogether?
|
||||
|
||||
Because there are hardware wallets that use a single phrase for the entire device, and operate multiple accounts on
|
||||
multiple networks using that. A completely different wordlist would make their life much harder when it comes to
|
||||
providing future Bizinikiwi support.
|
||||
@@ -0,0 +1,231 @@
|
||||
// Copyright (C) Parity Technologies (UK) Ltd.
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
#![cfg_attr(not(feature = "std"), no_std)]
|
||||
|
||||
#[cfg(not(feature = "std"))]
|
||||
extern crate alloc;
|
||||
#[cfg(not(feature = "std"))]
|
||||
use alloc::string::String;
|
||||
|
||||
use hmac::Hmac;
|
||||
use pbkdf2::pbkdf2;
|
||||
use schnorrkel::keys::MiniSecretKey;
|
||||
use sha2::Sha512;
|
||||
use zeroize::Zeroize;
|
||||
|
||||
#[derive(Clone, Copy, PartialEq, Eq, Debug)]
|
||||
pub enum Error {
|
||||
InvalidEntropy,
|
||||
}
|
||||
|
||||
/// `entropy` should be a byte array from a correctly recovered and checksumed BIP39.
|
||||
///
|
||||
/// This function accepts slices of different length for different word lengths:
|
||||
///
|
||||
/// + 16 bytes for 12 words.
|
||||
/// + 20 bytes for 15 words.
|
||||
/// + 24 bytes for 18 words.
|
||||
/// + 28 bytes for 21 words.
|
||||
/// + 32 bytes for 24 words.
|
||||
///
|
||||
/// Any other length will return an error.
|
||||
///
|
||||
/// `password` is analog to BIP39 seed generation itself, with an empty string being default.
|
||||
pub fn mini_secret_from_entropy(entropy: &[u8], password: &str) -> Result<MiniSecretKey, Error> {
|
||||
let seed = seed_from_entropy(entropy, password)?;
|
||||
Ok(MiniSecretKey::from_bytes(&seed[..32]).expect("Length is always correct; qed"))
|
||||
}
|
||||
|
||||
/// Similar to `mini_secret_from_entropy`, except that it provides the 64-byte seed directly.
|
||||
pub fn seed_from_entropy(entropy: &[u8], password: &str) -> Result<[u8; 64], Error> {
|
||||
if entropy.len() < 16 || entropy.len() > 32 || !entropy.len().is_multiple_of(4) {
|
||||
return Err(Error::InvalidEntropy);
|
||||
}
|
||||
|
||||
let mut salt = String::with_capacity(8 + password.len());
|
||||
salt.push_str("mnemonic");
|
||||
salt.push_str(password);
|
||||
|
||||
let mut seed = [0u8; 64];
|
||||
|
||||
pbkdf2::<Hmac<Sha512>>(entropy, salt.as_bytes(), 2048, &mut seed)
|
||||
.map_err(|_| Error::InvalidEntropy)?;
|
||||
|
||||
salt.zeroize();
|
||||
|
||||
Ok(seed)
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod test {
|
||||
use super::*;
|
||||
|
||||
#[cfg(not(feature = "std"))]
|
||||
use alloc::vec::Vec;
|
||||
|
||||
use bip39::{Language, Mnemonic};
|
||||
use rustc_hex::FromHex;
|
||||
|
||||
// phrase, entropy, seed, expanded secret_key
|
||||
//
|
||||
// ALL SEEDS GENERATED USING "Bizinikiwi" PASSWORD!
|
||||
static VECTORS: &[[&str; 3]] = &[
|
||||
[
|
||||
"abandon abandon abandon abandon abandon abandon abandon abandon abandon abandon abandon about",
|
||||
"00000000000000000000000000000000",
|
||||
"44e9d125f037ac1d51f0a7d3649689d422c2af8b1ec8e00d71db4d7bf6d127e33f50c3d5c84fa3e5399c72d6cbbbbc4a49bf76f76d952f479d74655a2ef2d453",
|
||||
],
|
||||
[
|
||||
"legal winner thank year wave sausage worth useful legal winner thank yellow",
|
||||
"7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f",
|
||||
"4313249608fe8ac10fd5886c92c4579007272cb77c21551ee5b8d60b780416850f1e26c1f4b8d88ece681cb058ab66d6182bc2ce5a03181f7b74c27576b5c8bf",
|
||||
],
|
||||
[
|
||||
"letter advice cage absurd amount doctor acoustic avoid letter advice cage above",
|
||||
"80808080808080808080808080808080",
|
||||
"27f3eb595928c60d5bc91a4d747da40ed236328183046892ed6cd5aa9ae38122acd1183adf09a89839acb1e6eaa7fb563cc958a3f9161248d5a036e0d0af533d",
|
||||
],
|
||||
[
|
||||
"zoo zoo zoo zoo zoo zoo zoo zoo zoo zoo zoo wrong",
|
||||
"ffffffffffffffffffffffffffffffff",
|
||||
"227d6256fd4f9ccaf06c45eaa4b2345969640462bbb00c5f51f43cb43418c7a753265f9b1e0c0822c155a9cabc769413ecc14553e135fe140fc50b6722c6b9df",
|
||||
],
|
||||
[
|
||||
"abandon abandon abandon abandon abandon abandon abandon abandon abandon abandon abandon abandon abandon abandon abandon abandon abandon agent",
|
||||
"000000000000000000000000000000000000000000000000",
|
||||
"44e9d125f037ac1d51f0a7d3649689d422c2af8b1ec8e00d71db4d7bf6d127e33f50c3d5c84fa3e5399c72d6cbbbbc4a49bf76f76d952f479d74655a2ef2d453",
|
||||
],
|
||||
[
|
||||
"legal winner thank year wave sausage worth useful legal winner thank year wave sausage worth useful legal will",
|
||||
"7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f",
|
||||
"cb1d50e14101024a88905a098feb1553d4306d072d7460e167a60ccb3439a6817a0afc59060f45d999ddebc05308714733c9e1e84f30feccddd4ad6f95c8a445",
|
||||
],
|
||||
[
|
||||
"letter advice cage absurd amount doctor acoustic avoid letter advice cage absurd amount doctor acoustic avoid letter always",
|
||||
"808080808080808080808080808080808080808080808080",
|
||||
"9ddecf32ce6bee77f867f3c4bb842d1f0151826a145cb4489598fe71ac29e3551b724f01052d1bc3f6d9514d6df6aa6d0291cfdf997a5afdb7b6a614c88ab36a",
|
||||
],
|
||||
[
|
||||
"zoo zoo zoo zoo zoo zoo zoo zoo zoo zoo zoo zoo zoo zoo zoo zoo zoo when",
|
||||
"ffffffffffffffffffffffffffffffffffffffffffffffff",
|
||||
"8971cb290e7117c64b63379c97ed3b5c6da488841bd9f95cdc2a5651ac89571e2c64d391d46e2475e8b043911885457cd23e99a28b5a18535fe53294dc8e1693",
|
||||
],
|
||||
[
|
||||
"abandon abandon abandon abandon abandon abandon abandon abandon abandon abandon abandon abandon abandon abandon abandon abandon abandon abandon abandon abandon abandon abandon abandon art",
|
||||
"0000000000000000000000000000000000000000000000000000000000000000",
|
||||
"44e9d125f037ac1d51f0a7d3649689d422c2af8b1ec8e00d71db4d7bf6d127e33f50c3d5c84fa3e5399c72d6cbbbbc4a49bf76f76d952f479d74655a2ef2d453",
|
||||
],
|
||||
[
|
||||
"legal winner thank year wave sausage worth useful legal winner thank year wave sausage worth useful legal winner thank year wave sausage worth title",
|
||||
"7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f7f",
|
||||
"3037276a5d05fcd7edf51869eb841bdde27c574dae01ac8cfb1ea476f6bea6ef57ab9afe14aea1df8a48f97ae25b37d7c8326e49289efb25af92ba5a25d09ed3",
|
||||
],
|
||||
[
|
||||
"letter advice cage absurd amount doctor acoustic avoid letter advice cage absurd amount doctor acoustic avoid letter advice cage absurd amount doctor acoustic bless",
|
||||
"8080808080808080808080808080808080808080808080808080808080808080",
|
||||
"2c9c6144a06ae5a855453d98c3dea470e2a8ffb78179c2e9eb15208ccca7d831c97ddafe844ab933131e6eb895f675ede2f4e39837bb5769d4e2bc11df58ac42",
|
||||
],
|
||||
[
|
||||
"zoo zoo zoo zoo zoo zoo zoo zoo zoo zoo zoo zoo zoo zoo zoo zoo zoo zoo zoo zoo zoo zoo zoo vote",
|
||||
"ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff",
|
||||
"047e89ef7739cbfe30da0ad32eb1720d8f62441dd4f139b981b8e2d0bd412ed4eb14b89b5098c49db2301d4e7df4e89c21e53f345138e56a5e7d63fae21c5939",
|
||||
],
|
||||
[
|
||||
"ozone drill grab fiber curtain grace pudding thank cruise elder eight picnic",
|
||||
"9e885d952ad362caeb4efe34a8e91bd2",
|
||||
"f4956be6960bc145cdab782e649a5056598fd07cd3f32ceb73421c3da27833241324dc2c8b0a4d847eee457e6d4c5429f5e625ece22abaa6a976e82f1ec5531d",
|
||||
],
|
||||
[
|
||||
"gravity machine north sort system female filter attitude volume fold club stay feature office ecology stable narrow fog",
|
||||
"6610b25967cdcca9d59875f5cb50b0ea75433311869e930b",
|
||||
"fbcc5229ade0c0ff018cb7a329c5459f91876e4dde2a97ddf03c832eab7f26124366a543f1485479c31a9db0d421bda82d7e1fe562e57f3533cb1733b001d84d",
|
||||
],
|
||||
[
|
||||
"hamster diagram private dutch cause delay private meat slide toddler razor book happy fancy gospel tennis maple dilemma loan word shrug inflict delay length",
|
||||
"68a79eaca2324873eacc50cb9c6eca8cc68ea5d936f98787c60c7ebc74e6ce7c",
|
||||
"7c60c555126c297deddddd59f8cdcdc9e3608944455824dd604897984b5cc369cad749803bb36eb8b786b570c9cdc8db275dbe841486676a6adf389f3be3f076",
|
||||
],
|
||||
[
|
||||
"scheme spot photo card baby mountain device kick cradle pact join borrow",
|
||||
"c0ba5a8e914111210f2bd131f3d5e08d",
|
||||
"c12157bf2506526c4bd1b79a056453b071361538e9e2c19c28ba2cfa39b5f23034b974e0164a1e8acd30f5b4c4de7d424fdb52c0116bfc6a965ba8205e6cc121",
|
||||
],
|
||||
[
|
||||
"horn tenant knee talent sponsor spell gate clip pulse soap slush warm silver nephew swap uncle crack brave",
|
||||
"6d9be1ee6ebd27a258115aad99b7317b9c8d28b6d76431c3",
|
||||
"23766723e970e6b79dec4d5e4fdd627fd27d1ee026eb898feb9f653af01ad22080c6f306d1061656d01c4fe9a14c05f991d2c7d8af8730780de4f94cd99bd819",
|
||||
],
|
||||
[
|
||||
"panda eyebrow bullet gorilla call smoke muffin taste mesh discover soft ostrich alcohol speed nation flash devote level hobby quick inner drive ghost inside",
|
||||
"9f6a2878b2520799a44ef18bc7df394e7061a224d2c33cd015b157d746869863",
|
||||
"f4c83c86617cb014d35cd87d38b5ef1c5d5c3d58a73ab779114438a7b358f457e0462c92bddab5a406fe0e6b97c71905cf19f925f356bc673ceb0e49792f4340",
|
||||
],
|
||||
[
|
||||
"cat swing flag economy stadium alone churn speed unique patch report train",
|
||||
"23db8160a31d3e0dca3688ed941adbf3",
|
||||
"719d4d4de0638a1705bf5237262458983da76933e718b2d64eb592c470f3c5d222e345cc795337bb3da393b94375ff4a56cfcd68d5ea25b577ee9384d35f4246",
|
||||
],
|
||||
[
|
||||
"light rule cinnamon wrap drastic word pride squirrel upgrade then income fatal apart sustain crack supply proud access",
|
||||
"8197a4a47f0425faeaa69deebc05ca29c0a5b5cc76ceacc0",
|
||||
"7ae1291db32d16457c248567f2b101e62c5549d2a64cd2b7605d503ec876d58707a8d663641e99663bc4f6cc9746f4852e75e7e54de5bc1bd3c299c9a113409e",
|
||||
],
|
||||
[
|
||||
"all hour make first leader extend hole alien behind guard gospel lava path output census museum junior mass reopen famous sing advance salt reform",
|
||||
"066dca1a2bb7e8a1db2832148ce9933eea0f3ac9548d793112d9a95c9407efad",
|
||||
"a911a5f4db0940b17ecb79c4dcf9392bf47dd18acaebdd4ef48799909ebb49672947cc15f4ef7e8ef47103a1a91a6732b821bda2c667e5b1d491c54788c69391",
|
||||
],
|
||||
[
|
||||
"vessel ladder alter error federal sibling chat ability sun glass valve picture",
|
||||
"f30f8c1da665478f49b001d94c5fc452",
|
||||
"4e2314ca7d9eebac6fe5a05a5a8d3546bc891785414d82207ac987926380411e559c885190d641ff7e686ace8c57db6f6e4333c1081e3d88d7141a74cf339c8f",
|
||||
],
|
||||
[
|
||||
"scissors invite lock maple supreme raw rapid void congress muscle digital elegant little brisk hair mango congress clump",
|
||||
"c10ec20dc3cd9f652c7fac2f1230f7a3c828389a14392f05",
|
||||
"7a83851102849edc5d2a3ca9d8044d0d4f00e5c4a292753ed3952e40808593251b0af1dd3c9ed9932d46e8608eb0b928216a6160bd4fc775a6e6fbd493d7c6b2",
|
||||
],
|
||||
[
|
||||
"void come effort suffer camp survey warrior heavy shoot primary clutch crush open amazing screen patrol group space point ten exist slush involve unfold",
|
||||
"f585c11aec520db57dd353c69554b21a89b20fb0650966fa0a9d6f74fd989d8f",
|
||||
"938ba18c3f521f19bd4a399c8425b02c716844325b1a65106b9d1593fbafe5e0b85448f523f91c48e331995ff24ae406757cff47d11f240847352b348ff436ed",
|
||||
]
|
||||
];
|
||||
|
||||
#[test]
|
||||
fn vectors_are_correct() {
|
||||
for vector in VECTORS {
|
||||
let phrase = vector[0];
|
||||
|
||||
let expected_entropy: Vec<u8> = vector[1].from_hex().unwrap();
|
||||
let expected_seed: Vec<u8> = vector[2].from_hex().unwrap();
|
||||
|
||||
let mnemonic = Mnemonic::parse_in(Language::English, phrase).unwrap();
|
||||
let seed = seed_from_entropy(&mnemonic.to_entropy(), "Bizinikiwi").unwrap();
|
||||
let secret = mini_secret_from_entropy(&mnemonic.to_entropy(), "Bizinikiwi")
|
||||
.unwrap()
|
||||
.to_bytes();
|
||||
|
||||
assert_eq!(
|
||||
mnemonic.to_entropy(),
|
||||
&expected_entropy[..],
|
||||
"Entropy is incorrect for {phrase}"
|
||||
);
|
||||
assert_eq!(&seed[..], &expected_seed[..], "Seed is incorrect for {phrase}");
|
||||
assert_eq!(&secret[..], &expected_seed[..32], "Secret is incorrect for {phrase}");
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,16 @@
|
||||
[package]
|
||||
name = "bizinikiwi-build-script-utils"
|
||||
version = "11.0.0"
|
||||
authors.workspace = true
|
||||
edition.workspace = true
|
||||
license = "Apache-2.0"
|
||||
homepage.workspace = true
|
||||
repository.workspace = true
|
||||
description = "Crate with utility functions for `build.rs` scripts."
|
||||
readme = "README.md"
|
||||
|
||||
[lints]
|
||||
workspace = true
|
||||
|
||||
[package.metadata.docs.rs]
|
||||
targets = ["x86_64-unknown-linux-gnu"]
|
||||
@@ -0,0 +1,3 @@
|
||||
Crate with utility functions for `build.rs` scripts.
|
||||
|
||||
License: Apache-2.0
|
||||
@@ -0,0 +1,131 @@
|
||||
// This file is part of Bizinikiwi.
|
||||
|
||||
// Copyright (C) Parity Technologies (UK) Ltd.
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
use std::{
|
||||
env, fs,
|
||||
fs::File,
|
||||
io,
|
||||
io::Read,
|
||||
path::{Path, PathBuf},
|
||||
};
|
||||
|
||||
/// Make sure the calling `build.rs` script is rerun when `.git/HEAD` or the ref of `.git/HEAD`
|
||||
/// changed.
|
||||
///
|
||||
/// The file is searched from the `CARGO_MANIFEST_DIR` upwards. If the file can not be found,
|
||||
/// a warning is generated.
|
||||
pub fn rerun_if_git_head_changed() {
|
||||
let mut manifest_dir = PathBuf::from(
|
||||
env::var("CARGO_MANIFEST_DIR").expect("`CARGO_MANIFEST_DIR` is always set by cargo."),
|
||||
);
|
||||
let manifest_dir_copy = manifest_dir.clone();
|
||||
|
||||
while manifest_dir.parent().is_some() {
|
||||
match get_git_paths(&manifest_dir) {
|
||||
Err(err) => {
|
||||
eprintln!("cargo:warning=Unable to read the Git repository: {}", err);
|
||||
|
||||
return;
|
||||
},
|
||||
Ok(None) => {},
|
||||
Ok(Some(paths)) => {
|
||||
for p in paths {
|
||||
println!("cargo:rerun-if-changed={}", p.display());
|
||||
}
|
||||
|
||||
return;
|
||||
},
|
||||
}
|
||||
|
||||
manifest_dir.pop();
|
||||
}
|
||||
|
||||
println!(
|
||||
"cargo:warning=Could not find `.git/HEAD` searching from `{}` upwards!",
|
||||
manifest_dir_copy.display(),
|
||||
);
|
||||
}
|
||||
|
||||
// Code taken from https://github.com/rustyhorde/vergen/blob/8d522db8c8e16e26c0fc9ea8e6b0247cbf5cca84/src/output/envvar.rs
|
||||
fn get_git_paths(path: &Path) -> Result<Option<Vec<PathBuf>>, io::Error> {
|
||||
let git_dir_or_file = path.join(".git");
|
||||
|
||||
if let Ok(metadata) = fs::metadata(&git_dir_or_file) {
|
||||
if metadata.is_dir() {
|
||||
// Echo the HEAD path
|
||||
let git_head_path = git_dir_or_file.join("HEAD");
|
||||
|
||||
// Determine where HEAD points and echo that path also.
|
||||
let mut f = File::open(&git_head_path)?;
|
||||
let mut git_head_contents = String::new();
|
||||
f.read_to_string(&mut git_head_contents)?;
|
||||
let ref_vec: Vec<&str> = git_head_contents.split(": ").collect();
|
||||
|
||||
if ref_vec.len() == 2 {
|
||||
let current_head_file = ref_vec[1];
|
||||
let git_refs_path = git_dir_or_file.join(current_head_file);
|
||||
|
||||
Ok(Some(vec![git_head_path, git_refs_path]))
|
||||
} else {
|
||||
Err(io::Error::new(
|
||||
io::ErrorKind::Other,
|
||||
"You are most likely in a detached HEAD state",
|
||||
))
|
||||
}
|
||||
} else if metadata.is_file() {
|
||||
// We are in a worktree, so find out where the actual worktrees/<name>/HEAD file is.
|
||||
let mut git_file = File::open(&git_dir_or_file)?;
|
||||
let mut git_contents = String::new();
|
||||
git_file.read_to_string(&mut git_contents)?;
|
||||
let dir_vec: Vec<&str> = git_contents.split(": ").collect();
|
||||
let git_path = dir_vec[1].trim();
|
||||
|
||||
// Echo the HEAD psth
|
||||
let git_head_path = PathBuf::from(git_path).join("HEAD");
|
||||
|
||||
// Find out what the full path to the .git dir is.
|
||||
let mut actual_git_dir = PathBuf::from(git_path);
|
||||
actual_git_dir.pop();
|
||||
actual_git_dir.pop();
|
||||
|
||||
// Determine where HEAD points and echo that path also.
|
||||
let mut f = File::open(&git_head_path)?;
|
||||
let mut git_head_contents = String::new();
|
||||
f.read_to_string(&mut git_head_contents)?;
|
||||
let ref_vec: Vec<&str> = git_head_contents.split(": ").collect();
|
||||
|
||||
if ref_vec.len() == 2 {
|
||||
let current_head_file = ref_vec[1];
|
||||
let git_refs_path = actual_git_dir.join(current_head_file);
|
||||
|
||||
Ok(Some(vec![git_head_path, git_refs_path]))
|
||||
} else {
|
||||
Err(io::Error::new(
|
||||
io::ErrorKind::Other,
|
||||
"You are most likely in a detached HEAD state",
|
||||
))
|
||||
}
|
||||
} else {
|
||||
Err(io::Error::new(
|
||||
io::ErrorKind::Other,
|
||||
"Invalid .git format (Not a directory or a file)",
|
||||
))
|
||||
}
|
||||
} else {
|
||||
Ok(None)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,24 @@
|
||||
// This file is part of Bizinikiwi.
|
||||
|
||||
// Copyright (C) Parity Technologies (UK) Ltd.
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
//! Crate with utility functions for `build.rs` scripts.
|
||||
|
||||
mod git;
|
||||
mod version;
|
||||
|
||||
pub use git::*;
|
||||
pub use version::*;
|
||||
@@ -0,0 +1,61 @@
|
||||
// This file is part of Bizinikiwi.
|
||||
|
||||
// Copyright (C) Parity Technologies (UK) Ltd.
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
use std::{borrow::Cow, process::Command};
|
||||
|
||||
/// Generate the `cargo:` key output
|
||||
pub fn generate_cargo_keys() {
|
||||
let commit = if let Ok(hash) = std::env::var("BIZINIKIWI_CLI_GIT_COMMIT_HASH") {
|
||||
Cow::from(hash.trim().to_owned())
|
||||
} else {
|
||||
// We deliberately set the length here to `11` to ensure that
|
||||
// the emitted hash is always of the same length; otherwise
|
||||
// it can (and will!) vary between different build environments.
|
||||
match Command::new("git").args(["rev-parse", "--short=11", "HEAD"]).output() {
|
||||
Ok(o) if o.status.success() => {
|
||||
let sha = String::from_utf8_lossy(&o.stdout).trim().to_owned();
|
||||
Cow::from(sha)
|
||||
},
|
||||
Ok(o) => {
|
||||
let stderr = String::from_utf8_lossy(&o.stderr).trim().to_owned();
|
||||
println!(
|
||||
"cargo:warning=Git command failed with status '{}' with message: '{}'",
|
||||
o.status, stderr,
|
||||
);
|
||||
Cow::from("unknown")
|
||||
},
|
||||
Err(err) => {
|
||||
println!("cargo:warning=Failed to execute git command: {}", err);
|
||||
Cow::from("unknown")
|
||||
},
|
||||
}
|
||||
};
|
||||
|
||||
println!("cargo:rustc-env=BIZINIKIWI_CLI_COMMIT_HASH={commit}");
|
||||
println!("cargo:rustc-env=BIZINIKIWI_CLI_IMPL_VERSION={}", get_version(&commit))
|
||||
}
|
||||
|
||||
fn get_version(impl_commit: &str) -> String {
|
||||
let commit_dash = if impl_commit.is_empty() { "" } else { "-" };
|
||||
|
||||
format!(
|
||||
"{}{}{}",
|
||||
std::env::var("CARGO_PKG_VERSION").unwrap_or_default(),
|
||||
commit_dash,
|
||||
impl_commit
|
||||
)
|
||||
}
|
||||
@@ -0,0 +1,20 @@
|
||||
[package]
|
||||
name = "fork-tree"
|
||||
version = "12.0.0"
|
||||
authors.workspace = true
|
||||
edition.workspace = true
|
||||
license = "Apache-2.0"
|
||||
homepage.workspace = true
|
||||
repository.workspace = true
|
||||
description = "Utility library for managing tree-like ordered data with logic for pruning the tree while finalizing nodes."
|
||||
documentation = "https://docs.rs/fork-tree"
|
||||
readme = "README.md"
|
||||
|
||||
[lints]
|
||||
workspace = true
|
||||
|
||||
[package.metadata.docs.rs]
|
||||
targets = ["x86_64-unknown-linux-gnu"]
|
||||
|
||||
[dependencies]
|
||||
codec = { features = ["derive"], workspace = true, default-features = true }
|
||||
@@ -0,0 +1,4 @@
|
||||
Utility library for managing tree-like ordered data with logic for pruning
|
||||
the tree while finalizing nodes.
|
||||
|
||||
License: Apache-2.0
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,121 @@
|
||||
[package]
|
||||
name = "pezframe-benchmarking-cli"
|
||||
version = "32.0.0"
|
||||
authors.workspace = true
|
||||
edition.workspace = true
|
||||
license = "Apache-2.0"
|
||||
homepage.workspace = true
|
||||
repository.workspace = true
|
||||
description = "CLI for benchmarking FRAME"
|
||||
readme = "README.md"
|
||||
|
||||
[lints]
|
||||
workspace = true
|
||||
|
||||
[package.metadata.docs.rs]
|
||||
targets = ["x86_64-unknown-linux-gnu"]
|
||||
|
||||
[dependencies]
|
||||
Inflector = { workspace = true }
|
||||
array-bytes = { workspace = true, default-features = true }
|
||||
chrono = { workspace = true }
|
||||
clap = { features = ["derive"], workspace = true }
|
||||
codec = { workspace = true, default-features = true }
|
||||
comfy-table = { workspace = true }
|
||||
pezcumulus-client-teyrchain-inherent = { workspace = true, default-features = true }
|
||||
pezcumulus-primitives-proof-size-hostfunction = { workspace = true, default-features = true }
|
||||
env_filter = { workspace = true }
|
||||
pezframe-benchmarking = { workspace = true, default-features = true }
|
||||
frame-storage-access-test-runtime = { workspace = true, default-features = true }
|
||||
pezframe-support = { workspace = true, default-features = true }
|
||||
pezframe-system = { workspace = true, default-features = true }
|
||||
gethostname = { workspace = true }
|
||||
handlebars = { workspace = true }
|
||||
itertools = { workspace = true }
|
||||
linked-hash-map = { workspace = true }
|
||||
log = { workspace = true, default-features = true }
|
||||
pezkuwi-primitives = { workspace = true, default-features = true }
|
||||
pezkuwi-teyrchain-primitives = { workspace = true, default-features = true }
|
||||
rand = { features = ["small_rng"], workspace = true, default-features = true }
|
||||
rand_pcg = { workspace = true }
|
||||
pezsc-block-builder = { workspace = true, default-features = true }
|
||||
pezsc-chain-spec = { workspace = true }
|
||||
pezsc-cli = { workspace = true, default-features = false }
|
||||
pezsc-client-api = { workspace = true, default-features = true }
|
||||
pezsc-client-db = { workspace = true, default-features = false }
|
||||
pezsc-executor = { workspace = true, default-features = true }
|
||||
pezsc-executor-common = { workspace = true }
|
||||
pezsc-executor-wasmtime = { workspace = true }
|
||||
pezsc-runtime-utilities = { workspace = true, default-features = true }
|
||||
pezsc-service = { workspace = true, default-features = false }
|
||||
pezsc-sysinfo = { workspace = true, default-features = true }
|
||||
serde = { workspace = true, default-features = true }
|
||||
serde_json = { workspace = true, default-features = true }
|
||||
pezsp-api = { workspace = true, default-features = true }
|
||||
pezsp-block-builder = { workspace = true, default-features = true }
|
||||
pezsp-blockchain = { workspace = true, default-features = true }
|
||||
pezsp-core = { workspace = true, default-features = true }
|
||||
pezsp-database = { workspace = true, default-features = true }
|
||||
pezsp-externalities = { workspace = true, default-features = true }
|
||||
pezsp-genesis-builder = { workspace = true, default-features = true }
|
||||
pezsp-inherents = { workspace = true, default-features = true }
|
||||
pezsp-io = { workspace = true, default-features = true }
|
||||
pezsp-keystore = { workspace = true, default-features = true }
|
||||
pezsp-runtime = { workspace = true, default-features = true }
|
||||
pezsp-runtime-interface = { workspace = true, default-features = true }
|
||||
pezsp-state-machine = { workspace = true, default-features = true }
|
||||
pezsp-storage = { workspace = true, default-features = true }
|
||||
pezsp-timestamp = { workspace = true, default-features = true }
|
||||
pezsp-transaction-pool = { workspace = true, default-features = true }
|
||||
pezsp-trie = { workspace = true, default-features = true }
|
||||
pezsp-version = { workspace = true, default-features = true }
|
||||
pezsp-wasm-interface = { workspace = true, default-features = true }
|
||||
subxt = { workspace = true, features = ["native"] }
|
||||
subxt-signer = { workspace = true, features = ["unstable-eth"] }
|
||||
thiserror = { workspace = true }
|
||||
thousands = { workspace = true }
|
||||
|
||||
[dev-dependencies]
|
||||
pezcumulus-test-runtime = { workspace = true, default-features = true }
|
||||
bizinikiwi-test-runtime = { workspace = true, default-features = true }
|
||||
zagros-runtime = { workspace = true, default-features = true }
|
||||
|
||||
[features]
|
||||
default = []
|
||||
runtime-benchmarks = [
|
||||
"pezcumulus-client-teyrchain-inherent/runtime-benchmarks",
|
||||
"pezcumulus-primitives-proof-size-hostfunction/runtime-benchmarks",
|
||||
"pezcumulus-test-runtime/runtime-benchmarks",
|
||||
"pezframe-benchmarking/runtime-benchmarks",
|
||||
"frame-storage-access-test-runtime/runtime-benchmarks",
|
||||
"pezframe-support/runtime-benchmarks",
|
||||
"pezframe-system/runtime-benchmarks",
|
||||
"pezkuwi-primitives/runtime-benchmarks",
|
||||
"pezkuwi-teyrchain-primitives/runtime-benchmarks",
|
||||
"pezsc-block-builder/runtime-benchmarks",
|
||||
"pezsc-chain-spec/runtime-benchmarks",
|
||||
"pezsc-cli/runtime-benchmarks",
|
||||
"pezsc-client-api/runtime-benchmarks",
|
||||
"pezsc-client-db/runtime-benchmarks",
|
||||
"pezsc-executor-wasmtime/runtime-benchmarks",
|
||||
"pezsc-executor/runtime-benchmarks",
|
||||
"pezsc-runtime-utilities/runtime-benchmarks",
|
||||
"pezsc-service/runtime-benchmarks",
|
||||
"pezsc-sysinfo/runtime-benchmarks",
|
||||
"pezsp-api/runtime-benchmarks",
|
||||
"pezsp-block-builder/runtime-benchmarks",
|
||||
"pezsp-blockchain/runtime-benchmarks",
|
||||
"pezsp-genesis-builder/runtime-benchmarks",
|
||||
"pezsp-inherents/runtime-benchmarks",
|
||||
"pezsp-io/runtime-benchmarks",
|
||||
"pezsp-runtime-interface/runtime-benchmarks",
|
||||
"pezsp-runtime/runtime-benchmarks",
|
||||
"pezsp-state-machine/runtime-benchmarks",
|
||||
"pezsp-timestamp/runtime-benchmarks",
|
||||
"pezsp-transaction-pool/runtime-benchmarks",
|
||||
"pezsp-trie/runtime-benchmarks",
|
||||
"pezsp-version/runtime-benchmarks",
|
||||
"bizinikiwi-test-runtime/runtime-benchmarks",
|
||||
"zagros-runtime/runtime-benchmarks",
|
||||
]
|
||||
rocksdb = ["pezsc-cli/rocksdb", "pezsc-client-db/rocksdb"]
|
||||
@@ -0,0 +1,98 @@
|
||||
# The FRAME Benchmarking CLI
|
||||
|
||||
This crate contains commands to benchmark various aspects of Bizinikiwi and the hardware.
|
||||
The goal is to have a comprehensive suite of benchmarks that cover all aspects of Bizinikiwi and the hardware that its
|
||||
running on.
|
||||
There exist fundamentally two ways to use this crate. A node-integrated CLI version, and a freestanding CLI. If you are
|
||||
only interested in pallet benchmarking, then skip ahead to the [Freestanding CLI](#freestanding-cli).
|
||||
|
||||
# Node Integrated CLI
|
||||
|
||||
Mostly all Bizinikiwi nodes will expose some commands for benchmarking. You can refer to the `pezstaging-node-cli` crate as
|
||||
an example on how to integrate those. Note that for solely benchmarking pallets, the freestanding CLI is more suitable.
|
||||
|
||||
## Usage
|
||||
|
||||
Here we invoke the root command on the `pezstaging-node-cli`. Most Bizinikiwi nodes should have a similar output, depending
|
||||
on their integration of these commands.
|
||||
|
||||
```sh
|
||||
$ cargo run -p pezstaging-node-cli --profile=production --features=runtime-benchmarks -- benchmark
|
||||
|
||||
Sub-commands concerned with benchmarking.
|
||||
|
||||
USAGE:
|
||||
bizinikiwi benchmark <SUBCOMMAND>
|
||||
|
||||
OPTIONS:
|
||||
-h, --help Print help information
|
||||
-V, --version Print version information
|
||||
|
||||
SUBCOMMANDS:
|
||||
block Benchmark the execution time of historic blocks
|
||||
machine Command to benchmark the hardware.
|
||||
overhead Benchmark the execution overhead per-block and per-extrinsic
|
||||
pallet Benchmark the extrinsic weight of FRAME Pallets
|
||||
storage Benchmark the storage speed of a chain snapshot
|
||||
```
|
||||
|
||||
All examples use the `production` profile for correctness which makes the compilation *very* slow; for testing you can
|
||||
use `--release`.
|
||||
For the final results the `production` profile and reference hardware should be used, otherwise the results are not
|
||||
comparable.
|
||||
|
||||
# Freestanding CLI
|
||||
|
||||
The freestanding is a standalone CLI that does not rely on any node integration. It can be used to benchmark pallets of
|
||||
any FRAME runtime that does not utilize 3rd party host functions.
|
||||
It currently only supports pallet benchmarking, since the other commands still rely on a node.
|
||||
|
||||
## Installation
|
||||
|
||||
Installing from local source repository:
|
||||
|
||||
```sh
|
||||
cargo install --locked --path bizinikiwi/utils/pezframe/omni-bencher --profile=production
|
||||
```
|
||||
|
||||
## Usage
|
||||
|
||||
The exposed pallet sub-command is identical as the node-integrated CLI. The only difference is that it needs to be prefixed
|
||||
with a `v1` to ensure drop-in compatibility.
|
||||
|
||||
First we need to ensure that there is a runtime available. As example we will build the zagros runtime:
|
||||
|
||||
```sh
|
||||
cargo build -p zagros-runtime --profile production --features runtime-benchmarks
|
||||
```
|
||||
|
||||
Now the benchmarking can be started with:
|
||||
|
||||
```sh
|
||||
frame-omni-bencher v1 \
|
||||
benchmark pallet \
|
||||
--runtime target/release/wbuild/zagros-runtime/zagros-runtime.compact.compressed.wasm \
|
||||
--pallet "pallet_balances" --extrinsic ""
|
||||
```
|
||||
|
||||
For the exact arguments of the `pallet` command, please refer to the [pallet] sub-module.
|
||||
|
||||
# Commands
|
||||
|
||||
The sub-commands of both CLIs have the same semantics and are documented in their respective sub-modules:
|
||||
|
||||
- [block] Compare the weight of a historic block to its actual resource usage
|
||||
- [machine] Gauges the speed of the hardware
|
||||
- [overhead] Creates weight files for the *Block*- and *Extrinsic*-base weights
|
||||
- [pallet] Creates weight files for a Pallet
|
||||
- [storage] Creates weight files for *Read* and *Write* storage operations
|
||||
|
||||
License: Apache-2.0
|
||||
|
||||
<!-- LINKS -->
|
||||
|
||||
[pallet]: ../../../frame/benchmarking/README.md
|
||||
[machine]: src/machine/README.md
|
||||
[storage]: src/storage/README.md
|
||||
[overhead]: src/overhead/README.md
|
||||
[block]: src/block/README.md
|
||||
@@ -0,0 +1,35 @@
|
||||
// This file is part of Bizinikiwi.
|
||||
|
||||
// Copyright (C) Parity Technologies (UK) Ltd.
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
use std::env;
|
||||
|
||||
/// Exposes build environment variables to the rust code.
|
||||
///
|
||||
/// - The build profile as `build_profile`
|
||||
/// - The optimization level as `build_opt_level`
|
||||
pub fn main() {
|
||||
if let Ok(opt_level) = env::var("OPT_LEVEL") {
|
||||
println!("cargo:rustc-cfg=build_opt_level={:?}", opt_level);
|
||||
} else {
|
||||
println!("cargo:rustc-cfg=build_opt_level={:?}", "unknown");
|
||||
}
|
||||
if let Ok(profile) = env::var("PROFILE") {
|
||||
println!("cargo:rustc-cfg=build_profile={:?}", profile);
|
||||
} else {
|
||||
println!("cargo:rustc-cfg=build_profile={:?}", "unknown");
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,113 @@
|
||||
# The `benchmark block` command
|
||||
|
||||
The whole benchmarking process in Bizinikiwi aims to predict the resource usage of an unexecuted block. This command
|
||||
measures how accurate this prediction was by executing a block and comparing the predicted weight to its actual resource
|
||||
usage. It can be used to measure the accuracy of the pallet benchmarking.
|
||||
|
||||
In the following it will be explained once for PezkuwiChain and once for Bizinikiwi.
|
||||
|
||||
## PezkuwiChain # 1
|
||||
<sup>(Also works for Kusama, zagros and pezkuwichain)</sup>
|
||||
|
||||
|
||||
Suppose you either have a synced PezkuwiChain node or downloaded a snapshot from [Polkachu]. This example uses a pruned
|
||||
ParityDB snapshot from the 2022-4-19 with the last block being 9939462. For pruned snapshots you need to know the number
|
||||
of the last block (to be improved [here]). Pruned snapshots normally store the last 256 blocks, archive nodes can use
|
||||
any block range.
|
||||
|
||||
In this example we will benchmark just the last 10 blocks:
|
||||
```sh
|
||||
cargo run --profile=production -- benchmark block --from 9939453 --to 9939462 --db paritydb
|
||||
```
|
||||
|
||||
Output:
|
||||
```pre
|
||||
Block 9939453 with 2 tx used 4.57% of its weight ( 26,458,801 of 579,047,053 ns)
|
||||
Block 9939454 with 3 tx used 4.80% of its weight ( 28,335,826 of 590,414,831 ns)
|
||||
Block 9939455 with 2 tx used 4.76% of its weight ( 27,889,567 of 586,484,595 ns)
|
||||
Block 9939456 with 2 tx used 4.65% of its weight ( 27,101,306 of 582,789,723 ns)
|
||||
Block 9939457 with 2 tx used 4.62% of its weight ( 26,908,882 of 582,789,723 ns)
|
||||
Block 9939458 with 2 tx used 4.78% of its weight ( 28,211,440 of 590,179,467 ns)
|
||||
Block 9939459 with 4 tx used 4.78% of its weight ( 27,866,077 of 583,260,451 ns)
|
||||
Block 9939460 with 3 tx used 4.72% of its weight ( 27,845,836 of 590,462,629 ns)
|
||||
Block 9939461 with 2 tx used 4.58% of its weight ( 26,685,119 of 582,789,723 ns)
|
||||
Block 9939462 with 2 tx used 4.60% of its weight ( 26,840,938 of 583,697,101 ns)
|
||||
```
|
||||
|
||||
### Output Interpretation
|
||||
|
||||
<sup>(Only results from reference hardware are relevant)</sup>
|
||||
|
||||
Each block is executed multiple times and the results are averaged. The percent number is the interesting part and
|
||||
indicates how much weight was used as compared to how much was predicted. The closer to 100% this is without exceeding
|
||||
100%, the better. If it exceeds 100%, the block is marked with "**OVER WEIGHT!**" to easier spot them. This is not good
|
||||
since then the benchmarking under-estimated the weight. This would mean that an honest validator would possibly not be
|
||||
able to keep up with importing blocks since users did not pay for enough weight. If that happens the validator could lag
|
||||
behind the chain and get slashed for missing deadlines. It is therefore important to investigate any overweight blocks.
|
||||
|
||||
In this example you can see an unexpected result; only < 5% of the weight was used! The measured blocks can be executed
|
||||
much faster than predicted. This means that the benchmarking process massively over-estimated the execution time. Since
|
||||
they are off by so much, it is an issue [`pezkuwi#5192`].
|
||||
|
||||
The ideal range for these results would be 85-100%.
|
||||
|
||||
## PezkuwiChain # 2
|
||||
|
||||
Let's take a more interesting example where the blocks use more of their predicted weight. Every day when validators pay
|
||||
out rewards, the blocks are nearly full. Using an archive node here is the easiest.
|
||||
|
||||
The PezkuwiChain blocks TODO-TODO for example contain large batch transactions for staking payout.
|
||||
|
||||
```sh
|
||||
cargo run --profile=production -- benchmark block --from TODO --to TODO --db paritydb
|
||||
```
|
||||
|
||||
```pre
|
||||
TODO
|
||||
```
|
||||
|
||||
## Bizinikiwi
|
||||
|
||||
It is also possible to try the procedure in Bizinikiwi, although it's a bit boring.
|
||||
|
||||
First you need to create some blocks with either a local or dev chain. This example will use the standard development
|
||||
spec. Pick a non existing directory where the chain data will be stored, eg `/tmp/dev`.
|
||||
```sh
|
||||
cargo run --profile=production -- --dev -d /tmp/dev
|
||||
```
|
||||
You should see after some seconds that it started to produce blocks:
|
||||
```pre
|
||||
…
|
||||
✨ Imported #1 (0x801d…9189)
|
||||
…
|
||||
```
|
||||
You can now kill the node with `Ctrl+C`. Then measure how long it takes to execute these blocks:
|
||||
```sh
|
||||
cargo run --profile=production -- benchmark block --from 1 --to 1 --dev -d /tmp/dev --pruning archive
|
||||
```
|
||||
This will benchmark the first block. If you killed the node at a later point, you can measure multiple blocks.
|
||||
```pre
|
||||
Block 1 with 1 tx used 72.04% of its weight ( 4,945,664 of 6,864,702 ns)
|
||||
```
|
||||
|
||||
In this example the block used ~72% of its weight. The benchmarking therefore over-estimated the effort to execute the
|
||||
block. Since this block is empty, its not very interesting.
|
||||
|
||||
## Arguments
|
||||
|
||||
- `--from` Number of the first block to measure (inclusive).
|
||||
- `--to` Number of the last block to measure (inclusive).
|
||||
- `--repeat` How often each block should be measured.
|
||||
- [`--db`]
|
||||
- [`--pruning`]
|
||||
|
||||
License: Apache-2.0
|
||||
|
||||
<!-- LINKS -->
|
||||
|
||||
[Polkachu]: https://polkachu.com/snapshots
|
||||
[here]: https://github.com/pezkuwichain/kurdistan-sdk/issues/6
|
||||
[pezkuwi#5192]: https://github.com/pezkuwichain/kurdistan-sdk/issues/154
|
||||
|
||||
[`--db`]: ../shared/README.md#arguments
|
||||
[`--pruning`]: ../shared/README.md#arguments
|
||||
@@ -0,0 +1,184 @@
|
||||
// This file is part of Bizinikiwi.
|
||||
|
||||
// Copyright (C) Parity Technologies (UK) Ltd.
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
//! Contains the core benchmarking logic.
|
||||
|
||||
use codec::DecodeAll;
|
||||
use pezframe_support::weights::constants::WEIGHT_REF_TIME_PER_NANOS;
|
||||
use pezframe_system::ConsumedWeight;
|
||||
use pezsc_block_builder::BlockBuilderApi;
|
||||
use pezsc_cli::{Error, Result};
|
||||
use pezsc_client_api::{
|
||||
Backend as ClientBackend, BlockBackend, HeaderBackend, StorageProvider, UsageProvider,
|
||||
};
|
||||
use pezsp_api::{ApiExt, Core, ProvideRuntimeApi};
|
||||
use pezsp_blockchain::Error::RuntimeApiError;
|
||||
use pezsp_runtime::{
|
||||
generic::BlockId,
|
||||
traits::{Block as BlockT, Header as HeaderT},
|
||||
DigestItem, OpaqueExtrinsic,
|
||||
};
|
||||
use pezsp_storage::StorageKey;
|
||||
|
||||
use clap::Args;
|
||||
use log::{info, warn};
|
||||
use serde::Serialize;
|
||||
use std::{fmt::Debug, marker::PhantomData, sync::Arc, time::Instant};
|
||||
use thousands::Separable;
|
||||
|
||||
use crate::shared::{StatSelect, Stats};
|
||||
|
||||
/// Log target for printing block weight info.
|
||||
const LOG_TARGET: &'static str = "benchmark::block::weight";
|
||||
|
||||
/// Parameters for modifying the benchmark behaviour.
|
||||
#[derive(Debug, Default, Serialize, Clone, PartialEq, Args)]
|
||||
pub struct BenchmarkParams {
|
||||
/// Number of the first block to consider.
|
||||
#[arg(long)]
|
||||
pub from: u32,
|
||||
|
||||
/// Last block number to consider.
|
||||
#[arg(long)]
|
||||
pub to: u32,
|
||||
|
||||
/// Number of times that the benchmark should be repeated for each block.
|
||||
#[arg(long, default_value_t = 10)]
|
||||
pub repeat: u32,
|
||||
}
|
||||
|
||||
/// Convenience closure for the [`Benchmark::run()`] function.
|
||||
pub struct Benchmark<Block, BA, C> {
|
||||
client: Arc<C>,
|
||||
params: BenchmarkParams,
|
||||
_p: PhantomData<(Block, BA, C)>,
|
||||
}
|
||||
|
||||
/// Helper for nano seconds.
|
||||
type NanoSeconds = u64;
|
||||
|
||||
impl<Block, BA, C> Benchmark<Block, BA, C>
|
||||
where
|
||||
Block: BlockT<Extrinsic = OpaqueExtrinsic>,
|
||||
BA: ClientBackend<Block>,
|
||||
C: ProvideRuntimeApi<Block>
|
||||
+ StorageProvider<Block, BA>
|
||||
+ UsageProvider<Block>
|
||||
+ BlockBackend<Block>
|
||||
+ HeaderBackend<Block>,
|
||||
C::Api: ApiExt<Block> + BlockBuilderApi<Block>,
|
||||
{
|
||||
/// Returns a new [`Self`] from the arguments.
|
||||
pub fn new(client: Arc<C>, params: BenchmarkParams) -> Self {
|
||||
Self { client, params, _p: PhantomData }
|
||||
}
|
||||
|
||||
/// Benchmark the execution speed of historic blocks and log the results.
|
||||
pub fn run(&self) -> Result<()> {
|
||||
if self.params.from == 0 {
|
||||
return Err("Cannot benchmark the genesis block".into());
|
||||
}
|
||||
|
||||
for i in self.params.from..=self.params.to {
|
||||
let block_num = BlockId::Number(i.into());
|
||||
let hash = self.client.expect_block_hash_from_id(&block_num)?;
|
||||
let consumed = self.consumed_weight(hash)?;
|
||||
|
||||
let block = self.client.block(hash)?.ok_or(format!("Block {} not found", block_num))?;
|
||||
let block = self.unsealed(block.block);
|
||||
let took = self.measure_block(&block, *block.header().parent_hash())?;
|
||||
|
||||
self.log_weight(i, block.extrinsics().len(), consumed, took);
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Return the average *execution* aka. *import* time of the block.
|
||||
fn measure_block(&self, block: &Block, parent_hash: Block::Hash) -> Result<NanoSeconds> {
|
||||
let mut record = Vec::<NanoSeconds>::default();
|
||||
// Interesting part here:
|
||||
// Execute the block multiple times and collect stats about its execution time.
|
||||
for _ in 0..self.params.repeat {
|
||||
let block = block.clone();
|
||||
let runtime_api = self.client.runtime_api();
|
||||
let start = Instant::now();
|
||||
|
||||
runtime_api
|
||||
.execute_block(parent_hash, block.into())
|
||||
.map_err(|e| Error::Client(RuntimeApiError(e)))?;
|
||||
|
||||
record.push(start.elapsed().as_nanos() as NanoSeconds);
|
||||
}
|
||||
|
||||
let took = Stats::new(&record)?.select(StatSelect::Average);
|
||||
Ok(took)
|
||||
}
|
||||
|
||||
/// Returns the total nanoseconds of a [`pezframe_system::ConsumedWeight`] for a block number.
|
||||
///
|
||||
/// This is the post-dispatch corrected weight and is only available
|
||||
/// after executing the block.
|
||||
fn consumed_weight(&self, block_hash: Block::Hash) -> Result<NanoSeconds> {
|
||||
// Hard-coded key for System::BlockWeight. It could also be passed in as argument
|
||||
// for the benchmark, but I think this should work as well.
|
||||
let hash = array_bytes::hex2bytes(
|
||||
"26aa394eea5630e07c48ae0c9558cef734abf5cb34d6244378cddbf18e849d96",
|
||||
)?;
|
||||
let key = StorageKey(hash);
|
||||
|
||||
let mut raw_weight = &self
|
||||
.client
|
||||
.storage(block_hash, &key)?
|
||||
.ok_or(format!("Could not find System::BlockWeight for block: {}", block_hash))?
|
||||
.0[..];
|
||||
|
||||
let weight = ConsumedWeight::decode_all(&mut raw_weight)?;
|
||||
// Should be divisible, but still use floats in case we ever change that.
|
||||
Ok((weight.total().ref_time() as f64 / WEIGHT_REF_TIME_PER_NANOS as f64).floor()
|
||||
as NanoSeconds)
|
||||
}
|
||||
|
||||
/// Prints the weight info of a block to the console.
|
||||
fn log_weight(&self, num: u32, num_ext: usize, consumed: NanoSeconds, took: NanoSeconds) {
|
||||
// The ratio of weight that the block used vs what it consumed.
|
||||
// This should in general not exceed 100% (minus outliers).
|
||||
let percent = (took as f64 / consumed as f64) * 100.0;
|
||||
|
||||
let msg = format!(
|
||||
"Block {} with {: >5} tx used {: >6.2}% of its weight ({: >14} of {: >14} ns)",
|
||||
num,
|
||||
num_ext,
|
||||
percent,
|
||||
took.separate_with_commas(),
|
||||
consumed.separate_with_commas()
|
||||
);
|
||||
|
||||
if took <= consumed {
|
||||
info!(target: LOG_TARGET, "{}", msg);
|
||||
} else {
|
||||
warn!(target: LOG_TARGET, "{} - OVER WEIGHT!", msg);
|
||||
}
|
||||
}
|
||||
|
||||
/// Removes the consensus seal from the block.
|
||||
fn unsealed(&self, block: Block) -> Block {
|
||||
let (mut header, exts) = block.deconstruct();
|
||||
header.digest_mut().logs.retain(|item| !matches!(item, DigestItem::Seal(_, _)));
|
||||
Block::new(header, exts)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,116 @@
|
||||
// This file is part of Bizinikiwi.
|
||||
|
||||
// Copyright (C) Parity Technologies (UK) Ltd.
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
//! Contains the [`BlockCmd`] as entry point for the CLI to execute
|
||||
//! the *block* benchmark.
|
||||
|
||||
use pezsc_block_builder::BlockBuilderApi;
|
||||
use pezsc_cli::{CliConfiguration, ImportParams, Result, SharedParams};
|
||||
use pezsc_client_api::{Backend as ClientBackend, BlockBackend, StorageProvider, UsageProvider};
|
||||
use pezsp_api::{ApiExt, ProvideRuntimeApi};
|
||||
use pezsp_blockchain::HeaderBackend;
|
||||
use pezsp_runtime::{traits::Block as BlockT, OpaqueExtrinsic};
|
||||
|
||||
use clap::Parser;
|
||||
use std::{fmt::Debug, sync::Arc};
|
||||
|
||||
use super::bench::{Benchmark, BenchmarkParams};
|
||||
|
||||
/// Benchmark the execution time of historic blocks.
|
||||
///
|
||||
/// This can be used to verify that blocks do not use more weight than they consumed
|
||||
/// in their `WeightInfo`. Example:
|
||||
///
|
||||
/// Let's say you are on a Bizinikiwi chain and want to verify that the first 3 blocks
|
||||
/// did not use more weight than declared which would otherwise be an issue.
|
||||
/// To test this with a dev node, first create one with a temp directory:
|
||||
///
|
||||
/// $ bizinikiwi --dev -d /tmp/my-dev --wasm-execution compiled
|
||||
///
|
||||
/// And wait some time to let it produce 3 blocks. Then benchmark them with:
|
||||
///
|
||||
/// $ bizinikiwi benchmark-block --from 1 --to 3 --dev -d /tmp/my-dev
|
||||
/// --wasm-execution compiled --pruning archive
|
||||
///
|
||||
/// The output will be similar to this:
|
||||
///
|
||||
/// Block 1 with 1 tx used 77.34% of its weight ( 5,308,964 of 6,864,645 ns)
|
||||
/// Block 2 with 1 tx used 77.99% of its weight ( 5,353,992 of 6,864,645 ns)
|
||||
/// Block 3 with 1 tx used 75.91% of its weight ( 5,305,938 of 6,989,645 ns)
|
||||
///
|
||||
/// The percent number is important and indicates how much weight
|
||||
/// was used as compared to the consumed weight.
|
||||
/// This number should be below 100% for reference hardware.
|
||||
#[derive(Debug, Parser)]
|
||||
pub struct BlockCmd {
|
||||
#[allow(missing_docs)]
|
||||
#[clap(flatten)]
|
||||
pub shared_params: SharedParams,
|
||||
|
||||
#[allow(missing_docs)]
|
||||
#[clap(flatten)]
|
||||
pub import_params: ImportParams,
|
||||
|
||||
#[allow(missing_docs)]
|
||||
#[clap(flatten)]
|
||||
pub params: BenchmarkParams,
|
||||
|
||||
/// Enable the Trie cache.
|
||||
///
|
||||
/// This should only be used for performance analysis and not for final results.
|
||||
#[arg(long)]
|
||||
pub enable_trie_cache: bool,
|
||||
}
|
||||
|
||||
impl BlockCmd {
|
||||
/// Benchmark the execution time of historic blocks and compare it to their consumed weight.
|
||||
///
|
||||
/// Output will be printed to console.
|
||||
pub fn run<Block, BA, C>(&self, client: Arc<C>) -> Result<()>
|
||||
where
|
||||
Block: BlockT<Extrinsic = OpaqueExtrinsic>,
|
||||
BA: ClientBackend<Block>,
|
||||
C: BlockBackend<Block>
|
||||
+ ProvideRuntimeApi<Block>
|
||||
+ StorageProvider<Block, BA>
|
||||
+ UsageProvider<Block>
|
||||
+ HeaderBackend<Block>,
|
||||
C::Api: ApiExt<Block> + BlockBuilderApi<Block>,
|
||||
{
|
||||
// Put everything in the benchmark type to have the generic types handy.
|
||||
Benchmark::new(client, self.params.clone()).run()
|
||||
}
|
||||
}
|
||||
|
||||
// Boilerplate
|
||||
impl CliConfiguration for BlockCmd {
|
||||
fn shared_params(&self) -> &SharedParams {
|
||||
&self.shared_params
|
||||
}
|
||||
|
||||
fn import_params(&self) -> Option<&ImportParams> {
|
||||
Some(&self.import_params)
|
||||
}
|
||||
|
||||
fn trie_cache_maximum_size(&self) -> Result<Option<usize>> {
|
||||
if self.enable_trie_cache {
|
||||
Ok(self.import_params().map(|x| x.trie_cache_maximum_size()).unwrap_or_default())
|
||||
} else {
|
||||
Ok(None)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,24 @@
|
||||
// This file is part of Bizinikiwi.
|
||||
|
||||
// Copyright (C) Parity Technologies (UK) Ltd.
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
//! Crate to benchmark the execution time of historic blocks
|
||||
//! and compare it to their consumed weight.
|
||||
|
||||
mod bench;
|
||||
mod cmd;
|
||||
|
||||
pub use cmd::BlockCmd;
|
||||
@@ -0,0 +1,234 @@
|
||||
// This file is part of Bizinikiwi.
|
||||
|
||||
// Copyright (C) Parity Technologies (UK) Ltd.
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
//! Contains the core benchmarking logic.
|
||||
|
||||
use pezsc_block_builder::{BlockBuilderApi, BlockBuilderBuilder, BuiltBlock};
|
||||
use pezsc_cli::{Error, Result};
|
||||
use pezsc_client_api::UsageProvider;
|
||||
use pezsp_api::{ApiExt, CallApiAt, Core, ProvideRuntimeApi};
|
||||
use pezsp_blockchain::{
|
||||
ApplyExtrinsicFailed::Validity,
|
||||
Error::{ApplyExtrinsicFailed, RuntimeApiError},
|
||||
};
|
||||
use pezsp_runtime::{
|
||||
traits::Block as BlockT,
|
||||
transaction_validity::{InvalidTransaction, TransactionValidityError},
|
||||
Digest, DigestItem, OpaqueExtrinsic,
|
||||
};
|
||||
|
||||
use super::ExtrinsicBuilder;
|
||||
use crate::shared::{StatSelect, Stats};
|
||||
use clap::Args;
|
||||
use codec::Encode;
|
||||
use log::info;
|
||||
use serde::Serialize;
|
||||
use pezsp_trie::proof_size_extension::ProofSizeExt;
|
||||
use std::{marker::PhantomData, sync::Arc, time::Instant};
|
||||
|
||||
/// Parameters to configure an *overhead* benchmark.
|
||||
#[derive(Debug, Default, Serialize, Clone, PartialEq, Args)]
|
||||
pub struct BenchmarkParams {
|
||||
/// Rounds of warmups before measuring.
|
||||
#[arg(long, default_value_t = 10)]
|
||||
pub warmup: u32,
|
||||
|
||||
/// How many times the benchmark should be repeated.
|
||||
#[arg(long, default_value_t = 100)]
|
||||
pub repeat: u32,
|
||||
|
||||
/// Maximal number of extrinsics that should be put into a block.
|
||||
///
|
||||
/// Only useful for debugging.
|
||||
#[arg(long)]
|
||||
pub max_ext_per_block: Option<u32>,
|
||||
}
|
||||
|
||||
/// The results of multiple runs in nano seconds.
|
||||
pub(crate) type BenchRecord = Vec<u64>;
|
||||
|
||||
/// Holds all objects needed to run the *overhead* benchmarks.
|
||||
pub(crate) struct Benchmark<Block, C> {
|
||||
client: Arc<C>,
|
||||
params: BenchmarkParams,
|
||||
inherent_data: pezsp_inherents::InherentData,
|
||||
digest_items: Vec<DigestItem>,
|
||||
record_proof: bool,
|
||||
_p: PhantomData<Block>,
|
||||
}
|
||||
|
||||
impl<Block, C> Benchmark<Block, C>
|
||||
where
|
||||
Block: BlockT<Extrinsic = OpaqueExtrinsic>,
|
||||
C: ProvideRuntimeApi<Block>
|
||||
+ CallApiAt<Block>
|
||||
+ UsageProvider<Block>
|
||||
+ pezsp_blockchain::HeaderBackend<Block>,
|
||||
C::Api: ApiExt<Block> + BlockBuilderApi<Block>,
|
||||
{
|
||||
/// Create a new [`Self`] from the arguments.
|
||||
pub fn new(
|
||||
client: Arc<C>,
|
||||
params: BenchmarkParams,
|
||||
inherent_data: pezsp_inherents::InherentData,
|
||||
digest_items: Vec<DigestItem>,
|
||||
record_proof: bool,
|
||||
) -> Self {
|
||||
Self { client, params, inherent_data, digest_items, record_proof, _p: PhantomData }
|
||||
}
|
||||
|
||||
/// Benchmark a block with only inherents.
|
||||
///
|
||||
/// Returns the Ref time stats and the proof size.
|
||||
pub fn bench_block(&self) -> Result<(Stats, u64)> {
|
||||
let (block, _, proof_size) = self.build_block(None)?;
|
||||
let record = self.measure_block(&block)?;
|
||||
|
||||
Ok((Stats::new(&record)?, proof_size))
|
||||
}
|
||||
|
||||
/// Benchmark the time of an extrinsic in a full block.
|
||||
///
|
||||
/// First benchmarks an empty block, analogous to `bench_block` and use it as baseline.
|
||||
/// Then benchmarks a full block built with the given `ext_builder` and subtracts the baseline
|
||||
/// from the result.
|
||||
/// This is necessary to account for the time the inherents use. Returns ref time stats and the
|
||||
/// proof size.
|
||||
pub fn bench_extrinsic(&self, ext_builder: &dyn ExtrinsicBuilder) -> Result<(Stats, u64)> {
|
||||
let (block, _, base_proof_size) = self.build_block(None)?;
|
||||
let base = self.measure_block(&block)?;
|
||||
let base_time = Stats::new(&base)?.select(StatSelect::Average);
|
||||
|
||||
let (block, num_ext, proof_size) = self.build_block(Some(ext_builder))?;
|
||||
let num_ext = num_ext.ok_or_else(|| Error::Input("Block was empty".into()))?;
|
||||
let mut records = self.measure_block(&block)?;
|
||||
|
||||
for r in &mut records {
|
||||
// Subtract the base time.
|
||||
*r = r.saturating_sub(base_time);
|
||||
// Divide by the number of extrinsics in the block.
|
||||
*r = ((*r as f64) / (num_ext as f64)).ceil() as u64;
|
||||
}
|
||||
|
||||
Ok((Stats::new(&records)?, proof_size.saturating_sub(base_proof_size)))
|
||||
}
|
||||
|
||||
/// Builds a block with some optional extrinsics.
|
||||
///
|
||||
/// Returns the block and the number of extrinsics in the block
|
||||
/// that are not inherents together with the proof size.
|
||||
/// Returns a block with only inherents if `ext_builder` is `None`.
|
||||
fn build_block(
|
||||
&self,
|
||||
ext_builder: Option<&dyn ExtrinsicBuilder>,
|
||||
) -> Result<(Block, Option<u64>, u64)> {
|
||||
let chain = self.client.usage_info().chain;
|
||||
let mut builder = BlockBuilderBuilder::new(&*self.client)
|
||||
.on_parent_block(chain.best_hash)
|
||||
.with_parent_block_number(chain.best_number)
|
||||
.with_inherent_digests(Digest { logs: self.digest_items.clone() })
|
||||
.with_proof_recording(self.record_proof)
|
||||
.build()?;
|
||||
|
||||
// Create and insert the inherents.
|
||||
let inherents = builder.create_inherents(self.inherent_data.clone())?;
|
||||
for inherent in inherents {
|
||||
builder.push(inherent)?;
|
||||
}
|
||||
|
||||
let num_ext = match ext_builder {
|
||||
Some(ext_builder) => {
|
||||
// Put as many extrinsics into the block as possible and count them.
|
||||
info!("Building block, this takes some time...");
|
||||
let mut num_ext = 0;
|
||||
for nonce in 0..self.max_ext_per_block() {
|
||||
let ext = ext_builder.build(nonce)?;
|
||||
match builder.push(ext.clone()) {
|
||||
Ok(()) => {},
|
||||
Err(ApplyExtrinsicFailed(Validity(TransactionValidityError::Invalid(
|
||||
InvalidTransaction::ExhaustsResources,
|
||||
)))) => break, // Block is full
|
||||
Err(e) => return Err(Error::Client(e)),
|
||||
}
|
||||
num_ext += 1;
|
||||
}
|
||||
if num_ext == 0 {
|
||||
return Err("A Block must hold at least one extrinsic".into());
|
||||
}
|
||||
info!("Extrinsics per block: {}", num_ext);
|
||||
Some(num_ext)
|
||||
},
|
||||
None => None,
|
||||
};
|
||||
|
||||
let BuiltBlock { block, proof, .. } = builder.build()?;
|
||||
|
||||
Ok((
|
||||
block,
|
||||
num_ext,
|
||||
proof
|
||||
.map(|p| p.encoded_size())
|
||||
.unwrap_or(0)
|
||||
.try_into()
|
||||
.map_err(|_| "Proof size is too large".to_string())?,
|
||||
))
|
||||
}
|
||||
|
||||
/// Measures the time that it take to execute a block or an extrinsic.
|
||||
fn measure_block(&self, block: &Block) -> Result<BenchRecord> {
|
||||
let mut record = BenchRecord::new();
|
||||
let genesis = self.client.info().genesis_hash;
|
||||
|
||||
let measure_block = || -> Result<u128> {
|
||||
let block = block.clone();
|
||||
let mut runtime_api = self.client.runtime_api();
|
||||
if self.record_proof {
|
||||
runtime_api.record_proof();
|
||||
let recorder = runtime_api
|
||||
.proof_recorder()
|
||||
.expect("Proof recording is enabled in the line above; qed.");
|
||||
runtime_api.register_extension(ProofSizeExt::new(recorder));
|
||||
}
|
||||
let start = Instant::now();
|
||||
|
||||
runtime_api
|
||||
.execute_block(genesis, block.into())
|
||||
.map_err(|e| Error::Client(RuntimeApiError(e)))?;
|
||||
|
||||
Ok(start.elapsed().as_nanos())
|
||||
};
|
||||
|
||||
info!("Running {} warmups...", self.params.warmup);
|
||||
for _ in 0..self.params.warmup {
|
||||
measure_block()?;
|
||||
}
|
||||
|
||||
info!("Executing block {} times", self.params.repeat);
|
||||
// Interesting part here:
|
||||
// Execute a block multiple times and record each execution time.
|
||||
for _ in 0..self.params.repeat {
|
||||
let elapsed = measure_block()?;
|
||||
record.push(elapsed as u64);
|
||||
}
|
||||
|
||||
Ok(record)
|
||||
}
|
||||
|
||||
fn max_ext_per_block(&self) -> u32 {
|
||||
self.params.max_ext_per_block.unwrap_or(u32::MAX)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,152 @@
|
||||
// This file is part of Bizinikiwi.
|
||||
|
||||
// Copyright (C) Parity Technologies (UK) Ltd.
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
use pezsc_block_builder::BlockBuilderApi;
|
||||
use pezsc_cli::{CliConfiguration, ImportParams, Result, SharedParams};
|
||||
use pezsc_client_api::UsageProvider;
|
||||
use pezsp_api::{ApiExt, CallApiAt, ProvideRuntimeApi};
|
||||
use pezsp_runtime::{traits::Block as BlockT, DigestItem, OpaqueExtrinsic};
|
||||
|
||||
use clap::{Args, Parser};
|
||||
use log::info;
|
||||
use serde::Serialize;
|
||||
use std::{fmt::Debug, sync::Arc};
|
||||
|
||||
use super::{
|
||||
bench::{Benchmark, BenchmarkParams},
|
||||
extrinsic_factory::ExtrinsicFactory,
|
||||
};
|
||||
|
||||
/// Benchmark the execution time of different extrinsics.
|
||||
///
|
||||
/// This is calculated by filling a block with a specific extrinsic and executing the block.
|
||||
/// The result time is then divided by the number of extrinsics in that block.
|
||||
///
|
||||
/// NOTE: The BlockExecutionWeight is ignored in this case since it
|
||||
// is very small compared to the total block execution time.
|
||||
#[derive(Debug, Parser)]
|
||||
pub struct ExtrinsicCmd {
|
||||
#[allow(missing_docs)]
|
||||
#[clap(flatten)]
|
||||
pub shared_params: SharedParams,
|
||||
|
||||
#[allow(missing_docs)]
|
||||
#[clap(flatten)]
|
||||
pub import_params: ImportParams,
|
||||
|
||||
#[allow(missing_docs)]
|
||||
#[clap(flatten)]
|
||||
pub params: ExtrinsicParams,
|
||||
}
|
||||
|
||||
/// The params for the [`ExtrinsicCmd`].
|
||||
#[derive(Debug, Default, Serialize, Clone, PartialEq, Args)]
|
||||
pub struct ExtrinsicParams {
|
||||
#[clap(flatten)]
|
||||
pub bench: BenchmarkParams,
|
||||
|
||||
/// List all available pallets and extrinsics.
|
||||
///
|
||||
/// The format is CSV with header `pallet, extrinsic`.
|
||||
#[arg(long)]
|
||||
pub list: bool,
|
||||
|
||||
/// Pallet name of the extrinsic to benchmark.
|
||||
#[arg(long, value_name = "PALLET", required_unless_present = "list")]
|
||||
pub pallet: Option<String>,
|
||||
|
||||
/// Extrinsic to benchmark.
|
||||
#[arg(long, value_name = "EXTRINSIC", required_unless_present = "list")]
|
||||
pub extrinsic: Option<String>,
|
||||
|
||||
/// Enable the Trie cache.
|
||||
///
|
||||
/// This should only be used for performance analysis and not for final results.
|
||||
#[arg(long)]
|
||||
pub enable_trie_cache: bool,
|
||||
}
|
||||
|
||||
impl ExtrinsicCmd {
|
||||
/// Benchmark the execution time of a specific type of extrinsic.
|
||||
///
|
||||
/// The output will be printed to console.
|
||||
pub fn run<Block, C>(
|
||||
&self,
|
||||
client: Arc<C>,
|
||||
inherent_data: pezsp_inherents::InherentData,
|
||||
digest_items: Vec<DigestItem>,
|
||||
ext_factory: &ExtrinsicFactory,
|
||||
) -> Result<()>
|
||||
where
|
||||
Block: BlockT<Extrinsic = OpaqueExtrinsic>,
|
||||
C: ProvideRuntimeApi<Block>
|
||||
+ CallApiAt<Block>
|
||||
+ UsageProvider<Block>
|
||||
+ pezsp_blockchain::HeaderBackend<Block>,
|
||||
C::Api: ApiExt<Block> + BlockBuilderApi<Block>,
|
||||
{
|
||||
// Short circuit if --list was specified.
|
||||
if self.params.list {
|
||||
let list: Vec<String> = ext_factory.0.iter().map(|b| b.name()).collect();
|
||||
info!(
|
||||
"Listing available extrinsics ({}):\npallet, extrinsic\n{}",
|
||||
list.len(),
|
||||
list.join("\n")
|
||||
);
|
||||
return Ok(());
|
||||
}
|
||||
|
||||
let pallet = self.params.pallet.clone().unwrap_or_default();
|
||||
let extrinsic = self.params.extrinsic.clone().unwrap_or_default();
|
||||
let ext_builder = match ext_factory.try_get(&pallet, &extrinsic) {
|
||||
Some(ext_builder) => ext_builder,
|
||||
None =>
|
||||
return Err("Unknown pallet or extrinsic. Use --list for a complete list.".into()),
|
||||
};
|
||||
|
||||
let bench =
|
||||
Benchmark::new(client, self.params.bench.clone(), inherent_data, digest_items, false);
|
||||
let stats = bench.bench_extrinsic(ext_builder)?;
|
||||
info!(
|
||||
"Executing a {}::{} extrinsic takes[ns]:\n{:?}",
|
||||
ext_builder.pallet(),
|
||||
ext_builder.extrinsic(),
|
||||
stats
|
||||
);
|
||||
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
// Boilerplate
|
||||
impl CliConfiguration for ExtrinsicCmd {
|
||||
fn shared_params(&self) -> &SharedParams {
|
||||
&self.shared_params
|
||||
}
|
||||
|
||||
fn import_params(&self) -> Option<&ImportParams> {
|
||||
Some(&self.import_params)
|
||||
}
|
||||
|
||||
fn trie_cache_maximum_size(&self) -> Result<Option<usize>> {
|
||||
if self.params.enable_trie_cache {
|
||||
Ok(self.import_params().map(|x| x.trie_cache_maximum_size()).unwrap_or_default())
|
||||
} else {
|
||||
Ok(None)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,70 @@
|
||||
// This file is part of Bizinikiwi.
|
||||
|
||||
// Copyright (C) Parity Technologies (UK) Ltd.
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
//! Provides the [`ExtrinsicFactory`] and the [`ExtrinsicBuilder`] types.
|
||||
//! Is used by the *overhead* and *extrinsic* benchmarks to build extrinsics.
|
||||
|
||||
use pezsp_runtime::OpaqueExtrinsic;
|
||||
|
||||
/// Helper to manage [`ExtrinsicBuilder`] instances.
|
||||
#[derive(Default)]
|
||||
pub struct ExtrinsicFactory(pub Vec<Box<dyn ExtrinsicBuilder>>);
|
||||
|
||||
impl ExtrinsicFactory {
|
||||
/// Returns a builder for a pallet and extrinsic name.
|
||||
///
|
||||
/// Is case in-sensitive.
|
||||
pub fn try_get(&self, pallet: &str, extrinsic: &str) -> Option<&dyn ExtrinsicBuilder> {
|
||||
let pallet = pallet.to_lowercase();
|
||||
let extrinsic = extrinsic.to_lowercase();
|
||||
|
||||
self.0
|
||||
.iter()
|
||||
.find(|b| b.pallet() == pallet && b.extrinsic() == extrinsic)
|
||||
.map(|b| b.as_ref())
|
||||
}
|
||||
}
|
||||
|
||||
/// Used by the benchmark to build signed extrinsics.
|
||||
///
|
||||
/// The built extrinsics only need to be valid in the first block
|
||||
/// who's parent block is the genesis block.
|
||||
/// This assumption simplifies the generation of the extrinsics.
|
||||
/// The signer should be one of the pre-funded dev accounts.
|
||||
pub trait ExtrinsicBuilder {
|
||||
/// Name of the pallet this builder is for.
|
||||
///
|
||||
/// Should be all lowercase.
|
||||
fn pallet(&self) -> &str;
|
||||
|
||||
/// Name of the extrinsic this builder is for.
|
||||
///
|
||||
/// Should be all lowercase.
|
||||
fn extrinsic(&self) -> &str;
|
||||
|
||||
/// Builds an extrinsic.
|
||||
///
|
||||
/// Will be called multiple times with increasing nonces.
|
||||
fn build(&self, nonce: u32) -> std::result::Result<OpaqueExtrinsic, &'static str>;
|
||||
}
|
||||
|
||||
impl dyn ExtrinsicBuilder + '_ {
|
||||
/// Name of this builder in CSV format: `pallet, extrinsic`.
|
||||
pub fn name(&self) -> String {
|
||||
format!("{}, {}", self.pallet(), self.extrinsic())
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,27 @@
|
||||
// This file is part of Bizinikiwi.
|
||||
|
||||
// Copyright (C) Parity Technologies (UK) Ltd.
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
//! Benchmark the time it takes to execute a specific extrinsic.
|
||||
//! This is a generalization of the *overhead* benchmark which can only measure `System::Remark`
|
||||
//! extrinsics.
|
||||
|
||||
pub mod bench;
|
||||
pub mod cmd;
|
||||
pub mod extrinsic_factory;
|
||||
|
||||
pub use cmd::ExtrinsicCmd;
|
||||
pub use extrinsic_factory::{ExtrinsicBuilder, ExtrinsicFactory};
|
||||
@@ -0,0 +1,124 @@
|
||||
// This file is part of Bizinikiwi.
|
||||
|
||||
// Copyright (C) Parity Technologies (UK) Ltd.
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
//! Contains the root [`BenchmarkCmd`] command and exports its sub-commands.
|
||||
|
||||
mod block;
|
||||
mod extrinsic;
|
||||
mod machine;
|
||||
mod overhead;
|
||||
mod pallet;
|
||||
mod shared;
|
||||
mod storage;
|
||||
|
||||
pub use block::BlockCmd;
|
||||
pub use extrinsic::{ExtrinsicBuilder, ExtrinsicCmd, ExtrinsicFactory};
|
||||
pub use machine::{MachineCmd, BIZINIKIWI_REFERENCE_HARDWARE};
|
||||
pub use overhead::{
|
||||
remark_builder::{DynamicRemarkBuilder, BizinikiwiRemarkBuilder},
|
||||
OpaqueBlock, OverheadCmd,
|
||||
};
|
||||
pub use pallet::PalletCmd;
|
||||
pub use pezsc_service::BasePath;
|
||||
pub use storage::StorageCmd;
|
||||
|
||||
use pezsc_cli::{CliConfiguration, DatabaseParams, ImportParams, PruningParams, Result, SharedParams};
|
||||
|
||||
/// The root `benchmarking` command.
|
||||
///
|
||||
/// Has no effect itself besides printing a help menu of the sub-commands.
|
||||
#[derive(Debug, clap::Subcommand)]
|
||||
pub enum BenchmarkCmd {
|
||||
Pallet(PalletCmd),
|
||||
Storage(StorageCmd),
|
||||
Overhead(OverheadCmd),
|
||||
Block(BlockCmd),
|
||||
Machine(MachineCmd),
|
||||
Extrinsic(ExtrinsicCmd),
|
||||
}
|
||||
|
||||
/// Unwraps a [`BenchmarkCmd`] into its concrete sub-command.
|
||||
macro_rules! unwrap_cmd {
|
||||
{
|
||||
$self:expr,
|
||||
$cmd:ident,
|
||||
$code:expr
|
||||
} => {
|
||||
match $self {
|
||||
BenchmarkCmd::Pallet($cmd) => $code,
|
||||
BenchmarkCmd::Storage($cmd) => $code,
|
||||
BenchmarkCmd::Overhead($cmd) => $code,
|
||||
BenchmarkCmd::Block($cmd) => $code,
|
||||
BenchmarkCmd::Machine($cmd) => $code,
|
||||
BenchmarkCmd::Extrinsic($cmd) => $code,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Forward the [`CliConfiguration`] trait implementation.
|
||||
///
|
||||
/// Each time a sub-command exposes a new config option, it must be added here.
|
||||
impl CliConfiguration for BenchmarkCmd {
|
||||
fn shared_params(&self) -> &SharedParams {
|
||||
unwrap_cmd! {
|
||||
self, cmd, cmd.shared_params()
|
||||
}
|
||||
}
|
||||
|
||||
fn import_params(&self) -> Option<&ImportParams> {
|
||||
unwrap_cmd! {
|
||||
self, cmd, cmd.import_params()
|
||||
}
|
||||
}
|
||||
|
||||
fn database_params(&self) -> Option<&DatabaseParams> {
|
||||
unwrap_cmd! {
|
||||
self, cmd, cmd.database_params()
|
||||
}
|
||||
}
|
||||
|
||||
fn base_path(&self) -> Result<Option<BasePath>> {
|
||||
let inner = unwrap_cmd! {
|
||||
self, cmd, cmd.base_path()
|
||||
};
|
||||
|
||||
// If the base path was not provided, benchmark command shall use temporary path. Otherwise
|
||||
// we may end up using shared path, which may be inappropriate for benchmarking.
|
||||
match inner {
|
||||
Ok(None) => Some(BasePath::new_temp_dir()).transpose().map_err(|e| e.into()),
|
||||
e => e,
|
||||
}
|
||||
}
|
||||
|
||||
fn pruning_params(&self) -> Option<&PruningParams> {
|
||||
unwrap_cmd! {
|
||||
self, cmd, cmd.pruning_params()
|
||||
}
|
||||
}
|
||||
|
||||
fn trie_cache_maximum_size(&self) -> Result<Option<usize>> {
|
||||
unwrap_cmd! {
|
||||
self, cmd, cmd.trie_cache_maximum_size()
|
||||
}
|
||||
}
|
||||
|
||||
fn chain_id(&self, is_dev: bool) -> Result<String> {
|
||||
unwrap_cmd! {
|
||||
self, cmd, cmd.chain_id(is_dev)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,83 @@
|
||||
# The `benchmark machine` command
|
||||
|
||||
Different Bizinikiwi chains can have different hardware requirements.
|
||||
It is therefore important to be able to quickly gauge if a piece of hardware fits a chains' requirements.
|
||||
The `benchmark machine` command archives this by measuring key metrics and making them comparable.
|
||||
|
||||
Invoking the command looks like this:
|
||||
```sh
|
||||
cargo run --profile=production -- benchmark machine --dev
|
||||
```
|
||||
|
||||
## Output
|
||||
|
||||
The output on reference hardware:
|
||||
|
||||
```pre
|
||||
+----------+----------------+---------------+--------------+-------------------+
|
||||
| Category | Function | Score | Minimum | Result |
|
||||
+----------+----------------+---------------+--------------+-------------------+
|
||||
| CPU | BLAKE2-256 | 1023.00 MiB/s | 1.00 GiB/s | ✅ Pass ( 99.4 %) |
|
||||
+----------+----------------+---------------+--------------+-------------------+
|
||||
| CPU | SR25519-Verify | 665.13 KiB/s | 666.00 KiB/s | ✅ Pass ( 99.9 %) |
|
||||
+----------+----------------+---------------+--------------+-------------------+
|
||||
| Memory | Copy | 14.39 GiB/s | 14.32 GiB/s | ✅ Pass (100.4 %) |
|
||||
+----------+----------------+---------------+--------------+-------------------+
|
||||
| Disk | Seq Write | 457.00 MiB/s | 450.00 MiB/s | ✅ Pass (101.6 %) |
|
||||
+----------+----------------+---------------+--------------+-------------------+
|
||||
| Disk | Rnd Write | 190.00 MiB/s | 200.00 MiB/s | ✅ Pass ( 95.0 %) |
|
||||
+----------+----------------+---------------+--------------+-------------------+
|
||||
```
|
||||
|
||||
The *score* is the average result of each benchmark. It always adheres to "higher is better".
|
||||
|
||||
The *category* indicate which part of the hardware was benchmarked:
|
||||
- **CPU** Processor intensive task
|
||||
- **Memory** RAM intensive task
|
||||
- **Disk** Hard drive intensive task
|
||||
|
||||
The *function* is the concrete benchmark that was run:
|
||||
- **BLAKE2-256** The throughput of the [Blake2-256] cryptographic hashing function with 32 KiB input. The [blake2_256
|
||||
function] is used in many places in Bizinikiwi. The throughput of a hash function strongly depends on the input size,
|
||||
therefore we settled to use a fixed input size for comparable results.
|
||||
- **SR25519 Verify** Sr25519 is an optimized version of the [Curve25519] signature scheme. Signature verification is
|
||||
used by Bizinikiwi when verifying extrinsics and blocks.
|
||||
- **Copy** The throughput of copying memory from one place in the RAM to another.
|
||||
- **Seq Write** The throughput of writing data to the storage location sequentially. It is important that the same disk
|
||||
is used that will later-on be used to store the chain data.
|
||||
- **Rnd Write** The throughput of writing data to the storage location in a random order. This is normally much slower
|
||||
than the sequential write.
|
||||
|
||||
The *score* needs to reach the *minimum* in order to pass the benchmark. This can be reduced with the `--tolerance`
|
||||
flag.
|
||||
|
||||
The *result* indicated if a specific benchmark was passed by the machine or not. The percent number is the relative
|
||||
score reached to the *minimum* that is needed. The `--tolerance` flag is taken into account for this decision. For
|
||||
example a benchmark that passes even with 95% since the *tolerance* was set to 10% would look like this: `✅ Pass ( 95.0
|
||||
%)`.
|
||||
|
||||
## Interpretation
|
||||
|
||||
Ideally all results show a `Pass` and the program exits with code 0. Currently some of the benchmarks can fail even on
|
||||
reference hardware; they are still being improved to make them more deterministic.
|
||||
Make sure to run nothing else on the machine when benchmarking it.
|
||||
You can re-run them multiple times to get more reliable results.
|
||||
|
||||
## Arguments
|
||||
|
||||
- `--tolerance` A percent number to reduce the *minimum* requirement. This should be used to ignore outliers of the
|
||||
benchmarks. The default value is 10%.
|
||||
- `--verify-duration` How long the verification benchmark should run.
|
||||
- `--disk-duration` How long the *read* and *write* benchmarks should run each.
|
||||
- `--allow-fail` Always exit the program with code 0.
|
||||
- `--chain` / `--dev` Specify the chain config to use. This will be used to compare the results with the requirements of
|
||||
the chain (WIP).
|
||||
- [`--base-path`]
|
||||
|
||||
License: Apache-2.0
|
||||
|
||||
<!-- LINKS -->
|
||||
[Blake2-256]: https://www.blake2.net/
|
||||
[blake2_256 function]: https://docs.rs/pezsp-crypto-hashing/latest/sp_crypto_hashing/fn.blake2_256.html
|
||||
[Curve25519]: https://en.wikipedia.org/wiki/Curve25519
|
||||
[`--base-path`]: ../shared/README.md#arguments
|
||||
@@ -0,0 +1,85 @@
|
||||
// This file is part of Bizinikiwi.
|
||||
|
||||
// Copyright (C) Parity Technologies (UK) Ltd.
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
//! Contains types to define hardware requirements.
|
||||
|
||||
use pezsc_sysinfo::Requirements;
|
||||
use std::sync::LazyLock;
|
||||
|
||||
/// The hardware requirements as measured on reference hardware.
|
||||
///
|
||||
/// These values are provided by Parity, however it is possible
|
||||
/// to use your own requirements if you are running a custom chain.
|
||||
pub static BIZINIKIWI_REFERENCE_HARDWARE: LazyLock<Requirements> = LazyLock::new(|| {
|
||||
let raw = include_bytes!("reference_hardware.json").as_slice();
|
||||
serde_json::from_slice(raw).expect("Hardcoded data is known good; qed")
|
||||
});
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use pezsc_sysinfo::{Metric, Requirement, Requirements, Throughput};
|
||||
|
||||
/// `BIZINIKIWI_REFERENCE_HARDWARE` can be decoded.
|
||||
#[test]
|
||||
fn json_static_data() {
|
||||
let raw = serde_json::to_string(&*BIZINIKIWI_REFERENCE_HARDWARE).unwrap();
|
||||
let decoded: Requirements = serde_json::from_str(&raw).unwrap();
|
||||
|
||||
assert_eq!(decoded, BIZINIKIWI_REFERENCE_HARDWARE.clone());
|
||||
}
|
||||
|
||||
/// The hard-coded values are correct.
|
||||
#[test]
|
||||
fn json_static_data_is_correct() {
|
||||
assert_eq!(
|
||||
*BIZINIKIWI_REFERENCE_HARDWARE,
|
||||
Requirements(vec![
|
||||
Requirement {
|
||||
metric: Metric::Blake2256,
|
||||
minimum: Throughput::from_mibs(1000.00),
|
||||
validator_only: false
|
||||
},
|
||||
Requirement {
|
||||
metric: Metric::Blake2256Parallel { num_cores: 8 },
|
||||
minimum: Throughput::from_mibs(1000.00),
|
||||
validator_only: true,
|
||||
},
|
||||
Requirement {
|
||||
metric: Metric::Sr25519Verify,
|
||||
minimum: Throughput::from_kibs(637.619999744),
|
||||
validator_only: false
|
||||
},
|
||||
Requirement {
|
||||
metric: Metric::MemCopy,
|
||||
minimum: Throughput::from_gibs(11.4925205078125003),
|
||||
validator_only: false,
|
||||
},
|
||||
Requirement {
|
||||
metric: Metric::DiskSeqWrite,
|
||||
minimum: Throughput::from_mibs(950.0),
|
||||
validator_only: false,
|
||||
},
|
||||
Requirement {
|
||||
metric: Metric::DiskRndWrite,
|
||||
minimum: Throughput::from_mibs(420.0),
|
||||
validator_only: false
|
||||
},
|
||||
])
|
||||
);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,242 @@
|
||||
// This file is part of Bizinikiwi.
|
||||
|
||||
// Copyright (C) Parity Technologies (UK) Ltd.
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
//! Contains the [`MachineCmd`] as entry point for the node
|
||||
//! and the core benchmarking logic.
|
||||
|
||||
pub mod hardware;
|
||||
|
||||
use std::{boxed::Box, fs, path::Path};
|
||||
|
||||
use clap::Parser;
|
||||
use comfy_table::{Row, Table};
|
||||
use log::{error, info, warn};
|
||||
|
||||
use pezsc_cli::{CliConfiguration, Result, SharedParams};
|
||||
use pezsc_service::Configuration;
|
||||
use pezsc_sysinfo::{
|
||||
benchmark_cpu, benchmark_cpu_parallelism, benchmark_disk_random_writes,
|
||||
benchmark_disk_sequential_writes, benchmark_memory, benchmark_sr25519_verify, ExecutionLimit,
|
||||
Metric, Requirement, Requirements, Throughput,
|
||||
};
|
||||
|
||||
use crate::shared::check_build_profile;
|
||||
pub use hardware::BIZINIKIWI_REFERENCE_HARDWARE;
|
||||
|
||||
/// Command to benchmark the hardware.
|
||||
///
|
||||
/// Runs multiple benchmarks and prints their output to console.
|
||||
/// Can be used to gauge if the hardware is fast enough to keep up with a chain's requirements.
|
||||
/// This command must be integrated by the client since the client can set compiler flags
|
||||
/// which influence the results.
|
||||
///
|
||||
/// You can use the `--base-path` flag to set a location for the disk benchmarks.
|
||||
#[derive(Debug, Parser)]
|
||||
pub struct MachineCmd {
|
||||
#[allow(missing_docs)]
|
||||
#[clap(flatten)]
|
||||
pub shared_params: SharedParams,
|
||||
|
||||
/// Do not return an error if any check fails.
|
||||
///
|
||||
/// Should only be used for debugging.
|
||||
#[arg(long)]
|
||||
pub allow_fail: bool,
|
||||
|
||||
/// Set a fault tolerance for passing a requirement.
|
||||
///
|
||||
/// 10% means that the test would pass even when only 90% score was archived.
|
||||
/// Can be used to mitigate outliers of the benchmarks.
|
||||
#[arg(long, default_value_t = 10.0, value_name = "PERCENT")]
|
||||
pub tolerance: f64,
|
||||
|
||||
/// Time limit for the verification benchmark.
|
||||
#[arg(long, default_value_t = 5.0, value_name = "SECONDS")]
|
||||
pub verify_duration: f32,
|
||||
|
||||
/// Time limit for the hash function benchmark.
|
||||
#[arg(long, default_value_t = 5.0, value_name = "SECONDS")]
|
||||
pub hash_duration: f32,
|
||||
|
||||
/// Time limit for the memory benchmark.
|
||||
#[arg(long, default_value_t = 5.0, value_name = "SECONDS")]
|
||||
pub memory_duration: f32,
|
||||
|
||||
/// Time limit for each disk benchmark.
|
||||
#[arg(long, default_value_t = 5.0, value_name = "SECONDS")]
|
||||
pub disk_duration: f32,
|
||||
}
|
||||
|
||||
/// Helper for the result of a concrete benchmark.
|
||||
struct BenchResult {
|
||||
/// Did the hardware pass the benchmark?
|
||||
passed: bool,
|
||||
|
||||
/// The absolute score that was archived.
|
||||
score: Throughput,
|
||||
|
||||
/// The score relative to the minimal required score.
|
||||
///
|
||||
/// Is in range [0, 1].
|
||||
rel_score: f64,
|
||||
}
|
||||
|
||||
/// Errors that can be returned by the this command.
|
||||
#[derive(Debug, thiserror::Error)]
|
||||
#[allow(missing_docs)]
|
||||
pub enum Error {
|
||||
#[error("One of the benchmarks had a score that was lower than its requirement")]
|
||||
UnmetRequirement,
|
||||
|
||||
#[error("The build profile is unfit for benchmarking: {0}")]
|
||||
BadBuildProfile(String),
|
||||
|
||||
#[error("Benchmark results are off by at least factor 100")]
|
||||
BadResults,
|
||||
}
|
||||
|
||||
impl MachineCmd {
|
||||
/// Execute the benchmark and print the results.
|
||||
pub fn run(&self, cfg: &Configuration, requirements: Requirements) -> Result<()> {
|
||||
self.validate_args()?;
|
||||
// Ensure that the dir exists since the node is not started to take care of it.
|
||||
let dir = cfg.database.path().ok_or("No DB directory provided")?;
|
||||
fs::create_dir_all(dir)?;
|
||||
|
||||
info!("Running machine benchmarks...");
|
||||
let mut results = Vec::new();
|
||||
for requirement in &requirements.0 {
|
||||
let result = self.run_benchmark(requirement, &dir)?;
|
||||
results.push(result);
|
||||
}
|
||||
self.print_summary(requirements, results)
|
||||
}
|
||||
|
||||
/// Benchmarks a specific metric of the hardware and judges the resulting score.
|
||||
fn run_benchmark(&self, requirement: &Requirement, dir: &Path) -> Result<BenchResult> {
|
||||
// Dispatch the concrete function from `sc-sysinfo`.
|
||||
|
||||
let score = self.measure(&requirement.metric, dir)?;
|
||||
let rel_score = score.as_bytes() / requirement.minimum.as_bytes();
|
||||
|
||||
// Sanity check if the result is off by factor >100x.
|
||||
if rel_score >= 100.0 || rel_score <= 0.01 {
|
||||
self.check_failed(Error::BadResults)?;
|
||||
}
|
||||
let passed = rel_score >= (1.0 - (self.tolerance / 100.0));
|
||||
Ok(BenchResult { passed, score, rel_score })
|
||||
}
|
||||
|
||||
/// Measures a metric of the hardware.
|
||||
fn measure(&self, metric: &Metric, dir: &Path) -> Result<Throughput> {
|
||||
let verify_limit = ExecutionLimit::from_secs_f32(self.verify_duration);
|
||||
let disk_limit = ExecutionLimit::from_secs_f32(self.disk_duration);
|
||||
let hash_limit = ExecutionLimit::from_secs_f32(self.hash_duration);
|
||||
let memory_limit = ExecutionLimit::from_secs_f32(self.memory_duration);
|
||||
|
||||
let score = match metric {
|
||||
Metric::Blake2256 => benchmark_cpu(hash_limit),
|
||||
Metric::Blake2256Parallel { num_cores } =>
|
||||
benchmark_cpu_parallelism(hash_limit, *num_cores),
|
||||
Metric::Sr25519Verify => benchmark_sr25519_verify(verify_limit),
|
||||
Metric::MemCopy => benchmark_memory(memory_limit),
|
||||
Metric::DiskSeqWrite => benchmark_disk_sequential_writes(disk_limit, dir)?,
|
||||
Metric::DiskRndWrite => benchmark_disk_random_writes(disk_limit, dir)?,
|
||||
};
|
||||
Ok(score)
|
||||
}
|
||||
|
||||
/// Prints a human-readable summary.
|
||||
fn print_summary(&self, requirements: Requirements, results: Vec<BenchResult>) -> Result<()> {
|
||||
// Use a table for nicer console output.
|
||||
let mut table = Table::new();
|
||||
table.set_header(["Category", "Function", "Score", "Minimum", "Result"]);
|
||||
// Count how many passed and how many failed.
|
||||
let (mut passed, mut failed) = (0, 0);
|
||||
for (requirement, result) in requirements.0.iter().zip(results.iter()) {
|
||||
if result.passed {
|
||||
passed += 1
|
||||
} else {
|
||||
failed += 1
|
||||
}
|
||||
|
||||
table.add_row(result.to_row(requirement));
|
||||
}
|
||||
// Print the table and a summary.
|
||||
info!(
|
||||
"\n{}\nFrom {} benchmarks in total, {} passed and {} failed ({:.0?}% fault tolerance).",
|
||||
table,
|
||||
passed + failed,
|
||||
passed,
|
||||
failed,
|
||||
self.tolerance
|
||||
);
|
||||
// Print the final result.
|
||||
if failed != 0 {
|
||||
info!("The hardware fails to meet the requirements");
|
||||
self.check_failed(Error::UnmetRequirement)?;
|
||||
} else {
|
||||
info!("The hardware meets the requirements ");
|
||||
}
|
||||
// Check that the results were not created by a bad build profile.
|
||||
if let Err(err) = check_build_profile() {
|
||||
self.check_failed(Error::BadBuildProfile(err))?;
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Returns `Ok` if [`self.allow_fail`] is set and otherwise the error argument.
|
||||
fn check_failed(&self, e: Error) -> Result<()> {
|
||||
if !self.allow_fail {
|
||||
error!("Failing since --allow-fail is not set");
|
||||
Err(pezsc_cli::Error::Application(Box::new(e)))
|
||||
} else {
|
||||
warn!("Ignoring error since --allow-fail is set: {:?}", e);
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
/// Validates the CLI arguments.
|
||||
fn validate_args(&self) -> Result<()> {
|
||||
if self.tolerance > 100.0 || self.tolerance < 0.0 {
|
||||
return Err("The --tolerance argument is out of range".into());
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
impl BenchResult {
|
||||
/// Format [`Self`] as row that can be printed in a table.
|
||||
fn to_row(&self, req: &Requirement) -> Row {
|
||||
let passed = if self.passed { "✅ Pass" } else { "❌ Fail" };
|
||||
vec![
|
||||
req.metric.category().into(),
|
||||
req.metric.name().into(),
|
||||
format!("{}", self.score),
|
||||
format!("{}", req.minimum),
|
||||
format!("{} ({: >5.1?} %)", passed, self.rel_score * 100.0),
|
||||
]
|
||||
.into()
|
||||
}
|
||||
}
|
||||
|
||||
// Boilerplate
|
||||
impl CliConfiguration for MachineCmd {
|
||||
fn shared_params(&self) -> &SharedParams {
|
||||
&self.shared_params
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,27 @@
|
||||
[
|
||||
{
|
||||
"metric": "Blake2256",
|
||||
"minimum": 1000.00
|
||||
},
|
||||
{
|
||||
"metric": {"Blake2256Parallel":{"num_cores":8}},
|
||||
"minimum": 1000.00,
|
||||
"validator_only": true
|
||||
},
|
||||
{
|
||||
"metric": "Sr25519Verify",
|
||||
"minimum": 0.622675781
|
||||
},
|
||||
{
|
||||
"metric": "MemCopy",
|
||||
"minimum": 11768.341
|
||||
},
|
||||
{
|
||||
"metric": "DiskSeqWrite",
|
||||
"minimum": 950.0
|
||||
},
|
||||
{
|
||||
"metric": "DiskRndWrite",
|
||||
"minimum": 420.0
|
||||
}
|
||||
]
|
||||
@@ -0,0 +1,146 @@
|
||||
# The `benchmark overhead` command
|
||||
|
||||
Each time an extrinsic or a block is executed, a fixed weight is charged as "execution overhead". This is necessary
|
||||
since the weight that is calculated by the pallet benchmarks does not include this overhead. The exact overhead to can
|
||||
vary per Bizinikiwi chain and needs to be calculated per chain. This command calculates the exact values of these
|
||||
overhead weights for any Bizinikiwi chain that supports it.
|
||||
|
||||
## How does it work?
|
||||
|
||||
The benchmark consists of two parts; the [`BlockExecutionWeight`] and the [`ExtrinsicBaseWeight`]. Both are executed
|
||||
sequentially when invoking the command.
|
||||
|
||||
## BlockExecutionWeight
|
||||
|
||||
The block execution weight is defined as the weight that it takes to execute an *empty block*. It is measured by
|
||||
constructing an empty block and measuring its executing time. The result are written to a `block_weights.rs` file which
|
||||
is created from a template. The file will contain the concrete weight value and various statistics about the
|
||||
measurements. For example:
|
||||
```rust
|
||||
/// Time to execute an empty block.
|
||||
/// Calculated by multiplying the *Average* with `1` and adding `0`.
|
||||
///
|
||||
/// Stats [NS]:
|
||||
/// Min, Max: 3_508_416, 3_680_498
|
||||
/// Average: 3_532_484
|
||||
/// Median: 3_522_111
|
||||
/// Std-Dev: 27070.23
|
||||
///
|
||||
/// Percentiles [NS]:
|
||||
/// 99th: 3_631_863
|
||||
/// 95th: 3_595_674
|
||||
/// 75th: 3_526_435
|
||||
pub const BlockExecutionWeight: Weight =
|
||||
Weight::from_parts(WEIGHT_REF_TIME_PER_NANOS.saturating_mul(3_532_484), 0);
|
||||
```
|
||||
|
||||
In this example it takes 3.5 ms to execute an empty block. That means that it always takes at least 3.5 ms to execute
|
||||
*any* block. This constant weight is therefore added to each block to ensure that Bizinikiwi budgets enough time to
|
||||
execute it.
|
||||
|
||||
## ExtrinsicBaseWeight
|
||||
|
||||
The extrinsic base weight is defined as the weight that it takes to execute an *empty* extrinsic. An *empty* extrinsic
|
||||
is also called a *NO-OP*. It does nothing and is the equivalent to the empty block form above. The benchmark now
|
||||
constructs a block which is filled with only NO-OP extrinsics. This block is then executed many times and the weights
|
||||
are measured. The result is divided by the number of extrinsics in that block and the results are written to
|
||||
`extrinsic_weights.rs`.
|
||||
|
||||
The relevant section in the output file looks like this:
|
||||
```rust
|
||||
/// Time to execute a NO-OP extrinsic, for example `System::remark`.
|
||||
/// Calculated by multiplying the *Average* with `1` and adding `0`.
|
||||
///
|
||||
/// Stats [NS]:
|
||||
/// Min, Max: 67_561, 69_855
|
||||
/// Average: 67_745
|
||||
/// Median: 67_701
|
||||
/// Std-Dev: 264.68
|
||||
///
|
||||
/// Percentiles [NS]:
|
||||
/// 99th: 68_758
|
||||
/// 95th: 67_843
|
||||
/// 75th: 67_749
|
||||
pub const ExtrinsicBaseWeight: Weight =
|
||||
Weight::from_parts(WEIGHT_REF_TIME_PER_NANOS.saturating_mul(67_745), 0);
|
||||
```
|
||||
|
||||
In this example it takes 67.7 µs to execute a NO-OP extrinsic. That means that it always takes at least 67.7 µs to
|
||||
execute *any* extrinsic. This constant weight is therefore added to each extrinsic to ensure that Bizinikiwi budgets
|
||||
enough time to execute it.
|
||||
|
||||
## Invocation
|
||||
|
||||
The base command looks like this (for debugging you can use `--release`):
|
||||
```sh
|
||||
cargo run --profile=production -- benchmark overhead --dev
|
||||
```
|
||||
|
||||
Output:
|
||||
```pre
|
||||
# BlockExecutionWeight
|
||||
Running 10 warmups...
|
||||
Executing block 100 times
|
||||
Per-block execution overhead [ns]:
|
||||
Total: 353248430
|
||||
Min: 3508416, Max: 3680498
|
||||
Average: 3532484, Median: 3522111, Stddev: 27070.23
|
||||
Percentiles 99th, 95th, 75th: 3631863, 3595674, 3526435
|
||||
Writing weights to "block_weights.rs"
|
||||
|
||||
# Setup
|
||||
Building block, this takes some time...
|
||||
Extrinsics per block: 12000
|
||||
|
||||
# ExtrinsicBaseWeight
|
||||
Running 10 warmups...
|
||||
Executing block 100 times
|
||||
Per-extrinsic execution overhead [ns]:
|
||||
Total: 6774590
|
||||
Min: 67561, Max: 69855
|
||||
Average: 67745, Median: 67701, Stddev: 264.68
|
||||
Percentiles 99th, 95th, 75th: 68758, 67843, 67749
|
||||
Writing weights to "extrinsic_weights.rs"
|
||||
```
|
||||
|
||||
The complete command for PezkuwiChain looks like this:
|
||||
```sh
|
||||
cargo run --profile=production -- benchmark overhead --chain=pezkuwi-dev --wasm-execution=compiled --weight-path=runtime/pezkuwi/constants/src/weights/
|
||||
```
|
||||
|
||||
This will overwrite the
|
||||
[block_weights.rs](https://github.com/paritytech/polkadot/blob/c254e5975711a6497af256f6831e9a6c752d28f5/runtime/polkadot/constants/src/weights/block_weights.rs)
|
||||
and
|
||||
[extrinsic_weights.rs](https://github.com/paritytech/polkadot/blob/c254e5975711a6497af256f6831e9a6c752d28f5/runtime/polkadot/constants/src/weights/extrinsic_weights.rs)
|
||||
files in the PezkuwiChain runtime directory. You can try the same for *pezkuwichain* and to see that the results slightly differ.
|
||||
👉 It is paramount to use `--profile=production` and `--wasm-execution=compiled` as the results are otherwise useless.
|
||||
|
||||
## Output Interpretation
|
||||
|
||||
Lower is better. The less weight the execution overhead needs, the better. Since the weights of the overhead is charged
|
||||
per extrinsic and per block, a larger weight results in less extrinsics per block. Minimizing this is important to have
|
||||
a large transaction throughput.
|
||||
|
||||
## Arguments
|
||||
|
||||
- `--chain` / `--dev` Set the chain specification.
|
||||
- `--weight-path` Set the output directory or file to write the weights to.
|
||||
- `--repeat` Set the repetitions of both benchmarks.
|
||||
- `--warmup` Set the rounds of warmup before measuring.
|
||||
- `--wasm-execution` Should be set to `compiled` for correct results.
|
||||
- [`--mul`](../shared/README.md#arguments)
|
||||
- [`--add`](../shared/README.md#arguments)
|
||||
- [`--metric`](../shared/README.md#arguments)
|
||||
- [`--weight-path`](../shared/README.md#arguments)
|
||||
- [`--header`](../shared/README.md#arguments)
|
||||
|
||||
License: Apache-2.0
|
||||
|
||||
<!-- LINKS -->
|
||||
[`ExtrinsicBaseWeight`]:
|
||||
https://github.com/paritytech/bizinikiwi/blob/580ebae17fa30082604f1c9720f6f4a1cfe95b50/frame/support/src/weights/extrinsic_weights.rs#L26
|
||||
[`BlockExecutionWeight`]:
|
||||
https://github.com/paritytech/bizinikiwi/blob/580ebae17fa30082604f1c9720f6f4a1cfe95b50/frame/support/src/weights/block_weights.rs#L26
|
||||
|
||||
[System::Remark]:
|
||||
https://github.com/paritytech/bizinikiwi/blob/580ebae17fa30082604f1c9720f6f4a1cfe95b50/frame/system/src/lib.rs#L382
|
||||
@@ -0,0 +1,783 @@
|
||||
// This file is part of Bizinikiwi.
|
||||
|
||||
// Copyright (C) Parity Technologies (UK) Ltd.
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
//! Contains the [`OverheadCmd`] as entry point for the CLI to execute
|
||||
//! the *overhead* benchmarks.
|
||||
|
||||
use crate::{
|
||||
extrinsic::{
|
||||
bench::{Benchmark, BenchmarkParams as ExtrinsicBenchmarkParams},
|
||||
ExtrinsicBuilder,
|
||||
},
|
||||
overhead::{
|
||||
command::ChainType::{Relaychain, Teyrchain, Unknown},
|
||||
fake_runtime_api,
|
||||
remark_builder::BizinikiwiRemarkBuilder,
|
||||
template::TemplateData,
|
||||
},
|
||||
shared::{
|
||||
genesis_state,
|
||||
genesis_state::{GenesisStateHandler, SpecGenesisSource},
|
||||
HostInfoParams, WeightParams,
|
||||
},
|
||||
};
|
||||
use clap::{error::ErrorKind, Args, CommandFactory, Parser};
|
||||
use codec::{Decode, Encode};
|
||||
use cumulus_client_teyrchain_inherent::MockValidationDataInherentDataProvider;
|
||||
use fake_runtime_api::RuntimeApi as FakeRuntimeApi;
|
||||
use pezframe_support::Deserialize;
|
||||
use genesis_state::WARN_SPEC_GENESIS_CTOR;
|
||||
use log::info;
|
||||
use pezkuwi_teyrchain_primitives::primitives::Id as ParaId;
|
||||
use pezsc_block_builder::BlockBuilderApi;
|
||||
use pezsc_chain_spec::{ChainSpec, ChainSpecExtension, GenesisBlockBuilder};
|
||||
use pezsc_cli::{CliConfiguration, Database, ImportParams, Result, SharedParams};
|
||||
use pezsc_client_api::{execution_extensions::ExecutionExtensions, UsageProvider};
|
||||
use pezsc_client_db::{BlocksPruning, DatabaseSettings};
|
||||
use pezsc_executor::WasmExecutor;
|
||||
use pezsc_runtime_utilities::fetch_latest_metadata_from_code_blob;
|
||||
use pezsc_service::{new_client, new_db_backend, BasePath, ClientConfig, TFullClient, TaskManager};
|
||||
use serde::Serialize;
|
||||
use serde_json::{json, Value};
|
||||
use pezsp_api::{ApiExt, CallApiAt, Core, ProvideRuntimeApi};
|
||||
use pezsp_blockchain::HeaderBackend;
|
||||
use pezsp_core::H256;
|
||||
use pezsp_inherents::{InherentData, InherentDataProvider};
|
||||
use pezsp_runtime::{
|
||||
generic,
|
||||
traits::{BlakeTwo256, Block as BlockT},
|
||||
DigestItem, OpaqueExtrinsic,
|
||||
};
|
||||
use pezsp_storage::Storage;
|
||||
use pezsp_wasm_interface::HostFunctions;
|
||||
use std::{
|
||||
fmt::{Debug, Display, Formatter},
|
||||
fs,
|
||||
path::PathBuf,
|
||||
sync::Arc,
|
||||
};
|
||||
use subxt::{client::RuntimeVersion, ext::futures, Metadata};
|
||||
|
||||
const DEFAULT_PARA_ID: u32 = 100;
|
||||
const LOG_TARGET: &'static str = "pezkuwi_sdk_frame::benchmark::overhead";
|
||||
|
||||
/// Benchmark the execution overhead per-block and per-extrinsic.
|
||||
#[derive(Debug, Parser)]
|
||||
pub struct OverheadCmd {
|
||||
#[allow(missing_docs)]
|
||||
#[clap(flatten)]
|
||||
pub shared_params: SharedParams,
|
||||
|
||||
#[allow(missing_docs)]
|
||||
#[clap(flatten)]
|
||||
pub import_params: ImportParams,
|
||||
|
||||
#[allow(missing_docs)]
|
||||
#[clap(flatten)]
|
||||
pub params: OverheadParams,
|
||||
}
|
||||
|
||||
/// Configures the benchmark, the post-processing and weight generation.
|
||||
#[derive(Debug, Default, Serialize, Clone, PartialEq, Args)]
|
||||
pub struct OverheadParams {
|
||||
#[allow(missing_docs)]
|
||||
#[clap(flatten)]
|
||||
pub weight: WeightParams,
|
||||
|
||||
#[allow(missing_docs)]
|
||||
#[clap(flatten)]
|
||||
pub bench: ExtrinsicBenchmarkParams,
|
||||
|
||||
#[allow(missing_docs)]
|
||||
#[clap(flatten)]
|
||||
pub hostinfo: HostInfoParams,
|
||||
|
||||
/// Add a header to the generated weight output file.
|
||||
///
|
||||
/// Good for adding LICENSE headers.
|
||||
#[arg(long, value_name = "PATH")]
|
||||
pub header: Option<PathBuf>,
|
||||
|
||||
/// Enable the Trie cache.
|
||||
///
|
||||
/// This should only be used for performance analysis and not for final results.
|
||||
#[arg(long)]
|
||||
pub enable_trie_cache: bool,
|
||||
|
||||
/// Optional runtime blob to use instead of the one from the genesis config.
|
||||
#[arg(
|
||||
long,
|
||||
value_name = "PATH",
|
||||
conflicts_with = "chain",
|
||||
required_if_eq("genesis_builder", "runtime")
|
||||
)]
|
||||
pub runtime: Option<PathBuf>,
|
||||
|
||||
/// The preset that we expect to find in the GenesisBuilder runtime API.
|
||||
///
|
||||
/// This can be useful when a runtime has a dedicated benchmarking preset instead of using the
|
||||
/// default one.
|
||||
#[arg(long, default_value = pezsp_genesis_builder::DEV_RUNTIME_PRESET)]
|
||||
pub genesis_builder_preset: String,
|
||||
|
||||
/// How to construct the genesis state.
|
||||
///
|
||||
/// Can be used together with `--chain` to determine whether the
|
||||
/// genesis state should be initialized with the values from the
|
||||
/// provided chain spec or a runtime-provided genesis preset.
|
||||
#[arg(long, value_enum, alias = "genesis-builder-policy")]
|
||||
pub genesis_builder: Option<GenesisBuilderPolicy>,
|
||||
|
||||
/// Teyrchain Id to use for teyrchains. If not specified, the benchmark code will choose
|
||||
/// a para-id and patch the state accordingly.
|
||||
#[arg(long)]
|
||||
pub para_id: Option<u32>,
|
||||
}
|
||||
|
||||
/// How the genesis state for benchmarking should be built.
|
||||
#[derive(clap::ValueEnum, Debug, Eq, PartialEq, Clone, Copy, Serialize)]
|
||||
#[clap(rename_all = "kebab-case")]
|
||||
pub enum GenesisBuilderPolicy {
|
||||
/// Let the runtime build the genesis state through its `BuildGenesisConfig` runtime API.
|
||||
/// This will use the `development` preset by default.
|
||||
Runtime,
|
||||
/// Use the runtime from the Spec file to build the genesis state.
|
||||
SpecRuntime,
|
||||
/// Use the spec file to build the genesis state. This fails when there is no spec.
|
||||
#[value(alias = "spec")]
|
||||
SpecGenesis,
|
||||
}
|
||||
|
||||
/// Type of a benchmark.
|
||||
#[derive(Serialize, Clone, PartialEq, Copy)]
|
||||
pub(crate) enum BenchmarkType {
|
||||
/// Measure the per-extrinsic execution overhead.
|
||||
Extrinsic,
|
||||
/// Measure the per-block execution overhead.
|
||||
Block,
|
||||
}
|
||||
|
||||
/// Hostfunctions that are typically used by teyrchains.
|
||||
pub type TeyrchainHostFunctions = (
|
||||
cumulus_primitives_proof_size_hostfunction::storage_proof_size::HostFunctions,
|
||||
pezsp_io::BizinikiwiHostFunctions,
|
||||
);
|
||||
|
||||
pub type BlockNumber = u32;
|
||||
|
||||
/// Typical block header.
|
||||
pub type Header = generic::Header<BlockNumber, BlakeTwo256>;
|
||||
|
||||
/// Typical block type using `OpaqueExtrinsic`.
|
||||
pub type OpaqueBlock = generic::Block<Header, OpaqueExtrinsic>;
|
||||
|
||||
/// Client type used throughout the benchmarking code.
|
||||
type OverheadClient<Block, HF> = TFullClient<Block, FakeRuntimeApi, WasmExecutor<HF>>;
|
||||
|
||||
/// Creates inherent data for a given teyrchain ID.
|
||||
///
|
||||
/// This function constructs the inherent data required for block execution,
|
||||
/// including the relay chain state and validation data. Not all of these
|
||||
/// inherents are required for every chain. The runtime will pick the ones
|
||||
/// it requires based on their identifier.
|
||||
fn create_inherent_data<Client: UsageProvider<Block> + HeaderBackend<Block>, Block: BlockT>(
|
||||
client: &Arc<Client>,
|
||||
chain_type: &ChainType,
|
||||
) -> InherentData {
|
||||
let genesis = client.usage_info().chain.best_hash;
|
||||
let header = client.header(genesis).unwrap().unwrap();
|
||||
|
||||
let mut inherent_data = InherentData::new();
|
||||
|
||||
// Para inherent can only makes sense when we are handling a teyrchain.
|
||||
if let Teyrchain(para_id) = chain_type {
|
||||
let teyrchain_validation_data_provider = MockValidationDataInherentDataProvider::<()> {
|
||||
para_id: ParaId::from(*para_id),
|
||||
current_para_block_head: Some(header.encode().into()),
|
||||
relay_offset: 0,
|
||||
..Default::default()
|
||||
};
|
||||
let _ = futures::executor::block_on(
|
||||
teyrchain_validation_data_provider.provide_inherent_data(&mut inherent_data),
|
||||
);
|
||||
}
|
||||
|
||||
// Teyrchain inherent that is used on relay chains to perform teyrchain validation.
|
||||
let para_inherent = pezkuwi_primitives::InherentData {
|
||||
bitfields: Vec::new(),
|
||||
backed_candidates: Vec::new(),
|
||||
disputes: Vec::new(),
|
||||
parent_header: header,
|
||||
};
|
||||
|
||||
// Timestamp inherent that is very common in bizinikiwi chains.
|
||||
let timestamp = pezsp_timestamp::InherentDataProvider::new(std::time::Duration::default().into());
|
||||
|
||||
let _ = futures::executor::block_on(timestamp.provide_inherent_data(&mut inherent_data));
|
||||
let _ =
|
||||
inherent_data.put_data(pezkuwi_primitives::TEYRCHAINS_INHERENT_IDENTIFIER, ¶_inherent);
|
||||
|
||||
inherent_data
|
||||
}
|
||||
|
||||
/// Identifies what kind of chain we are dealing with.
|
||||
///
|
||||
/// Chains containing the `TeyrchainSystem` and `TeyrchainInfo` pallet are considered teyrchains.
|
||||
/// Chains containing the `ParaInherent` pallet are considered relay chains.
|
||||
fn identify_chain(metadata: &Metadata, para_id: Option<u32>) -> ChainType {
|
||||
let teyrchain_info_exists = metadata.pezpallet_by_name("TeyrchainInfo").is_some();
|
||||
let teyrchain_system_exists = metadata.pezpallet_by_name("TeyrchainSystem").is_some();
|
||||
let para_inherent_exists = metadata.pezpallet_by_name("ParaInherent").is_some();
|
||||
|
||||
log::debug!("{} TeyrchainSystem", if teyrchain_system_exists { "✅" } else { "❌" });
|
||||
log::debug!("{} TeyrchainInfo", if teyrchain_info_exists { "✅" } else { "❌" });
|
||||
log::debug!("{} ParaInherent", if para_inherent_exists { "✅" } else { "❌" });
|
||||
|
||||
let chain_type = if teyrchain_system_exists && teyrchain_info_exists {
|
||||
Teyrchain(para_id.unwrap_or(DEFAULT_PARA_ID))
|
||||
} else if para_inherent_exists {
|
||||
Relaychain
|
||||
} else {
|
||||
Unknown
|
||||
};
|
||||
|
||||
log::info!(target: LOG_TARGET, "Identified Chain type from metadata: {}", chain_type);
|
||||
|
||||
chain_type
|
||||
}
|
||||
|
||||
#[derive(Deserialize, Serialize, Clone, ChainSpecExtension)]
|
||||
pub struct TeyrchainExtension {
|
||||
/// The id of the Teyrchain.
|
||||
pub para_id: Option<u32>,
|
||||
}
|
||||
|
||||
impl OverheadCmd {
|
||||
fn state_handler_from_cli<HF: HostFunctions>(
|
||||
&self,
|
||||
chain_spec_from_api: Option<Box<dyn ChainSpec>>,
|
||||
) -> Result<(GenesisStateHandler, Option<u32>)> {
|
||||
let genesis_builder_to_source = || match self.params.genesis_builder {
|
||||
Some(GenesisBuilderPolicy::Runtime) | Some(GenesisBuilderPolicy::SpecRuntime) =>
|
||||
SpecGenesisSource::Runtime(self.params.genesis_builder_preset.clone()),
|
||||
Some(GenesisBuilderPolicy::SpecGenesis) | None => {
|
||||
log::warn!(target: LOG_TARGET, "{WARN_SPEC_GENESIS_CTOR}");
|
||||
SpecGenesisSource::SpecJson
|
||||
},
|
||||
};
|
||||
|
||||
// First handle chain-spec passed in via API parameter.
|
||||
if let Some(chain_spec) = chain_spec_from_api {
|
||||
log::debug!(target: LOG_TARGET, "Initializing state handler with chain-spec from API: {:?}", chain_spec);
|
||||
|
||||
let source = genesis_builder_to_source();
|
||||
return Ok((GenesisStateHandler::ChainSpec(chain_spec, source), self.params.para_id));
|
||||
};
|
||||
|
||||
// Handle chain-spec passed in via CLI.
|
||||
if let Some(chain_spec_path) = &self.shared_params.chain {
|
||||
log::debug!(target: LOG_TARGET,
|
||||
"Initializing state handler with chain-spec from path: {:?}",
|
||||
chain_spec_path
|
||||
);
|
||||
let (chain_spec, para_id_from_chain_spec) =
|
||||
genesis_state::chain_spec_from_path::<HF>(chain_spec_path.to_string().into())?;
|
||||
|
||||
let source = genesis_builder_to_source();
|
||||
|
||||
return Ok((
|
||||
GenesisStateHandler::ChainSpec(chain_spec, source),
|
||||
self.params.para_id.or(para_id_from_chain_spec),
|
||||
));
|
||||
};
|
||||
|
||||
// Check for runtimes. In general, we make sure that `--runtime` and `--chain` are
|
||||
// incompatible on the CLI level.
|
||||
if let Some(runtime_path) = &self.params.runtime {
|
||||
log::debug!(target: LOG_TARGET, "Initializing state handler with runtime from path: {:?}", runtime_path);
|
||||
|
||||
let runtime_blob = fs::read(runtime_path)?;
|
||||
return Ok((
|
||||
GenesisStateHandler::Runtime(
|
||||
runtime_blob,
|
||||
Some(self.params.genesis_builder_preset.clone()),
|
||||
),
|
||||
self.params.para_id,
|
||||
));
|
||||
};
|
||||
|
||||
Err("Neither a runtime nor a chain-spec were specified".to_string().into())
|
||||
}
|
||||
|
||||
fn check_args(
|
||||
&self,
|
||||
chain_spec: &Option<Box<dyn ChainSpec>>,
|
||||
) -> std::result::Result<(), (ErrorKind, String)> {
|
||||
if chain_spec.is_none() &&
|
||||
self.params.runtime.is_none() &&
|
||||
self.shared_params.chain.is_none()
|
||||
{
|
||||
return Err((
|
||||
ErrorKind::MissingRequiredArgument,
|
||||
"Provide either a runtime via `--runtime` or a chain spec via `--chain`"
|
||||
.to_string(),
|
||||
));
|
||||
}
|
||||
|
||||
match self.params.genesis_builder {
|
||||
Some(GenesisBuilderPolicy::SpecGenesis | GenesisBuilderPolicy::SpecRuntime) =>
|
||||
if chain_spec.is_none() && self.shared_params.chain.is_none() {
|
||||
return Err((
|
||||
ErrorKind::MissingRequiredArgument,
|
||||
"Provide a chain spec via `--chain`.".to_string(),
|
||||
));
|
||||
},
|
||||
_ => {},
|
||||
};
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Run the overhead benchmark with the default extrinsic builder.
|
||||
///
|
||||
/// This will use [BizinikiwiRemarkBuilder] to build the extrinsic. It is
|
||||
/// designed to match common configurations found in bizinikiwi chains.
|
||||
pub fn run_with_default_builder_and_spec<Block, ExtraHF>(
|
||||
&self,
|
||||
chain_spec: Option<Box<dyn ChainSpec>>,
|
||||
) -> Result<()>
|
||||
where
|
||||
Block: BlockT<Extrinsic = OpaqueExtrinsic, Hash = H256>,
|
||||
ExtraHF: HostFunctions,
|
||||
{
|
||||
self.run_with_extrinsic_builder_and_spec::<Block, ExtraHF>(
|
||||
Box::new(|metadata, hash, version| {
|
||||
let genesis = subxt::utils::H256::from(hash.to_fixed_bytes());
|
||||
Box::new(BizinikiwiRemarkBuilder::new(metadata, genesis, version)) as Box<_>
|
||||
}),
|
||||
chain_spec,
|
||||
)
|
||||
}
|
||||
|
||||
/// Run the benchmark overhead command.
|
||||
///
|
||||
/// The provided [ExtrinsicBuilder] will be used to build extrinsics for
|
||||
/// block-building. It is expected that the provided implementation builds
|
||||
/// a `System::remark` extrinsic.
|
||||
pub fn run_with_extrinsic_builder_and_spec<Block, ExtraHF>(
|
||||
&self,
|
||||
ext_builder_provider: Box<
|
||||
dyn FnOnce(Metadata, Block::Hash, RuntimeVersion) -> Box<dyn ExtrinsicBuilder>,
|
||||
>,
|
||||
chain_spec: Option<Box<dyn ChainSpec>>,
|
||||
) -> Result<()>
|
||||
where
|
||||
Block: BlockT<Extrinsic = OpaqueExtrinsic>,
|
||||
ExtraHF: HostFunctions,
|
||||
{
|
||||
if let Err((error_kind, msg)) = self.check_args(&chain_spec) {
|
||||
let mut cmd = OverheadCmd::command();
|
||||
cmd.error(error_kind, msg).exit();
|
||||
};
|
||||
|
||||
let (state_handler, para_id) =
|
||||
self.state_handler_from_cli::<(TeyrchainHostFunctions, ExtraHF)>(chain_spec)?;
|
||||
|
||||
let executor = WasmExecutor::<(TeyrchainHostFunctions, ExtraHF)>::builder()
|
||||
.with_allow_missing_host_functions(true)
|
||||
.build();
|
||||
|
||||
let opaque_metadata =
|
||||
fetch_latest_metadata_from_code_blob(&executor, state_handler.get_code_bytes()?)
|
||||
.map_err(|_| {
|
||||
<&str as Into<pezsc_cli::Error>>::into("Unable to fetch latest stable metadata")
|
||||
})?;
|
||||
let metadata = subxt::Metadata::decode(&mut (*opaque_metadata).as_slice())?;
|
||||
|
||||
// At this point we know what kind of chain we are dealing with.
|
||||
let chain_type = identify_chain(&metadata, para_id);
|
||||
|
||||
// If we are dealing with a teyrchain, make sure that the para id in genesis will
|
||||
// match what we expect.
|
||||
let genesis_patcher = match chain_type {
|
||||
Teyrchain(para_id) =>
|
||||
Some(Box::new(move |value| patch_genesis(value, Some(para_id))) as Box<_>),
|
||||
_ => None,
|
||||
};
|
||||
|
||||
let client = self.build_client_components::<Block, (TeyrchainHostFunctions, ExtraHF)>(
|
||||
state_handler.build_storage::<(TeyrchainHostFunctions, ExtraHF)>(genesis_patcher)?,
|
||||
executor,
|
||||
&chain_type,
|
||||
)?;
|
||||
|
||||
let inherent_data = create_inherent_data(&client, &chain_type);
|
||||
|
||||
let (ext_builder, runtime_name) = {
|
||||
let genesis = client.usage_info().chain.best_hash;
|
||||
let version = client.runtime_api().version(genesis).unwrap();
|
||||
let runtime_name = version.spec_name;
|
||||
let runtime_version = RuntimeVersion {
|
||||
spec_version: version.spec_version,
|
||||
transaction_version: version.transaction_version,
|
||||
};
|
||||
|
||||
(ext_builder_provider(metadata, genesis, runtime_version), runtime_name)
|
||||
};
|
||||
|
||||
self.run(
|
||||
runtime_name.to_string(),
|
||||
client,
|
||||
inherent_data,
|
||||
Default::default(),
|
||||
&*ext_builder,
|
||||
chain_type.requires_proof_recording(),
|
||||
)
|
||||
}
|
||||
|
||||
/// Run the benchmark overhead command.
|
||||
pub fn run_with_extrinsic_builder<Block, ExtraHF>(
|
||||
&self,
|
||||
ext_builder_provider: Box<
|
||||
dyn FnOnce(Metadata, Block::Hash, RuntimeVersion) -> Box<dyn ExtrinsicBuilder>,
|
||||
>,
|
||||
) -> Result<()>
|
||||
where
|
||||
Block: BlockT<Extrinsic = OpaqueExtrinsic>,
|
||||
ExtraHF: HostFunctions,
|
||||
{
|
||||
self.run_with_extrinsic_builder_and_spec::<Block, ExtraHF>(ext_builder_provider, None)
|
||||
}
|
||||
|
||||
fn build_client_components<Block, HF>(
|
||||
&self,
|
||||
genesis_storage: Storage,
|
||||
executor: WasmExecutor<HF>,
|
||||
chain_type: &ChainType,
|
||||
) -> Result<Arc<OverheadClient<Block, HF>>>
|
||||
where
|
||||
Block: BlockT,
|
||||
HF: HostFunctions,
|
||||
{
|
||||
let extensions = ExecutionExtensions::new(None, Arc::new(executor.clone()));
|
||||
|
||||
let base_path = match &self.shared_params.base_path {
|
||||
None => BasePath::new_temp_dir()?,
|
||||
Some(path) => BasePath::from(path.clone()),
|
||||
};
|
||||
|
||||
let database_source = self.database_config(
|
||||
&base_path.path().to_path_buf(),
|
||||
self.database_cache_size()?.unwrap_or(1024),
|
||||
self.database()?.unwrap_or(Database::Auto),
|
||||
)?;
|
||||
|
||||
let backend = new_db_backend(DatabaseSettings {
|
||||
trie_cache_maximum_size: self.trie_cache_maximum_size()?,
|
||||
state_pruning: None,
|
||||
blocks_pruning: BlocksPruning::KeepAll,
|
||||
source: database_source,
|
||||
metrics_registry: None,
|
||||
})?;
|
||||
|
||||
let genesis_block_builder = GenesisBlockBuilder::new_with_storage(
|
||||
genesis_storage,
|
||||
true,
|
||||
backend.clone(),
|
||||
executor.clone(),
|
||||
)?;
|
||||
|
||||
let tokio_runtime = pezsc_cli::build_runtime()?;
|
||||
let task_manager = TaskManager::new(tokio_runtime.handle().clone(), None)
|
||||
.map_err(|_| "Unable to build task manager")?;
|
||||
|
||||
let client: Arc<OverheadClient<Block, HF>> = Arc::new(new_client(
|
||||
backend.clone(),
|
||||
executor,
|
||||
genesis_block_builder,
|
||||
Default::default(),
|
||||
Default::default(),
|
||||
extensions,
|
||||
Box::new(task_manager.spawn_handle()),
|
||||
None,
|
||||
None,
|
||||
ClientConfig {
|
||||
offchain_worker_enabled: false,
|
||||
offchain_indexing_api: false,
|
||||
wasm_runtime_overrides: None,
|
||||
no_genesis: false,
|
||||
wasm_runtime_substitutes: Default::default(),
|
||||
enable_import_proof_recording: chain_type.requires_proof_recording(),
|
||||
},
|
||||
)?);
|
||||
|
||||
Ok(client)
|
||||
}
|
||||
|
||||
/// Measure the per-block and per-extrinsic execution overhead.
|
||||
///
|
||||
/// Writes the results to console and into two instances of the
|
||||
/// `weights.hbs` template, one for each benchmark.
|
||||
pub fn run<Block, C>(
|
||||
&self,
|
||||
chain_name: String,
|
||||
client: Arc<C>,
|
||||
inherent_data: pezsp_inherents::InherentData,
|
||||
digest_items: Vec<DigestItem>,
|
||||
ext_builder: &dyn ExtrinsicBuilder,
|
||||
should_record_proof: bool,
|
||||
) -> Result<()>
|
||||
where
|
||||
Block: BlockT<Extrinsic = OpaqueExtrinsic>,
|
||||
C: ProvideRuntimeApi<Block>
|
||||
+ CallApiAt<Block>
|
||||
+ UsageProvider<Block>
|
||||
+ pezsp_blockchain::HeaderBackend<Block>,
|
||||
C::Api: ApiExt<Block> + BlockBuilderApi<Block>,
|
||||
{
|
||||
if ext_builder.pallet() != "system" || ext_builder.extrinsic() != "remark" {
|
||||
return Err(format!("The extrinsic builder is required to build `System::Remark` extrinsics but builds `{}` extrinsics instead", ext_builder.name()).into());
|
||||
}
|
||||
|
||||
let bench = Benchmark::new(
|
||||
client,
|
||||
self.params.bench.clone(),
|
||||
inherent_data,
|
||||
digest_items,
|
||||
should_record_proof,
|
||||
);
|
||||
|
||||
// per-block execution overhead
|
||||
{
|
||||
let (stats, proof_size) = bench.bench_block()?;
|
||||
info!(target: LOG_TARGET, "Per-block execution overhead [ns]:\n{:?}", stats);
|
||||
let template = TemplateData::new(
|
||||
BenchmarkType::Block,
|
||||
&chain_name,
|
||||
&self.params,
|
||||
&stats,
|
||||
proof_size,
|
||||
)?;
|
||||
template.write(&self.params.weight.weight_path)?;
|
||||
}
|
||||
// per-extrinsic execution overhead
|
||||
{
|
||||
let (stats, proof_size) = bench.bench_extrinsic(ext_builder)?;
|
||||
info!(target: LOG_TARGET, "Per-extrinsic execution overhead [ns]:\n{:?}", stats);
|
||||
let template = TemplateData::new(
|
||||
BenchmarkType::Extrinsic,
|
||||
&chain_name,
|
||||
&self.params,
|
||||
&stats,
|
||||
proof_size,
|
||||
)?;
|
||||
template.write(&self.params.weight.weight_path)?;
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
impl BenchmarkType {
|
||||
/// Short name of the benchmark type.
|
||||
pub(crate) fn short_name(&self) -> &'static str {
|
||||
match self {
|
||||
Self::Extrinsic => "extrinsic",
|
||||
Self::Block => "block",
|
||||
}
|
||||
}
|
||||
|
||||
/// Long name of the benchmark type.
|
||||
pub(crate) fn long_name(&self) -> &'static str {
|
||||
match self {
|
||||
Self::Extrinsic => "ExtrinsicBase",
|
||||
Self::Block => "BlockExecution",
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Clone, PartialEq, Debug)]
|
||||
enum ChainType {
|
||||
Teyrchain(u32),
|
||||
Relaychain,
|
||||
Unknown,
|
||||
}
|
||||
|
||||
impl Display for ChainType {
|
||||
fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
|
||||
match self {
|
||||
ChainType::Teyrchain(id) => write!(f, "Teyrchain(paraid = {})", id),
|
||||
ChainType::Relaychain => write!(f, "Relaychain"),
|
||||
ChainType::Unknown => write!(f, "Unknown"),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl ChainType {
|
||||
fn requires_proof_recording(&self) -> bool {
|
||||
match self {
|
||||
Teyrchain(_) => true,
|
||||
Relaychain => false,
|
||||
Unknown => false,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Patch the teyrchain id into the genesis config. This is necessary since the inherents
|
||||
/// also contain a teyrchain id and they need to match.
|
||||
fn patch_genesis(mut input_value: Value, para_id: Option<u32>) -> Value {
|
||||
// If we identified a teyrchain we should patch a teyrchain id into the genesis config.
|
||||
// This ensures compatibility with the inherents that we provide to successfully build a
|
||||
// block.
|
||||
if let Some(para_id) = para_id {
|
||||
pezsc_chain_spec::json_patch::merge(
|
||||
&mut input_value,
|
||||
json!({
|
||||
"teyrchainInfo": {
|
||||
"teyrchainId": para_id,
|
||||
}
|
||||
}),
|
||||
);
|
||||
log::debug!(target: LOG_TARGET, "Genesis Config Json");
|
||||
log::debug!(target: LOG_TARGET, "{}", input_value);
|
||||
}
|
||||
input_value
|
||||
}
|
||||
|
||||
// Boilerplate
|
||||
impl CliConfiguration for OverheadCmd {
|
||||
fn shared_params(&self) -> &SharedParams {
|
||||
&self.shared_params
|
||||
}
|
||||
|
||||
fn import_params(&self) -> Option<&ImportParams> {
|
||||
Some(&self.import_params)
|
||||
}
|
||||
|
||||
fn base_path(&self) -> Result<Option<BasePath>> {
|
||||
Ok(Some(BasePath::new_temp_dir()?))
|
||||
}
|
||||
|
||||
fn trie_cache_maximum_size(&self) -> Result<Option<usize>> {
|
||||
if self.params.enable_trie_cache {
|
||||
Ok(self.import_params().map(|x| x.trie_cache_maximum_size()).unwrap_or_default())
|
||||
} else {
|
||||
Ok(None)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use crate::{
|
||||
overhead::command::{identify_chain, ChainType, TeyrchainHostFunctions, DEFAULT_PARA_ID},
|
||||
OverheadCmd,
|
||||
};
|
||||
use clap::Parser;
|
||||
use codec::Decode;
|
||||
use pezsc_executor::WasmExecutor;
|
||||
|
||||
#[test]
|
||||
fn test_chain_type_relaychain() {
|
||||
let executor: WasmExecutor<TeyrchainHostFunctions> = WasmExecutor::builder().build();
|
||||
let code_bytes = zagros_runtime::WASM_BINARY
|
||||
.expect("To run this test, build the wasm binary of zagros-runtime")
|
||||
.to_vec();
|
||||
let opaque_metadata =
|
||||
super::fetch_latest_metadata_from_code_blob(&executor, code_bytes.into()).unwrap();
|
||||
let metadata = subxt::Metadata::decode(&mut (*opaque_metadata).as_slice()).unwrap();
|
||||
let chain_type = identify_chain(&metadata, None);
|
||||
assert_eq!(chain_type, ChainType::Relaychain);
|
||||
assert_eq!(chain_type.requires_proof_recording(), false);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_chain_type_teyrchain() {
|
||||
let executor: WasmExecutor<TeyrchainHostFunctions> = WasmExecutor::builder().build();
|
||||
let code_bytes = cumulus_test_runtime::WASM_BINARY
|
||||
.expect("To run this test, build the wasm binary of pezcumulus-test-runtime")
|
||||
.to_vec();
|
||||
let opaque_metadata =
|
||||
super::fetch_latest_metadata_from_code_blob(&executor, code_bytes.into()).unwrap();
|
||||
let metadata = subxt::Metadata::decode(&mut (*opaque_metadata).as_slice()).unwrap();
|
||||
let chain_type = identify_chain(&metadata, Some(100));
|
||||
assert_eq!(chain_type, ChainType::Teyrchain(100));
|
||||
assert!(chain_type.requires_proof_recording());
|
||||
assert_eq!(identify_chain(&metadata, None), ChainType::Teyrchain(DEFAULT_PARA_ID));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_chain_type_custom() {
|
||||
let executor: WasmExecutor<TeyrchainHostFunctions> = WasmExecutor::builder().build();
|
||||
let code_bytes = bizinikiwi_test_runtime::WASM_BINARY
|
||||
.expect("To run this test, build the wasm binary of bizinikiwi-test-runtime")
|
||||
.to_vec();
|
||||
let opaque_metadata =
|
||||
super::fetch_latest_metadata_from_code_blob(&executor, code_bytes.into()).unwrap();
|
||||
let metadata = subxt::Metadata::decode(&mut (*opaque_metadata).as_slice()).unwrap();
|
||||
let chain_type = identify_chain(&metadata, None);
|
||||
assert_eq!(chain_type, ChainType::Unknown);
|
||||
assert_eq!(chain_type.requires_proof_recording(), false);
|
||||
}
|
||||
|
||||
fn cli_succeed(args: &[&str]) -> Result<(), clap::Error> {
|
||||
let cmd = OverheadCmd::try_parse_from(args)?;
|
||||
assert!(cmd.check_args(&None).is_ok());
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn cli_fail(args: &[&str]) {
|
||||
let cmd = OverheadCmd::try_parse_from(args);
|
||||
if let Ok(cmd) = cmd {
|
||||
assert!(cmd.check_args(&None).is_err());
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_cli_conflicts() -> Result<(), clap::Error> {
|
||||
// Runtime tests
|
||||
cli_succeed(&["test", "--runtime", "path/to/runtime", "--genesis-builder", "runtime"])?;
|
||||
cli_succeed(&["test", "--runtime", "path/to/runtime"])?;
|
||||
cli_succeed(&[
|
||||
"test",
|
||||
"--runtime",
|
||||
"path/to/runtime",
|
||||
"--genesis-builder-preset",
|
||||
"preset",
|
||||
])?;
|
||||
cli_fail(&["test", "--runtime", "path/to/spec", "--genesis-builder", "spec"]);
|
||||
cli_fail(&["test", "--runtime", "path/to/spec", "--genesis-builder", "spec-genesis"]);
|
||||
cli_fail(&["test", "--runtime", "path/to/spec", "--genesis-builder", "spec-runtime"]);
|
||||
|
||||
// Spec tests
|
||||
cli_succeed(&["test", "--chain", "path/to/spec"])?;
|
||||
cli_succeed(&["test", "--chain", "path/to/spec", "--genesis-builder", "spec"])?;
|
||||
cli_succeed(&["test", "--chain", "path/to/spec", "--genesis-builder", "spec-genesis"])?;
|
||||
cli_succeed(&["test", "--chain", "path/to/spec", "--genesis-builder", "spec-runtime"])?;
|
||||
cli_fail(&["test", "--chain", "path/to/spec", "--genesis-builder", "none"]);
|
||||
cli_fail(&["test", "--chain", "path/to/spec", "--genesis-builder", "runtime"]);
|
||||
cli_fail(&[
|
||||
"test",
|
||||
"--chain",
|
||||
"path/to/spec",
|
||||
"--genesis-builder",
|
||||
"runtime",
|
||||
"--genesis-builder-preset",
|
||||
"preset",
|
||||
]);
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,112 @@
|
||||
// This file is part of Bizinikiwi.
|
||||
|
||||
// Copyright (C) Parity Technologies (UK) Ltd.
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
//! A fake runtime struct that allows us to instantiate a client.
|
||||
//! Has all the required runtime APIs implemented to satisfy trait bounds,
|
||||
//! but the methods are never called since we use WASM exclusively.
|
||||
|
||||
use pezsp_core::OpaqueMetadata;
|
||||
use pezsp_runtime::{
|
||||
generic,
|
||||
traits::{BlakeTwo256, Block as BlockT},
|
||||
transaction_validity::{TransactionSource, TransactionValidity},
|
||||
ApplyExtrinsicResult, OpaqueExtrinsic,
|
||||
};
|
||||
|
||||
/// Block number
|
||||
#[allow(dead_code)]
|
||||
type BlockNumber = u32;
|
||||
/// Opaque block header type.
|
||||
#[allow(dead_code)]
|
||||
type Header = generic::Header<BlockNumber, BlakeTwo256>;
|
||||
/// Opaque block type.
|
||||
#[allow(dead_code)]
|
||||
type Block = generic::Block<Header, OpaqueExtrinsic>;
|
||||
|
||||
#[allow(unused)]
|
||||
pub struct Runtime;
|
||||
|
||||
pezsp_api::impl_runtime_apis! {
|
||||
impl pezsp_api::Core<Block> for Runtime {
|
||||
fn version() -> pezsp_version::RuntimeVersion {
|
||||
unimplemented!()
|
||||
}
|
||||
|
||||
fn execute_block(_: <Block as BlockT>::LazyBlock) {
|
||||
unimplemented!()
|
||||
}
|
||||
|
||||
fn initialize_block(_: &<Block as BlockT>::Header) -> pezsp_runtime::ExtrinsicInclusionMode {
|
||||
unimplemented!()
|
||||
}
|
||||
}
|
||||
|
||||
impl pezsp_api::Metadata<Block> for Runtime {
|
||||
fn metadata() -> OpaqueMetadata {
|
||||
unimplemented!()
|
||||
}
|
||||
|
||||
fn metadata_at_version(_: u32) -> Option<OpaqueMetadata> {
|
||||
unimplemented!()
|
||||
}
|
||||
|
||||
fn metadata_versions() -> Vec<u32> {
|
||||
unimplemented!()
|
||||
}
|
||||
}
|
||||
impl pezsp_block_builder::BlockBuilder<Block> for Runtime {
|
||||
fn apply_extrinsic(_: <Block as BlockT>::Extrinsic) -> ApplyExtrinsicResult {
|
||||
unimplemented!()
|
||||
}
|
||||
|
||||
fn finalize_block() -> <Block as BlockT>::Header {
|
||||
unimplemented!()
|
||||
}
|
||||
|
||||
fn inherent_extrinsics(_: pezsp_inherents::InherentData) -> Vec<<Block as BlockT>::Extrinsic> {
|
||||
unimplemented!()
|
||||
}
|
||||
|
||||
fn check_inherents(_: <Block as BlockT>::LazyBlock, _: pezsp_inherents::InherentData) -> pezsp_inherents::CheckInherentsResult {
|
||||
unimplemented!()
|
||||
}
|
||||
}
|
||||
|
||||
impl pezsp_transaction_pool::runtime_api::TaggedTransactionQueue<Block> for Runtime {
|
||||
fn validate_transaction(
|
||||
_: TransactionSource,
|
||||
_: <Block as BlockT>::Extrinsic,
|
||||
_: <Block as BlockT>::Hash,
|
||||
) -> TransactionValidity {
|
||||
unimplemented!()
|
||||
}
|
||||
}
|
||||
|
||||
impl pezsp_genesis_builder::GenesisBuilder<Block> for Runtime {
|
||||
fn build_state(_: Vec<u8>) -> pezsp_genesis_builder::Result {
|
||||
unimplemented!()
|
||||
}
|
||||
|
||||
fn get_preset(_id: &Option<pezsp_genesis_builder::PresetId>) -> Option<Vec<u8>> {
|
||||
unimplemented!()
|
||||
}
|
||||
|
||||
fn preset_names() -> Vec<pezsp_genesis_builder::PresetId> {
|
||||
unimplemented!()
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,24 @@
|
||||
// This file is part of Bizinikiwi.
|
||||
|
||||
// Copyright (C) Parity Technologies (UK) Ltd.
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
pub mod command;
|
||||
pub mod template;
|
||||
|
||||
mod fake_runtime_api;
|
||||
pub mod remark_builder;
|
||||
|
||||
pub use command::{OpaqueBlock, OverheadCmd};
|
||||
@@ -0,0 +1,127 @@
|
||||
// This file is part of Bizinikiwi.
|
||||
|
||||
// Copyright (C) Parity Technologies (UK) Ltd.
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
use crate::extrinsic::ExtrinsicBuilder;
|
||||
use codec::{Decode, Encode};
|
||||
use pezsc_client_api::UsageProvider;
|
||||
use pezsp_api::{ApiExt, Core, Metadata, ProvideRuntimeApi};
|
||||
use pezsp_runtime::{traits::Block as BlockT, OpaqueExtrinsic};
|
||||
use std::sync::Arc;
|
||||
use subxt::{
|
||||
client::RuntimeVersion as SubxtRuntimeVersion,
|
||||
config::{bizinikiwi::BizinikiwiExtrinsicParamsBuilder, HashFor},
|
||||
Config, OfflineClient, BizinikiwiConfig,
|
||||
};
|
||||
|
||||
pub type BizinikiwiRemarkBuilder = DynamicRemarkBuilder<BizinikiwiConfig>;
|
||||
|
||||
/// Remark builder that can be used to build simple extrinsics for
|
||||
/// FRAME-based runtimes.
|
||||
pub struct DynamicRemarkBuilder<C: Config> {
|
||||
offline_client: OfflineClient<C>,
|
||||
}
|
||||
|
||||
impl<C: Config> DynamicRemarkBuilder<C> {
|
||||
/// Initializes a new remark builder from a client.
|
||||
///
|
||||
/// This will first fetch metadata and runtime version from the runtime and then
|
||||
/// construct an offline client that provides the extrinsics.
|
||||
pub fn new_from_client<Client, Block>(client: Arc<Client>) -> pezsc_cli::Result<Self>
|
||||
where
|
||||
Block: BlockT,
|
||||
Client: UsageProvider<Block> + ProvideRuntimeApi<Block>,
|
||||
Client::Api: Metadata<Block> + Core<Block>,
|
||||
{
|
||||
let genesis = client.usage_info().chain.best_hash;
|
||||
let api = client.runtime_api();
|
||||
|
||||
let Ok(Some(metadata_api_version)) = api.api_version::<dyn Metadata<Block>>(genesis) else {
|
||||
return Err("Unable to fetch metadata runtime API version.".to_string().into());
|
||||
};
|
||||
|
||||
log::debug!("Found metadata API version {}.", metadata_api_version);
|
||||
let opaque_metadata = if metadata_api_version > 1 {
|
||||
let Ok(supported_metadata_versions) = api.metadata_versions(genesis) else {
|
||||
return Err("Unable to fetch metadata versions".to_string().into());
|
||||
};
|
||||
|
||||
let latest = supported_metadata_versions
|
||||
.into_iter()
|
||||
.max()
|
||||
.ok_or("No stable metadata versions supported".to_string())?;
|
||||
|
||||
api.metadata_at_version(genesis, latest)
|
||||
.map_err(|e| format!("Unable to fetch metadata: {:?}", e))?
|
||||
.ok_or("Unable to decode metadata".to_string())?
|
||||
} else {
|
||||
// Fall back to using the non-versioned metadata API.
|
||||
api.metadata(genesis)
|
||||
.map_err(|e| format!("Unable to fetch metadata: {:?}", e))?
|
||||
};
|
||||
|
||||
let version = api.version(genesis).unwrap();
|
||||
let runtime_version = SubxtRuntimeVersion {
|
||||
spec_version: version.spec_version,
|
||||
transaction_version: version.transaction_version,
|
||||
};
|
||||
let metadata = subxt::Metadata::decode(&mut (*opaque_metadata).as_slice())?;
|
||||
let genesis = HashFor::<C>::decode(&mut &genesis.encode()[..])
|
||||
.map_err(|_| "Incompatible hash types?")?;
|
||||
|
||||
Ok(Self { offline_client: OfflineClient::new(genesis, runtime_version, metadata) })
|
||||
}
|
||||
}
|
||||
|
||||
impl<C: Config> DynamicRemarkBuilder<C> {
|
||||
/// Constructs a new remark builder.
|
||||
pub fn new(
|
||||
metadata: subxt::Metadata,
|
||||
genesis_hash: HashFor<C>,
|
||||
runtime_version: SubxtRuntimeVersion,
|
||||
) -> Self {
|
||||
Self { offline_client: OfflineClient::new(genesis_hash, runtime_version, metadata) }
|
||||
}
|
||||
}
|
||||
|
||||
impl ExtrinsicBuilder for DynamicRemarkBuilder<BizinikiwiConfig> {
|
||||
fn pallet(&self) -> &str {
|
||||
"system"
|
||||
}
|
||||
|
||||
fn extrinsic(&self) -> &str {
|
||||
"remark"
|
||||
}
|
||||
|
||||
fn build(&self, nonce: u32) -> std::result::Result<OpaqueExtrinsic, &'static str> {
|
||||
let signer = subxt_signer::sr25519::dev::alice();
|
||||
let dynamic_tx = subxt::dynamic::tx("System", "remark", vec![Vec::<u8>::new()]);
|
||||
|
||||
let params = BizinikiwiExtrinsicParamsBuilder::new().nonce(nonce.into()).build();
|
||||
|
||||
// Default transaction parameters assume a nonce of 0.
|
||||
let transaction = self
|
||||
.offline_client
|
||||
.tx()
|
||||
.create_partial_offline(&dynamic_tx, params)
|
||||
.unwrap()
|
||||
.sign(&signer);
|
||||
let mut encoded = transaction.into_encoded();
|
||||
|
||||
OpaqueExtrinsic::try_from_encoded_extrinsic(&mut encoded)
|
||||
.map_err(|_| "Unable to construct OpaqueExtrinsic")
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,128 @@
|
||||
// This file is part of Bizinikiwi.
|
||||
|
||||
// Copyright (C) Parity Technologies (UK) Ltd.
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
//! Converts a benchmark result into [`TemplateData`] and writes
|
||||
//! it into the `weights.hbs` template.
|
||||
|
||||
use pezsc_cli::Result;
|
||||
|
||||
use handlebars::Handlebars;
|
||||
use log::info;
|
||||
use serde::Serialize;
|
||||
use std::{env, fs, path::PathBuf};
|
||||
|
||||
use crate::{
|
||||
overhead::command::{BenchmarkType, OverheadParams},
|
||||
shared::{Stats, UnderscoreHelper},
|
||||
};
|
||||
|
||||
static VERSION: &str = env!("CARGO_PKG_VERSION");
|
||||
static TEMPLATE: &str = include_str!("./weights.hbs");
|
||||
|
||||
/// Data consumed by Handlebar to fill out the `weights.hbs` template.
|
||||
#[derive(Serialize, Debug, Clone)]
|
||||
pub(crate) struct TemplateData {
|
||||
/// Short name of the benchmark. Can be "block" or "extrinsic".
|
||||
long_name: String,
|
||||
/// Long name of the benchmark. Can be "BlockExecution" or "ExtrinsicBase".
|
||||
short_name: String,
|
||||
/// Name of the runtime. Taken from the chain spec.
|
||||
runtime_name: String,
|
||||
/// Version of the benchmarking CLI used.
|
||||
version: String,
|
||||
/// Date that the template was filled out.
|
||||
date: String,
|
||||
/// Hostname of the machine that executed the benchmarks.
|
||||
hostname: String,
|
||||
/// CPU name of the machine that executed the benchmarks.
|
||||
cpuname: String,
|
||||
/// Header for the generated file.
|
||||
header: String,
|
||||
/// Command line arguments that were passed to the CLI.
|
||||
args: Vec<String>,
|
||||
/// Params of the executed command.
|
||||
params: OverheadParams,
|
||||
/// Stats about the benchmark result.
|
||||
stats: Stats,
|
||||
/// The resulting ref time weight.
|
||||
ref_time: u64,
|
||||
/// The size of the proof weight.
|
||||
proof_size: u64,
|
||||
}
|
||||
|
||||
impl TemplateData {
|
||||
/// Returns a new [`Self`] from the given params.
|
||||
pub(crate) fn new(
|
||||
t: BenchmarkType,
|
||||
chain_name: &String,
|
||||
params: &OverheadParams,
|
||||
stats: &Stats,
|
||||
proof_size: u64,
|
||||
) -> Result<Self> {
|
||||
let ref_time = params.weight.calc_weight(stats)?;
|
||||
let header = params
|
||||
.header
|
||||
.as_ref()
|
||||
.map(|p| std::fs::read_to_string(p))
|
||||
.transpose()?
|
||||
.unwrap_or_default();
|
||||
|
||||
Ok(TemplateData {
|
||||
short_name: t.short_name().into(),
|
||||
long_name: t.long_name().into(),
|
||||
runtime_name: chain_name.to_owned(),
|
||||
version: VERSION.into(),
|
||||
date: chrono::Utc::now().format("%Y-%m-%d (Y/M/D)").to_string(),
|
||||
hostname: params.hostinfo.hostname(),
|
||||
cpuname: params.hostinfo.cpuname(),
|
||||
header,
|
||||
args: env::args().collect::<Vec<String>>(),
|
||||
params: params.clone(),
|
||||
stats: stats.clone(),
|
||||
ref_time,
|
||||
proof_size,
|
||||
})
|
||||
}
|
||||
|
||||
/// Fill out the `weights.hbs` HBS template with its own data.
|
||||
/// Writes the result to `path` which can be a directory or a file.
|
||||
pub fn write(&self, path: &Option<PathBuf>) -> Result<()> {
|
||||
let mut handlebars = Handlebars::new();
|
||||
// Format large integers with underscores.
|
||||
handlebars.register_helper("underscore", Box::new(UnderscoreHelper));
|
||||
// Don't HTML escape any characters.
|
||||
handlebars.register_escape_fn(|s| -> String { s.to_string() });
|
||||
|
||||
let out_path = self.build_path(path)?;
|
||||
let mut fd = fs::File::create(&out_path)?;
|
||||
info!("Writing weights to {:?}", fs::canonicalize(&out_path)?);
|
||||
handlebars
|
||||
.render_template_to_write(TEMPLATE, &self, &mut fd)
|
||||
.map_err(|e| format!("HBS template write: {:?}", e).into())
|
||||
}
|
||||
|
||||
/// Build a path for the weight file.
|
||||
fn build_path(&self, weight_out: &Option<PathBuf>) -> Result<PathBuf> {
|
||||
let mut path = weight_out.clone().unwrap_or_else(|| PathBuf::from("."));
|
||||
|
||||
if !path.is_dir() {
|
||||
return Err("Need directory as --weight-path".into());
|
||||
}
|
||||
path.push(format!("{}_weights.rs", self.short_name));
|
||||
Ok(path)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,76 @@
|
||||
{{header}}
|
||||
//! THIS FILE WAS AUTO-GENERATED USING THE BIZINIKIWI BENCHMARK CLI VERSION {{version}}
|
||||
//! DATE: {{date}}
|
||||
//! HOSTNAME: `{{hostname}}`, CPU: `{{cpuname}}`
|
||||
//!
|
||||
//! SHORT-NAME: `{{short_name}}`, LONG-NAME: `{{long_name}}`, RUNTIME: `{{runtime_name}}`
|
||||
//! WARMUPS: `{{params.bench.warmup}}`, REPEAT: `{{params.bench.repeat}}`
|
||||
//! WEIGHT-PATH: `{{params.weight.weight_path}}`
|
||||
//! WEIGHT-METRIC: `{{params.weight.weight_metric}}`, WEIGHT-MUL: `{{params.weight.weight_mul}}`, WEIGHT-ADD: `{{params.weight.weight_add}}`
|
||||
|
||||
// Executed Command:
|
||||
{{#each args as |arg|}}
|
||||
// {{arg}}
|
||||
{{/each}}
|
||||
|
||||
use sp_core::parameter_types;
|
||||
use sp_weights::{constants::WEIGHT_REF_TIME_PER_NANOS, Weight};
|
||||
|
||||
parameter_types! {
|
||||
{{#if (eq short_name "block")}}
|
||||
/// Weight of executing an empty block.
|
||||
{{else}}
|
||||
/// Weight of executing a NO-OP extrinsic, for example `System::remark`.
|
||||
{{/if}}
|
||||
/// Calculated by multiplying the *{{params.weight.weight_metric}}* with `{{params.weight.weight_mul}}` and adding `{{params.weight.weight_add}}`.
|
||||
///
|
||||
/// Stats nanoseconds:
|
||||
/// Min, Max: {{underscore stats.min}}, {{underscore stats.max}}
|
||||
/// Average: {{underscore stats.avg}}
|
||||
/// Median: {{underscore stats.median}}
|
||||
/// Std-Dev: {{stats.stddev}}
|
||||
///
|
||||
/// Percentiles nanoseconds:
|
||||
/// 99th: {{underscore stats.p99}}
|
||||
/// 95th: {{underscore stats.p95}}
|
||||
/// 75th: {{underscore stats.p75}}
|
||||
pub const {{long_name}}Weight: Weight =
|
||||
Weight::from_parts(WEIGHT_REF_TIME_PER_NANOS.saturating_mul({{underscore ref_time}}), {{underscore proof_size}});
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod test_weights {
|
||||
use sp_weights::constants;
|
||||
|
||||
/// Checks that the weight exists and is sane.
|
||||
// NOTE: If this test fails but you are sure that the generated values are fine,
|
||||
// you can delete it.
|
||||
#[test]
|
||||
fn sane() {
|
||||
let w = super::{{long_name}}Weight::get();
|
||||
|
||||
{{#if (eq short_name "block")}}
|
||||
// At least 100 µs.
|
||||
assert!(
|
||||
w.ref_time() >= 100u64 * constants::WEIGHT_REF_TIME_PER_MICROS,
|
||||
"Weight should be at least 100 µs."
|
||||
);
|
||||
// At most 50 ms.
|
||||
assert!(
|
||||
w.ref_time() <= 50u64 * constants::WEIGHT_REF_TIME_PER_MILLIS,
|
||||
"Weight should be at most 50 ms."
|
||||
);
|
||||
{{else}}
|
||||
// At least 10 µs.
|
||||
assert!(
|
||||
w.ref_time() >= 10u64 * constants::WEIGHT_REF_TIME_PER_MICROS,
|
||||
"Weight should be at least 10 µs."
|
||||
);
|
||||
// At most 1 ms.
|
||||
assert!(
|
||||
w.ref_time() <= constants::WEIGHT_REF_TIME_PER_MILLIS,
|
||||
"Weight should be at most 1 ms."
|
||||
);
|
||||
{{/if}}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,3 @@
|
||||
The pallet command is explained in [frame/benchmarking](../../../../../frame/benchmarking/README.md).
|
||||
|
||||
License: Apache-2.0
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,89 @@
|
||||
// This file is part of Bizinikiwi.
|
||||
|
||||
// Copyright (C) Parity Technologies (UK) Ltd.
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
use super::LOG_TARGET;
|
||||
use pezsp_core::{LogLevelFilter, RuntimeInterfaceLogLevel};
|
||||
use pezsp_runtime_interface::{
|
||||
pass_by::{PassAs, PassFatPointerAndRead, ReturnAs},
|
||||
runtime_interface,
|
||||
};
|
||||
use std::cell::OnceCell;
|
||||
|
||||
thread_local! {
|
||||
/// Log level filter that the runtime will use.
|
||||
///
|
||||
/// Must be initialized by the host before invoking the runtime executor. You may use `init` for
|
||||
/// this or set it manually. The that can be set are either levels directly or filter like
|
||||
// `warn,runtime=info`.
|
||||
pub static RUNTIME_LOG: OnceCell<env_filter::Filter> = OnceCell::new();
|
||||
}
|
||||
|
||||
/// Init runtime logger with the following priority (high to low):
|
||||
/// - CLI argument
|
||||
/// - Environment variable
|
||||
/// - Default logger settings
|
||||
pub fn init(arg: Option<String>) {
|
||||
let filter_str = arg.unwrap_or_else(|| {
|
||||
if let Ok(env) = std::env::var("RUNTIME_LOG") {
|
||||
env
|
||||
} else {
|
||||
log::max_level().to_string()
|
||||
}
|
||||
});
|
||||
|
||||
let filter = env_filter::Builder::new()
|
||||
.try_parse(&filter_str)
|
||||
.expect("Invalid runtime log filter")
|
||||
.build();
|
||||
|
||||
RUNTIME_LOG.with(|cell| {
|
||||
cell.set(filter).expect("Can be set by host");
|
||||
log::info!(target: LOG_TARGET, "Initialized runtime log filter to '{}'", filter_str);
|
||||
});
|
||||
}
|
||||
|
||||
/// Alternative implementation to `pezsp_runtime_interface::logging::HostFunctions` for benchmarking.
|
||||
#[runtime_interface]
|
||||
pub trait Logging {
|
||||
#[allow(dead_code)]
|
||||
fn log(
|
||||
level: PassAs<RuntimeInterfaceLogLevel, u8>,
|
||||
target: PassFatPointerAndRead<&str>,
|
||||
message: PassFatPointerAndRead<&[u8]>,
|
||||
) {
|
||||
let Ok(message) = core::str::from_utf8(message) else {
|
||||
log::error!(target: LOG_TARGET, "Runtime tried to log invalid UTF-8 data");
|
||||
return;
|
||||
};
|
||||
|
||||
let level = log::Level::from(level);
|
||||
let metadata = log::MetadataBuilder::new().level(level).target(target).build();
|
||||
|
||||
if RUNTIME_LOG.with(|filter| filter.get().expect("Must be set by host").enabled(&metadata))
|
||||
{
|
||||
log::log!(target: target, level, "{}", message);
|
||||
}
|
||||
}
|
||||
|
||||
#[allow(dead_code)]
|
||||
fn max_level() -> ReturnAs<LogLevelFilter, u8> {
|
||||
RUNTIME_LOG
|
||||
// .filter() gives us the max level of this filter
|
||||
.with(|filter| filter.get().expect("Must be set by host").filter())
|
||||
.into()
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,308 @@
|
||||
// This file is part of Bizinikiwi.
|
||||
|
||||
// Copyright (C) Parity Technologies (UK) Ltd.
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
mod command;
|
||||
mod logging;
|
||||
mod types;
|
||||
mod writer;
|
||||
|
||||
use crate::shared::HostInfoParams;
|
||||
use clap::ValueEnum;
|
||||
use pezframe_support::Serialize;
|
||||
use pezsc_cli::{
|
||||
WasmExecutionMethod, WasmtimeInstantiationStrategy, DEFAULT_WASMTIME_INSTANTIATION_STRATEGY,
|
||||
DEFAULT_WASM_EXECUTION_METHOD,
|
||||
};
|
||||
use std::{fmt::Debug, path::PathBuf};
|
||||
|
||||
/// Logging target
|
||||
const LOG_TARGET: &'static str = "frame::benchmark::pallet";
|
||||
|
||||
// Add a more relaxed parsing for pallet names by allowing pallet directory names with `-` to be
|
||||
// used like crate names with `_`
|
||||
fn parse_pallet_name(pallet: &str) -> std::result::Result<String, String> {
|
||||
Ok(pallet.replace("-", "_"))
|
||||
}
|
||||
|
||||
/// List options for available benchmarks.
|
||||
#[derive(Debug, Clone, Copy, ValueEnum)]
|
||||
pub enum ListOutput {
|
||||
/// List all available pallets and extrinsics.
|
||||
All,
|
||||
/// List all available pallets only.
|
||||
Pallets,
|
||||
}
|
||||
|
||||
/// Benchmark the extrinsic weight of FRAME Pallets.
|
||||
#[derive(Debug, clap::Parser)]
|
||||
pub struct PalletCmd {
|
||||
/// Select a FRAME Pallets to benchmark, or `*` for all (in which case `extrinsic` must be
|
||||
/// `*`).
|
||||
#[arg(short, long, alias = "pallet", num_args = 1.., value_delimiter = ',', value_parser = parse_pallet_name, required_unless_present_any = ["list", "json_input", "all"], default_value_if("all", "true", Some("*".into())))]
|
||||
pub pallets: Vec<String>,
|
||||
|
||||
/// Select an extrinsic inside the pallet to benchmark, or `*` or 'all' for all.
|
||||
#[arg(short, long, required_unless_present_any = ["list", "json_input", "all"], default_value_if("all", "true", Some("*".into())))]
|
||||
pub extrinsic: Option<String>,
|
||||
|
||||
/// Comma separated list of pallets that should be excluded from the benchmark.
|
||||
#[arg(long, value_parser, num_args = 1.., value_delimiter = ',')]
|
||||
pub exclude_pallets: Vec<String>,
|
||||
|
||||
/// Comma separated list of `pallet::extrinsic` combinations that should not be run.
|
||||
///
|
||||
/// Example: `pezframe_system::remark,pezpallet_balances::transfer_keep_alive`
|
||||
#[arg(long, value_parser, num_args = 1.., value_delimiter = ',')]
|
||||
pub exclude_extrinsics: Vec<String>,
|
||||
|
||||
/// Run benchmarks for all pallets and extrinsics.
|
||||
///
|
||||
/// This is equivalent to running `--pallet * --extrinsic *`.
|
||||
#[arg(long)]
|
||||
pub all: bool,
|
||||
|
||||
/// Select how many samples we should take across the variable components.
|
||||
#[arg(short, long, default_value_t = 50)]
|
||||
pub steps: u32,
|
||||
|
||||
/// Indicates lowest values for each of the component ranges.
|
||||
#[arg(long = "low", value_delimiter = ',')]
|
||||
pub lowest_range_values: Vec<u32>,
|
||||
|
||||
/// Indicates highest values for each of the component ranges.
|
||||
#[arg(long = "high", value_delimiter = ',')]
|
||||
pub highest_range_values: Vec<u32>,
|
||||
|
||||
/// Select how many repetitions of this benchmark should run from within the wasm.
|
||||
#[arg(short, long, default_value_t = 20)]
|
||||
pub repeat: u32,
|
||||
|
||||
/// Select how many repetitions of this benchmark should run from the client.
|
||||
///
|
||||
/// NOTE: Using this alone may give slower results, but will afford you maximum Wasm memory.
|
||||
#[arg(long, default_value_t = 1)]
|
||||
pub external_repeat: u32,
|
||||
|
||||
/// Print the raw results in JSON format.
|
||||
#[arg(long = "json")]
|
||||
pub json_output: bool,
|
||||
|
||||
/// Write the raw results in JSON format into the given file.
|
||||
#[arg(long, conflicts_with = "json_output")]
|
||||
pub json_file: Option<PathBuf>,
|
||||
|
||||
/// Don't print the median-slopes linear regression analysis.
|
||||
#[arg(long)]
|
||||
pub no_median_slopes: bool,
|
||||
|
||||
/// Don't print the min-squares linear regression analysis.
|
||||
#[arg(long)]
|
||||
pub no_min_squares: bool,
|
||||
|
||||
/// Output the benchmarks to a Rust file at the given path.
|
||||
#[arg(long)]
|
||||
pub output: Option<PathBuf>,
|
||||
|
||||
/// Add a header file to your outputted benchmarks.
|
||||
#[arg(long)]
|
||||
pub header: Option<PathBuf>,
|
||||
|
||||
/// Path to Handlebars template file used for outputting benchmark results. (Optional)
|
||||
#[arg(long)]
|
||||
pub template: Option<PathBuf>,
|
||||
|
||||
#[allow(missing_docs)]
|
||||
#[clap(flatten)]
|
||||
pub hostinfo_params: HostInfoParams,
|
||||
|
||||
/// Which analysis function to use when outputting benchmarks:
|
||||
/// * min-squares (default)
|
||||
/// * median-slopes
|
||||
/// * max (max of min squares and median slopes for each value)
|
||||
#[arg(long)]
|
||||
pub output_analysis: Option<String>,
|
||||
|
||||
/// Which analysis function to use when analyzing measured proof sizes.
|
||||
#[arg(long, default_value("median-slopes"))]
|
||||
pub output_pov_analysis: Option<String>,
|
||||
|
||||
/// The PoV estimation mode of a benchmark if no `pov_mode` attribute is present.
|
||||
#[arg(long, default_value("max-encoded-len"), value_enum)]
|
||||
pub default_pov_mode: command::PovEstimationMode,
|
||||
|
||||
/// Ignore the error when PoV modes reference unknown storage items or pallets.
|
||||
#[arg(long)]
|
||||
pub ignore_unknown_pov_mode: bool,
|
||||
|
||||
/// Set the heap pages while running benchmarks. If not set, the default value from the client
|
||||
/// is used.
|
||||
#[arg(long)]
|
||||
pub heap_pages: Option<u64>,
|
||||
|
||||
/// Disable verification logic when running benchmarks.
|
||||
#[arg(long)]
|
||||
pub no_verify: bool,
|
||||
|
||||
/// Display and run extra benchmarks that would otherwise not be needed for weight
|
||||
/// construction.
|
||||
#[arg(long)]
|
||||
pub extra: bool,
|
||||
|
||||
#[allow(missing_docs)]
|
||||
#[clap(flatten)]
|
||||
pub shared_params: pezsc_cli::SharedParams,
|
||||
|
||||
/// Method for executing Wasm runtime code.
|
||||
#[arg(
|
||||
long = "wasm-execution",
|
||||
value_name = "METHOD",
|
||||
value_enum,
|
||||
ignore_case = true,
|
||||
default_value_t = DEFAULT_WASM_EXECUTION_METHOD,
|
||||
)]
|
||||
pub wasm_method: WasmExecutionMethod,
|
||||
|
||||
/// The WASM instantiation method to use.
|
||||
///
|
||||
/// Only has an effect when `wasm-execution` is set to `compiled`.
|
||||
#[arg(
|
||||
long = "wasm-instantiation-strategy",
|
||||
value_name = "STRATEGY",
|
||||
default_value_t = DEFAULT_WASMTIME_INSTANTIATION_STRATEGY,
|
||||
value_enum,
|
||||
)]
|
||||
pub wasmtime_instantiation_strategy: WasmtimeInstantiationStrategy,
|
||||
|
||||
/// Optional runtime blob to use instead of the one from the genesis config.
|
||||
#[arg(long, conflicts_with = "chain", required_if_eq("genesis_builder", "runtime"))]
|
||||
pub runtime: Option<PathBuf>,
|
||||
|
||||
/// Set the runtime log level.
|
||||
///
|
||||
/// This will overwrite the `RUNTIME_LOG` environment variable. If neither is set, the CLI
|
||||
/// default set by `RUST_LOG` setting is used.
|
||||
#[arg(long)]
|
||||
pub runtime_log: Option<String>,
|
||||
|
||||
/// Do not fail if there are unknown but also unused host functions in the runtime.
|
||||
#[arg(long)]
|
||||
pub allow_missing_host_functions: bool,
|
||||
|
||||
/// How to construct the genesis state.
|
||||
///
|
||||
/// Uses `GenesisBuilderPolicy::Spec` by default.
|
||||
#[arg(long, value_enum, alias = "genesis-builder-policy")]
|
||||
pub genesis_builder: Option<GenesisBuilderPolicy>,
|
||||
|
||||
/// The preset that we expect to find in the GenesisBuilder runtime API.
|
||||
///
|
||||
/// This can be useful when a runtime has a dedicated benchmarking preset instead of using the
|
||||
/// default one.
|
||||
#[arg(long, default_value = pezsp_genesis_builder::DEV_RUNTIME_PRESET)]
|
||||
pub genesis_builder_preset: String,
|
||||
|
||||
/// DEPRECATED: This argument has no effect.
|
||||
#[arg(long = "execution")]
|
||||
pub execution: Option<String>,
|
||||
|
||||
/// Limit the memory the database cache can use.
|
||||
#[arg(long = "db-cache", value_name = "MiB", default_value_t = 1024)]
|
||||
pub database_cache_size: u32,
|
||||
|
||||
/// List and print available benchmarks in a csv-friendly format.
|
||||
///
|
||||
/// NOTE: `num_args` and `require_equals` are required to allow `--list`
|
||||
#[arg(long, value_enum, ignore_case = true, num_args = 0..=1, require_equals = true, default_missing_value("All"))]
|
||||
pub list: Option<ListOutput>,
|
||||
|
||||
/// Don't include csv header when listing benchmarks.
|
||||
#[arg(long, requires("list"))]
|
||||
pub no_csv_header: bool,
|
||||
|
||||
/// If enabled, the storage info is not displayed in the output next to the analysis.
|
||||
///
|
||||
/// This is independent of the storage info appearing in the *output file*. Use a Handlebar
|
||||
/// template for that purpose.
|
||||
#[arg(long)]
|
||||
pub no_storage_info: bool,
|
||||
|
||||
/// The assumed default maximum size of any `StorageMap`.
|
||||
///
|
||||
/// When the maximum size of a map is not defined by the runtime developer,
|
||||
/// this value is used as a worst case scenario. It will affect the calculated worst case
|
||||
/// PoV size for accessing a value in a map, since the PoV will need to include the trie
|
||||
/// nodes down to the underlying value.
|
||||
#[clap(long = "map-size", default_value = "1000000")]
|
||||
pub worst_case_map_values: u32,
|
||||
|
||||
/// Adjust the PoV estimation by adding additional trie layers to it.
|
||||
///
|
||||
/// This should be set to `log16(n)` where `n` is the number of top-level storage items in the
|
||||
/// runtime, eg. `StorageMap`s and `StorageValue`s. A value of 2 to 3 is usually sufficient.
|
||||
/// Each layer will result in an additional 495 bytes PoV per distinct top-level access.
|
||||
/// Therefore multiple `StorageMap` accesses only suffer from this increase once. The exact
|
||||
/// number of storage items depends on the runtime and the deployed pallets.
|
||||
#[clap(long, default_value = "2")]
|
||||
pub additional_trie_layers: u8,
|
||||
|
||||
/// A path to a `.json` file with existing benchmark results generated with `--json` or
|
||||
/// `--json-file`. When specified the benchmarks are not actually executed, and the data for
|
||||
/// the analysis is read from this file.
|
||||
#[arg(long)]
|
||||
pub json_input: Option<PathBuf>,
|
||||
|
||||
/// Allow overwriting a single file with multiple results.
|
||||
///
|
||||
/// This exists only to restore legacy behaviour. It should never actually be needed.
|
||||
#[arg(long)]
|
||||
pub unsafe_overwrite_results: bool,
|
||||
|
||||
/// Do not print a summary at the end of the run.
|
||||
///
|
||||
/// These summaries can be very long when benchmarking multiple pallets at once. For CI
|
||||
/// use-cases, this option reduces the noise.
|
||||
#[arg(long)]
|
||||
quiet: bool,
|
||||
|
||||
/// Do not enable proof recording during time benchmarking.
|
||||
///
|
||||
/// By default, proof recording is enabled during benchmark execution. This can slightly
|
||||
/// inflate the resulting time weights. For teyrchains using PoV-reclaim, this is typically the
|
||||
/// correct setting. Chains that ignore the proof size dimension of weight (e.g. relay chain,
|
||||
/// solo-chains) can disable proof recording to get more accurate results.
|
||||
#[arg(long)]
|
||||
disable_proof_recording: bool,
|
||||
}
|
||||
|
||||
/// How the genesis state for benchmarking should be built.
|
||||
#[derive(clap::ValueEnum, Debug, Eq, PartialEq, Clone, Copy, Serialize)]
|
||||
#[clap(rename_all = "kebab-case")]
|
||||
pub enum GenesisBuilderPolicy {
|
||||
/// Do not provide any genesis state.
|
||||
///
|
||||
/// Benchmarks are advised to function with this, since they should setup their own required
|
||||
/// state. However, to keep backwards compatibility, this is not the default.
|
||||
None,
|
||||
/// Let the runtime build the genesis state through its `BuildGenesisConfig` runtime API.
|
||||
/// This will use the `development` preset by default.
|
||||
Runtime,
|
||||
/// Use the runtime from the Spec file to build the genesis state.
|
||||
SpecRuntime,
|
||||
/// Use the spec file to build the genesis state. This fails when there is no spec.
|
||||
#[value(alias = "spec")]
|
||||
SpecGenesis,
|
||||
}
|
||||
@@ -0,0 +1,69 @@
|
||||
{{header}}
|
||||
//! Autogenerated weights for `{{pallet}}`
|
||||
//!
|
||||
//! THIS FILE WAS AUTO-GENERATED USING THE BIZINIKIWI BENCHMARK CLI VERSION {{version}}
|
||||
//! DATE: {{date}}, STEPS: `{{cmd.steps}}`, REPEAT: `{{cmd.repeat}}`, LOW RANGE: `{{cmd.lowest_range_values}}`, HIGH RANGE: `{{cmd.highest_range_values}}`
|
||||
//! WORST CASE MAP SIZE: `{{cmd.worst_case_map_values}}`
|
||||
//! HOSTNAME: `{{hostname}}`, CPU: `{{cpuname}}`
|
||||
//! WASM-EXECUTION: `{{cmd.wasm_execution}}`, CHAIN: `{{cmd.chain}}`, DB CACHE: {{cmd.db_cache}}
|
||||
|
||||
// Executed Command:
|
||||
{{#each args as |arg|}}
|
||||
// {{arg}}
|
||||
{{/each}}
|
||||
|
||||
#![cfg_attr(rustfmt, rustfmt_skip)]
|
||||
#![allow(unused_parens)]
|
||||
#![allow(unused_imports)]
|
||||
#![allow(missing_docs)]
|
||||
|
||||
use frame_support::{traits::Get, weights::Weight};
|
||||
use core::marker::PhantomData;
|
||||
|
||||
/// Weight functions for `{{pallet}}`.
|
||||
pub struct WeightInfo<T>(PhantomData<T>);
|
||||
{{#if (eq pallet "frame_system_extensions")}}
|
||||
impl<T: frame_system::Config> frame_system::ExtensionsWeightInfo for WeightInfo<T> {
|
||||
{{else}}
|
||||
impl<T: frame_system::Config> {{pallet}}::WeightInfo for WeightInfo<T> {
|
||||
{{/if}}
|
||||
{{#each benchmarks as |benchmark|}}
|
||||
{{#each benchmark.comments as |comment|}}
|
||||
/// {{comment}}
|
||||
{{/each}}
|
||||
{{#each benchmark.component_ranges as |range|}}
|
||||
/// The range of component `{{range.name}}` is `[{{range.min}}, {{range.max}}]`.
|
||||
{{/each}}
|
||||
fn {{benchmark.name~}}
|
||||
(
|
||||
{{~#each benchmark.components as |c| ~}}
|
||||
{{~#if (not c.is_used)}}_{{/if}}{{c.name}}: u32, {{/each~}}
|
||||
) -> Weight {
|
||||
// Proof Size summary in bytes:
|
||||
// Measured: `{{benchmark.base_recorded_proof_size}}{{#each benchmark.component_recorded_proof_size as |cp|}} + {{cp.name}} * ({{cp.slope}} ±{{underscore cp.error}}){{/each}}`
|
||||
// Estimated: `{{benchmark.base_calculated_proof_size}}{{#each benchmark.component_calculated_proof_size as |cp|}} + {{cp.name}} * ({{cp.slope}} ±{{underscore cp.error}}){{/each}}`
|
||||
// Minimum execution time: {{underscore benchmark.min_execution_time}}_000 picoseconds.
|
||||
Weight::from_parts({{underscore benchmark.base_weight}}, 0)
|
||||
.saturating_add(Weight::from_parts(0, {{benchmark.base_calculated_proof_size}}))
|
||||
{{#each benchmark.component_weight as |cw|}}
|
||||
// Standard Error: {{underscore cw.error}}
|
||||
.saturating_add(Weight::from_parts({{underscore cw.slope}}, 0).saturating_mul({{cw.name}}.into()))
|
||||
{{/each}}
|
||||
{{#if (ne benchmark.base_reads "0")}}
|
||||
.saturating_add(T::DbWeight::get().reads({{benchmark.base_reads}}))
|
||||
{{/if}}
|
||||
{{#each benchmark.component_reads as |cr|}}
|
||||
.saturating_add(T::DbWeight::get().reads(({{cr.slope}}_u64).saturating_mul({{cr.name}}.into())))
|
||||
{{/each}}
|
||||
{{#if (ne benchmark.base_writes "0")}}
|
||||
.saturating_add(T::DbWeight::get().writes({{benchmark.base_writes}}))
|
||||
{{/if}}
|
||||
{{#each benchmark.component_writes as |cw|}}
|
||||
.saturating_add(T::DbWeight::get().writes(({{cw.slope}}_u64).saturating_mul({{cw.name}}.into())))
|
||||
{{/each}}
|
||||
{{#each benchmark.component_calculated_proof_size as |cp|}}
|
||||
.saturating_add(Weight::from_parts(0, {{cp.slope}}).saturating_mul({{cp.name}}.into()))
|
||||
{{/each}}
|
||||
}
|
||||
{{/each}}
|
||||
}
|
||||
@@ -0,0 +1,63 @@
|
||||
// This file is part of Bizinikiwi.
|
||||
|
||||
// Copyright (C) Parity Technologies (UK) Ltd.
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
//! Various types used by this crate.
|
||||
|
||||
use pezsc_cli::Result;
|
||||
use pezsp_core::traits::{RuntimeCode, WrappedRuntimeCode};
|
||||
use pezsp_runtime::traits::Hash;
|
||||
|
||||
/// A runtime blob that was either fetched from genesis storage or loaded from a file.
|
||||
// NOTE: This enum is only needed for the annoying lifetime bounds on `RuntimeCode`. Otherwise we
|
||||
// could just directly return the blob.
|
||||
pub enum FetchedCode<'a, B, H> {
|
||||
FromGenesis { state: pezsp_state_machine::backend::BackendRuntimeCode<'a, B, H> },
|
||||
FromFile { wrapped_code: WrappedRuntimeCode<'a>, heap_pages: Option<u64>, hash: Vec<u8> },
|
||||
}
|
||||
|
||||
impl<'a, B, H> FetchedCode<'a, B, H>
|
||||
where
|
||||
H: Hash,
|
||||
B: pezsc_client_api::StateBackend<H>,
|
||||
{
|
||||
/// The runtime blob.
|
||||
pub fn code(&'a self) -> Result<RuntimeCode<'a>> {
|
||||
match self {
|
||||
Self::FromGenesis { state } => state.runtime_code().map_err(Into::into),
|
||||
Self::FromFile { wrapped_code, heap_pages, hash } => Ok(RuntimeCode {
|
||||
code_fetcher: wrapped_code,
|
||||
heap_pages: *heap_pages,
|
||||
hash: hash.clone(),
|
||||
}),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Maps a (pallet, benchmark) to its component ranges.
|
||||
pub(crate) type ComponentRangeMap =
|
||||
std::collections::HashMap<(String, String), Vec<ComponentRange>>;
|
||||
|
||||
/// The inclusive range of a component.
|
||||
#[derive(serde::Serialize, Debug, Clone, Eq, PartialEq)]
|
||||
pub(crate) struct ComponentRange {
|
||||
/// Name of the component.
|
||||
pub(crate) name: String,
|
||||
/// Minimal valid value of the component.
|
||||
pub(crate) min: u32,
|
||||
/// Maximal valid value of the component.
|
||||
pub(crate) max: u32,
|
||||
}
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,19 @@
|
||||
# Shared code
|
||||
|
||||
Contains code that is shared among multiple sub-commands.
|
||||
|
||||
## Arguments
|
||||
|
||||
- `--mul` Multiply the result with a factor. Can be used to manually adjust for future chain growth.
|
||||
- `--add` Add a value to the result. Can be used to manually offset the results.
|
||||
- `--metric` Set the metric to use for calculating the final weight from the raw data. Defaults to `average`.
|
||||
- `--weight-path` Set the file or directory to write the weight files to.
|
||||
- `--db` The database backend to use. This depends on your snapshot.
|
||||
- `--pruning` Set the pruning mode of the node. Some benchmarks require you to set this to `archive`.
|
||||
- `--base-path` The location on the disk that should be used for the benchmarks. You can try this on different disks or
|
||||
even on a mounted RAM-disk. It is important to use the same location that will later-on be used to store the chain
|
||||
data to get the correct results.
|
||||
- `--header` Optional file header which will be prepended to the weight output file. Can be used for adding LICENSE
|
||||
headers.
|
||||
|
||||
License: Apache-2.0
|
||||
@@ -0,0 +1,141 @@
|
||||
// This file is part of Bizinikiwi.
|
||||
|
||||
// Copyright (C) Parity Technologies (UK) Ltd.
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
use crate::overhead::command::TeyrchainExtension;
|
||||
use pezsc_chain_spec::{ChainSpec, GenericChainSpec, GenesisConfigBuilderRuntimeCaller};
|
||||
use pezsc_cli::Result;
|
||||
use serde_json::Value;
|
||||
use pezsp_storage::{well_known_keys::CODE, Storage};
|
||||
use pezsp_wasm_interface::HostFunctions;
|
||||
use std::{borrow::Cow, path::PathBuf};
|
||||
|
||||
/// When the runtime could not build the genesis storage.
|
||||
const ERROR_CANNOT_BUILD_GENESIS: &str = "The runtime returned \
|
||||
an error when trying to build the genesis storage. Please ensure that all pallets \
|
||||
define a genesis config that can be built. This can be tested with: \
|
||||
https://github.com/pezkuwichain/kurdistan-sdk/issues/115";
|
||||
|
||||
/// Warn when using the chain spec to generate the genesis state.
|
||||
pub const WARN_SPEC_GENESIS_CTOR: &'static str = "Using the chain spec instead of the runtime to \
|
||||
generate the genesis state is deprecated. Please remove the `--chain`/`--dev`/`--local` argument, \
|
||||
point `--runtime` to your runtime blob and set `--genesis-builder=runtime`. This warning may \
|
||||
become a hard error any time after December 2024.";
|
||||
|
||||
/// Defines how the chain specification shall be used to build the genesis storage.
|
||||
pub enum SpecGenesisSource {
|
||||
/// Use preset provided by the runtime embedded in the chain specification.
|
||||
Runtime(String),
|
||||
/// Use provided chain-specification JSON file.
|
||||
SpecJson,
|
||||
/// Use default storage.
|
||||
None,
|
||||
}
|
||||
|
||||
/// Defines how the genesis storage shall be built.
|
||||
pub enum GenesisStateHandler {
|
||||
ChainSpec(Box<dyn ChainSpec>, SpecGenesisSource),
|
||||
Runtime(Vec<u8>, Option<String>),
|
||||
}
|
||||
|
||||
impl GenesisStateHandler {
|
||||
/// Populate the genesis storage.
|
||||
///
|
||||
/// If the raw storage is derived from a named genesis preset, `json_patcher` is can be used to
|
||||
/// inject values into the preset.
|
||||
pub fn build_storage<HF: HostFunctions>(
|
||||
&self,
|
||||
json_patcher: Option<Box<dyn FnOnce(Value) -> Value + 'static>>,
|
||||
) -> Result<Storage> {
|
||||
match self {
|
||||
GenesisStateHandler::ChainSpec(chain_spec, source) => match source {
|
||||
SpecGenesisSource::Runtime(preset) => {
|
||||
let mut storage = chain_spec.build_storage()?;
|
||||
let code_bytes = storage
|
||||
.top
|
||||
.remove(CODE)
|
||||
.ok_or("chain spec genesis does not contain code")?;
|
||||
genesis_from_code::<HF>(code_bytes.as_slice(), preset, json_patcher)
|
||||
},
|
||||
SpecGenesisSource::SpecJson => chain_spec
|
||||
.build_storage()
|
||||
.map_err(|e| format!("{ERROR_CANNOT_BUILD_GENESIS}\nError: {e}").into()),
|
||||
SpecGenesisSource::None => Ok(Storage::default()),
|
||||
},
|
||||
GenesisStateHandler::Runtime(code_bytes, Some(preset)) =>
|
||||
genesis_from_code::<HF>(code_bytes.as_slice(), preset, json_patcher),
|
||||
GenesisStateHandler::Runtime(_, None) => Ok(Storage::default()),
|
||||
}
|
||||
}
|
||||
|
||||
/// Get the runtime code blob.
|
||||
pub fn get_code_bytes(&self) -> Result<Cow<'_, [u8]>> {
|
||||
match self {
|
||||
GenesisStateHandler::ChainSpec(chain_spec, _) => {
|
||||
let mut storage = chain_spec.build_storage()?;
|
||||
storage
|
||||
.top
|
||||
.remove(CODE)
|
||||
.map(|code| Cow::from(code))
|
||||
.ok_or("chain spec genesis does not contain code".into())
|
||||
},
|
||||
GenesisStateHandler::Runtime(code_bytes, _) => Ok(code_bytes.into()),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
pub fn chain_spec_from_path<HF: HostFunctions>(
|
||||
chain: PathBuf,
|
||||
) -> Result<(Box<dyn ChainSpec>, Option<u32>)> {
|
||||
let spec = GenericChainSpec::<TeyrchainExtension, HF>::from_json_file(chain)
|
||||
.map_err(|e| format!("Unable to load chain spec: {:?}", e))?;
|
||||
|
||||
let para_id_from_chain_spec = spec.extensions().para_id;
|
||||
Ok((Box::new(spec), para_id_from_chain_spec))
|
||||
}
|
||||
|
||||
fn genesis_from_code<EHF: HostFunctions>(
|
||||
code: &[u8],
|
||||
genesis_builder_preset: &String,
|
||||
storage_patcher: Option<Box<dyn FnOnce(Value) -> Value>>,
|
||||
) -> Result<Storage> {
|
||||
let genesis_config_caller = GenesisConfigBuilderRuntimeCaller::<(
|
||||
pezsp_io::BizinikiwiHostFunctions,
|
||||
pezframe_benchmarking::benchmarking::HostFunctions,
|
||||
EHF,
|
||||
)>::new(code);
|
||||
|
||||
let mut preset_json = genesis_config_caller.get_named_preset(Some(genesis_builder_preset))?;
|
||||
if let Some(patcher) = storage_patcher {
|
||||
preset_json = patcher(preset_json);
|
||||
}
|
||||
|
||||
let mut storage =
|
||||
genesis_config_caller.get_storage_for_patch(preset_json).inspect_err(|e| {
|
||||
let presets = genesis_config_caller.preset_names().unwrap_or_default();
|
||||
log::error!(
|
||||
"Please pick one of the available presets with \
|
||||
`--genesis-builder-preset=<PRESET>`. Available presets ({}): {:?}. Error: {:?}",
|
||||
presets.len(),
|
||||
presets,
|
||||
e
|
||||
);
|
||||
})?;
|
||||
|
||||
storage.top.insert(CODE.into(), code.into());
|
||||
|
||||
Ok(storage)
|
||||
}
|
||||
@@ -0,0 +1,135 @@
|
||||
// This file is part of Bizinikiwi.
|
||||
|
||||
// Copyright (C) Parity Technologies (UK) Ltd.
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
//! Code that is shared among all benchmarking sub-commands.
|
||||
|
||||
pub mod genesis_state;
|
||||
pub mod record;
|
||||
pub mod stats;
|
||||
pub mod weight_params;
|
||||
|
||||
pub use record::BenchRecord;
|
||||
pub use stats::{StatSelect, Stats};
|
||||
pub use weight_params::WeightParams;
|
||||
|
||||
use clap::Args;
|
||||
use rand::prelude::*;
|
||||
use pezsc_sysinfo::gather_sysinfo;
|
||||
use serde::Serialize;
|
||||
|
||||
/// A Handlebars helper to add an underscore after every 3rd character,
|
||||
/// i.e. a separator for large numbers.
|
||||
#[derive(Clone, Copy)]
|
||||
pub struct UnderscoreHelper;
|
||||
|
||||
impl handlebars::HelperDef for UnderscoreHelper {
|
||||
fn call<'reg: 'rc, 'rc>(
|
||||
&self,
|
||||
h: &handlebars::Helper,
|
||||
_: &handlebars::Handlebars,
|
||||
_: &handlebars::Context,
|
||||
_rc: &mut handlebars::RenderContext,
|
||||
out: &mut dyn handlebars::Output,
|
||||
) -> handlebars::HelperResult {
|
||||
use handlebars::JsonRender;
|
||||
let param = h.param(0).unwrap();
|
||||
let underscore_param = underscore(param.value().render());
|
||||
out.write(&underscore_param)?;
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
/// Add an underscore after every 3rd character, i.e. a separator for large numbers.
|
||||
fn underscore<Number>(i: Number) -> String
|
||||
where
|
||||
Number: std::string::ToString,
|
||||
{
|
||||
let mut s = String::new();
|
||||
let i_str = i.to_string();
|
||||
let a = i_str.chars().rev().enumerate();
|
||||
for (idx, val) in a {
|
||||
if idx != 0 && idx % 3 == 0 {
|
||||
s.insert(0, '_');
|
||||
}
|
||||
s.insert(0, val);
|
||||
}
|
||||
s
|
||||
}
|
||||
|
||||
/// Returns an rng and the seed that was used to create it.
|
||||
///
|
||||
/// Uses a random seed if none is provided.
|
||||
pub fn new_rng(seed: Option<u64>) -> (impl rand::Rng, u64) {
|
||||
let seed = seed.unwrap_or(rand::thread_rng().gen::<u64>());
|
||||
(rand_pcg::Pcg64::seed_from_u64(seed), seed)
|
||||
}
|
||||
|
||||
/// Returns an error if a debug profile is detected.
|
||||
///
|
||||
/// The rust compiler only exposes the binary information whether
|
||||
/// or not we are in a `debug` build.
|
||||
/// This means that `release` and `production` cannot be told apart.
|
||||
/// This function additionally checks for OPT-LEVEL = 3.
|
||||
pub fn check_build_profile() -> Result<(), String> {
|
||||
if cfg!(build_profile = "debug") {
|
||||
Err("Detected a `debug` profile".into())
|
||||
} else if !cfg!(build_opt_level = "3") {
|
||||
Err("The optimization level is not set to 3".into())
|
||||
} else {
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
/// Parameters to configure how the host info will be determined.
|
||||
#[derive(Debug, Default, Serialize, Clone, PartialEq, Args)]
|
||||
#[command(rename_all = "kebab-case")]
|
||||
pub struct HostInfoParams {
|
||||
/// Manually override the hostname to use.
|
||||
#[arg(long)]
|
||||
pub hostname_override: Option<String>,
|
||||
|
||||
/// Specify a fallback hostname if no-one could be detected automatically.
|
||||
///
|
||||
/// Note: This only exists to make the `hostname` function infallible.
|
||||
#[arg(long, default_value = "<UNKNOWN>")]
|
||||
pub hostname_fallback: String,
|
||||
|
||||
/// Specify a fallback CPU name if no-one could be detected automatically.
|
||||
///
|
||||
/// Note: This only exists to make the `cpuname` function infallible.
|
||||
#[arg(long, default_value = "<UNKNOWN>")]
|
||||
pub cpuname_fallback: String,
|
||||
}
|
||||
|
||||
impl HostInfoParams {
|
||||
/// Returns the hostname of the machine.
|
||||
///
|
||||
/// Can be used to track on which machine a benchmark was run.
|
||||
pub fn hostname(&self) -> String {
|
||||
self.hostname_override
|
||||
.clone()
|
||||
.or(gethostname::gethostname().into_string().ok())
|
||||
.unwrap_or(self.hostname_fallback.clone())
|
||||
}
|
||||
|
||||
/// Returns the CPU name of the current machine.
|
||||
///
|
||||
/// Can be used to track on which machine a benchmark was run.
|
||||
pub fn cpuname(&self) -> String {
|
||||
gather_sysinfo().cpu.unwrap_or(self.cpuname_fallback.clone())
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,72 @@
|
||||
// This file is part of Bizinikiwi.
|
||||
|
||||
// Copyright (C) Parity Technologies (UK) Ltd.
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
//! Defines the [`BenchRecord`] and its facilities for computing [`super::Stats`].
|
||||
|
||||
use pezsc_cli::Result;
|
||||
use pezsc_service::Configuration;
|
||||
|
||||
use log::info;
|
||||
use serde::Serialize;
|
||||
use std::{fs, path::PathBuf, time::Duration};
|
||||
|
||||
use super::Stats;
|
||||
|
||||
/// Raw output of a Storage benchmark.
|
||||
#[derive(Debug, Default, Clone, Serialize)]
|
||||
pub struct BenchRecord {
|
||||
/// Multi-Map of value sizes and the time that it took to access them.
|
||||
ns_per_size: Vec<(u64, u64)>,
|
||||
}
|
||||
|
||||
impl BenchRecord {
|
||||
/// Appends a new record. Uses safe casts.
|
||||
pub fn append(&mut self, size: usize, d: Duration) -> Result<()> {
|
||||
let size: u64 = size.try_into().map_err(|e| format!("Size overflow u64: {}", e))?;
|
||||
let ns: u64 = d
|
||||
.as_nanos()
|
||||
.try_into()
|
||||
.map_err(|e| format!("Nanoseconds overflow u64: {}", e))?;
|
||||
self.ns_per_size.push((size, ns));
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Returns the statistics for *time* and *value size*.
|
||||
pub fn calculate_stats(self) -> Result<(Stats, Stats)> {
|
||||
let (size, time): (Vec<_>, Vec<_>) = self.ns_per_size.into_iter().unzip();
|
||||
let size = Stats::new(&size)?;
|
||||
let time = Stats::new(&time)?;
|
||||
Ok((time, size)) // The swap of time/size here is intentional.
|
||||
}
|
||||
|
||||
/// Unless a path is specified, saves the raw results in a json file in the current directory.
|
||||
/// Prefixes it with the DB name and suffixed with `path_suffix`.
|
||||
pub fn save_json(&self, cfg: &Configuration, out_path: &PathBuf, suffix: &str) -> Result<()> {
|
||||
let mut path = PathBuf::from(out_path);
|
||||
if path.is_dir() || path.as_os_str().is_empty() {
|
||||
path.push(&format!("{}_{}", cfg.database, suffix).to_lowercase());
|
||||
path.set_extension("json");
|
||||
}
|
||||
|
||||
let json = serde_json::to_string_pretty(&self)
|
||||
.map_err(|e| format!("Serializing as JSON: {:?}", e))?;
|
||||
|
||||
fs::write(&path, json)?;
|
||||
info!("Raw data written to {:?}", fs::canonicalize(&path)?);
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,188 @@
|
||||
// This file is part of Bizinikiwi.
|
||||
|
||||
// Copyright (C) Parity Technologies (UK) Ltd.
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
//! Handles statistics that were generated from benchmarking results and
|
||||
//! that can be used to fill out weight templates.
|
||||
|
||||
use pezsc_cli::Result;
|
||||
|
||||
use serde::Serialize;
|
||||
use std::{fmt, result, str::FromStr};
|
||||
|
||||
/// Various statistics that help to gauge the quality of the produced weights.
|
||||
/// Will be written to the weight file and printed to console.
|
||||
#[derive(Serialize, Default, Clone)]
|
||||
pub struct Stats {
|
||||
/// Sum of all values.
|
||||
pub sum: u64,
|
||||
/// Minimal observed value.
|
||||
pub min: u64,
|
||||
/// Maximal observed value.
|
||||
pub max: u64,
|
||||
|
||||
/// Average of all values.
|
||||
pub avg: u64,
|
||||
/// Median of all values.
|
||||
pub median: u64,
|
||||
/// Standard derivation of all values.
|
||||
pub stddev: f64,
|
||||
|
||||
/// 99th percentile. At least 99% of all values are below this threshold.
|
||||
pub p99: u64,
|
||||
/// 95th percentile. At least 95% of all values are below this threshold.
|
||||
pub p95: u64,
|
||||
/// 75th percentile. At least 75% of all values are below this threshold.
|
||||
pub p75: u64,
|
||||
}
|
||||
|
||||
/// Selects a specific field from a [`Stats`] object.
|
||||
/// Not all fields are available.
|
||||
#[derive(Debug, Clone, Copy, Serialize, PartialEq)]
|
||||
pub enum StatSelect {
|
||||
/// Select the maximum.
|
||||
Maximum,
|
||||
/// Select the average.
|
||||
Average,
|
||||
/// Select the median.
|
||||
Median,
|
||||
/// Select the 99th percentile.
|
||||
P99Percentile,
|
||||
/// Select the 95th percentile.
|
||||
P95Percentile,
|
||||
/// Select the 75th percentile.
|
||||
P75Percentile,
|
||||
}
|
||||
|
||||
impl Stats {
|
||||
/// Calculates statistics and returns them.
|
||||
pub fn new(xs: &Vec<u64>) -> Result<Self> {
|
||||
if xs.is_empty() {
|
||||
return Err("Empty input is invalid".into());
|
||||
}
|
||||
let (avg, stddev) = Self::avg_and_stddev(xs);
|
||||
|
||||
Ok(Self {
|
||||
sum: xs.iter().sum(),
|
||||
min: *xs.iter().min().expect("Checked for non-empty above"),
|
||||
max: *xs.iter().max().expect("Checked for non-empty above"),
|
||||
|
||||
avg: avg as u64,
|
||||
median: Self::percentile(xs.clone(), 0.50),
|
||||
stddev: (stddev * 100.0).round() / 100.0, // round to 1/100
|
||||
|
||||
p99: Self::percentile(xs.clone(), 0.99),
|
||||
p95: Self::percentile(xs.clone(), 0.95),
|
||||
p75: Self::percentile(xs.clone(), 0.75),
|
||||
})
|
||||
}
|
||||
|
||||
/// Returns the selected stat.
|
||||
pub fn select(&self, s: StatSelect) -> u64 {
|
||||
match s {
|
||||
StatSelect::Maximum => self.max,
|
||||
StatSelect::Average => self.avg,
|
||||
StatSelect::Median => self.median,
|
||||
StatSelect::P99Percentile => self.p99,
|
||||
StatSelect::P95Percentile => self.p95,
|
||||
StatSelect::P75Percentile => self.p75,
|
||||
}
|
||||
}
|
||||
|
||||
/// Returns the *average* and the *standard derivation*.
|
||||
fn avg_and_stddev(xs: &Vec<u64>) -> (f64, f64) {
|
||||
let avg = xs.iter().map(|x| *x as f64).sum::<f64>() / xs.len() as f64;
|
||||
let variance = xs.iter().map(|x| (*x as f64 - avg).powi(2)).sum::<f64>() / xs.len() as f64;
|
||||
(avg, variance.sqrt())
|
||||
}
|
||||
|
||||
/// Returns the specified percentile for the given data.
|
||||
/// This is best effort since it ignores the interpolation case.
|
||||
fn percentile(mut xs: Vec<u64>, p: f64) -> u64 {
|
||||
xs.sort();
|
||||
let index = (xs.len() as f64 * p).ceil() as usize - 1;
|
||||
xs[index.clamp(0, xs.len() - 1)]
|
||||
}
|
||||
}
|
||||
|
||||
impl fmt::Debug for Stats {
|
||||
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
|
||||
writeln!(f, "Total: {}", self.sum)?;
|
||||
writeln!(f, "Min: {}, Max: {}", self.min, self.max)?;
|
||||
writeln!(f, "Average: {}, Median: {}, Stddev: {}", self.avg, self.median, self.stddev)?;
|
||||
write!(f, "Percentiles 99th, 95th, 75th: {}, {}, {}", self.p99, self.p95, self.p75)
|
||||
}
|
||||
}
|
||||
|
||||
impl Default for StatSelect {
|
||||
/// Returns the `Average` selector.
|
||||
fn default() -> Self {
|
||||
Self::Average
|
||||
}
|
||||
}
|
||||
|
||||
impl FromStr for StatSelect {
|
||||
type Err = &'static str;
|
||||
|
||||
fn from_str(day: &str) -> result::Result<Self, Self::Err> {
|
||||
match day.to_lowercase().as_str() {
|
||||
"max" => Ok(Self::Maximum),
|
||||
"average" => Ok(Self::Average),
|
||||
"median" => Ok(Self::Median),
|
||||
"p99" => Ok(Self::P99Percentile),
|
||||
"p95" => Ok(Self::P95Percentile),
|
||||
"p75" => Ok(Self::P75Percentile),
|
||||
_ => Err("String was not a StatSelect"),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod test_stats {
|
||||
use super::Stats;
|
||||
use rand::{seq::SliceRandom, thread_rng};
|
||||
|
||||
#[test]
|
||||
fn stats_correct() {
|
||||
let mut data: Vec<u64> = (1..=100).collect();
|
||||
data.shuffle(&mut thread_rng());
|
||||
let stats = Stats::new(&data).unwrap();
|
||||
|
||||
assert_eq!(stats.sum, 5050);
|
||||
assert_eq!(stats.min, 1);
|
||||
assert_eq!(stats.max, 100);
|
||||
|
||||
assert_eq!(stats.avg, 50);
|
||||
assert_eq!(stats.median, 50); // 50.5 to be exact.
|
||||
assert_eq!(stats.stddev, 28.87); // Rounded with 1/100 precision.
|
||||
|
||||
assert_eq!(stats.p99, 99);
|
||||
assert_eq!(stats.p95, 95);
|
||||
assert_eq!(stats.p75, 75);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn no_panic_short_lengths() {
|
||||
// Empty input does error.
|
||||
assert!(Stats::new(&vec![]).is_err());
|
||||
|
||||
// Different small input lengths are fine.
|
||||
for l in 1..10 {
|
||||
let data = (0..=l).collect();
|
||||
assert!(Stats::new(&data).is_ok());
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,95 @@
|
||||
// This file is part of Bizinikiwi.
|
||||
|
||||
// Copyright (C) Parity Technologies (UK) Ltd.
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
//! Calculates a weight from the [`super::Stats`] of a benchmark result.
|
||||
|
||||
use pezsc_cli::Result;
|
||||
|
||||
use clap::Args;
|
||||
use serde::Serialize;
|
||||
use std::path::PathBuf;
|
||||
|
||||
use super::{StatSelect, Stats};
|
||||
|
||||
/// Configures the weight generation.
|
||||
#[derive(Debug, Default, Serialize, Clone, PartialEq, Args)]
|
||||
pub struct WeightParams {
|
||||
/// File or directory to write the *weight* files to.
|
||||
///
|
||||
/// For Bizinikiwi this should be `frame/support/src/weights`.
|
||||
#[arg(long)]
|
||||
pub weight_path: Option<PathBuf>,
|
||||
|
||||
/// Select a specific metric to calculate the final weight output.
|
||||
#[arg(long = "metric", default_value = "average")]
|
||||
pub weight_metric: StatSelect,
|
||||
|
||||
/// Multiply the resulting weight with the given factor. Must be positive.
|
||||
///
|
||||
/// Is applied before `weight_add`.
|
||||
#[arg(long = "mul", default_value_t = 1.0)]
|
||||
pub weight_mul: f64,
|
||||
|
||||
/// Add the given offset to the resulting weight.
|
||||
///
|
||||
/// Is applied after `weight_mul`.
|
||||
#[arg(long = "add", default_value_t = 0)]
|
||||
pub weight_add: u64,
|
||||
}
|
||||
|
||||
/// Calculates the final weight by multiplying the selected metric with
|
||||
/// `weight_mul` and adding `weight_add`.
|
||||
/// Does not use safe casts and can overflow.
|
||||
impl WeightParams {
|
||||
pub fn calc_weight(&self, stat: &Stats) -> Result<u64> {
|
||||
if self.weight_mul.is_sign_negative() || !self.weight_mul.is_normal() {
|
||||
return Err("invalid floating number for `weight_mul`".into());
|
||||
}
|
||||
let s = stat.select(self.weight_metric) as f64;
|
||||
let w = s.mul_add(self.weight_mul, self.weight_add as f64).ceil();
|
||||
Ok(w as u64) // No safe cast here since there is no `From<f64>` for `u64`.
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod test_weight_params {
|
||||
use super::WeightParams;
|
||||
use crate::shared::{StatSelect, Stats};
|
||||
|
||||
#[test]
|
||||
fn calc_weight_works() {
|
||||
let stats = Stats { avg: 113, ..Default::default() };
|
||||
let params = WeightParams {
|
||||
weight_metric: StatSelect::Average,
|
||||
weight_mul: 0.75,
|
||||
weight_add: 3,
|
||||
..Default::default()
|
||||
};
|
||||
|
||||
let want = (113.0f64 * 0.75 + 3.0).ceil() as u64; // Ceil for overestimation.
|
||||
let got = params.calc_weight(&stats).unwrap();
|
||||
assert_eq!(want, got);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn calc_weight_detects_negative_mul() {
|
||||
let stats = Stats::default();
|
||||
let params = WeightParams { weight_mul: -0.75, ..Default::default() };
|
||||
|
||||
assert!(params.calc_weight(&stats).is_err());
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,111 @@
|
||||
# The `benchmark storage` command
|
||||
|
||||
The cost of storage operations in a Bizinikiwi chain depends on the current chain state.
|
||||
It is therefore important to regularly update these weights as the chain grows.
|
||||
This sub-command measures the cost of storage operations for a concrete snapshot.
|
||||
|
||||
For the Bizinikiwi node it looks like this (for debugging you can use `--release`):
|
||||
```sh
|
||||
cargo run --profile=production -- benchmark storage --dev --state-version=1
|
||||
```
|
||||
|
||||
Running the command on Bizinikiwi itself is not verify meaningful, since the genesis state of the `--dev` chain spec is
|
||||
used.
|
||||
|
||||
The output for the PezkuwiChain client with a recent chain snapshot will give you a better impression. A recent snapshot can
|
||||
be downloaded from [PezkuwiChain Snapshots].
|
||||
Then run (remove the `--db=paritydb` if you have a RocksDB snapshot):
|
||||
```sh
|
||||
cargo run --profile=production -- benchmark storage --dev --state-version=0 --db=paritydb --weight-path runtime/pezkuwi/constants/src/weights
|
||||
```
|
||||
|
||||
This takes a while since reads and writes all keys from the snapshot:
|
||||
```pre
|
||||
# The 'read' benchmark
|
||||
Preparing keys from block BlockId::Number(9939462)
|
||||
Reading 1379083 keys
|
||||
Time summary [ns]:
|
||||
Total: 19668919930
|
||||
Min: 6450, Max: 1217259
|
||||
Average: 14262, Median: 14190, Stddev: 3035.79
|
||||
Percentiles 99th, 95th, 75th: 18270, 16190, 14819
|
||||
Value size summary:
|
||||
Total: 265702275
|
||||
Min: 1, Max: 1381859
|
||||
Average: 192, Median: 80, Stddev: 3427.53
|
||||
Percentiles 99th, 95th, 75th: 3368, 383, 80
|
||||
|
||||
# The 'write' benchmark
|
||||
Preparing keys from block BlockId::Number(9939462)
|
||||
Writing 1379083 keys
|
||||
Time summary [ns]:
|
||||
Total: 98393809781
|
||||
Min: 12969, Max: 13282577
|
||||
Average: 71347, Median: 69499, Stddev: 25145.27
|
||||
Percentiles 99th, 95th, 75th: 135839, 106129, 79239
|
||||
Value size summary:
|
||||
Total: 265702275
|
||||
Min: 1, Max: 1381859
|
||||
Average: 192, Median: 80, Stddev: 3427.53
|
||||
Percentiles 99th, 95th, 75th: 3368, 383, 80
|
||||
|
||||
Writing weights to "paritydb_weights.rs"
|
||||
```
|
||||
You will see that the [paritydb_weights.rs] files was modified and now contains new weights. The exact command for
|
||||
PezkuwiChain can be seen at the top of the file.
|
||||
This uses the most recent block from your snapshot which is printed at the top.
|
||||
The value size summary tells us that the pruned PezkuwiChain chain state is ~253 MiB in size.
|
||||
Reading a value on average takes (in this examples) 14.3 µs and writing 71.3 µs.
|
||||
The interesting part in the generated weight file tells us the weight constants and some statistics about the
|
||||
measurements:
|
||||
```rust
|
||||
/// Time to read one storage item.
|
||||
/// Calculated by multiplying the *Average* of all values with `1.1` and adding `0`.
|
||||
///
|
||||
/// Stats [NS]:
|
||||
/// Min, Max: 4_611, 1_217_259
|
||||
/// Average: 14_262
|
||||
/// Median: 14_190
|
||||
/// Std-Dev: 3035.79
|
||||
///
|
||||
/// Percentiles [NS]:
|
||||
/// 99th: 18_270
|
||||
/// 95th: 16_190
|
||||
/// 75th: 14_819
|
||||
read: 14_262 * constants::WEIGHT_REF_TIME_PER_NANOS,
|
||||
|
||||
/// Time to write one storage item.
|
||||
/// Calculated by multiplying the *Average* of all values with `1.1` and adding `0`.
|
||||
///
|
||||
/// Stats [NS]:
|
||||
/// Min, Max: 12_969, 13_282_577
|
||||
/// Average: 71_347This works under the assumption that the *average* read a
|
||||
/// Median: 69_499
|
||||
/// Std-Dev: 25145.27
|
||||
///
|
||||
/// Percentiles [NS]:
|
||||
/// 99th: 135_839
|
||||
/// 95th: 106_129
|
||||
/// 75th: 79_239
|
||||
write: 71_347 * constants::WEIGHT_REF_TIME_PER_NANOS,
|
||||
```
|
||||
|
||||
## Arguments
|
||||
|
||||
- `--db` Specify which database backend to use. This greatly influences the results.
|
||||
- `--state-version` Set the version of the state encoding that this snapshot uses. Should be set to `1` for Bizinikiwi
|
||||
`--dev` and `0` for PezkuwiChain et al. Using the wrong version can corrupt the snapshot.
|
||||
- [`--mul`](../shared/README.md#arguments)
|
||||
- [`--add`](../shared/README.md#arguments)
|
||||
- [`--metric`](../shared/README.md#arguments)
|
||||
- [`--weight-path`](../shared/README.md#arguments)
|
||||
- `--json-read-path` Write the raw 'read' results to this file or directory.
|
||||
- `--json-write-path` Write the raw 'write' results to this file or directory.
|
||||
- [`--header`](../shared/README.md#arguments)
|
||||
|
||||
License: Apache-2.0
|
||||
|
||||
<!-- LINKS -->
|
||||
[PezkuwiChain Snapshots]: https://snapshots.polkadot.io
|
||||
[paritydb_weights.rs]:
|
||||
https://github.com/paritytech/polkadot/blob/c254e5975711a6497af256f6831e9a6c752d28f5/runtime/polkadot/constants/src/weights/paritydb_weights.rs#L60
|
||||
@@ -0,0 +1,310 @@
|
||||
// This file is part of Bizinikiwi.
|
||||
|
||||
// Copyright (C) Parity Technologies (UK) Ltd.
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
use pezsc_cli::{CliConfiguration, DatabaseParams, PruningParams, Result, SharedParams};
|
||||
use pezsc_client_api::{Backend as ClientBackend, StorageProvider, UsageProvider};
|
||||
use pezsc_client_db::DbHash;
|
||||
use pezsc_service::Configuration;
|
||||
use pezsp_api::CallApiAt;
|
||||
use pezsp_blockchain::HeaderBackend;
|
||||
use pezsp_database::{ColumnId, Database};
|
||||
use pezsp_runtime::traits::{Block as BlockT, HashingFor};
|
||||
use pezsp_state_machine::Storage;
|
||||
use pezsp_storage::{ChildInfo, ChildType, PrefixedStorageKey, StateVersion};
|
||||
|
||||
use clap::{Args, Parser, ValueEnum};
|
||||
use log::info;
|
||||
use rand::prelude::*;
|
||||
use serde::Serialize;
|
||||
use pezsp_runtime::generic::BlockId;
|
||||
use std::{fmt::Debug, path::PathBuf, sync::Arc};
|
||||
|
||||
use super::template::TemplateData;
|
||||
use crate::shared::{new_rng, HostInfoParams, WeightParams};
|
||||
|
||||
/// The mode in which to run the storage benchmark.
|
||||
#[derive(Default, Debug, Clone, Copy, PartialEq, Eq, Serialize, ValueEnum)]
|
||||
pub enum StorageBenchmarkMode {
|
||||
/// Run the benchmark for block import.
|
||||
#[default]
|
||||
ImportBlock,
|
||||
/// Run the benchmark for block validation.
|
||||
ValidateBlock,
|
||||
}
|
||||
|
||||
/// Benchmark the storage speed of a chain snapshot.
|
||||
#[derive(Debug, Parser)]
|
||||
pub struct StorageCmd {
|
||||
#[allow(missing_docs)]
|
||||
#[clap(flatten)]
|
||||
pub shared_params: SharedParams,
|
||||
|
||||
#[allow(missing_docs)]
|
||||
#[clap(flatten)]
|
||||
pub database_params: DatabaseParams,
|
||||
|
||||
#[allow(missing_docs)]
|
||||
#[clap(flatten)]
|
||||
pub pruning_params: PruningParams,
|
||||
|
||||
#[allow(missing_docs)]
|
||||
#[clap(flatten)]
|
||||
pub params: StorageParams,
|
||||
}
|
||||
|
||||
/// Parameters for modifying the benchmark behaviour and the post processing of the results.
|
||||
#[derive(Debug, Default, Serialize, Clone, PartialEq, Args)]
|
||||
pub struct StorageParams {
|
||||
#[allow(missing_docs)]
|
||||
#[clap(flatten)]
|
||||
pub weight_params: WeightParams,
|
||||
|
||||
#[allow(missing_docs)]
|
||||
#[clap(flatten)]
|
||||
pub hostinfo: HostInfoParams,
|
||||
|
||||
/// Skip the `read` benchmark.
|
||||
#[arg(long)]
|
||||
pub skip_read: bool,
|
||||
|
||||
/// Skip the `write` benchmark.
|
||||
#[arg(long)]
|
||||
pub skip_write: bool,
|
||||
|
||||
/// Specify the Handlebars template to use for outputting benchmark results.
|
||||
#[arg(long)]
|
||||
pub template_path: Option<PathBuf>,
|
||||
|
||||
/// Add a header to the generated weight output file.
|
||||
///
|
||||
/// Good for adding LICENSE headers.
|
||||
#[arg(long, value_name = "PATH")]
|
||||
pub header: Option<PathBuf>,
|
||||
|
||||
/// Path to write the raw 'read' results in JSON format to. Can be a file or directory.
|
||||
#[arg(long)]
|
||||
pub json_read_path: Option<PathBuf>,
|
||||
|
||||
/// Path to write the raw 'write' results in JSON format to. Can be a file or directory.
|
||||
#[arg(long)]
|
||||
pub json_write_path: Option<PathBuf>,
|
||||
|
||||
/// Rounds of warmups before measuring.
|
||||
#[arg(long, default_value_t = 1)]
|
||||
pub warmups: u32,
|
||||
|
||||
/// The `StateVersion` to use. Bizinikiwi `--dev` should use `V1` and Pezkuwi `V0`.
|
||||
/// Selecting the wrong version can corrupt the DB.
|
||||
#[arg(long, value_parser = clap::value_parser!(u8).range(0..=1))]
|
||||
pub state_version: u8,
|
||||
|
||||
/// Trie cache size in bytes.
|
||||
///
|
||||
/// Providing `0` will disable the cache.
|
||||
#[arg(long, value_name = "Bytes", default_value_t = 67108864)]
|
||||
pub trie_cache_size: usize,
|
||||
|
||||
/// Enable the Trie cache.
|
||||
///
|
||||
/// This should only be used for performance analysis and not for final results.
|
||||
#[arg(long)]
|
||||
pub enable_trie_cache: bool,
|
||||
|
||||
/// Include child trees in benchmark.
|
||||
#[arg(long)]
|
||||
pub include_child_trees: bool,
|
||||
|
||||
/// Disable PoV recorder.
|
||||
///
|
||||
/// The recorder has impact on performance when benchmarking with the TrieCache enabled.
|
||||
/// If the chain is recording a proof while building/importing a block, the pov recorder
|
||||
/// should be activated.
|
||||
///
|
||||
/// Hence, when generating weights for a teyrchain this should be activated and when generating
|
||||
/// weights for a standalone chain this should be deactivated.
|
||||
#[arg(long, default_value = "false")]
|
||||
pub disable_pov_recorder: bool,
|
||||
|
||||
/// The batch size for the read/write benchmark.
|
||||
///
|
||||
/// Since the write size needs to also include the cost of computing the storage root, which is
|
||||
/// done once at the end of the block, the batch size is used to simulate multiple writes in a
|
||||
/// block.
|
||||
#[arg(long, default_value_t = 100_000)]
|
||||
pub batch_size: usize,
|
||||
|
||||
/// The mode in which to run the storage benchmark.
|
||||
///
|
||||
/// PoV recorder must be activated to provide a storage proof for block validation at runtime.
|
||||
#[arg(long, value_enum, default_value_t = StorageBenchmarkMode::ImportBlock)]
|
||||
pub mode: StorageBenchmarkMode,
|
||||
|
||||
/// Number of rounds to execute block validation during the benchmark.
|
||||
///
|
||||
/// We need to run the benchmark several times to avoid fluctuations during runtime setup.
|
||||
/// This is only used when `mode` is `validate-block`.
|
||||
#[arg(long, default_value_t = 20)]
|
||||
pub validate_block_rounds: u32,
|
||||
}
|
||||
|
||||
impl StorageParams {
|
||||
pub fn is_import_block_mode(&self) -> bool {
|
||||
matches!(self.mode, StorageBenchmarkMode::ImportBlock)
|
||||
}
|
||||
|
||||
pub fn is_validate_block_mode(&self) -> bool {
|
||||
matches!(self.mode, StorageBenchmarkMode::ValidateBlock)
|
||||
}
|
||||
}
|
||||
|
||||
impl StorageCmd {
|
||||
/// Calls into the Read and Write benchmarking functions.
|
||||
/// Processes the output and writes it into files and stdout.
|
||||
pub fn run<Block, BA, C>(
|
||||
&self,
|
||||
cfg: Configuration,
|
||||
client: Arc<C>,
|
||||
db: (Arc<dyn Database<DbHash>>, ColumnId),
|
||||
storage: Arc<dyn Storage<HashingFor<Block>>>,
|
||||
shared_trie_cache: Option<pezsp_trie::cache::SharedTrieCache<HashingFor<Block>>>,
|
||||
) -> Result<()>
|
||||
where
|
||||
BA: ClientBackend<Block>,
|
||||
Block: BlockT<Hash = DbHash>,
|
||||
C: UsageProvider<Block>
|
||||
+ StorageProvider<Block, BA>
|
||||
+ HeaderBackend<Block>
|
||||
+ CallApiAt<Block>,
|
||||
{
|
||||
let mut template = TemplateData::new(&cfg, &self.params)?;
|
||||
|
||||
let block_id = BlockId::<Block>::Number(client.usage_info().chain.best_number);
|
||||
template.set_block_number(block_id.to_string());
|
||||
|
||||
if !self.params.skip_read {
|
||||
self.bench_warmup(&client)?;
|
||||
let record = self.bench_read(client.clone(), shared_trie_cache.clone())?;
|
||||
if let Some(path) = &self.params.json_read_path {
|
||||
record.save_json(&cfg, path, "read")?;
|
||||
}
|
||||
let stats = record.calculate_stats()?;
|
||||
info!("Time summary [ns]:\n{:?}\nValue size summary:\n{:?}", stats.0, stats.1);
|
||||
template.set_stats(Some(stats), None)?;
|
||||
}
|
||||
|
||||
if !self.params.skip_write {
|
||||
self.bench_warmup(&client)?;
|
||||
let record = self.bench_write(client, db, storage, shared_trie_cache)?;
|
||||
if let Some(path) = &self.params.json_write_path {
|
||||
record.save_json(&cfg, path, "write")?;
|
||||
}
|
||||
let stats = record.calculate_stats()?;
|
||||
info!("Time summary [ns]:\n{:?}\nValue size summary:\n{:?}", stats.0, stats.1);
|
||||
template.set_stats(None, Some(stats))?;
|
||||
}
|
||||
|
||||
template.write(&self.params.weight_params.weight_path, &self.params.template_path)
|
||||
}
|
||||
|
||||
/// Returns the specified state version.
|
||||
pub(crate) fn state_version(&self) -> StateVersion {
|
||||
match self.params.state_version {
|
||||
0 => StateVersion::V0,
|
||||
1 => StateVersion::V1,
|
||||
_ => unreachable!("Clap set to only allow 0 and 1"),
|
||||
}
|
||||
}
|
||||
|
||||
/// Returns Some if child node and None if regular
|
||||
pub(crate) fn is_child_key(&self, key: Vec<u8>) -> Option<ChildInfo> {
|
||||
if let Some((ChildType::ParentKeyId, storage_key)) =
|
||||
ChildType::from_prefixed_key(&PrefixedStorageKey::new(key))
|
||||
{
|
||||
return Some(ChildInfo::new_default(storage_key));
|
||||
}
|
||||
None
|
||||
}
|
||||
|
||||
/// Run some rounds of the (read) benchmark as warmup.
|
||||
/// See `pezframe_benchmarking_cli::storage::read::bench_read` for detailed comments.
|
||||
fn bench_warmup<B, BA, C>(&self, client: &Arc<C>) -> Result<()>
|
||||
where
|
||||
C: UsageProvider<B> + StorageProvider<B, BA>,
|
||||
B: BlockT + Debug,
|
||||
BA: ClientBackend<B>,
|
||||
{
|
||||
let hash = client.usage_info().chain.best_hash;
|
||||
let mut keys: Vec<_> = client.storage_keys(hash, None, None)?.collect();
|
||||
let (mut rng, _) = new_rng(None);
|
||||
keys.shuffle(&mut rng);
|
||||
|
||||
for i in 0..self.params.warmups {
|
||||
info!("Warmup round {}/{}", i + 1, self.params.warmups);
|
||||
let mut child_nodes = Vec::new();
|
||||
|
||||
for key in keys.as_slice() {
|
||||
let _ = client
|
||||
.storage(hash, &key)
|
||||
.expect("Checked above to exist")
|
||||
.ok_or("Value unexpectedly empty");
|
||||
|
||||
if let Some(info) = self
|
||||
.params
|
||||
.include_child_trees
|
||||
.then(|| self.is_child_key(key.clone().0))
|
||||
.flatten()
|
||||
{
|
||||
// child tree key
|
||||
for ck in client.child_storage_keys(hash, info.clone(), None, None)? {
|
||||
child_nodes.push((ck.clone(), info.clone()));
|
||||
}
|
||||
}
|
||||
}
|
||||
for (key, info) in child_nodes.as_slice() {
|
||||
client
|
||||
.child_storage(hash, info, key)
|
||||
.expect("Checked above to exist")
|
||||
.ok_or("Value unexpectedly empty")?;
|
||||
}
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
// Boilerplate
|
||||
impl CliConfiguration for StorageCmd {
|
||||
fn shared_params(&self) -> &SharedParams {
|
||||
&self.shared_params
|
||||
}
|
||||
|
||||
fn database_params(&self) -> Option<&DatabaseParams> {
|
||||
Some(&self.database_params)
|
||||
}
|
||||
|
||||
fn pruning_params(&self) -> Option<&PruningParams> {
|
||||
Some(&self.pruning_params)
|
||||
}
|
||||
|
||||
fn trie_cache_maximum_size(&self) -> Result<Option<usize>> {
|
||||
if self.params.enable_trie_cache && self.params.trie_cache_size > 0 {
|
||||
Ok(Some(self.params.trie_cache_size))
|
||||
} else {
|
||||
Ok(None)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,57 @@
|
||||
// This file is part of Bizinikiwi.
|
||||
|
||||
// Copyright (C) Parity Technologies (UK) Ltd.
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
pub mod cmd;
|
||||
pub mod read;
|
||||
pub mod template;
|
||||
pub mod write;
|
||||
|
||||
pub use cmd::StorageCmd;
|
||||
|
||||
/// Empirically, the maximum batch size for block validation should be no more than 10,000.
|
||||
/// Bigger sizes may cause problems with runtime memory allocation.
|
||||
pub(crate) const MAX_BATCH_SIZE_FOR_BLOCK_VALIDATION: usize = 10_000;
|
||||
|
||||
pub(crate) fn get_wasm_module() -> Box<dyn pezsc_executor_common::wasm_runtime::WasmModule> {
|
||||
let blob = pezsc_executor_common::runtime_blob::RuntimeBlob::uncompress_if_needed(
|
||||
frame_storage_access_test_runtime::WASM_BINARY
|
||||
.expect("You need to build the WASM binaries to run the benchmark!"),
|
||||
)
|
||||
.expect("Failed to create runtime blob");
|
||||
let config = pezsc_executor_wasmtime::Config {
|
||||
allow_missing_func_imports: true,
|
||||
cache_path: None,
|
||||
semantics: pezsc_executor_wasmtime::Semantics {
|
||||
heap_alloc_strategy: pezsc_executor_common::wasm_runtime::HeapAllocStrategy::Dynamic {
|
||||
maximum_pages: Some(4096),
|
||||
},
|
||||
instantiation_strategy: pezsc_executor::WasmtimeInstantiationStrategy::PoolingCopyOnWrite,
|
||||
deterministic_stack_limit: None,
|
||||
canonicalize_nans: false,
|
||||
parallel_compilation: false,
|
||||
wasm_multi_value: false,
|
||||
wasm_bulk_memory: false,
|
||||
wasm_reference_types: false,
|
||||
wasm_simd: false,
|
||||
},
|
||||
};
|
||||
|
||||
Box::new(
|
||||
pezsc_executor_wasmtime::create_runtime::<pezsp_io::BizinikiwiHostFunctions>(blob, config)
|
||||
.expect("Unable to create wasm module."),
|
||||
)
|
||||
}
|
||||
@@ -0,0 +1,263 @@
|
||||
// This file is part of Bizinikiwi.
|
||||
|
||||
// Copyright (C) Parity Technologies (UK) Ltd.
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
use codec::Encode;
|
||||
use frame_storage_access_test_runtime::StorageAccessParams;
|
||||
use log::{debug, info};
|
||||
use rand::prelude::*;
|
||||
use pezsc_cli::{Error, Result};
|
||||
use pezsc_client_api::{Backend as ClientBackend, StorageProvider, UsageProvider};
|
||||
use pezsp_api::CallApiAt;
|
||||
use pezsp_runtime::traits::{Block as BlockT, HashingFor, Header as HeaderT};
|
||||
use pezsp_state_machine::{backend::AsTrieBackend, Backend};
|
||||
use pezsp_storage::ChildInfo;
|
||||
use pezsp_trie::StorageProof;
|
||||
use std::{fmt::Debug, sync::Arc, time::Instant};
|
||||
|
||||
use super::{cmd::StorageCmd, get_wasm_module, MAX_BATCH_SIZE_FOR_BLOCK_VALIDATION};
|
||||
use crate::shared::{new_rng, BenchRecord};
|
||||
|
||||
impl StorageCmd {
|
||||
/// Benchmarks the time it takes to read a single Storage item.
|
||||
/// Uses the latest state that is available for the given client.
|
||||
pub(crate) fn bench_read<B, BA, C>(
|
||||
&self,
|
||||
client: Arc<C>,
|
||||
_shared_trie_cache: Option<pezsp_trie::cache::SharedTrieCache<HashingFor<B>>>,
|
||||
) -> Result<BenchRecord>
|
||||
where
|
||||
C: UsageProvider<B> + StorageProvider<B, BA> + CallApiAt<B>,
|
||||
B: BlockT + Debug,
|
||||
BA: ClientBackend<B>,
|
||||
<<B as BlockT>::Header as HeaderT>::Number: From<u32>,
|
||||
{
|
||||
if self.params.is_validate_block_mode() && self.params.disable_pov_recorder {
|
||||
return Err("PoV recorder must be activated to provide a storage proof for block validation at runtime. Remove `--disable-pov-recorder` from the command line.".into());
|
||||
}
|
||||
if self.params.is_validate_block_mode() &&
|
||||
self.params.batch_size > MAX_BATCH_SIZE_FOR_BLOCK_VALIDATION
|
||||
{
|
||||
return Err(format!("Batch size is too large. This may cause problems with runtime memory allocation. Better set `--batch-size {}` or less.", MAX_BATCH_SIZE_FOR_BLOCK_VALIDATION).into());
|
||||
}
|
||||
|
||||
let mut record = BenchRecord::default();
|
||||
let best_hash = client.usage_info().chain.best_hash;
|
||||
|
||||
info!("Preparing keys from block {}", best_hash);
|
||||
// Load all keys and randomly shuffle them.
|
||||
let mut keys: Vec<_> = client.storage_keys(best_hash, None, None)?.collect();
|
||||
let (mut rng, _) = new_rng(None);
|
||||
keys.shuffle(&mut rng);
|
||||
if keys.is_empty() {
|
||||
return Err("Can't process benchmarking with empty storage".into());
|
||||
}
|
||||
|
||||
let mut child_nodes = Vec::new();
|
||||
// Interesting part here:
|
||||
// Read all the keys in the database and measure the time it takes to access each.
|
||||
info!("Reading {} keys", keys.len());
|
||||
|
||||
// Read using the same TrieBackend and recorder for up to `batch_size` keys.
|
||||
// This would allow us to measure the amortized cost of reading a key.
|
||||
let state = client
|
||||
.state_at(best_hash)
|
||||
.map_err(|_err| Error::Input("State not found".into()))?;
|
||||
// We reassign the backend and recorder for every batch size.
|
||||
// Using a new recorder for every read vs using the same for the entire batch
|
||||
// produces significant different results. Since in the real use case we use a
|
||||
// single recorder per block, simulate the same behavior by creating a new
|
||||
// recorder every batch size, so that the amortized cost of reading a key is
|
||||
// measured in conditions closer to the real world.
|
||||
let (mut backend, mut recorder) = self.create_backend::<B, C>(&state);
|
||||
|
||||
let mut read_in_batch = 0;
|
||||
let mut on_validation_batch = vec![];
|
||||
let mut on_validation_size = 0;
|
||||
|
||||
let last_key = keys.last().expect("Checked above to be non-empty");
|
||||
for key in keys.as_slice() {
|
||||
match (self.params.include_child_trees, self.is_child_key(key.clone().0)) {
|
||||
(true, Some(info)) => {
|
||||
// child tree key
|
||||
for ck in client.child_storage_keys(best_hash, info.clone(), None, None)? {
|
||||
child_nodes.push((ck, info.clone()));
|
||||
}
|
||||
},
|
||||
_ => {
|
||||
// regular key
|
||||
on_validation_batch.push((key.0.clone(), None));
|
||||
let start = Instant::now();
|
||||
let v = backend
|
||||
.storage(key.0.as_ref())
|
||||
.expect("Checked above to exist")
|
||||
.ok_or("Value unexpectedly empty")?;
|
||||
on_validation_size += v.len();
|
||||
if self.params.is_import_block_mode() {
|
||||
record.append(v.len(), start.elapsed())?;
|
||||
}
|
||||
},
|
||||
}
|
||||
read_in_batch += 1;
|
||||
let is_batch_full = read_in_batch >= self.params.batch_size || key == last_key;
|
||||
|
||||
// Read keys on block validation
|
||||
if is_batch_full && self.params.is_validate_block_mode() {
|
||||
let root = backend.root();
|
||||
let storage_proof = recorder
|
||||
.clone()
|
||||
.map(|r| r.drain_storage_proof())
|
||||
.expect("Storage proof must exist for block validation");
|
||||
let elapsed = measure_block_validation::<B>(
|
||||
*root,
|
||||
storage_proof,
|
||||
on_validation_batch.clone(),
|
||||
self.params.validate_block_rounds,
|
||||
);
|
||||
record.append(on_validation_size / on_validation_batch.len(), elapsed)?;
|
||||
|
||||
on_validation_batch = vec![];
|
||||
on_validation_size = 0;
|
||||
}
|
||||
|
||||
// Reload recorder
|
||||
if is_batch_full {
|
||||
(backend, recorder) = self.create_backend::<B, C>(&state);
|
||||
read_in_batch = 0;
|
||||
}
|
||||
}
|
||||
|
||||
if self.params.include_child_trees && !child_nodes.is_empty() {
|
||||
child_nodes.shuffle(&mut rng);
|
||||
|
||||
info!("Reading {} child keys", child_nodes.len());
|
||||
let (last_child_key, last_child_info) =
|
||||
child_nodes.last().expect("Checked above to be non-empty");
|
||||
for (key, info) in child_nodes.as_slice() {
|
||||
on_validation_batch.push((key.0.clone(), Some(info.clone())));
|
||||
let start = Instant::now();
|
||||
let v = backend
|
||||
.child_storage(info, key.0.as_ref())
|
||||
.expect("Checked above to exist")
|
||||
.ok_or("Value unexpectedly empty")?;
|
||||
on_validation_size += v.len();
|
||||
if self.params.is_import_block_mode() {
|
||||
record.append(v.len(), start.elapsed())?;
|
||||
}
|
||||
read_in_batch += 1;
|
||||
let is_batch_full = read_in_batch >= self.params.batch_size ||
|
||||
(last_child_key == key && last_child_info == info);
|
||||
|
||||
// Read child keys on block validation
|
||||
if is_batch_full && self.params.is_validate_block_mode() {
|
||||
let root = backend.root();
|
||||
let storage_proof = recorder
|
||||
.clone()
|
||||
.map(|r| r.drain_storage_proof())
|
||||
.expect("Storage proof must exist for block validation");
|
||||
let elapsed = measure_block_validation::<B>(
|
||||
*root,
|
||||
storage_proof,
|
||||
on_validation_batch.clone(),
|
||||
self.params.validate_block_rounds,
|
||||
);
|
||||
record.append(on_validation_size / on_validation_batch.len(), elapsed)?;
|
||||
|
||||
on_validation_batch = vec![];
|
||||
on_validation_size = 0;
|
||||
}
|
||||
|
||||
// Reload recorder
|
||||
if is_batch_full {
|
||||
(backend, recorder) = self.create_backend::<B, C>(&state);
|
||||
read_in_batch = 0;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
Ok(record)
|
||||
}
|
||||
|
||||
fn create_backend<'a, B, C>(
|
||||
&self,
|
||||
state: &'a C::StateBackend,
|
||||
) -> (
|
||||
pezsp_state_machine::TrieBackend<
|
||||
&'a <C::StateBackend as AsTrieBackend<HashingFor<B>>>::TrieBackendStorage,
|
||||
HashingFor<B>,
|
||||
&'a pezsp_trie::cache::LocalTrieCache<HashingFor<B>>,
|
||||
>,
|
||||
Option<pezsp_trie::recorder::Recorder<HashingFor<B>>>,
|
||||
)
|
||||
where
|
||||
C: CallApiAt<B>,
|
||||
B: BlockT + Debug,
|
||||
{
|
||||
let recorder = (!self.params.disable_pov_recorder).then(|| Default::default());
|
||||
let backend = pezsp_state_machine::TrieBackendBuilder::wrap(state.as_trie_backend())
|
||||
.with_optional_recorder(recorder.clone())
|
||||
.build();
|
||||
|
||||
(backend, recorder)
|
||||
}
|
||||
}
|
||||
|
||||
fn measure_block_validation<B: BlockT + Debug>(
|
||||
root: B::Hash,
|
||||
storage_proof: StorageProof,
|
||||
on_validation_batch: Vec<(Vec<u8>, Option<ChildInfo>)>,
|
||||
rounds: u32,
|
||||
) -> std::time::Duration {
|
||||
debug!(
|
||||
"POV: len {:?} {:?}",
|
||||
storage_proof.len(),
|
||||
storage_proof.clone().encoded_compact_size::<HashingFor<B>>(root)
|
||||
);
|
||||
let batch_size = on_validation_batch.len();
|
||||
let wasm_module = get_wasm_module();
|
||||
let mut instance = wasm_module.new_instance().expect("Failed to create wasm instance");
|
||||
let params = StorageAccessParams::<B>::new_read(root, storage_proof, on_validation_batch);
|
||||
let dry_run_encoded = params.as_dry_run().encode();
|
||||
let encoded = params.encode();
|
||||
|
||||
let mut durations_in_nanos = Vec::new();
|
||||
|
||||
for i in 1..=rounds {
|
||||
info!("validate_block with {} keys, round {}/{}", batch_size, i, rounds);
|
||||
|
||||
// Dry run to get the time it takes without storage access
|
||||
let dry_run_start = Instant::now();
|
||||
instance
|
||||
.call_export("validate_block", &dry_run_encoded)
|
||||
.expect("Failed to call validate_block");
|
||||
let dry_run_elapsed = dry_run_start.elapsed();
|
||||
debug!("validate_block dry-run time {:?}", dry_run_elapsed);
|
||||
|
||||
let start = Instant::now();
|
||||
instance
|
||||
.call_export("validate_block", &encoded)
|
||||
.expect("Failed to call validate_block");
|
||||
let elapsed = start.elapsed();
|
||||
debug!("validate_block time {:?}", elapsed);
|
||||
|
||||
durations_in_nanos
|
||||
.push(elapsed.saturating_sub(dry_run_elapsed).as_nanos() as u64 / batch_size as u64);
|
||||
}
|
||||
|
||||
std::time::Duration::from_nanos(
|
||||
durations_in_nanos.iter().sum::<u64>() / durations_in_nanos.len() as u64,
|
||||
)
|
||||
}
|
||||
@@ -0,0 +1,153 @@
|
||||
// This file is part of Bizinikiwi.
|
||||
|
||||
// Copyright (C) Parity Technologies (UK) Ltd.
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
use pezsc_cli::Result;
|
||||
use pezsc_service::Configuration;
|
||||
|
||||
use log::info;
|
||||
use serde::Serialize;
|
||||
use std::{env, fs, path::PathBuf};
|
||||
|
||||
use super::cmd::StorageParams;
|
||||
use crate::shared::{Stats, UnderscoreHelper};
|
||||
|
||||
static VERSION: &str = env!("CARGO_PKG_VERSION");
|
||||
static TEMPLATE: &str = include_str!("./weights.hbs");
|
||||
|
||||
/// Data consumed by Handlebar to fill out the `weights.hbs` template.
|
||||
#[derive(Serialize, Default, Debug, Clone)]
|
||||
pub(crate) struct TemplateData {
|
||||
/// Name of the database used.
|
||||
db_name: String,
|
||||
/// Block number that was used.
|
||||
block_number: String,
|
||||
/// Name of the runtime. Taken from the chain spec.
|
||||
runtime_name: String,
|
||||
/// Version of the benchmarking CLI used.
|
||||
version: String,
|
||||
/// Date that the template was filled out.
|
||||
date: String,
|
||||
/// Hostname of the machine that executed the benchmarks.
|
||||
hostname: String,
|
||||
/// CPU name of the machine that executed the benchmarks.
|
||||
cpuname: String,
|
||||
/// Header for the generated file.
|
||||
header: String,
|
||||
/// Command line arguments that were passed to the CLI.
|
||||
args: Vec<String>,
|
||||
/// Storage params of the executed command.
|
||||
params: StorageParams,
|
||||
/// The weight for one `read`.
|
||||
read_weight: u64,
|
||||
/// The weight for one `write`.
|
||||
write_weight: u64,
|
||||
/// Stats about a `read` benchmark. Contains *time* and *value size* stats.
|
||||
/// The *value size* stats are currently not used in the template.
|
||||
read: Option<(Stats, Stats)>,
|
||||
/// Stats about a `write` benchmark. Contains *time* and *value size* stats.
|
||||
/// The *value size* stats are currently not used in the template.
|
||||
write: Option<(Stats, Stats)>,
|
||||
}
|
||||
|
||||
impl TemplateData {
|
||||
/// Returns a new [`Self`] from the given configuration.
|
||||
pub fn new(cfg: &Configuration, params: &StorageParams) -> Result<Self> {
|
||||
let header = params
|
||||
.header
|
||||
.as_ref()
|
||||
.map(|p| std::fs::read_to_string(p))
|
||||
.transpose()?
|
||||
.unwrap_or_default();
|
||||
|
||||
Ok(TemplateData {
|
||||
db_name: if params.is_validate_block_mode() {
|
||||
String::from("InMemoryDb")
|
||||
} else {
|
||||
format!("{}", cfg.database)
|
||||
},
|
||||
runtime_name: cfg.chain_spec.name().into(),
|
||||
version: VERSION.into(),
|
||||
date: chrono::Utc::now().format("%Y-%m-%d (Y/M/D)").to_string(),
|
||||
hostname: params.hostinfo.hostname(),
|
||||
cpuname: params.hostinfo.cpuname(),
|
||||
header,
|
||||
args: env::args().collect::<Vec<String>>(),
|
||||
params: params.clone(),
|
||||
..Default::default()
|
||||
})
|
||||
}
|
||||
|
||||
/// Sets the stats and calculates the final weights.
|
||||
pub fn set_stats(
|
||||
&mut self,
|
||||
read: Option<(Stats, Stats)>,
|
||||
write: Option<(Stats, Stats)>,
|
||||
) -> Result<()> {
|
||||
if let Some(read) = read {
|
||||
self.read_weight = self.params.weight_params.calc_weight(&read.0)?;
|
||||
self.read = Some(read);
|
||||
}
|
||||
if let Some(write) = write {
|
||||
self.write_weight = self.params.weight_params.calc_weight(&write.0)?;
|
||||
self.write = Some(write);
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Sets the block id that was used.
|
||||
pub fn set_block_number(&mut self, block_number: String) {
|
||||
self.block_number = block_number
|
||||
}
|
||||
|
||||
/// Fills out the `weights.hbs` or specified HBS template with its own data.
|
||||
/// Writes the result to `path` which can be a directory or file.
|
||||
pub fn write(&self, path: &Option<PathBuf>, hbs_template: &Option<PathBuf>) -> Result<()> {
|
||||
let mut handlebars = handlebars::Handlebars::new();
|
||||
// Format large integers with underscore.
|
||||
handlebars.register_helper("underscore", Box::new(UnderscoreHelper));
|
||||
// Don't HTML escape any characters.
|
||||
handlebars.register_escape_fn(|s| -> String { s.to_string() });
|
||||
// Use custom template if provided.
|
||||
let template = match hbs_template {
|
||||
Some(template) if template.is_file() => fs::read_to_string(template)?,
|
||||
Some(_) => return Err("Handlebars template is not a valid file!".into()),
|
||||
None => TEMPLATE.to_string(),
|
||||
};
|
||||
|
||||
let out_path = self.build_path(path);
|
||||
let mut fd = fs::File::create(&out_path)?;
|
||||
info!("Writing weights to {:?}", fs::canonicalize(&out_path)?);
|
||||
|
||||
handlebars
|
||||
.render_template_to_write(&template, &self, &mut fd)
|
||||
.map_err(|e| format!("HBS template write: {:?}", e).into())
|
||||
}
|
||||
|
||||
/// Builds a path for the weight file.
|
||||
fn build_path(&self, weight_out: &Option<PathBuf>) -> PathBuf {
|
||||
let mut path = match weight_out {
|
||||
Some(p) => PathBuf::from(p),
|
||||
None => PathBuf::new(),
|
||||
};
|
||||
|
||||
if path.is_dir() || path.as_os_str().is_empty() {
|
||||
path.push(format!("{}_weights", self.db_name.to_lowercase()));
|
||||
path.set_extension("rs");
|
||||
}
|
||||
path
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,99 @@
|
||||
{{header}}
|
||||
//! THIS FILE WAS AUTO-GENERATED USING THE BIZINIKIWI BENCHMARK CLI VERSION {{version}}
|
||||
//! DATE: {{date}}
|
||||
//! HOSTNAME: `{{hostname}}`, CPU: `{{cpuname}}`
|
||||
//!
|
||||
//! DATABASE: `{{db_name}}`, RUNTIME: `{{runtime_name}}`
|
||||
//! BLOCK-NUM: `{{block_number}}`
|
||||
//! SKIP-WRITE: `{{params.skip_write}}`, SKIP-READ: `{{params.skip_read}}`, WARMUPS: `{{params.warmups}}`
|
||||
//! STATE-VERSION: `V{{params.state_version}}`, STATE-CACHE-SIZE: `{{params.state_cache_size}}`
|
||||
//! WEIGHT-PATH: `{{params.weight_params.weight_path}}`
|
||||
//! METRIC: `{{params.weight_params.weight_metric}}`, WEIGHT-MUL: `{{params.weight_params.weight_mul}}`, WEIGHT-ADD: `{{params.weight_params.weight_add}}`
|
||||
|
||||
// Executed Command:
|
||||
{{#each args as |arg|}}
|
||||
// {{arg}}
|
||||
{{/each}}
|
||||
|
||||
/// Storage DB weights for the `{{runtime_name}}` runtime and `{{db_name}}`.
|
||||
pub mod constants {
|
||||
use frame_support::weights::constants;
|
||||
use sp_core::parameter_types;
|
||||
use sp_weights::RuntimeDbWeight;
|
||||
|
||||
parameter_types! {
|
||||
{{#if (eq db_name "InMemoryDb")}}
|
||||
/// `InMemoryDb` weights are measured in the context of the validation functions.
|
||||
/// To avoid submitting overweight blocks to the relay chain this is the configuration
|
||||
/// parachains should use.
|
||||
{{else if (eq db_name "ParityDb")}}
|
||||
/// `ParityDB` can be enabled with a feature flag, but is still experimental. These weights
|
||||
/// are available for brave runtime engineers who may want to try this out as default.
|
||||
{{else}}
|
||||
/// By default, Bizinikiwi uses `RocksDB`, so this will be the weight used throughout
|
||||
/// the runtime.
|
||||
{{/if}}
|
||||
pub const {{db_name}}Weight: RuntimeDbWeight = RuntimeDbWeight {
|
||||
/// Time to read one storage item.
|
||||
/// Calculated by multiplying the *{{params.weight_params.weight_metric}}* of all values with `{{params.weight_params.weight_mul}}` and adding `{{params.weight_params.weight_add}}`.
|
||||
///
|
||||
/// Stats nanoseconds:
|
||||
/// Min, Max: {{underscore read.0.min}}, {{underscore read.0.max}}
|
||||
/// Average: {{underscore read.0.avg}}
|
||||
/// Median: {{underscore read.0.median}}
|
||||
/// Std-Dev: {{read.0.stddev}}
|
||||
///
|
||||
/// Percentiles nanoseconds:
|
||||
/// 99th: {{underscore read.0.p99}}
|
||||
/// 95th: {{underscore read.0.p95}}
|
||||
/// 75th: {{underscore read.0.p75}}
|
||||
read: {{underscore read_weight}} * constants::WEIGHT_REF_TIME_PER_NANOS,
|
||||
|
||||
/// Time to write one storage item.
|
||||
/// Calculated by multiplying the *{{params.weight_params.weight_metric}}* of all values with `{{params.weight_params.weight_mul}}` and adding `{{params.weight_params.weight_add}}`.
|
||||
///
|
||||
/// Stats nanoseconds:
|
||||
/// Min, Max: {{underscore write.0.min}}, {{underscore write.0.max}}
|
||||
/// Average: {{underscore write.0.avg}}
|
||||
/// Median: {{underscore write.0.median}}
|
||||
/// Std-Dev: {{write.0.stddev}}
|
||||
///
|
||||
/// Percentiles nanoseconds:
|
||||
/// 99th: {{underscore write.0.p99}}
|
||||
/// 95th: {{underscore write.0.p95}}
|
||||
/// 75th: {{underscore write.0.p75}}
|
||||
write: {{underscore write_weight}} * constants::WEIGHT_REF_TIME_PER_NANOS,
|
||||
};
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod test_db_weights {
|
||||
use super::constants::{{db_name}}Weight as W;
|
||||
use sp_weights::constants;
|
||||
|
||||
/// Checks that all weights exist and have sane values.
|
||||
// NOTE: If this test fails but you are sure that the generated values are fine,
|
||||
// you can delete it.
|
||||
#[test]
|
||||
fn bound() {
|
||||
// At least 1 µs.
|
||||
assert!(
|
||||
W::get().reads(1).ref_time() >= constants::WEIGHT_REF_TIME_PER_MICROS,
|
||||
"Read weight should be at least 1 µs."
|
||||
);
|
||||
assert!(
|
||||
W::get().writes(1).ref_time() >= constants::WEIGHT_REF_TIME_PER_MICROS,
|
||||
"Write weight should be at least 1 µs."
|
||||
);
|
||||
// At most 1 ms.
|
||||
assert!(
|
||||
W::get().reads(1).ref_time() <= constants::WEIGHT_REF_TIME_PER_MILLIS,
|
||||
"Read weight should be at most 1 ms."
|
||||
);
|
||||
assert!(
|
||||
W::get().writes(1).ref_time() <= constants::WEIGHT_REF_TIME_PER_MILLIS,
|
||||
"Write weight should be at most 1 ms."
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,436 @@
|
||||
// This file is part of Bizinikiwi.
|
||||
|
||||
// Copyright (C) Parity Technologies (UK) Ltd.
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
use codec::Encode;
|
||||
use frame_storage_access_test_runtime::StorageAccessParams;
|
||||
use log::{debug, info, trace, warn};
|
||||
use rand::prelude::*;
|
||||
use pezsc_cli::Result;
|
||||
use pezsc_client_api::{Backend as ClientBackend, StorageProvider, UsageProvider};
|
||||
use pezsc_client_db::{DbHash, DbState, DbStateBuilder};
|
||||
use pezsp_blockchain::HeaderBackend;
|
||||
use pezsp_database::{ColumnId, Transaction};
|
||||
use pezsp_runtime::traits::{Block as BlockT, HashingFor, Header as HeaderT};
|
||||
use pezsp_state_machine::Backend as StateBackend;
|
||||
use pezsp_storage::{ChildInfo, StateVersion};
|
||||
use pezsp_trie::{recorder::Recorder, PrefixedMemoryDB};
|
||||
use std::{
|
||||
fmt::Debug,
|
||||
sync::Arc,
|
||||
time::{Duration, Instant},
|
||||
};
|
||||
|
||||
use super::{cmd::StorageCmd, get_wasm_module, MAX_BATCH_SIZE_FOR_BLOCK_VALIDATION};
|
||||
use crate::shared::{new_rng, BenchRecord};
|
||||
|
||||
impl StorageCmd {
|
||||
/// Benchmarks the time it takes to write a single Storage item.
|
||||
///
|
||||
/// Uses the latest state that is available for the given client.
|
||||
///
|
||||
/// Unlike reading benchmark, where we read every single key, here we write a batch of keys in
|
||||
/// one time. So writing a remaining keys with the size much smaller than batch size can
|
||||
/// dramatically distort the results. To avoid this, we skip the remaining keys.
|
||||
pub(crate) fn bench_write<Block, BA, H, C>(
|
||||
&self,
|
||||
client: Arc<C>,
|
||||
(db, state_col): (Arc<dyn pezsp_database::Database<DbHash>>, ColumnId),
|
||||
storage: Arc<dyn pezsp_state_machine::Storage<HashingFor<Block>>>,
|
||||
shared_trie_cache: Option<pezsp_trie::cache::SharedTrieCache<HashingFor<Block>>>,
|
||||
) -> Result<BenchRecord>
|
||||
where
|
||||
Block: BlockT<Header = H, Hash = DbHash> + Debug,
|
||||
H: HeaderT<Hash = DbHash>,
|
||||
BA: ClientBackend<Block>,
|
||||
C: UsageProvider<Block> + HeaderBackend<Block> + StorageProvider<Block, BA>,
|
||||
{
|
||||
if self.params.is_validate_block_mode() && self.params.disable_pov_recorder {
|
||||
return Err("PoV recorder must be activated to provide a storage proof for block validation at runtime. Remove `--disable-pov-recorder`.".into());
|
||||
}
|
||||
if self.params.is_validate_block_mode() &&
|
||||
self.params.batch_size > MAX_BATCH_SIZE_FOR_BLOCK_VALIDATION
|
||||
{
|
||||
return Err(format!("Batch size is too large. This may cause problems with runtime memory allocation. Better set `--batch-size {}` or less.", MAX_BATCH_SIZE_FOR_BLOCK_VALIDATION).into());
|
||||
}
|
||||
|
||||
// Store the time that it took to write each value.
|
||||
let mut record = BenchRecord::default();
|
||||
|
||||
let best_hash = client.usage_info().chain.best_hash;
|
||||
let header = client.header(best_hash)?.ok_or("Header not found")?;
|
||||
let original_root = *header.state_root();
|
||||
|
||||
let (trie, _) = self.create_trie_backend::<Block, H>(
|
||||
original_root,
|
||||
&storage,
|
||||
shared_trie_cache.as_ref(),
|
||||
);
|
||||
|
||||
info!("Preparing keys from block {}", best_hash);
|
||||
// Load all KV pairs and randomly shuffle them.
|
||||
let mut kvs: Vec<_> = trie.pairs(Default::default())?.collect();
|
||||
let (mut rng, _) = new_rng(None);
|
||||
kvs.shuffle(&mut rng);
|
||||
if kvs.is_empty() {
|
||||
return Err("Can't process benchmarking with empty storage".into());
|
||||
}
|
||||
|
||||
info!("Writing {} keys in batches of {}", kvs.len(), self.params.batch_size);
|
||||
let remainder = kvs.len() % self.params.batch_size;
|
||||
if self.params.is_validate_block_mode() && remainder != 0 {
|
||||
info!("Remaining `{remainder}` keys will be skipped");
|
||||
}
|
||||
|
||||
let mut child_nodes = Vec::new();
|
||||
let mut batched_keys = Vec::new();
|
||||
// Generate all random values first; Make sure there are no collisions with existing
|
||||
// db entries, so we can rollback all additions without corrupting existing entries.
|
||||
|
||||
for key_value in kvs {
|
||||
let (k, original_v) = key_value?;
|
||||
match (self.params.include_child_trees, self.is_child_key(k.to_vec())) {
|
||||
(true, Some(info)) => {
|
||||
let child_keys = client
|
||||
.child_storage_keys(best_hash, info.clone(), None, None)?
|
||||
.collect::<Vec<_>>();
|
||||
child_nodes.push((child_keys, info.clone()));
|
||||
},
|
||||
_ => {
|
||||
// regular key
|
||||
let mut new_v = vec![0; original_v.len()];
|
||||
|
||||
loop {
|
||||
// Create a random value to overwrite with.
|
||||
// NOTE: We use a possibly higher entropy than the original value,
|
||||
// could be improved but acts as an over-estimation which is fine for now.
|
||||
rng.fill_bytes(&mut new_v[..]);
|
||||
if check_new_value::<Block>(
|
||||
db.clone(),
|
||||
&trie,
|
||||
&k.to_vec(),
|
||||
&new_v,
|
||||
self.state_version(),
|
||||
state_col,
|
||||
None,
|
||||
) {
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
batched_keys.push((k.to_vec(), new_v.to_vec()));
|
||||
if batched_keys.len() < self.params.batch_size {
|
||||
continue;
|
||||
}
|
||||
|
||||
// Write each value in one commit.
|
||||
let (size, duration) = if self.params.is_validate_block_mode() {
|
||||
self.measure_per_key_amortised_validate_block_write_cost::<Block, H>(
|
||||
original_root,
|
||||
&storage,
|
||||
shared_trie_cache.as_ref(),
|
||||
batched_keys.clone(),
|
||||
None,
|
||||
)?
|
||||
} else {
|
||||
self.measure_per_key_amortised_import_block_write_cost::<Block, H>(
|
||||
original_root,
|
||||
&storage,
|
||||
shared_trie_cache.as_ref(),
|
||||
db.clone(),
|
||||
batched_keys.clone(),
|
||||
self.state_version(),
|
||||
state_col,
|
||||
None,
|
||||
)?
|
||||
};
|
||||
record.append(size, duration)?;
|
||||
batched_keys.clear();
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
if self.params.include_child_trees && !child_nodes.is_empty() {
|
||||
info!("Writing {} child keys", child_nodes.iter().map(|(c, _)| c.len()).sum::<usize>());
|
||||
for (mut child_keys, info) in child_nodes {
|
||||
if child_keys.len() < self.params.batch_size {
|
||||
warn!(
|
||||
"{} child keys will be skipped because it's less than batch size",
|
||||
child_keys.len()
|
||||
);
|
||||
continue;
|
||||
}
|
||||
|
||||
child_keys.shuffle(&mut rng);
|
||||
|
||||
for key in child_keys {
|
||||
if let Some(original_v) = client
|
||||
.child_storage(best_hash, &info, &key)
|
||||
.expect("Checked above to exist")
|
||||
{
|
||||
let mut new_v = vec![0; original_v.0.len()];
|
||||
|
||||
loop {
|
||||
rng.fill_bytes(&mut new_v[..]);
|
||||
if check_new_value::<Block>(
|
||||
db.clone(),
|
||||
&trie,
|
||||
&key.0,
|
||||
&new_v,
|
||||
self.state_version(),
|
||||
state_col,
|
||||
Some(&info),
|
||||
) {
|
||||
break;
|
||||
}
|
||||
}
|
||||
batched_keys.push((key.0, new_v.to_vec()));
|
||||
if batched_keys.len() < self.params.batch_size {
|
||||
continue;
|
||||
}
|
||||
|
||||
let (size, duration) = if self.params.is_validate_block_mode() {
|
||||
self.measure_per_key_amortised_validate_block_write_cost::<Block, H>(
|
||||
original_root,
|
||||
&storage,
|
||||
shared_trie_cache.as_ref(),
|
||||
batched_keys.clone(),
|
||||
None,
|
||||
)?
|
||||
} else {
|
||||
self.measure_per_key_amortised_import_block_write_cost::<Block, H>(
|
||||
original_root,
|
||||
&storage,
|
||||
shared_trie_cache.as_ref(),
|
||||
db.clone(),
|
||||
batched_keys.clone(),
|
||||
self.state_version(),
|
||||
state_col,
|
||||
Some(&info),
|
||||
)?
|
||||
};
|
||||
record.append(size, duration)?;
|
||||
batched_keys.clear();
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
Ok(record)
|
||||
}
|
||||
|
||||
fn create_trie_backend<Block, H>(
|
||||
&self,
|
||||
original_root: Block::Hash,
|
||||
storage: &Arc<dyn pezsp_state_machine::Storage<HashingFor<Block>>>,
|
||||
shared_trie_cache: Option<&pezsp_trie::cache::SharedTrieCache<HashingFor<Block>>>,
|
||||
) -> (DbState<HashingFor<Block>>, Option<Recorder<HashingFor<Block>>>)
|
||||
where
|
||||
Block: BlockT<Header = H, Hash = DbHash> + Debug,
|
||||
H: HeaderT<Hash = DbHash>,
|
||||
{
|
||||
let recorder = (!self.params.disable_pov_recorder).then(|| Default::default());
|
||||
let trie = DbStateBuilder::<HashingFor<Block>>::new(storage.clone(), original_root)
|
||||
.with_optional_cache(shared_trie_cache.map(|c| c.local_cache_trusted()))
|
||||
.with_optional_recorder(recorder.clone())
|
||||
.build();
|
||||
|
||||
(trie, recorder)
|
||||
}
|
||||
|
||||
/// Measures write benchmark
|
||||
/// if `child_info` exist then it means this is a child tree key
|
||||
fn measure_per_key_amortised_import_block_write_cost<Block, H>(
|
||||
&self,
|
||||
original_root: Block::Hash,
|
||||
storage: &Arc<dyn pezsp_state_machine::Storage<HashingFor<Block>>>,
|
||||
shared_trie_cache: Option<&pezsp_trie::cache::SharedTrieCache<HashingFor<Block>>>,
|
||||
db: Arc<dyn pezsp_database::Database<DbHash>>,
|
||||
changes: Vec<(Vec<u8>, Vec<u8>)>,
|
||||
version: StateVersion,
|
||||
col: ColumnId,
|
||||
child_info: Option<&ChildInfo>,
|
||||
) -> Result<(usize, Duration)>
|
||||
where
|
||||
Block: BlockT<Header = H, Hash = DbHash> + Debug,
|
||||
H: HeaderT<Hash = DbHash>,
|
||||
{
|
||||
let batch_size = changes.len();
|
||||
let average_len = changes.iter().map(|(_, v)| v.len()).sum::<usize>() / batch_size;
|
||||
// For every batched write use a different trie instance and recorder, so we
|
||||
// don't benefit from past runs.
|
||||
let (trie, _recorder) =
|
||||
self.create_trie_backend::<Block, H>(original_root, storage, shared_trie_cache);
|
||||
|
||||
let start = Instant::now();
|
||||
// Create a TX that will modify the Trie in the DB and
|
||||
// calculate the root hash of the Trie after the modification.
|
||||
let replace = changes
|
||||
.iter()
|
||||
.map(|(key, new_v)| (key.as_ref(), Some(new_v.as_ref())))
|
||||
.collect::<Vec<_>>();
|
||||
let stx = match child_info {
|
||||
Some(info) => trie.child_storage_root(info, replace.iter().cloned(), version).2,
|
||||
None => trie.storage_root(replace.iter().cloned(), version).1,
|
||||
};
|
||||
// Only the keep the insertions, since we do not want to benchmark pruning.
|
||||
let tx = convert_tx::<Block>(db.clone(), stx.clone(), false, col);
|
||||
db.commit(tx).map_err(|e| format!("Writing to the Database: {}", e))?;
|
||||
let result = (average_len, start.elapsed() / batch_size as u32);
|
||||
|
||||
// Now undo the changes by removing what was added.
|
||||
let tx = convert_tx::<Block>(db.clone(), stx.clone(), true, col);
|
||||
db.commit(tx).map_err(|e| format!("Writing to the Database: {}", e))?;
|
||||
|
||||
Ok(result)
|
||||
}
|
||||
|
||||
/// Measures write benchmark on block validation
|
||||
/// if `child_info` exist then it means this is a child tree key
|
||||
fn measure_per_key_amortised_validate_block_write_cost<Block, H>(
|
||||
&self,
|
||||
original_root: Block::Hash,
|
||||
storage: &Arc<dyn pezsp_state_machine::Storage<HashingFor<Block>>>,
|
||||
shared_trie_cache: Option<&pezsp_trie::cache::SharedTrieCache<HashingFor<Block>>>,
|
||||
changes: Vec<(Vec<u8>, Vec<u8>)>,
|
||||
maybe_child_info: Option<&ChildInfo>,
|
||||
) -> Result<(usize, Duration)>
|
||||
where
|
||||
Block: BlockT<Header = H, Hash = DbHash> + Debug,
|
||||
H: HeaderT<Hash = DbHash>,
|
||||
{
|
||||
let batch_size = changes.len();
|
||||
let average_len = changes.iter().map(|(_, v)| v.len()).sum::<usize>() / batch_size;
|
||||
let (trie, recorder) =
|
||||
self.create_trie_backend::<Block, H>(original_root, storage, shared_trie_cache);
|
||||
for (key, _) in changes.iter() {
|
||||
let _v = trie
|
||||
.storage(key)
|
||||
.expect("Checked above to exist")
|
||||
.ok_or("Value unexpectedly empty")?;
|
||||
}
|
||||
let storage_proof = recorder
|
||||
.map(|r| r.drain_storage_proof())
|
||||
.expect("Storage proof must exist for block validation");
|
||||
let root = trie.root();
|
||||
debug!(
|
||||
"POV: len {:?} {:?}",
|
||||
storage_proof.len(),
|
||||
storage_proof.clone().encoded_compact_size::<HashingFor<Block>>(*root)
|
||||
);
|
||||
let params = StorageAccessParams::<Block>::new_write(
|
||||
*root,
|
||||
storage_proof,
|
||||
(changes, maybe_child_info.cloned()),
|
||||
);
|
||||
|
||||
let mut durations_in_nanos = Vec::new();
|
||||
let wasm_module = get_wasm_module();
|
||||
let mut instance = wasm_module.new_instance().expect("Failed to create wasm instance");
|
||||
let dry_run_encoded = params.as_dry_run().encode();
|
||||
let encoded = params.encode();
|
||||
|
||||
for i in 1..=self.params.validate_block_rounds {
|
||||
info!(
|
||||
"validate_block with {} keys, round {}/{}",
|
||||
batch_size, i, self.params.validate_block_rounds
|
||||
);
|
||||
|
||||
// Dry run to get the time it takes without storage access
|
||||
let dry_run_start = Instant::now();
|
||||
instance
|
||||
.call_export("validate_block", &dry_run_encoded)
|
||||
.expect("Failed to call validate_block");
|
||||
let dry_run_elapsed = dry_run_start.elapsed();
|
||||
debug!("validate_block dry-run time {:?}", dry_run_elapsed);
|
||||
|
||||
let start = Instant::now();
|
||||
instance
|
||||
.call_export("validate_block", &encoded)
|
||||
.expect("Failed to call validate_block");
|
||||
let elapsed = start.elapsed();
|
||||
debug!("validate_block time {:?}", elapsed);
|
||||
|
||||
durations_in_nanos.push(
|
||||
elapsed.saturating_sub(dry_run_elapsed).as_nanos() as u64 / batch_size as u64,
|
||||
);
|
||||
}
|
||||
|
||||
let result = (
|
||||
average_len,
|
||||
std::time::Duration::from_nanos(
|
||||
durations_in_nanos.iter().sum::<u64>() / durations_in_nanos.len() as u64,
|
||||
),
|
||||
);
|
||||
|
||||
Ok(result)
|
||||
}
|
||||
}
|
||||
|
||||
/// Converts a Trie transaction into a DB transaction.
|
||||
/// Removals are ignored and will not be included in the final tx.
|
||||
/// `invert_inserts` replaces all inserts with removals.
|
||||
fn convert_tx<B: BlockT>(
|
||||
db: Arc<dyn pezsp_database::Database<DbHash>>,
|
||||
mut tx: PrefixedMemoryDB<HashingFor<B>>,
|
||||
invert_inserts: bool,
|
||||
col: ColumnId,
|
||||
) -> Transaction<DbHash> {
|
||||
let mut ret = Transaction::<DbHash>::default();
|
||||
|
||||
for (mut k, (v, rc)) in tx.drain().into_iter() {
|
||||
if rc > 0 {
|
||||
db.sanitize_key(&mut k);
|
||||
if invert_inserts {
|
||||
ret.remove(col, &k);
|
||||
} else {
|
||||
ret.set(col, &k, &v);
|
||||
}
|
||||
}
|
||||
// < 0 means removal - ignored.
|
||||
// 0 means no modification.
|
||||
}
|
||||
ret
|
||||
}
|
||||
|
||||
/// Checks if a new value causes any collision in tree updates
|
||||
/// returns true if there is no collision
|
||||
/// if `child_info` exist then it means this is a child tree key
|
||||
fn check_new_value<Block: BlockT>(
|
||||
db: Arc<dyn pezsp_database::Database<DbHash>>,
|
||||
trie: &DbState<HashingFor<Block>>,
|
||||
key: &Vec<u8>,
|
||||
new_v: &Vec<u8>,
|
||||
version: StateVersion,
|
||||
col: ColumnId,
|
||||
child_info: Option<&ChildInfo>,
|
||||
) -> bool {
|
||||
let new_kv = vec![(key.as_ref(), Some(new_v.as_ref()))];
|
||||
let mut stx = match child_info {
|
||||
Some(info) => trie.child_storage_root(info, new_kv.iter().cloned(), version).2,
|
||||
None => trie.storage_root(new_kv.iter().cloned(), version).1,
|
||||
};
|
||||
for (mut k, (_, rc)) in stx.drain().into_iter() {
|
||||
if rc > 0 {
|
||||
db.sanitize_key(&mut k);
|
||||
if db.get(col, &k).is_some() {
|
||||
trace!("Benchmark-store key creation: Key collision detected, retry");
|
||||
return false;
|
||||
}
|
||||
}
|
||||
}
|
||||
true
|
||||
}
|
||||
@@ -0,0 +1,33 @@
|
||||
[package]
|
||||
name = "generate-bags"
|
||||
version = "28.0.0"
|
||||
authors.workspace = true
|
||||
edition.workspace = true
|
||||
license = "Apache-2.0"
|
||||
homepage.workspace = true
|
||||
repository.workspace = true
|
||||
description = "Bag threshold generation script for pezpallet-bag-list"
|
||||
|
||||
[lints]
|
||||
workspace = true
|
||||
|
||||
[dependencies]
|
||||
# FRAME
|
||||
pezframe-election-provider-support = { workspace = true, default-features = true }
|
||||
pezframe-support = { workspace = true, default-features = true }
|
||||
pezframe-system = { workspace = true, default-features = true }
|
||||
pezpallet-staking = { workspace = true, default-features = true }
|
||||
pezsp-staking = { workspace = true, default-features = true }
|
||||
|
||||
# third party
|
||||
chrono = { workspace = true }
|
||||
num-format = { workspace = true }
|
||||
|
||||
[features]
|
||||
runtime-benchmarks = [
|
||||
"pezframe-election-provider-support/runtime-benchmarks",
|
||||
"pezframe-support/runtime-benchmarks",
|
||||
"pezframe-system/runtime-benchmarks",
|
||||
"pezpallet-staking/runtime-benchmarks",
|
||||
"pezsp-staking/runtime-benchmarks",
|
||||
]
|
||||
@@ -0,0 +1,26 @@
|
||||
[package]
|
||||
name = "node-runtime-generate-bags"
|
||||
version = "3.0.0"
|
||||
authors.workspace = true
|
||||
edition.workspace = true
|
||||
license = "Apache-2.0"
|
||||
homepage.workspace = true
|
||||
repository.workspace = true
|
||||
description = "Bag threshold generation script for pezpallet-bag-list and kitchensink-runtime."
|
||||
publish = false
|
||||
|
||||
[lints]
|
||||
workspace = true
|
||||
|
||||
[dependencies]
|
||||
generate-bags = { workspace = true, default-features = true }
|
||||
kitchensink-runtime = { workspace = true }
|
||||
|
||||
# third-party
|
||||
clap = { features = ["derive"], workspace = true }
|
||||
|
||||
[features]
|
||||
runtime-benchmarks = [
|
||||
"generate-bags/runtime-benchmarks",
|
||||
"kitchensink-runtime/runtime-benchmarks",
|
||||
]
|
||||
@@ -0,0 +1,52 @@
|
||||
// This file is part of Bizinikiwi.
|
||||
|
||||
// Copyright (C) Parity Technologies (UK) Ltd.
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
//! Make the set of bag thresholds to be used with pezpallet-bags-list.
|
||||
|
||||
use clap::Parser;
|
||||
use generate_bags::generate_thresholds;
|
||||
use std::path::PathBuf;
|
||||
|
||||
#[derive(Debug, Parser)]
|
||||
// #[clap(author, version, about)]
|
||||
struct Opt {
|
||||
/// How many bags to generate.
|
||||
#[arg(long, default_value_t = 200)]
|
||||
n_bags: usize,
|
||||
|
||||
/// Where to write the output.
|
||||
output: PathBuf,
|
||||
|
||||
/// The total issuance of the currency used to create `VoteWeight`.
|
||||
#[arg(short, long)]
|
||||
total_issuance: u128,
|
||||
|
||||
/// The minimum account balance (i.e. existential deposit) for the currency used to create
|
||||
/// `VoteWeight`.
|
||||
#[arg(short, long)]
|
||||
minimum_balance: u128,
|
||||
}
|
||||
|
||||
fn main() -> Result<(), std::io::Error> {
|
||||
let Opt { n_bags, output, total_issuance, minimum_balance } = Opt::parse();
|
||||
generate_thresholds::<kitchensink_runtime::Runtime>(
|
||||
n_bags,
|
||||
&output,
|
||||
total_issuance,
|
||||
minimum_balance,
|
||||
)
|
||||
}
|
||||
@@ -0,0 +1,262 @@
|
||||
// This file is part of Bizinikiwi.
|
||||
|
||||
// Copyright (C) Parity Technologies (UK) Ltd.
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
//! Support code to ease the process of generating bag thresholds.
|
||||
//!
|
||||
//! NOTE: this assume the runtime implements [`pezpallet_staking::Config`], as it requires an
|
||||
//! implementation of the traits [`pezframe_support::traits::Currency`] and `CurrencyToVote`.
|
||||
//!
|
||||
//! The process of adding bags to a runtime requires only four steps.
|
||||
//!
|
||||
//! 1. Update the runtime definition.
|
||||
//!
|
||||
//! ```ignore
|
||||
//! parameter_types!{
|
||||
//! pub const BagThresholds: &'static [u64] = &[];
|
||||
//! }
|
||||
//!
|
||||
//! impl pezpallet_bags_list::Config for Runtime {
|
||||
//! // <snip>
|
||||
//! type BagThresholds = BagThresholds;
|
||||
//! }
|
||||
//! ```
|
||||
//!
|
||||
//! 2. Write a little program to generate the definitions. This program exists only to hook together
|
||||
//! the runtime definitions with the various calculations here. Take a look at
|
||||
//! _utils/pezframe/generate_bags/node-runtime_ for an example.
|
||||
//!
|
||||
//! 3. Run that program:
|
||||
//!
|
||||
//! ```sh,notrust
|
||||
//! $ cargo run -p node-runtime-generate-bags -- --total-issuance 1234 --minimum-balance 1
|
||||
//! output.rs ```
|
||||
//!
|
||||
//! 4. Update the runtime definition.
|
||||
//!
|
||||
//! ```diff,notrust
|
||||
//! + mod output;
|
||||
//! - pub const BagThresholds: &'static [u64] = &[];
|
||||
//! + pub const BagThresholds: &'static [u64] = &output::THRESHOLDS;
|
||||
//! ```
|
||||
|
||||
use pezframe_election_provider_support::VoteWeight;
|
||||
use pezframe_support::traits::Get;
|
||||
use std::{
|
||||
io::Write,
|
||||
path::{Path, PathBuf},
|
||||
};
|
||||
|
||||
/// Compute the existential weight for the specified configuration.
|
||||
///
|
||||
/// Note that this value depends on the current issuance, a quantity known to change over time.
|
||||
/// This makes the project of computing a static value suitable for inclusion in a static,
|
||||
/// generated file _excitingly unstable_.
|
||||
fn existential_weight<T: pezpallet_staking::Config>(
|
||||
total_issuance: u128,
|
||||
minimum_balance: u128,
|
||||
) -> VoteWeight {
|
||||
use pezsp_staking::currency_to_vote::CurrencyToVote;
|
||||
|
||||
T::CurrencyToVote::to_vote(
|
||||
minimum_balance
|
||||
.try_into()
|
||||
.map_err(|_| "failed to convert minimum_balance to type Balance")
|
||||
.unwrap(),
|
||||
total_issuance
|
||||
.try_into()
|
||||
.map_err(|_| "failed to convert total_issuance to type Balance")
|
||||
.unwrap(),
|
||||
)
|
||||
}
|
||||
|
||||
/// Return the path to a header file used in this repository if is exists.
|
||||
///
|
||||
/// Just searches the git working directory root for files matching certain patterns; it's
|
||||
/// pretty naive.
|
||||
fn path_to_header_file() -> Option<PathBuf> {
|
||||
let mut workdir: &Path = &std::env::current_dir().ok()?;
|
||||
while !workdir.join(".git").exists() {
|
||||
workdir = workdir.parent()?;
|
||||
}
|
||||
|
||||
for file_name in &["HEADER-APACHE2", "HEADER-GPL3", "HEADER", "file_header.txt"] {
|
||||
let path = workdir.join(file_name);
|
||||
if path.exists() {
|
||||
return Some(path);
|
||||
}
|
||||
}
|
||||
None
|
||||
}
|
||||
|
||||
/// Create an underscore formatter: a formatter which inserts `_` every 3 digits of a number.
|
||||
fn underscore_formatter() -> num_format::CustomFormat {
|
||||
num_format::CustomFormat::builder()
|
||||
.grouping(num_format::Grouping::Standard)
|
||||
.separator("_")
|
||||
.build()
|
||||
.expect("format described here meets all constraints")
|
||||
}
|
||||
|
||||
/// Compute the constant ratio for the thresholds.
|
||||
///
|
||||
/// This ratio ensures that each bag, with the possible exceptions of certain small ones and the
|
||||
/// final one, is a constant multiple of the previous, while fully occupying the `VoteWeight`
|
||||
/// space.
|
||||
pub fn constant_ratio(existential_weight: VoteWeight, n_bags: usize) -> f64 {
|
||||
((VoteWeight::MAX as f64 / existential_weight as f64).ln() / ((n_bags - 1) as f64)).exp()
|
||||
}
|
||||
|
||||
/// Compute the list of bag thresholds.
|
||||
///
|
||||
/// Returns a list of exactly `n_bags` elements, except in the case of overflow.
|
||||
/// The first element is always `existential_weight`.
|
||||
/// The last element is always `VoteWeight::MAX`.
|
||||
///
|
||||
/// All other elements are computed from the previous according to the formula
|
||||
/// `threshold[k + 1] = (threshold[k] * ratio).max(threshold[k] + 1);`
|
||||
pub fn thresholds(
|
||||
existential_weight: VoteWeight,
|
||||
constant_ratio: f64,
|
||||
n_bags: usize,
|
||||
) -> Vec<VoteWeight> {
|
||||
const WEIGHT_LIMIT: f64 = VoteWeight::MAX as f64;
|
||||
|
||||
let mut thresholds = Vec::with_capacity(n_bags);
|
||||
|
||||
if n_bags > 1 {
|
||||
thresholds.push(existential_weight);
|
||||
}
|
||||
|
||||
while n_bags > 0 && thresholds.len() < n_bags - 1 {
|
||||
let last = thresholds.last().copied().unwrap_or(existential_weight);
|
||||
let successor = (last as f64 * constant_ratio).round().max(last as f64 + 1.0);
|
||||
if successor < WEIGHT_LIMIT {
|
||||
thresholds.push(successor as VoteWeight);
|
||||
} else {
|
||||
eprintln!("unexpectedly exceeded weight limit; breaking threshold generation loop");
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
thresholds.push(VoteWeight::MAX);
|
||||
|
||||
debug_assert_eq!(thresholds.len(), n_bags);
|
||||
debug_assert!(n_bags == 0 || thresholds[0] == existential_weight);
|
||||
debug_assert!(n_bags == 0 || thresholds[thresholds.len() - 1] == VoteWeight::MAX);
|
||||
|
||||
thresholds
|
||||
}
|
||||
|
||||
/// Write a thresholds module to the path specified.
|
||||
///
|
||||
/// Parameters:
|
||||
/// - `n_bags` the number of bags to generate.
|
||||
/// - `output` the path to write to; should terminate with a Rust module name, i.e.
|
||||
/// `foo/bar/thresholds.rs`.
|
||||
/// - `total_issuance` the total amount of the currency in the network.
|
||||
/// - `minimum_balance` the minimum balance of the currency required for an account to exist (i.e.
|
||||
/// existential deposit).
|
||||
///
|
||||
/// This generated module contains, in order:
|
||||
///
|
||||
/// - The contents of the header file in this repository's root, if found.
|
||||
/// - Module documentation noting that this is autogenerated and when.
|
||||
/// - Some associated constants.
|
||||
/// - The constant array of thresholds.
|
||||
pub fn generate_thresholds<T: pezpallet_staking::Config>(
|
||||
n_bags: usize,
|
||||
output: &Path,
|
||||
total_issuance: u128,
|
||||
minimum_balance: u128,
|
||||
) -> Result<(), std::io::Error> {
|
||||
// ensure the file is accessible
|
||||
if let Some(parent) = output.parent() {
|
||||
if !parent.exists() {
|
||||
std::fs::create_dir_all(parent)?;
|
||||
}
|
||||
}
|
||||
|
||||
// copy the header file
|
||||
if let Some(header_path) = path_to_header_file() {
|
||||
std::fs::copy(header_path, output)?;
|
||||
}
|
||||
|
||||
// open an append buffer
|
||||
let file = std::fs::OpenOptions::new().create(true).append(true).open(output)?;
|
||||
let mut buf = std::io::BufWriter::new(file);
|
||||
|
||||
// create underscore formatter and format buffer
|
||||
let mut num_buf = num_format::Buffer::new();
|
||||
let format = underscore_formatter();
|
||||
|
||||
// module docs
|
||||
let now = chrono::Utc::now();
|
||||
writeln!(buf)?;
|
||||
writeln!(buf, "//! Autogenerated bag thresholds.")?;
|
||||
writeln!(buf, "//!")?;
|
||||
writeln!(buf, "//! Generated on {}", now.to_rfc3339())?;
|
||||
writeln!(buf, "//! Arguments")?;
|
||||
writeln!(buf, "//! Total issuance: {}", &total_issuance)?;
|
||||
writeln!(buf, "//! Minimum balance: {}", &minimum_balance)?;
|
||||
|
||||
writeln!(
|
||||
buf,
|
||||
"//! for the {} runtime.",
|
||||
<T as pezframe_system::Config>::Version::get().spec_name,
|
||||
)?;
|
||||
|
||||
let existential_weight = existential_weight::<T>(total_issuance, minimum_balance);
|
||||
num_buf.write_formatted(&existential_weight, &format);
|
||||
writeln!(buf)?;
|
||||
writeln!(buf, "/// Existential weight for this runtime.")?;
|
||||
writeln!(buf, "#[cfg(any(test, feature = \"std\"))]")?;
|
||||
writeln!(buf, "#[allow(unused)]")?;
|
||||
writeln!(buf, "pub const EXISTENTIAL_WEIGHT: u64 = {};", num_buf.as_str())?;
|
||||
|
||||
// constant ratio
|
||||
let constant_ratio = constant_ratio(existential_weight, n_bags);
|
||||
writeln!(buf)?;
|
||||
writeln!(buf, "/// Constant ratio between bags for this runtime.")?;
|
||||
writeln!(buf, "#[cfg(any(test, feature = \"std\"))]")?;
|
||||
writeln!(buf, "#[allow(unused)]")?;
|
||||
writeln!(buf, "pub const CONSTANT_RATIO: f64 = {:.16};", constant_ratio)?;
|
||||
|
||||
// thresholds
|
||||
let thresholds = thresholds(existential_weight, constant_ratio, n_bags);
|
||||
writeln!(buf)?;
|
||||
writeln!(buf, "/// Upper thresholds delimiting the bag list.")?;
|
||||
writeln!(buf, "pub const THRESHOLDS: [u64; {}] = [", thresholds.len())?;
|
||||
for threshold in &thresholds {
|
||||
num_buf.write_formatted(threshold, &format);
|
||||
// u64::MAX, with spacers every 3 digits, is 26 characters wide
|
||||
writeln!(buf, " {:>26},", num_buf.as_str())?;
|
||||
}
|
||||
writeln!(buf, "];")?;
|
||||
|
||||
// thresholds balance
|
||||
writeln!(buf)?;
|
||||
writeln!(buf, "/// Upper thresholds delimiting the bag list.")?;
|
||||
writeln!(buf, "pub const THRESHOLDS_BALANCES: [u128; {}] = [", thresholds.len())?;
|
||||
for threshold in thresholds {
|
||||
num_buf.write_formatted(&threshold, &format);
|
||||
// u64::MAX, with spacers every 3 digits, is 26 characters wide
|
||||
writeln!(buf, " {:>26},", num_buf.as_str())?;
|
||||
}
|
||||
writeln!(buf, "];")?;
|
||||
|
||||
Ok(())
|
||||
}
|
||||
@@ -0,0 +1,40 @@
|
||||
[package]
|
||||
name = "frame-omni-bencher"
|
||||
version = "0.1.0"
|
||||
description = "Freestanding benchmark runner for any Pezkuwi runtime."
|
||||
authors.workspace = true
|
||||
edition.workspace = true
|
||||
repository.workspace = true
|
||||
license.workspace = true
|
||||
readme = "README.md"
|
||||
|
||||
[lints]
|
||||
workspace = true
|
||||
|
||||
[dependencies]
|
||||
clap = { features = ["derive"], workspace = true }
|
||||
pezcumulus-primitives-proof-size-hostfunction = { workspace = true, default-features = true }
|
||||
pezframe-benchmarking-cli = { workspace = true }
|
||||
pezsc-cli = { workspace = true, default-features = true }
|
||||
pezsp-runtime = { workspace = true, default-features = true }
|
||||
pezsp-statement-store = { workspace = true, default-features = true }
|
||||
tracing-subscriber = { workspace = true }
|
||||
|
||||
[dev-dependencies]
|
||||
assert_cmd = { workspace = true }
|
||||
pezcumulus-test-runtime = { workspace = true }
|
||||
pezsc-chain-spec = { workspace = true }
|
||||
pezsp-genesis-builder = { workspace = true, default-features = true }
|
||||
tempfile = { workspace = true }
|
||||
|
||||
[features]
|
||||
runtime-benchmarks = [
|
||||
"pezcumulus-primitives-proof-size-hostfunction/runtime-benchmarks",
|
||||
"pezcumulus-test-runtime/runtime-benchmarks",
|
||||
"pezframe-benchmarking-cli/runtime-benchmarks",
|
||||
"pezsc-chain-spec/runtime-benchmarks",
|
||||
"pezsc-cli/runtime-benchmarks",
|
||||
"pezsp-genesis-builder/runtime-benchmarks",
|
||||
"pezsp-runtime/runtime-benchmarks",
|
||||
"pezsp-statement-store/runtime-benchmarks",
|
||||
]
|
||||
@@ -0,0 +1,77 @@
|
||||
# PezkuwiChain Omni Benchmarking CLI
|
||||
|
||||
The PezkuwiChain Omni benchmarker allows to benchmark the extrinsics of any PezkuwiChain runtime. It is
|
||||
meant to replace the current manual integration of the `benchmark pallet` into every teyrchain node.
|
||||
This reduces duplicate code and makes maintenance for builders easier. The CLI is currently only
|
||||
able to benchmark extrinsics. In the future it is planned to extend this to some other areas.
|
||||
|
||||
General FRAME runtimes could also be used with this benchmarker, as long as they don't utilize any
|
||||
host functions that are not part of the PezkuwiChain host specification.
|
||||
|
||||
## Installation
|
||||
|
||||
Directly via crates.io:
|
||||
|
||||
```sh
|
||||
cargo install frame-omni-bencher --profile=production --locked
|
||||
```
|
||||
|
||||
from GitHub:
|
||||
|
||||
```sh
|
||||
cargo install --git https://github.com/pezkuwichain/pezkuwi-sdk frame-omni-bencher --profile=production --locked
|
||||
```
|
||||
|
||||
or locally from the sources:
|
||||
|
||||
```sh
|
||||
cargo install --path bizinikiwi/utils/pezframe/omni-bencher --profile=production
|
||||
```
|
||||
|
||||
Check the installed version and print the docs:
|
||||
|
||||
```sh
|
||||
frame-omni-bencher --help
|
||||
```
|
||||
|
||||
## Usage
|
||||
|
||||
First we need to ensure that there is a runtime available. As example we will build the zagros
|
||||
runtime:
|
||||
|
||||
```sh
|
||||
cargo build -p zagros-runtime --profile production --features runtime-benchmarks
|
||||
```
|
||||
|
||||
Now as an example, we benchmark the `balances` pallet:
|
||||
|
||||
```sh
|
||||
frame-omni-bencher v1 benchmark pallet \
|
||||
--runtime target/release/wbuild/zagros-runtime/zagros-runtime.compact.compressed.wasm \
|
||||
--pallet "pallet_balances" --extrinsic ""
|
||||
```
|
||||
|
||||
The `--steps`, `--repeat`, `--heap-pages` and `--wasm-execution` arguments have sane defaults and do
|
||||
not need be passed explicitly anymore.
|
||||
|
||||
### Generate weights (templates)
|
||||
|
||||
To render Rust weight files from benchmark results, pass an output path. Optionally you can pass a
|
||||
custom header and a Handlebars template (defaults are provided):
|
||||
|
||||
```sh
|
||||
frame-omni-bencher v1 benchmark pallet \
|
||||
--runtime target/release/wbuild/zagros-runtime/zagros-runtime.compact.compressed.wasm \
|
||||
--pallet "pallet_balances" --extrinsic "*" \
|
||||
--output ./weights/ \
|
||||
--header ./HEADER.rs \
|
||||
--template ./template.hbs
|
||||
```
|
||||
|
||||
This uses the same flags as the node-integrated benchmarking CLI. The output can be a directory or a
|
||||
file path; when a directory is given, a file name is generated per pallet/instance.
|
||||
|
||||
## Backwards Compatibility
|
||||
|
||||
The exposed pallet sub-command is identical as the node-integrated CLI. The only difference is that
|
||||
it needs to be prefixed with a `v1` to ensure drop-in compatibility.
|
||||
@@ -0,0 +1,148 @@
|
||||
// This file is part of Bizinikiwi.
|
||||
|
||||
// Copyright (C) Parity Technologies (UK) Ltd.
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
use clap::Parser;
|
||||
use pezframe_benchmarking_cli::{BenchmarkCmd, OpaqueBlock};
|
||||
use pezsc_cli::Result;
|
||||
use pezsp_runtime::traits::BlakeTwo256;
|
||||
|
||||
/// # Pezkuwi Omni Benchmarking CLI
|
||||
///
|
||||
/// The Pezkuwi Omni benchmarker allows to benchmark the extrinsics of any Pezkuwi runtime. It is
|
||||
/// meant to replace the current manual integration of the `benchmark pallet` into every teyrchain
|
||||
/// node. This reduces duplicate code and makes maintenance for builders easier. The CLI is
|
||||
/// currently only able to benchmark extrinsics. In the future it is planned to extend this to some
|
||||
/// other areas.
|
||||
///
|
||||
/// General FRAME runtimes could also be used with this benchmarker, as long as they don't utilize
|
||||
/// any host functions that are not part of the Pezkuwi host specification.
|
||||
///
|
||||
/// ## Installation
|
||||
///
|
||||
/// Directly via crates.io:
|
||||
///
|
||||
/// ```sh
|
||||
/// cargo install frame-omni-bencher --profile=production
|
||||
/// ```
|
||||
///
|
||||
/// from GitHub:
|
||||
///
|
||||
/// ```sh
|
||||
/// cargo install --git https://github.com/pezkuwichain/pezkuwi-sdk frame-omni-bencher --profile=production
|
||||
/// ```
|
||||
///
|
||||
/// or locally from the sources:
|
||||
///
|
||||
/// ```sh
|
||||
/// cargo install --path bizinikiwi/utils/pezframe/omni-bencher --profile=production
|
||||
/// ```
|
||||
///
|
||||
/// Check the installed version and print the docs:
|
||||
///
|
||||
/// ```sh
|
||||
/// frame-omni-bencher --help
|
||||
/// ```
|
||||
///
|
||||
/// ## Usage
|
||||
///
|
||||
/// First we need to ensure that there is a runtime available. As example we will build the Zagros
|
||||
/// runtime:
|
||||
///
|
||||
/// ```sh
|
||||
/// cargo build -p zagros-runtime --profile production --features runtime-benchmarks
|
||||
/// ```
|
||||
///
|
||||
/// Now as an example, we benchmark the `balances` pallet:
|
||||
///
|
||||
/// ```sh
|
||||
/// frame-omni-bencher v1 benchmark pallet \
|
||||
/// --runtime target/release/wbuild/zagros-runtime/zagros-runtime.compact.compressed.wasm \
|
||||
/// --pallet "pezpallet_balances" --extrinsic ""
|
||||
/// ```
|
||||
///
|
||||
/// For the exact arguments of the `pallet` command, please refer to the `pallet` sub-module.
|
||||
///
|
||||
/// ## Backwards Compatibility
|
||||
///
|
||||
/// The exposed pallet sub-command is identical as the node-integrated CLI. The only difference is
|
||||
/// that it needs to be prefixed with a `v1` to ensure drop-in compatibility.
|
||||
#[derive(Parser, Debug)]
|
||||
#[clap(author, version, about, verbatim_doc_comment)]
|
||||
pub struct Command {
|
||||
#[command(subcommand)]
|
||||
sub: SubCommand,
|
||||
}
|
||||
|
||||
/// Root-level subcommands.
|
||||
#[derive(Debug, clap::Subcommand)]
|
||||
pub enum SubCommand {
|
||||
/// Compatibility syntax with the old benchmark runner.
|
||||
V1(V1Command),
|
||||
// NOTE: Here we can add new commands in a forward-compatible way. For example when
|
||||
// transforming the CLI from a monolithic design to a data driven pipeline, there could be
|
||||
// commands like `measure`, `analyze` and `render`.
|
||||
}
|
||||
|
||||
/// A command that conforms to the legacy `benchmark` argument syntax.
|
||||
#[derive(Parser, Debug)]
|
||||
pub struct V1Command {
|
||||
#[command(subcommand)]
|
||||
sub: V1SubCommand,
|
||||
}
|
||||
|
||||
/// The `v1 benchmark` subcommand.
|
||||
#[derive(Debug, clap::Subcommand)]
|
||||
pub enum V1SubCommand {
|
||||
Benchmark(V1BenchmarkCommand),
|
||||
}
|
||||
|
||||
/// Subcommands for `v1 benchmark`.
|
||||
#[derive(Parser, Debug)]
|
||||
pub struct V1BenchmarkCommand {
|
||||
#[command(subcommand)]
|
||||
sub: BenchmarkCmd,
|
||||
}
|
||||
|
||||
type HostFunctions = (
|
||||
pezsp_statement_store::runtime_api::HostFunctions,
|
||||
cumulus_primitives_proof_size_hostfunction::storage_proof_size::HostFunctions,
|
||||
);
|
||||
|
||||
impl Command {
|
||||
pub fn run(self) -> Result<()> {
|
||||
match self.sub {
|
||||
SubCommand::V1(V1Command { sub }) => sub.run(),
|
||||
}
|
||||
}
|
||||
}
|
||||
impl V1SubCommand {
|
||||
pub fn run(self) -> Result<()> {
|
||||
match self {
|
||||
V1SubCommand::Benchmark(V1BenchmarkCommand { sub }) => match sub {
|
||||
BenchmarkCmd::Pallet(pallet) => {
|
||||
pallet.run_with_spec::<BlakeTwo256, HostFunctions>(None)
|
||||
},
|
||||
BenchmarkCmd::Overhead(overhead_cmd) =>
|
||||
overhead_cmd.run_with_default_builder_and_spec::<OpaqueBlock, HostFunctions>(None),
|
||||
_ =>
|
||||
return Err(
|
||||
"Only the `v1 benchmark pallet` and `v1 benchmark overhead` command is currently supported".into()
|
||||
),
|
||||
},
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,47 @@
|
||||
// This file is part of Bizinikiwi.
|
||||
|
||||
// Copyright (C) Parity Technologies (UK) Ltd.
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
mod command;
|
||||
|
||||
use clap::Parser;
|
||||
use pezsc_cli::Result;
|
||||
use tracing_subscriber::EnvFilter;
|
||||
|
||||
fn main() -> Result<()> {
|
||||
setup_logger();
|
||||
|
||||
command::Command::parse().run()
|
||||
}
|
||||
|
||||
/// Setup logging with `info` as default level. Can be set via `RUST_LOG` env.
|
||||
fn setup_logger() {
|
||||
// Disable these log targets because they are spammy.
|
||||
let unwanted_targets =
|
||||
&["cranelift_codegen", "wasm_cranelift", "wasmtime_jit", "wasmtime_cranelift", "wasm_jit"];
|
||||
|
||||
let mut env_filter =
|
||||
EnvFilter::try_from_default_env().unwrap_or_else(|_| EnvFilter::new("info"));
|
||||
|
||||
for target in unwanted_targets {
|
||||
env_filter = env_filter.add_directive(format!("{}=off", target).parse().unwrap());
|
||||
}
|
||||
|
||||
tracing_subscriber::fmt()
|
||||
.with_env_filter(env_filter)
|
||||
.with_writer(std::io::stderr)
|
||||
.init();
|
||||
}
|
||||
@@ -0,0 +1,167 @@
|
||||
// This file is part of Bizinikiwi.
|
||||
|
||||
// Copyright (C) Parity Technologies (UK) Ltd.
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
use assert_cmd::cargo::cargo_bin;
|
||||
use std::{
|
||||
fs,
|
||||
path::{Path, PathBuf},
|
||||
process::{Command, ExitStatus},
|
||||
};
|
||||
|
||||
#[test]
|
||||
fn benchmark_overhead_runtime_works() -> std::result::Result<(), String> {
|
||||
let tmp_dir = tempfile::tempdir().expect("Should be able to create tmp dir.");
|
||||
let base_path = tmp_dir.path();
|
||||
let wasm = cumulus_test_runtime::WASM_BINARY.ok_or("WASM binary not available".to_string())?;
|
||||
let runtime_path = base_path.join("runtime.wasm");
|
||||
let _ =
|
||||
fs::write(&runtime_path, wasm).map_err(|e| format!("Unable to write runtime file: {}", e));
|
||||
|
||||
// Invoke `benchmark overhead` with all options to make sure that they are valid.
|
||||
let status = std::process::Command::new(cargo_bin("frame-omni-bencher"))
|
||||
.args(["v1", "benchmark", "overhead", "--runtime", runtime_path.to_str().unwrap()])
|
||||
.arg("-d")
|
||||
.arg(base_path)
|
||||
.arg("--weight-path")
|
||||
.arg(base_path)
|
||||
.args(["--warmup", "5", "--repeat", "5"])
|
||||
// Exotic para id to see that we are actually patching.
|
||||
.args(["--para-id", "666"])
|
||||
.args(["--add", "100", "--mul", "1.2", "--metric", "p75"])
|
||||
// Only put 5 extrinsics into the block otherwise it takes forever to build it
|
||||
// especially for a non-release builds.
|
||||
.args(["--max-ext-per-block", "5"])
|
||||
.status()
|
||||
.map_err(|e| format!("command failed: {:?}", e))?;
|
||||
|
||||
assert_benchmark_success(status, base_path)
|
||||
}
|
||||
#[test]
|
||||
fn benchmark_overhead_chain_spec_works() -> std::result::Result<(), String> {
|
||||
let tmp_dir = tempfile::tempdir().expect("Should be able to create tmp dir.");
|
||||
let (base_path, chain_spec_path) = setup_chain_spec(tmp_dir.path(), false)?;
|
||||
|
||||
let status = create_benchmark_spec_command(&base_path, &chain_spec_path)
|
||||
.args(["--genesis-builder-policy", "spec-runtime"])
|
||||
.args(["--para-id", "666"])
|
||||
.status()
|
||||
.map_err(|e| format!("command failed: {:?}", e))?;
|
||||
|
||||
assert_benchmark_success(status, &base_path)
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn benchmark_overhead_chain_spec_works_plain_spec() -> std::result::Result<(), String> {
|
||||
let tmp_dir = tempfile::tempdir().expect("Should be able to create tmp dir.");
|
||||
let (base_path, chain_spec_path) = setup_chain_spec(tmp_dir.path(), false)?;
|
||||
|
||||
let status = create_benchmark_spec_command(&base_path, &chain_spec_path)
|
||||
.args(["--genesis-builder-policy", "spec"])
|
||||
.args(["--para-id", "100"])
|
||||
.status()
|
||||
.map_err(|e| format!("command failed: {:?}", e))?;
|
||||
|
||||
assert_benchmark_success(status, &base_path)
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn benchmark_overhead_chain_spec_works_raw() -> std::result::Result<(), String> {
|
||||
let tmp_dir = tempfile::tempdir().expect("Should be able to create tmp dir.");
|
||||
let (base_path, chain_spec_path) = setup_chain_spec(tmp_dir.path(), true)?;
|
||||
|
||||
let status = create_benchmark_spec_command(&base_path, &chain_spec_path)
|
||||
.args(["--genesis-builder-policy", "spec"])
|
||||
.args(["--para-id", "100"])
|
||||
.status()
|
||||
.map_err(|e| format!("command failed: {:?}", e))?;
|
||||
|
||||
assert_benchmark_success(status, &base_path)
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn benchmark_overhead_chain_spec_fails_wrong_para_id() -> std::result::Result<(), String> {
|
||||
let tmp_dir = tempfile::tempdir().expect("Should be able to create tmp dir.");
|
||||
let (base_path, chain_spec_path) = setup_chain_spec(tmp_dir.path(), false)?;
|
||||
|
||||
let status = create_benchmark_spec_command(&base_path, &chain_spec_path)
|
||||
.args(["--genesis-builder-policy", "spec"])
|
||||
.args(["--para-id", "666"])
|
||||
.status()
|
||||
.map_err(|e| format!("command failed: {:?}", e))?;
|
||||
|
||||
if status.success() {
|
||||
return Err("Command should have failed!".into());
|
||||
}
|
||||
|
||||
// Weight files should not have been created
|
||||
assert!(!base_path.join("block_weights.rs").exists());
|
||||
assert!(!base_path.join("extrinsic_weights.rs").exists());
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Sets up a temporary directory and creates a chain spec file
|
||||
fn setup_chain_spec(tmp_dir: &Path, raw: bool) -> Result<(PathBuf, PathBuf), String> {
|
||||
let base_path = tmp_dir.to_path_buf();
|
||||
let chain_spec_path = base_path.join("chain_spec.json");
|
||||
|
||||
let wasm = cumulus_test_runtime::WASM_BINARY.ok_or("WASM binary not available".to_string())?;
|
||||
|
||||
let mut properties = pezsc_chain_spec::Properties::new();
|
||||
properties.insert("tokenSymbol".into(), "UNIT".into());
|
||||
properties.insert("tokenDecimals".into(), 12.into());
|
||||
|
||||
let chain_spec = pezsc_chain_spec::GenericChainSpec::<()>::builder(wasm, Default::default())
|
||||
.with_name("some-chain")
|
||||
.with_id("some-id")
|
||||
.with_properties(properties)
|
||||
.with_chain_type(pezsc_chain_spec::ChainType::Development)
|
||||
.with_genesis_config_preset_name(pezsp_genesis_builder::LOCAL_TESTNET_RUNTIME_PRESET)
|
||||
.build();
|
||||
|
||||
let json = chain_spec.as_json(raw).unwrap();
|
||||
fs::write(&chain_spec_path, json)
|
||||
.map_err(|e| format!("Unable to write chain-spec file: {}", e))?;
|
||||
|
||||
Ok((base_path, chain_spec_path))
|
||||
}
|
||||
|
||||
/// Creates a Command for the benchmark with common arguments
|
||||
fn create_benchmark_spec_command(base_path: &Path, chain_spec_path: &Path) -> Command {
|
||||
let mut cmd = Command::new(cargo_bin("frame-omni-bencher"));
|
||||
cmd.args(["v1", "benchmark", "overhead", "--chain", chain_spec_path.to_str().unwrap()])
|
||||
.arg("-d")
|
||||
.arg(base_path)
|
||||
.arg("--weight-path")
|
||||
.arg(base_path)
|
||||
.args(["--warmup", "5", "--repeat", "5"])
|
||||
.args(["--add", "100", "--mul", "1.2", "--metric", "p75"])
|
||||
// Only put 5 extrinsics into the block otherwise it takes forever to build it
|
||||
.args(["--max-ext-per-block", "5"]);
|
||||
cmd
|
||||
}
|
||||
|
||||
/// Checks if the benchmark completed successfully and created weight files
|
||||
fn assert_benchmark_success(status: ExitStatus, base_path: &Path) -> Result<(), String> {
|
||||
if !status.success() {
|
||||
return Err("Command failed".into());
|
||||
}
|
||||
|
||||
// Weight files have been created
|
||||
assert!(base_path.join("block_weights.rs").exists());
|
||||
assert!(base_path.join("extrinsic_weights.rs").exists());
|
||||
Ok(())
|
||||
}
|
||||
@@ -0,0 +1,47 @@
|
||||
[package]
|
||||
name = "frame-remote-externalities"
|
||||
version = "0.35.0"
|
||||
authors.workspace = true
|
||||
edition.workspace = true
|
||||
license = "Apache-2.0"
|
||||
homepage.workspace = true
|
||||
repository.workspace = true
|
||||
description = "An externalities provided environment that can load itself from remote nodes or cached files"
|
||||
|
||||
[lints]
|
||||
workspace = true
|
||||
|
||||
[package.metadata.docs.rs]
|
||||
targets = ["x86_64-unknown-linux-gnu"]
|
||||
|
||||
[dependencies]
|
||||
codec = { workspace = true, default-features = true }
|
||||
futures = { workspace = true }
|
||||
indicatif = { workspace = true }
|
||||
jsonrpsee = { features = ["http-client"], workspace = true }
|
||||
log = { workspace = true, default-features = true }
|
||||
serde = { workspace = true, default-features = true }
|
||||
pezsp-core = { workspace = true, default-features = true }
|
||||
pezsp-crypto-hashing = { workspace = true, default-features = true }
|
||||
pezsp-io = { workspace = true, default-features = true }
|
||||
pezsp-runtime = { workspace = true, default-features = true }
|
||||
pezsp-state-machine = { workspace = true, default-features = true }
|
||||
spinners = { workspace = true }
|
||||
bizinikiwi-rpc-client = { workspace = true, default-features = true }
|
||||
tokio = { features = [
|
||||
"macros",
|
||||
"rt-multi-thread",
|
||||
], workspace = true, default-features = true }
|
||||
tokio-retry = { workspace = true }
|
||||
|
||||
[dev-dependencies]
|
||||
pezsp-tracing = { workspace = true, default-features = true }
|
||||
|
||||
[features]
|
||||
remote-test = []
|
||||
runtime-benchmarks = [
|
||||
"pezsp-io/runtime-benchmarks",
|
||||
"pezsp-runtime/runtime-benchmarks",
|
||||
"pezsp-state-machine/runtime-benchmarks",
|
||||
"bizinikiwi-rpc-client/runtime-benchmarks",
|
||||
]
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,86 @@
|
||||
// This file is part of Bizinikiwi.
|
||||
|
||||
// Copyright (C) Parity Technologies (UK) Ltd.
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
use std::{
|
||||
future::Future,
|
||||
io::{self, IsTerminal},
|
||||
time::Instant,
|
||||
};
|
||||
|
||||
use spinners::{Spinner, Spinners};
|
||||
|
||||
use super::Result;
|
||||
|
||||
// A simple helper to time a operation with a nice spinner, start message, and end message.
|
||||
//
|
||||
// The spinner is only displayed when stdout is a terminal.
|
||||
pub(super) fn with_elapsed<F, R, EndMsg>(f: F, start_msg: &str, end_msg: EndMsg) -> Result<R>
|
||||
where
|
||||
F: FnOnce() -> Result<R>,
|
||||
EndMsg: FnOnce(&R) -> String,
|
||||
{
|
||||
let timer = Instant::now();
|
||||
let mut maybe_sp = start(start_msg);
|
||||
|
||||
Ok(end(f()?, timer, maybe_sp.as_mut(), end_msg))
|
||||
}
|
||||
|
||||
// A simple helper to time an async operation with a nice spinner, start message, and end message.
|
||||
//
|
||||
// The spinner is only displayed when stdout is a terminal.
|
||||
pub(super) async fn with_elapsed_async<F, Fut, R, EndMsg>(
|
||||
f: F,
|
||||
start_msg: &str,
|
||||
end_msg: EndMsg,
|
||||
) -> Result<R>
|
||||
where
|
||||
F: FnOnce() -> Fut,
|
||||
Fut: Future<Output = Result<R>>,
|
||||
EndMsg: FnOnce(&R) -> String,
|
||||
{
|
||||
let timer = Instant::now();
|
||||
let mut maybe_sp = start(start_msg);
|
||||
|
||||
Ok(end(f().await?, timer, maybe_sp.as_mut(), end_msg))
|
||||
}
|
||||
|
||||
fn start(start_msg: &str) -> Option<Spinner> {
|
||||
let msg = format!("⏳ {start_msg}");
|
||||
|
||||
if io::stdout().is_terminal() {
|
||||
Some(Spinner::new(Spinners::Dots, msg))
|
||||
} else {
|
||||
println!("{msg}");
|
||||
|
||||
None
|
||||
}
|
||||
}
|
||||
|
||||
fn end<T, EndMsg>(val: T, timer: Instant, maybe_sp: Option<&mut Spinner>, end_msg: EndMsg) -> T
|
||||
where
|
||||
EndMsg: FnOnce(&T) -> String,
|
||||
{
|
||||
let msg = format!("✅ {} in {:.2}s", end_msg(&val), timer.elapsed().as_secs_f32());
|
||||
|
||||
if let Some(sp) = maybe_sp {
|
||||
sp.stop_with_message(msg);
|
||||
} else {
|
||||
println!("{msg}");
|
||||
}
|
||||
|
||||
val
|
||||
}
|
||||
Binary file not shown.
@@ -0,0 +1,37 @@
|
||||
[package]
|
||||
name = "bizinikiwi-rpc-client"
|
||||
version = "0.33.0"
|
||||
authors.workspace = true
|
||||
edition.workspace = true
|
||||
license = "Apache-2.0"
|
||||
homepage.workspace = true
|
||||
repository.workspace = true
|
||||
description = "Shared JSON-RPC client"
|
||||
|
||||
[lints]
|
||||
workspace = true
|
||||
|
||||
[package.metadata.docs.rs]
|
||||
targets = ["x86_64-unknown-linux-gnu"]
|
||||
|
||||
[dependencies]
|
||||
async-trait = { workspace = true }
|
||||
jsonrpsee = { features = ["ws-client"], workspace = true }
|
||||
log = { workspace = true, default-features = true }
|
||||
pezsc-rpc-api = { workspace = true, default-features = true }
|
||||
serde = { workspace = true, default-features = true }
|
||||
pezsp-runtime = { workspace = true, default-features = true }
|
||||
|
||||
[dev-dependencies]
|
||||
pezsp-core = { workspace = true, default-features = true }
|
||||
tokio = { features = [
|
||||
"macros",
|
||||
"rt-multi-thread",
|
||||
"sync",
|
||||
], workspace = true, default-features = true }
|
||||
|
||||
[features]
|
||||
runtime-benchmarks = [
|
||||
"pezsc-rpc-api/runtime-benchmarks",
|
||||
"pezsp-runtime/runtime-benchmarks",
|
||||
]
|
||||
@@ -0,0 +1,271 @@
|
||||
// This file is part of Bizinikiwi.
|
||||
|
||||
// Copyright (C) Parity Technologies (UK) Ltd.
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
//! # Shared JSON-RPC client related code and abstractions.
|
||||
//!
|
||||
//! It exposes a `WebSocket JSON-RPC` client that implements the RPC interface in [`sc-rpc-api`]
|
||||
//! along with some abstractions.
|
||||
//!
|
||||
//! ## Usage
|
||||
//!
|
||||
//! ```no_run
|
||||
//! # use bizinikiwi_rpc_client::{ws_client, StateApi};
|
||||
//! # use pezsp_core::{H256, storage::StorageKey};
|
||||
//!
|
||||
//! #[tokio::main]
|
||||
//! async fn main() {
|
||||
//!
|
||||
//! let client = ws_client("ws://127.0.0.1:9944").await.unwrap();
|
||||
//! client.storage(StorageKey(vec![]), Some(H256::zero())).await.unwrap();
|
||||
//!
|
||||
//! // if all type params are not known you need to provide type params
|
||||
//! StateApi::<H256>::storage(&client, StorageKey(vec![]), None).await.unwrap();
|
||||
//! }
|
||||
//! ```
|
||||
|
||||
use async_trait::async_trait;
|
||||
use serde::de::DeserializeOwned;
|
||||
use pezsp_runtime::traits::{Block as BlockT, Header as HeaderT};
|
||||
use std::collections::VecDeque;
|
||||
|
||||
pub use jsonrpsee::{
|
||||
core::{
|
||||
client::{ClientT, Error, Subscription, SubscriptionClientT},
|
||||
params::BatchRequestBuilder,
|
||||
RpcResult,
|
||||
},
|
||||
rpc_params,
|
||||
ws_client::{WsClient, WsClientBuilder},
|
||||
};
|
||||
pub use pezsc_rpc_api::{
|
||||
author::AuthorApiClient as AuthorApi, chain::ChainApiClient as ChainApi,
|
||||
child_state::ChildStateApiClient as ChildStateApi, dev::DevApiClient as DevApi,
|
||||
offchain::OffchainApiClient as OffchainApi, state::StateApiClient as StateApi,
|
||||
system::SystemApiClient as SystemApi,
|
||||
};
|
||||
|
||||
/// Create a new `WebSocket` connection with shared settings.
|
||||
pub async fn ws_client(uri: impl AsRef<str>) -> Result<WsClient, String> {
|
||||
WsClientBuilder::default()
|
||||
.max_request_size(u32::MAX)
|
||||
.max_response_size(u32::MAX)
|
||||
.request_timeout(std::time::Duration::from_secs(60 * 10))
|
||||
.connection_timeout(std::time::Duration::from_secs(60))
|
||||
.max_buffer_capacity_per_subscription(1024)
|
||||
.build(uri)
|
||||
.await
|
||||
.map_err(|e| format!("`WsClientBuilder` failed to build: {:?}", e))
|
||||
}
|
||||
|
||||
/// Abstraction over RPC calling for headers.
|
||||
#[async_trait]
|
||||
pub trait HeaderProvider<Block: BlockT>
|
||||
where
|
||||
Block::Header: HeaderT,
|
||||
{
|
||||
/// Awaits for the header of the block with hash `hash`.
|
||||
async fn get_header(&self, hash: Block::Hash) -> Block::Header;
|
||||
}
|
||||
|
||||
#[async_trait]
|
||||
impl<Block: BlockT> HeaderProvider<Block> for WsClient
|
||||
where
|
||||
Block::Header: DeserializeOwned,
|
||||
{
|
||||
async fn get_header(&self, hash: Block::Hash) -> Block::Header {
|
||||
ChainApi::<(), Block::Hash, Block::Header, ()>::header(self, Some(hash))
|
||||
.await
|
||||
.unwrap()
|
||||
.unwrap()
|
||||
}
|
||||
}
|
||||
|
||||
/// Abstraction over RPC subscription for finalized headers.
|
||||
#[async_trait]
|
||||
pub trait HeaderSubscription<Block: BlockT>
|
||||
where
|
||||
Block::Header: HeaderT,
|
||||
{
|
||||
/// Await for the next finalized header from the subscription.
|
||||
///
|
||||
/// Returns `None` if either the subscription has been closed or there was an error when reading
|
||||
/// an object from the client.
|
||||
async fn next_header(&mut self) -> Option<Block::Header>;
|
||||
}
|
||||
|
||||
#[async_trait]
|
||||
impl<Block: BlockT> HeaderSubscription<Block> for Subscription<Block::Header>
|
||||
where
|
||||
Block::Header: DeserializeOwned,
|
||||
{
|
||||
async fn next_header(&mut self) -> Option<Block::Header> {
|
||||
match self.next().await {
|
||||
Some(Ok(header)) => Some(header),
|
||||
None => {
|
||||
log::warn!("subscription closed");
|
||||
None
|
||||
},
|
||||
Some(Err(why)) => {
|
||||
log::warn!("subscription returned error: {:?}. Probably decoding has failed.", why);
|
||||
None
|
||||
},
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Stream of all finalized headers.
|
||||
///
|
||||
/// Returned headers are guaranteed to be ordered. There are no missing headers (even if some of
|
||||
/// them lack justification).
|
||||
pub struct FinalizedHeaders<
|
||||
'a,
|
||||
Block: BlockT,
|
||||
HP: HeaderProvider<Block>,
|
||||
HS: HeaderSubscription<Block>,
|
||||
> {
|
||||
header_provider: &'a HP,
|
||||
subscription: HS,
|
||||
fetched_headers: VecDeque<Block::Header>,
|
||||
last_returned: Option<<Block::Header as HeaderT>::Hash>,
|
||||
}
|
||||
|
||||
impl<'a, Block: BlockT, HP: HeaderProvider<Block>, HS: HeaderSubscription<Block>>
|
||||
FinalizedHeaders<'a, Block, HP, HS>
|
||||
where
|
||||
<Block as BlockT>::Header: DeserializeOwned,
|
||||
{
|
||||
pub fn new(header_provider: &'a HP, subscription: HS) -> Self {
|
||||
Self {
|
||||
header_provider,
|
||||
subscription,
|
||||
fetched_headers: VecDeque::new(),
|
||||
last_returned: None,
|
||||
}
|
||||
}
|
||||
|
||||
/// Reads next finalized header from the subscription. If some headers (without justification)
|
||||
/// have been skipped, fetches them as well. Returns number of headers that have been fetched.
|
||||
///
|
||||
/// All fetched headers are stored in `self.fetched_headers`.
|
||||
async fn fetch(&mut self) -> usize {
|
||||
let last_finalized = match self.subscription.next_header().await {
|
||||
Some(header) => header,
|
||||
None => return 0,
|
||||
};
|
||||
|
||||
self.fetched_headers.push_front(last_finalized.clone());
|
||||
|
||||
let mut last_finalized_parent = *last_finalized.parent_hash();
|
||||
let last_returned = self.last_returned.unwrap_or(last_finalized_parent);
|
||||
|
||||
while last_finalized_parent != last_returned {
|
||||
let parent_header = self.header_provider.get_header(last_finalized_parent).await;
|
||||
self.fetched_headers.push_front(parent_header.clone());
|
||||
last_finalized_parent = *parent_header.parent_hash();
|
||||
}
|
||||
|
||||
self.fetched_headers.len()
|
||||
}
|
||||
|
||||
/// Get the next finalized header.
|
||||
pub async fn next(&mut self) -> Option<Block::Header> {
|
||||
if self.fetched_headers.is_empty() {
|
||||
self.fetch().await;
|
||||
}
|
||||
|
||||
if let Some(header) = self.fetched_headers.pop_front() {
|
||||
self.last_returned = Some(header.hash());
|
||||
Some(header)
|
||||
} else {
|
||||
None
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use pezsp_runtime::testing::{Block as TBlock, Header, MockCallU64, TestXt, H256};
|
||||
use std::sync::Arc;
|
||||
use tokio::sync::Mutex;
|
||||
|
||||
type UncheckedXt = TestXt<MockCallU64, ()>;
|
||||
type Block = TBlock<UncheckedXt>;
|
||||
type BlockNumber = u64;
|
||||
type Hash = H256;
|
||||
|
||||
struct MockHeaderProvider(pub Arc<Mutex<VecDeque<BlockNumber>>>);
|
||||
|
||||
fn headers() -> Vec<Header> {
|
||||
let mut headers = vec![Header::new_from_number(0)];
|
||||
for n in 1..11 {
|
||||
headers.push(Header {
|
||||
parent_hash: headers.last().unwrap().hash(),
|
||||
..Header::new_from_number(n)
|
||||
})
|
||||
}
|
||||
headers
|
||||
}
|
||||
|
||||
#[async_trait]
|
||||
impl HeaderProvider<Block> for MockHeaderProvider {
|
||||
async fn get_header(&self, _hash: Hash) -> Header {
|
||||
let height = self.0.lock().await.pop_front().unwrap();
|
||||
headers()[height as usize].clone()
|
||||
}
|
||||
}
|
||||
|
||||
struct MockHeaderSubscription(pub VecDeque<BlockNumber>);
|
||||
|
||||
#[async_trait]
|
||||
impl HeaderSubscription<Block> for MockHeaderSubscription {
|
||||
async fn next_header(&mut self) -> Option<Header> {
|
||||
self.0.pop_front().map(|h| headers()[h as usize].clone())
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn finalized_headers_works_when_every_block_comes_from_subscription() {
|
||||
let heights = vec![4, 5, 6, 7];
|
||||
|
||||
let provider = MockHeaderProvider(Default::default());
|
||||
let subscription = MockHeaderSubscription(heights.clone().into());
|
||||
let mut headers = FinalizedHeaders::new(&provider, subscription);
|
||||
|
||||
for h in heights {
|
||||
assert_eq!(h, headers.next().await.unwrap().number);
|
||||
}
|
||||
assert_eq!(None, headers.next().await);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn finalized_headers_come_from_subscription_and_provider_if_in_need() {
|
||||
let all_heights = 3..11;
|
||||
let heights_in_subscription = vec![3, 4, 6, 10];
|
||||
// Consecutive headers will be requested in the reversed order.
|
||||
let heights_not_in_subscription = vec![5, 9, 8, 7];
|
||||
|
||||
let provider = MockHeaderProvider(Arc::new(Mutex::new(heights_not_in_subscription.into())));
|
||||
let subscription = MockHeaderSubscription(heights_in_subscription.into());
|
||||
let mut headers = FinalizedHeaders::new(&provider, subscription);
|
||||
|
||||
for h in all_heights {
|
||||
assert_eq!(h, headers.next().await.unwrap().number);
|
||||
}
|
||||
assert_eq!(None, headers.next().await);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,45 @@
|
||||
[package]
|
||||
name = "bizinikiwi-state-trie-migration-rpc"
|
||||
version = "27.0.0"
|
||||
authors.workspace = true
|
||||
edition.workspace = true
|
||||
license = "Apache-2.0"
|
||||
homepage.workspace = true
|
||||
repository.workspace = true
|
||||
description = "Node-specific RPC methods for interaction with state trie migration."
|
||||
readme = "README.md"
|
||||
|
||||
[lints]
|
||||
workspace = true
|
||||
|
||||
[package.metadata.docs.rs]
|
||||
targets = ["x86_64-unknown-linux-gnu"]
|
||||
|
||||
[dependencies]
|
||||
codec = { workspace = true }
|
||||
serde = { features = ["derive"], workspace = true, default-features = true }
|
||||
|
||||
pezsp-core = { workspace = true, default-features = true }
|
||||
pezsp-state-machine = { workspace = true, default-features = true }
|
||||
pezsp-trie = { workspace = true, default-features = true }
|
||||
trie-db = { workspace = true, default-features = true }
|
||||
|
||||
jsonrpsee = { features = [
|
||||
"client-core",
|
||||
"macros",
|
||||
"server-core",
|
||||
], workspace = true }
|
||||
|
||||
# Bizinikiwi Dependencies
|
||||
pezsc-client-api = { workspace = true, default-features = true }
|
||||
pezsc-rpc-api = { workspace = true, default-features = true }
|
||||
pezsp-runtime = { workspace = true, default-features = true }
|
||||
|
||||
[features]
|
||||
runtime-benchmarks = [
|
||||
"pezsc-client-api/runtime-benchmarks",
|
||||
"pezsc-rpc-api/runtime-benchmarks",
|
||||
"pezsp-runtime/runtime-benchmarks",
|
||||
"pezsp-state-machine/runtime-benchmarks",
|
||||
"pezsp-trie/runtime-benchmarks",
|
||||
]
|
||||
@@ -0,0 +1,3 @@
|
||||
Node-specific RPC methods for interaction with trie migration.
|
||||
|
||||
License: Apache-2.0
|
||||
@@ -0,0 +1,185 @@
|
||||
// This file is part of Bizinikiwi.
|
||||
|
||||
// Copyright (C) Parity Technologies (UK) Ltd.
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
//! Rpc for state migration.
|
||||
|
||||
use jsonrpsee::{
|
||||
core::RpcResult,
|
||||
proc_macros::rpc,
|
||||
types::error::{ErrorCode, ErrorObject, ErrorObjectOwned},
|
||||
Extensions,
|
||||
};
|
||||
use pezsc_client_api::TrieCacheContext;
|
||||
use pezsc_rpc_api::check_if_safe;
|
||||
use serde::{Deserialize, Serialize};
|
||||
use pezsp_runtime::traits::Block as BlockT;
|
||||
use std::sync::Arc;
|
||||
|
||||
use pezsp_core::{
|
||||
storage::{ChildInfo, ChildType, PrefixedStorageKey},
|
||||
Hasher,
|
||||
};
|
||||
use pezsp_state_machine::backend::AsTrieBackend;
|
||||
use pezsp_trie::{
|
||||
trie_types::{TrieDB, TrieDBBuilder},
|
||||
KeySpacedDB, Trie,
|
||||
};
|
||||
use trie_db::{
|
||||
node::{NodePlan, ValuePlan},
|
||||
TrieDBNodeIterator,
|
||||
};
|
||||
|
||||
fn count_migrate<'a, H: Hasher>(
|
||||
storage: &'a dyn trie_db::HashDBRef<H, Vec<u8>>,
|
||||
root: &'a H::Out,
|
||||
) -> std::result::Result<(u64, u64, TrieDB<'a, 'a, H>), String> {
|
||||
let mut nb = 0u64;
|
||||
let mut total_nb = 0u64;
|
||||
let trie = TrieDBBuilder::new(storage, root).build();
|
||||
let iter_node =
|
||||
TrieDBNodeIterator::new(&trie).map_err(|e| format!("TrieDB node iterator error: {}", e))?;
|
||||
for node in iter_node {
|
||||
let node = node.map_err(|e| format!("TrieDB node iterator error: {}", e))?;
|
||||
match node.2.node_plan() {
|
||||
NodePlan::Leaf { value, .. } | NodePlan::NibbledBranch { value: Some(value), .. } => {
|
||||
total_nb += 1;
|
||||
if let ValuePlan::Inline(range) = value {
|
||||
if (range.end - range.start) as u32 >=
|
||||
pezsp_core::storage::TRIE_VALUE_NODE_THRESHOLD
|
||||
{
|
||||
nb += 1;
|
||||
}
|
||||
}
|
||||
},
|
||||
_ => (),
|
||||
}
|
||||
}
|
||||
Ok((nb, total_nb, trie))
|
||||
}
|
||||
|
||||
/// Check trie migration status.
|
||||
pub fn migration_status<H, B>(backend: &B) -> std::result::Result<MigrationStatusResult, String>
|
||||
where
|
||||
H: Hasher,
|
||||
H::Out: codec::Codec,
|
||||
B: AsTrieBackend<H>,
|
||||
{
|
||||
let trie_backend = backend.as_trie_backend();
|
||||
let essence = trie_backend.essence();
|
||||
let (top_remaining_to_migrate, total_top, trie) = count_migrate(essence, essence.root())?;
|
||||
|
||||
let mut child_remaining_to_migrate = 0;
|
||||
let mut total_child = 0;
|
||||
let mut child_roots: Vec<(ChildInfo, Vec<u8>)> = Vec::new();
|
||||
// get all child trie roots
|
||||
for key_value in trie.iter().map_err(|e| format!("TrieDB node iterator error: {}", e))? {
|
||||
let (key, value) = key_value.map_err(|e| format!("TrieDB node iterator error: {}", e))?;
|
||||
if key[..].starts_with(pezsp_core::storage::well_known_keys::DEFAULT_CHILD_STORAGE_KEY_PREFIX)
|
||||
{
|
||||
let prefixed_key = PrefixedStorageKey::new(key);
|
||||
let (_type, unprefixed) = ChildType::from_prefixed_key(&prefixed_key).unwrap();
|
||||
child_roots.push((ChildInfo::new_default(unprefixed), value));
|
||||
}
|
||||
}
|
||||
for (child_info, root) in child_roots {
|
||||
let mut child_root = H::Out::default();
|
||||
let storage = KeySpacedDB::new(essence, child_info.keyspace());
|
||||
|
||||
child_root.as_mut()[..].copy_from_slice(&root[..]);
|
||||
let (nb, total_top, _) = count_migrate(&storage, &child_root)?;
|
||||
child_remaining_to_migrate += nb;
|
||||
total_child += total_top;
|
||||
}
|
||||
|
||||
Ok(MigrationStatusResult {
|
||||
top_remaining_to_migrate,
|
||||
child_remaining_to_migrate,
|
||||
total_top,
|
||||
total_child,
|
||||
})
|
||||
}
|
||||
|
||||
/// Current state migration status.
|
||||
#[derive(Serialize, Deserialize, Clone)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
#[serde(deny_unknown_fields)]
|
||||
pub struct MigrationStatusResult {
|
||||
/// Number of top items that should migrate.
|
||||
pub top_remaining_to_migrate: u64,
|
||||
/// Number of child items that should migrate.
|
||||
pub child_remaining_to_migrate: u64,
|
||||
/// Number of top items that we will iterate on.
|
||||
pub total_top: u64,
|
||||
/// Number of child items that we will iterate on.
|
||||
pub total_child: u64,
|
||||
}
|
||||
|
||||
/// Migration RPC methods.
|
||||
#[rpc(server)]
|
||||
pub trait StateMigrationApi<BlockHash> {
|
||||
/// Check current migration state.
|
||||
///
|
||||
/// This call is performed locally without submitting any transactions. Thus executing this
|
||||
/// won't change any state. Nonetheless it is a VERY costly call that should be
|
||||
/// only exposed to trusted peers.
|
||||
#[method(name = "state_trieMigrationStatus", with_extensions)]
|
||||
fn call(&self, at: Option<BlockHash>) -> RpcResult<MigrationStatusResult>;
|
||||
}
|
||||
|
||||
/// An implementation of state migration specific RPC methods.
|
||||
pub struct StateMigration<C, B, BA> {
|
||||
client: Arc<C>,
|
||||
backend: Arc<BA>,
|
||||
_marker: std::marker::PhantomData<(B, BA)>,
|
||||
}
|
||||
|
||||
impl<C, B, BA> StateMigration<C, B, BA> {
|
||||
/// Create new state migration rpc for the given reference to the client.
|
||||
pub fn new(client: Arc<C>, backend: Arc<BA>) -> Self {
|
||||
StateMigration { client, backend, _marker: Default::default() }
|
||||
}
|
||||
}
|
||||
|
||||
impl<C, B, BA> StateMigrationApiServer<<B as BlockT>::Hash> for StateMigration<C, B, BA>
|
||||
where
|
||||
B: BlockT,
|
||||
C: Send + Sync + 'static + pezsc_client_api::HeaderBackend<B>,
|
||||
BA: 'static + pezsc_client_api::backend::Backend<B>,
|
||||
{
|
||||
fn call(
|
||||
&self,
|
||||
ext: &Extensions,
|
||||
at: Option<<B as BlockT>::Hash>,
|
||||
) -> RpcResult<MigrationStatusResult> {
|
||||
check_if_safe(ext)?;
|
||||
|
||||
let hash = at.unwrap_or_else(|| self.client.info().best_hash);
|
||||
let state = self
|
||||
.backend
|
||||
.state_at(hash, TrieCacheContext::Untrusted)
|
||||
.map_err(error_into_rpc_err)?;
|
||||
migration_status(&state).map_err(error_into_rpc_err)
|
||||
}
|
||||
}
|
||||
|
||||
fn error_into_rpc_err(err: impl std::fmt::Display) -> ErrorObjectOwned {
|
||||
ErrorObject::owned(
|
||||
ErrorCode::InternalError.code(),
|
||||
"Error while checking migration state",
|
||||
Some(err.to_string()),
|
||||
)
|
||||
}
|
||||
@@ -0,0 +1,39 @@
|
||||
[package]
|
||||
name = "bizinikiwi-frame-rpc-support"
|
||||
version = "29.0.0"
|
||||
authors.workspace = true
|
||||
edition.workspace = true
|
||||
license = "Apache-2.0"
|
||||
homepage.workspace = true
|
||||
repository.workspace = true
|
||||
description = "Bizinikiwi RPC for FRAME's support"
|
||||
|
||||
[lints]
|
||||
workspace = true
|
||||
|
||||
[package.metadata.docs.rs]
|
||||
targets = ["x86_64-unknown-linux-gnu"]
|
||||
|
||||
[dependencies]
|
||||
codec = { workspace = true, default-features = true }
|
||||
pezframe-support = { workspace = true, default-features = true }
|
||||
jsonrpsee = { features = ["jsonrpsee-types"], workspace = true }
|
||||
pezsc-rpc-api = { workspace = true, default-features = true }
|
||||
scale-info = { workspace = true, default-features = true }
|
||||
serde = { workspace = true, default-features = true }
|
||||
pezsp-storage = { workspace = true, default-features = true }
|
||||
|
||||
[dev-dependencies]
|
||||
pezframe-system = { workspace = true, default-features = true }
|
||||
jsonrpsee = { features = ["jsonrpsee-types", "ws-client"], workspace = true }
|
||||
pezsp-core = { workspace = true, default-features = true }
|
||||
pezsp-runtime = { workspace = true, default-features = true }
|
||||
tokio = { workspace = true, default-features = true }
|
||||
|
||||
[features]
|
||||
runtime-benchmarks = [
|
||||
"pezframe-support/runtime-benchmarks",
|
||||
"pezframe-system/runtime-benchmarks",
|
||||
"pezsc-rpc-api/runtime-benchmarks",
|
||||
"pezsp-runtime/runtime-benchmarks",
|
||||
]
|
||||
@@ -0,0 +1,4 @@
|
||||
Combines [sc_rpc_api::state::StateClient] with [frame_support::storage::generator] traits
|
||||
to provide strongly typed chain state queries over rpc.
|
||||
|
||||
License: Apache-2.0
|
||||
@@ -0,0 +1,186 @@
|
||||
// This file is part of Bizinikiwi.
|
||||
|
||||
// Copyright (C) Parity Technologies (UK) Ltd.
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
//! Combines [pezsc_rpc_api::state::StateApiClient] with [pezframe_support::storage::generator] traits
|
||||
//! to provide strongly typed chain state queries over rpc.
|
||||
|
||||
#![warn(missing_docs)]
|
||||
|
||||
use codec::{DecodeAll, FullCodec, FullEncode};
|
||||
use core::marker::PhantomData;
|
||||
use pezframe_support::storage::generator::{StorageDoubleMap, StorageMap, StorageValue};
|
||||
use jsonrpsee::core::ClientError as RpcError;
|
||||
use pezsc_rpc_api::state::StateApiClient;
|
||||
use serde::{de::DeserializeOwned, Serialize};
|
||||
use pezsp_storage::{StorageData, StorageKey};
|
||||
|
||||
/// A typed query on chain state usable from an RPC client.
|
||||
///
|
||||
/// ```no_run
|
||||
/// # use jsonrpsee::core::ClientError as RpcError;
|
||||
/// # use jsonrpsee::ws_client::WsClientBuilder;
|
||||
/// # use codec::Encode;
|
||||
/// # use pezframe_support::{construct_runtime, derive_impl, traits::ConstU32};
|
||||
/// # use bizinikiwi_frame_rpc_support::StorageQuery;
|
||||
/// # use pezsc_rpc_api::state::StateApiClient;
|
||||
/// # use pezsp_runtime::{traits::{BlakeTwo256, IdentityLookup}, testing::Header};
|
||||
/// #
|
||||
/// # construct_runtime!(
|
||||
/// # pub enum TestRuntime
|
||||
/// # {
|
||||
/// # System: pezframe_system,
|
||||
/// # Test: pezpallet_test,
|
||||
/// # }
|
||||
/// # );
|
||||
/// #
|
||||
/// # type Hash = pezsp_core::H256;
|
||||
/// #
|
||||
/// # #[derive_impl(pezframe_system::config_preludes::TestDefaultConfig)]
|
||||
/// # impl pezframe_system::Config for TestRuntime {
|
||||
/// # type BaseCallFilter = ();
|
||||
/// # type BlockWeights = ();
|
||||
/// # type BlockLength = ();
|
||||
/// # type RuntimeOrigin = RuntimeOrigin;
|
||||
/// # type RuntimeCall = RuntimeCall;
|
||||
/// # type Nonce = u64;
|
||||
/// # type Hash = Hash;
|
||||
/// # type Hashing = BlakeTwo256;
|
||||
/// # type AccountId = u64;
|
||||
/// # type Lookup = IdentityLookup<Self::AccountId>;
|
||||
/// # type Block = pezframe_system::mocking::MockBlock<TestRuntime>;
|
||||
/// # type RuntimeEvent = RuntimeEvent;
|
||||
/// # type RuntimeTask = RuntimeTask;
|
||||
/// # type BlockHashCount = ();
|
||||
/// # type DbWeight = ();
|
||||
/// # type Version = ();
|
||||
/// # type PalletInfo = PalletInfo;
|
||||
/// # type AccountData = ();
|
||||
/// # type OnNewAccount = ();
|
||||
/// # type OnKilledAccount = ();
|
||||
/// # type SystemWeightInfo = ();
|
||||
/// # type SS58Prefix = ();
|
||||
/// # type OnSetCode = ();
|
||||
/// # type MaxConsumers = ConstU32<16>;
|
||||
/// # }
|
||||
/// #
|
||||
/// # impl pezpallet_test::Config for TestRuntime {}
|
||||
/// #
|
||||
///
|
||||
/// pub type Loc = (i64, i64, i64);
|
||||
/// pub type Block = u8;
|
||||
///
|
||||
/// // Note that all fields are marked pub.
|
||||
/// pub use self::pezpallet_test::*;
|
||||
///
|
||||
/// #[pezframe_support::pallet]
|
||||
/// mod pezpallet_test {
|
||||
/// use super::*;
|
||||
/// use pezframe_support::pezpallet_prelude::*;
|
||||
///
|
||||
/// #[pallet::pallet]
|
||||
/// pub struct Pallet<T>(_);
|
||||
///
|
||||
/// #[pallet::config]
|
||||
/// pub trait Config: pezframe_system::Config {}
|
||||
///
|
||||
/// #[pallet::storage]
|
||||
/// pub type LastActionId<T> = StorageValue<_, u64, ValueQuery>;
|
||||
///
|
||||
/// #[pallet::storage]
|
||||
/// pub type Voxels<T> = StorageMap<_, Blake2_128Concat, Loc, Block>;
|
||||
///
|
||||
/// #[pallet::storage]
|
||||
/// pub type Actions<T> = StorageMap<_, Blake2_128Concat, u64, Loc>;
|
||||
///
|
||||
/// #[pallet::storage]
|
||||
/// pub type Prefab<T> = StorageDoubleMap<
|
||||
/// _,
|
||||
/// Blake2_128Concat, u128,
|
||||
/// Blake2_128Concat, (i8, i8, i8), Block
|
||||
/// >;
|
||||
/// }
|
||||
///
|
||||
/// #[tokio::main]
|
||||
/// async fn main() -> Result<(), RpcError> {
|
||||
/// let cl = WsClientBuilder::default().build("ws://[::1]:9944").await?;
|
||||
///
|
||||
/// let q = StorageQuery::value::<LastActionId<TestRuntime>>();
|
||||
/// let hash = None::<Hash>;
|
||||
/// let _: Option<u64> = q.get(&cl, hash).await?;
|
||||
///
|
||||
/// let q = StorageQuery::map::<Voxels<TestRuntime>, _>((0, 0, 0));
|
||||
/// let _: Option<Block> = q.get(&cl, hash).await?;
|
||||
///
|
||||
/// let q = StorageQuery::map::<Actions<TestRuntime>, _>(12);
|
||||
/// let _: Option<Loc> = q.get(&cl, hash).await?;
|
||||
///
|
||||
/// let q = StorageQuery::double_map::<Prefab<TestRuntime>, _, _>(3, (0, 0, 0));
|
||||
/// let _: Option<Block> = q.get(&cl, hash).await?;
|
||||
///
|
||||
/// Ok(())
|
||||
/// }
|
||||
/// ```
|
||||
#[derive(Clone, Eq, PartialEq, Ord, PartialOrd, Hash, Debug)]
|
||||
pub struct StorageQuery<V> {
|
||||
key: StorageKey,
|
||||
_spook: PhantomData<V>,
|
||||
}
|
||||
|
||||
impl<V: FullCodec> StorageQuery<V> {
|
||||
/// Create a storage query for a StorageValue.
|
||||
pub fn value<St: StorageValue<V>>() -> Self {
|
||||
Self { key: StorageKey(St::storage_value_final_key().to_vec()), _spook: PhantomData }
|
||||
}
|
||||
|
||||
/// Create a storage query for a value in a StorageMap.
|
||||
pub fn map<St: StorageMap<K, V>, K: FullEncode>(key: K) -> Self {
|
||||
Self { key: StorageKey(St::storage_map_final_key(key)), _spook: PhantomData }
|
||||
}
|
||||
|
||||
/// Create a storage query for a value in a StorageDoubleMap.
|
||||
pub fn double_map<St: StorageDoubleMap<K1, K2, V>, K1: FullEncode, K2: FullEncode>(
|
||||
key1: K1,
|
||||
key2: K2,
|
||||
) -> Self {
|
||||
Self { key: StorageKey(St::storage_double_map_final_key(key1, key2)), _spook: PhantomData }
|
||||
}
|
||||
|
||||
/// Send this query over RPC, await the typed result.
|
||||
///
|
||||
/// Hash should be `<YourRuntime as pezframe_system::Config>::Hash`.
|
||||
///
|
||||
/// # Arguments
|
||||
///
|
||||
/// state_client represents a connection to the RPC server.
|
||||
///
|
||||
/// block_index indicates the block for which state will be queried. A value of None indicates
|
||||
/// the latest block.
|
||||
pub async fn get<Hash, StateClient>(
|
||||
self,
|
||||
state_client: &StateClient,
|
||||
block_index: Option<Hash>,
|
||||
) -> Result<Option<V>, RpcError>
|
||||
where
|
||||
Hash: Send + Sync + 'static + DeserializeOwned + Serialize,
|
||||
StateClient: StateApiClient<Hash> + Sync,
|
||||
{
|
||||
let opt: Option<StorageData> = state_client.storage(self.key, block_index).await?;
|
||||
opt.map(|encoded| V::decode_all(&mut &encoded.0[..]))
|
||||
.transpose()
|
||||
.map_err(|decode_err| RpcError::Custom(decode_err.to_string()))
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,55 @@
|
||||
[package]
|
||||
name = "bizinikiwi-frame-rpc-system"
|
||||
version = "28.0.0"
|
||||
authors.workspace = true
|
||||
edition.workspace = true
|
||||
license = "Apache-2.0"
|
||||
homepage.workspace = true
|
||||
repository.workspace = true
|
||||
description = "FRAME's system exposed over Bizinikiwi RPC"
|
||||
readme = "README.md"
|
||||
|
||||
[lints]
|
||||
workspace = true
|
||||
|
||||
[package.metadata.docs.rs]
|
||||
targets = ["x86_64-unknown-linux-gnu"]
|
||||
|
||||
[dependencies]
|
||||
codec = { workspace = true, default-features = true }
|
||||
docify = { workspace = true }
|
||||
pezframe-system-rpc-runtime-api = { workspace = true, default-features = true }
|
||||
futures = { workspace = true }
|
||||
jsonrpsee = { features = [
|
||||
"client-core",
|
||||
"macros",
|
||||
"server-core",
|
||||
], workspace = true }
|
||||
log = { workspace = true, default-features = true }
|
||||
pezsc-rpc-api = { workspace = true, default-features = true }
|
||||
pezsc-transaction-pool-api = { workspace = true, default-features = true }
|
||||
pezsp-api = { workspace = true, default-features = true }
|
||||
pezsp-block-builder = { workspace = true, default-features = true }
|
||||
pezsp-blockchain = { workspace = true, default-features = true }
|
||||
pezsp-core = { workspace = true, default-features = true }
|
||||
pezsp-runtime = { workspace = true, default-features = true }
|
||||
|
||||
[dev-dependencies]
|
||||
assert_matches = { workspace = true }
|
||||
pezsc-transaction-pool = { workspace = true, default-features = true }
|
||||
pezsp-tracing = { workspace = true, default-features = true }
|
||||
bizinikiwi-test-runtime-client = { workspace = true }
|
||||
tokio = { workspace = true, default-features = true }
|
||||
|
||||
[features]
|
||||
runtime-benchmarks = [
|
||||
"pezframe-system-rpc-runtime-api/runtime-benchmarks",
|
||||
"pezsc-rpc-api/runtime-benchmarks",
|
||||
"pezsc-transaction-pool-api/runtime-benchmarks",
|
||||
"pezsc-transaction-pool/runtime-benchmarks",
|
||||
"pezsp-api/runtime-benchmarks",
|
||||
"pezsp-block-builder/runtime-benchmarks",
|
||||
"pezsp-blockchain/runtime-benchmarks",
|
||||
"pezsp-runtime/runtime-benchmarks",
|
||||
"bizinikiwi-test-runtime-client/runtime-benchmarks",
|
||||
]
|
||||
@@ -0,0 +1,3 @@
|
||||
System FRAME specific RPC methods.
|
||||
|
||||
License: Apache-2.0
|
||||
@@ -0,0 +1,377 @@
|
||||
// This file is part of Bizinikiwi.
|
||||
|
||||
// Copyright (C) Parity Technologies (UK) Ltd.
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
//! System FRAME specific RPC methods.
|
||||
|
||||
use std::{fmt::Display, sync::Arc};
|
||||
|
||||
use codec::{self, Codec, Decode, Encode};
|
||||
use jsonrpsee::{
|
||||
core::{async_trait, RpcResult},
|
||||
proc_macros::rpc,
|
||||
types::error::ErrorObject,
|
||||
Extensions,
|
||||
};
|
||||
|
||||
use pezsc_transaction_pool_api::{InPoolTransaction, TransactionPool};
|
||||
use pezsp_api::ApiExt;
|
||||
use pezsp_block_builder::BlockBuilder;
|
||||
use pezsp_blockchain::HeaderBackend;
|
||||
use pezsp_core::{hexdisplay::HexDisplay, Bytes};
|
||||
use pezsp_runtime::{legacy, traits};
|
||||
|
||||
pub use pezframe_system_rpc_runtime_api::AccountNonceApi;
|
||||
|
||||
/// System RPC methods.
|
||||
#[docify::export]
|
||||
#[rpc(client, server)]
|
||||
pub trait SystemApi<BlockHash, AccountId, Nonce> {
|
||||
/// Returns the next valid index (aka nonce) for given account.
|
||||
///
|
||||
/// This method takes into consideration all pending transactions
|
||||
/// currently in the pool and if no transactions are found in the pool
|
||||
/// it fallbacks to query the index from the runtime (aka. state nonce).
|
||||
#[method(name = "system_accountNextIndex", aliases = ["account_nextIndex"])]
|
||||
async fn nonce(&self, account: AccountId) -> RpcResult<Nonce>;
|
||||
|
||||
/// Dry run an extrinsic at a given block. Return SCALE encoded ApplyExtrinsicResult.
|
||||
#[method(name = "system_dryRun", aliases = ["system_dryRunAt"], with_extensions)]
|
||||
async fn dry_run(&self, extrinsic: Bytes, at: Option<BlockHash>) -> RpcResult<Bytes>;
|
||||
}
|
||||
|
||||
/// Error type of this RPC api.
|
||||
pub enum Error {
|
||||
/// The transaction was not decodable.
|
||||
DecodeError,
|
||||
/// The call to runtime failed.
|
||||
RuntimeError,
|
||||
}
|
||||
|
||||
impl From<Error> for i32 {
|
||||
fn from(e: Error) -> i32 {
|
||||
match e {
|
||||
Error::RuntimeError => 1,
|
||||
Error::DecodeError => 2,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// An implementation of System-specific RPC methods on full client.
|
||||
pub struct System<P: TransactionPool, C, B> {
|
||||
client: Arc<C>,
|
||||
pool: Arc<P>,
|
||||
_marker: std::marker::PhantomData<B>,
|
||||
}
|
||||
|
||||
impl<P: TransactionPool, C, B> System<P, C, B> {
|
||||
/// Create new `FullSystem` given client and transaction pool.
|
||||
pub fn new(client: Arc<C>, pool: Arc<P>) -> Self {
|
||||
Self { client, pool, _marker: Default::default() }
|
||||
}
|
||||
}
|
||||
|
||||
#[async_trait]
|
||||
impl<P, C, Block, AccountId, Nonce>
|
||||
SystemApiServer<<Block as traits::Block>::Hash, AccountId, Nonce> for System<P, C, Block>
|
||||
where
|
||||
C: pezsp_api::ProvideRuntimeApi<Block>,
|
||||
C: HeaderBackend<Block>,
|
||||
C: Send + Sync + 'static,
|
||||
C::Api: AccountNonceApi<Block, AccountId, Nonce>,
|
||||
C::Api: BlockBuilder<Block>,
|
||||
P: TransactionPool + 'static,
|
||||
Block: traits::Block,
|
||||
AccountId: Clone + Display + Codec + Send + 'static,
|
||||
Nonce: Clone + Display + Codec + Send + traits::AtLeast32Bit + 'static,
|
||||
{
|
||||
async fn nonce(&self, account: AccountId) -> RpcResult<Nonce> {
|
||||
let api = self.client.runtime_api();
|
||||
let best = self.client.info().best_hash;
|
||||
|
||||
let nonce = api.account_nonce(best, account.clone()).map_err(|e| {
|
||||
ErrorObject::owned(
|
||||
Error::RuntimeError.into(),
|
||||
"Unable to query nonce.",
|
||||
Some(e.to_string()),
|
||||
)
|
||||
})?;
|
||||
Ok(adjust_nonce(&*self.pool, account, nonce))
|
||||
}
|
||||
|
||||
async fn dry_run(
|
||||
&self,
|
||||
ext: &Extensions,
|
||||
extrinsic: Bytes,
|
||||
at: Option<<Block as traits::Block>::Hash>,
|
||||
) -> RpcResult<Bytes> {
|
||||
pezsc_rpc_api::check_if_safe(ext)?;
|
||||
|
||||
let api = self.client.runtime_api();
|
||||
let best_hash = at.unwrap_or_else(||
|
||||
// If the block hash is not supplied assume the best block.
|
||||
self.client.info().best_hash);
|
||||
|
||||
let uxt: <Block as traits::Block>::Extrinsic =
|
||||
Decode::decode(&mut &*extrinsic).map_err(|e| {
|
||||
ErrorObject::owned(
|
||||
Error::DecodeError.into(),
|
||||
"Unable to dry run extrinsic",
|
||||
Some(e.to_string()),
|
||||
)
|
||||
})?;
|
||||
|
||||
let api_version = api
|
||||
.api_version::<dyn BlockBuilder<Block>>(best_hash)
|
||||
.map_err(|e| {
|
||||
ErrorObject::owned(
|
||||
Error::RuntimeError.into(),
|
||||
"Unable to dry run extrinsic.",
|
||||
Some(e.to_string()),
|
||||
)
|
||||
})?
|
||||
.ok_or_else(|| {
|
||||
ErrorObject::owned(
|
||||
Error::RuntimeError.into(),
|
||||
"Unable to dry run extrinsic.",
|
||||
Some(format!("Could not find `BlockBuilder` api for block `{:?}`.", best_hash)),
|
||||
)
|
||||
})?;
|
||||
|
||||
let result = if api_version < 6 {
|
||||
#[allow(deprecated)]
|
||||
api.apply_extrinsic_before_version_6(best_hash, uxt)
|
||||
.map(legacy::byte_sized_error::convert_to_latest)
|
||||
.map_err(|e| {
|
||||
ErrorObject::owned(
|
||||
Error::RuntimeError.into(),
|
||||
"Unable to dry run extrinsic.",
|
||||
Some(e.to_string()),
|
||||
)
|
||||
})?
|
||||
} else {
|
||||
api.apply_extrinsic(best_hash, uxt).map_err(|e| {
|
||||
ErrorObject::owned(
|
||||
Error::RuntimeError.into(),
|
||||
"Unable to dry run extrinsic.",
|
||||
Some(e.to_string()),
|
||||
)
|
||||
})?
|
||||
};
|
||||
|
||||
Ok(Encode::encode(&result).into())
|
||||
}
|
||||
}
|
||||
|
||||
/// Adjust account nonce from state, so that tx with the nonce will be
|
||||
/// placed after all ready txpool transactions.
|
||||
fn adjust_nonce<P, AccountId, Nonce>(pool: &P, account: AccountId, nonce: Nonce) -> Nonce
|
||||
where
|
||||
P: TransactionPool,
|
||||
AccountId: Clone + std::fmt::Display + Encode,
|
||||
Nonce: Clone + std::fmt::Display + Encode + traits::AtLeast32Bit + 'static,
|
||||
{
|
||||
log::debug!(target: "rpc", "State nonce for {}: {}", account, nonce);
|
||||
// Now we need to query the transaction pool
|
||||
// and find transactions originating from the same sender.
|
||||
//
|
||||
// Since extrinsics are opaque to us, we look for them using
|
||||
// `provides` tag. And increment the nonce if we find a transaction
|
||||
// that matches the current one.
|
||||
let mut current_nonce = nonce.clone();
|
||||
let mut current_tag = (account.clone(), nonce).encode();
|
||||
for tx in pool.ready() {
|
||||
log::debug!(
|
||||
target: "rpc",
|
||||
"Current nonce to {}, checking {} vs {:?}",
|
||||
current_nonce,
|
||||
HexDisplay::from(¤t_tag),
|
||||
tx.provides().iter().map(|x| format!("{}", HexDisplay::from(x))).collect::<Vec<_>>(),
|
||||
);
|
||||
// since transactions in `ready()` need to be ordered by nonce
|
||||
// it's fine to continue with current iterator.
|
||||
if tx.provides().get(0) == Some(¤t_tag) {
|
||||
current_nonce += traits::One::one();
|
||||
current_tag = (account.clone(), current_nonce.clone()).encode();
|
||||
}
|
||||
}
|
||||
|
||||
current_nonce
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
use assert_matches::assert_matches;
|
||||
use futures::executor::block_on;
|
||||
use pezsc_rpc_api::DenyUnsafe;
|
||||
use pezsc_transaction_pool::BasicPool;
|
||||
use pezsp_runtime::{
|
||||
transaction_validity::{InvalidTransaction, TransactionValidityError},
|
||||
ApplyExtrinsicResult,
|
||||
};
|
||||
use bizinikiwi_test_runtime_client::{runtime::Transfer, Sr25519Keyring};
|
||||
|
||||
fn deny_unsafe() -> Extensions {
|
||||
let mut ext = Extensions::new();
|
||||
ext.insert(DenyUnsafe::Yes);
|
||||
ext
|
||||
}
|
||||
|
||||
fn allow_unsafe() -> Extensions {
|
||||
let mut ext = Extensions::new();
|
||||
ext.insert(DenyUnsafe::No);
|
||||
ext
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn should_return_next_nonce_for_some_account() {
|
||||
pezsp_tracing::try_init_simple();
|
||||
|
||||
// given
|
||||
let client = Arc::new(bizinikiwi_test_runtime_client::new());
|
||||
let spawner = pezsp_core::testing::TaskExecutor::new();
|
||||
let pool = Arc::from(BasicPool::new_full(
|
||||
Default::default(),
|
||||
true.into(),
|
||||
None,
|
||||
spawner,
|
||||
client.clone(),
|
||||
));
|
||||
|
||||
let source = pezsp_runtime::transaction_validity::TransactionSource::External;
|
||||
let new_transaction = |nonce: u64| {
|
||||
let t = Transfer {
|
||||
from: Sr25519Keyring::Alice.into(),
|
||||
to: Sr25519Keyring::Bob.into(),
|
||||
amount: 5,
|
||||
nonce,
|
||||
};
|
||||
t.into_unchecked_extrinsic()
|
||||
};
|
||||
let hash_of_block0 = client.info().genesis_hash;
|
||||
// Populate the pool
|
||||
let ext0 = new_transaction(0);
|
||||
block_on(pool.submit_one(hash_of_block0, source, ext0)).unwrap();
|
||||
let ext1 = new_transaction(1);
|
||||
block_on(pool.submit_one(hash_of_block0, source, ext1)).unwrap();
|
||||
|
||||
let accounts = System::new(client, pool);
|
||||
|
||||
// when
|
||||
let nonce = accounts.nonce(Sr25519Keyring::Alice.into()).await;
|
||||
|
||||
// then
|
||||
assert_eq!(nonce.unwrap(), 2);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn dry_run_should_deny_unsafe() {
|
||||
pezsp_tracing::try_init_simple();
|
||||
|
||||
// given
|
||||
let client = Arc::new(bizinikiwi_test_runtime_client::new());
|
||||
let spawner = pezsp_core::testing::TaskExecutor::new();
|
||||
let pool = Arc::from(BasicPool::new_full(
|
||||
Default::default(),
|
||||
true.into(),
|
||||
None,
|
||||
spawner,
|
||||
client.clone(),
|
||||
));
|
||||
|
||||
let accounts = System::new(client, pool);
|
||||
|
||||
// when
|
||||
let res = accounts.dry_run(&deny_unsafe(), vec![].into(), None).await;
|
||||
assert_matches!(res, Err(e) => {
|
||||
assert!(e.message().contains("RPC call is unsafe to be called externally"));
|
||||
});
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn dry_run_should_work() {
|
||||
pezsp_tracing::try_init_simple();
|
||||
|
||||
// given
|
||||
let client = Arc::new(bizinikiwi_test_runtime_client::new());
|
||||
let spawner = pezsp_core::testing::TaskExecutor::new();
|
||||
let pool = Arc::from(BasicPool::new_full(
|
||||
Default::default(),
|
||||
true.into(),
|
||||
None,
|
||||
spawner,
|
||||
client.clone(),
|
||||
));
|
||||
|
||||
let accounts = System::new(client, pool);
|
||||
|
||||
let tx = Transfer {
|
||||
from: Sr25519Keyring::Alice.into(),
|
||||
to: Sr25519Keyring::Bob.into(),
|
||||
amount: 5,
|
||||
nonce: 0,
|
||||
}
|
||||
.into_unchecked_extrinsic();
|
||||
|
||||
// when
|
||||
let bytes = accounts
|
||||
.dry_run(&allow_unsafe(), tx.encode().into(), None)
|
||||
.await
|
||||
.expect("Call is successful");
|
||||
|
||||
// then
|
||||
let apply_res: ApplyExtrinsicResult = Decode::decode(&mut bytes.as_ref()).unwrap();
|
||||
assert_eq!(apply_res, Ok(Ok(())));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn dry_run_should_indicate_error() {
|
||||
pezsp_tracing::try_init_simple();
|
||||
|
||||
// given
|
||||
let client = Arc::new(bizinikiwi_test_runtime_client::new());
|
||||
let spawner = pezsp_core::testing::TaskExecutor::new();
|
||||
let pool = Arc::from(BasicPool::new_full(
|
||||
Default::default(),
|
||||
true.into(),
|
||||
None,
|
||||
spawner,
|
||||
client.clone(),
|
||||
));
|
||||
|
||||
let accounts = System::new(client, pool);
|
||||
|
||||
let tx = Transfer {
|
||||
from: Sr25519Keyring::Alice.into(),
|
||||
to: Sr25519Keyring::Bob.into(),
|
||||
amount: 5,
|
||||
nonce: 100,
|
||||
}
|
||||
.into_unchecked_extrinsic();
|
||||
|
||||
// when
|
||||
let bytes = accounts
|
||||
.dry_run(&allow_unsafe(), tx.encode().into(), None)
|
||||
.await
|
||||
.expect("Call is successful");
|
||||
|
||||
// then
|
||||
let apply_res: ApplyExtrinsicResult = Decode::decode(&mut bytes.as_ref()).unwrap();
|
||||
assert_eq!(apply_res, Err(TransactionValidityError::Invalid(InvalidTransaction::Future)));
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,45 @@
|
||||
[package]
|
||||
name = "frame-storage-access-test-runtime"
|
||||
description = "A runtime for testing storage access on block validation"
|
||||
version = "0.1.0"
|
||||
authors.workspace = true
|
||||
edition.workspace = true
|
||||
build = "build.rs"
|
||||
license = "Apache-2.0"
|
||||
homepage.workspace = true
|
||||
repository.workspace = true
|
||||
publish = true
|
||||
|
||||
[lints]
|
||||
workspace = true
|
||||
|
||||
[dependencies]
|
||||
codec = { features = ["derive"], workspace = true }
|
||||
pezcumulus-pezpallet-teyrchain-system = { workspace = true, optional = true }
|
||||
pezsp-core = { workspace = true }
|
||||
pezsp-runtime = { workspace = true }
|
||||
pezsp-state-machine = { workspace = true }
|
||||
pezsp-trie = { workspace = true }
|
||||
|
||||
[build-dependencies]
|
||||
bizinikiwi-wasm-builder = { optional = true, workspace = true, default-features = true }
|
||||
|
||||
[features]
|
||||
default = ["std"]
|
||||
no_std = []
|
||||
std = [
|
||||
"codec/std",
|
||||
"pezcumulus-pezpallet-teyrchain-system/std",
|
||||
"pezsp-core/std",
|
||||
"pezsp-runtime/std",
|
||||
"pezsp-state-machine/std",
|
||||
"pezsp-trie/std",
|
||||
"bizinikiwi-wasm-builder",
|
||||
]
|
||||
runtime-benchmarks = [
|
||||
"pezcumulus-pezpallet-teyrchain-system/runtime-benchmarks",
|
||||
"pezsp-runtime/runtime-benchmarks",
|
||||
"pezsp-state-machine/runtime-benchmarks",
|
||||
"pezsp-trie/runtime-benchmarks",
|
||||
"bizinikiwi-wasm-builder?/runtime-benchmarks",
|
||||
]
|
||||
@@ -0,0 +1,28 @@
|
||||
// This file is part of Bizinikiwi.
|
||||
|
||||
// Copyright (C) Parity Technologies (UK) Ltd.
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
fn main() {
|
||||
#[cfg(feature = "std")]
|
||||
{
|
||||
bizinikiwi_wasm_builder::WasmBuilder::new()
|
||||
.with_current_project()
|
||||
.export_heap_base()
|
||||
.import_memory()
|
||||
.disable_runtime_version_section_check()
|
||||
.build();
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,181 @@
|
||||
// This file is part of Bizinikiwi.
|
||||
|
||||
// Copyright (C) Parity Technologies (UK) Ltd.
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
//! Test runtime to benchmark storage access on block validation
|
||||
|
||||
#![cfg_attr(not(feature = "std"), no_std)]
|
||||
|
||||
extern crate alloc;
|
||||
|
||||
use alloc::vec::Vec;
|
||||
use codec::{Decode, Encode};
|
||||
use pezsp_core::storage::ChildInfo;
|
||||
use pezsp_runtime::traits;
|
||||
use pezsp_trie::StorageProof;
|
||||
|
||||
#[cfg(all(not(feature = "std"), feature = "runtime-benchmarks"))]
|
||||
use {
|
||||
cumulus_pallet_teyrchain_system::validate_block::{
|
||||
trie_cache::CacheProvider, trie_recorder::SizeOnlyRecorderProvider,
|
||||
},
|
||||
pezsp_core::storage::StateVersion,
|
||||
pezsp_runtime::{generic, OpaqueExtrinsic},
|
||||
pezsp_state_machine::{Backend, TrieBackendBuilder},
|
||||
};
|
||||
|
||||
// Include the WASM binary
|
||||
#[cfg(feature = "std")]
|
||||
include!(concat!(env!("OUT_DIR"), "/wasm_binary.rs"));
|
||||
|
||||
/// Parameters for benchmarking storage access on block validation.
|
||||
///
|
||||
/// On dry-run, the storage access is not performed to measure the cost of the runtime call.
|
||||
#[derive(Decode, Clone)]
|
||||
#[cfg_attr(feature = "std", derive(Encode))]
|
||||
pub struct StorageAccessParams<B: traits::Block> {
|
||||
pub state_root: B::Hash,
|
||||
pub storage_proof: StorageProof,
|
||||
pub payload: StorageAccessPayload,
|
||||
/// On dry-run, we don't read/write to the storage.
|
||||
pub is_dry_run: bool,
|
||||
}
|
||||
|
||||
/// Payload for benchmarking read and write operations on block validation.
|
||||
#[derive(Debug, Clone, Decode, Encode)]
|
||||
pub enum StorageAccessPayload {
|
||||
// Storage keys with optional child info.
|
||||
Read(Vec<(Vec<u8>, Option<ChildInfo>)>),
|
||||
// Storage key-value pairs with optional child info.
|
||||
Write((Vec<(Vec<u8>, Vec<u8>)>, Option<ChildInfo>)),
|
||||
}
|
||||
|
||||
impl<B: traits::Block> StorageAccessParams<B> {
|
||||
/// Create a new params for reading from the storage.
|
||||
pub fn new_read(
|
||||
state_root: B::Hash,
|
||||
storage_proof: StorageProof,
|
||||
payload: Vec<(Vec<u8>, Option<ChildInfo>)>,
|
||||
) -> Self {
|
||||
Self {
|
||||
state_root,
|
||||
storage_proof,
|
||||
payload: StorageAccessPayload::Read(payload),
|
||||
is_dry_run: false,
|
||||
}
|
||||
}
|
||||
|
||||
/// Create a new params for writing to the storage.
|
||||
pub fn new_write(
|
||||
state_root: B::Hash,
|
||||
storage_proof: StorageProof,
|
||||
payload: (Vec<(Vec<u8>, Vec<u8>)>, Option<ChildInfo>),
|
||||
) -> Self {
|
||||
Self {
|
||||
state_root,
|
||||
storage_proof,
|
||||
payload: StorageAccessPayload::Write(payload),
|
||||
is_dry_run: false,
|
||||
}
|
||||
}
|
||||
|
||||
/// Create a dry-run version of the params.
|
||||
pub fn as_dry_run(&self) -> Self {
|
||||
Self {
|
||||
state_root: self.state_root,
|
||||
storage_proof: self.storage_proof.clone(),
|
||||
payload: self.payload.clone(),
|
||||
is_dry_run: true,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Imitates `cumulus_pallet_teyrchain_system::validate_block::implementation::validate_block`
|
||||
///
|
||||
/// Only performs the storage access, this is used to benchmark the storage access cost.
|
||||
#[doc(hidden)]
|
||||
#[cfg(all(not(feature = "std"), feature = "runtime-benchmarks"))]
|
||||
pub fn proceed_storage_access<B: traits::Block>(mut params: &[u8]) {
|
||||
let StorageAccessParams { state_root, storage_proof, payload, is_dry_run } =
|
||||
StorageAccessParams::<B>::decode(&mut params)
|
||||
.expect("Invalid arguments to `validate_block`.");
|
||||
|
||||
let db = storage_proof.into_memory_db();
|
||||
let recorder = SizeOnlyRecorderProvider::<traits::HashingFor<B>>::default();
|
||||
let cache_provider = CacheProvider::new();
|
||||
let backend = TrieBackendBuilder::new_with_cache(db, state_root, cache_provider)
|
||||
.with_recorder(recorder)
|
||||
.build();
|
||||
|
||||
if is_dry_run {
|
||||
return;
|
||||
}
|
||||
|
||||
match payload {
|
||||
StorageAccessPayload::Read(keys) =>
|
||||
for (key, maybe_child_info) in keys {
|
||||
match maybe_child_info {
|
||||
Some(child_info) => {
|
||||
let _ = backend
|
||||
.child_storage(&child_info, key.as_ref())
|
||||
.expect("Key not found")
|
||||
.ok_or("Value unexpectedly empty");
|
||||
},
|
||||
None => {
|
||||
let _ = backend
|
||||
.storage(key.as_ref())
|
||||
.expect("Key not found")
|
||||
.ok_or("Value unexpectedly empty");
|
||||
},
|
||||
}
|
||||
},
|
||||
StorageAccessPayload::Write((changes, maybe_child_info)) => {
|
||||
let delta = changes.iter().map(|(key, value)| (key.as_ref(), Some(value.as_ref())));
|
||||
match maybe_child_info {
|
||||
Some(child_info) => {
|
||||
backend.child_storage_root(&child_info, delta, StateVersion::V1);
|
||||
},
|
||||
None => {
|
||||
backend.storage_root(delta, StateVersion::V1);
|
||||
},
|
||||
}
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
/// Wasm binary unwrapped. If built with `SKIP_WASM_BUILD`, the function panics.
|
||||
#[cfg(feature = "std")]
|
||||
pub fn wasm_binary_unwrap() -> &'static [u8] {
|
||||
WASM_BINARY.expect(
|
||||
"Development wasm binary is not available. Unset SKIP_WASM_BUILD and compile the runtime again.",
|
||||
)
|
||||
}
|
||||
|
||||
#[cfg(enable_alloc_error_handler)]
|
||||
#[alloc_error_handler]
|
||||
#[no_mangle]
|
||||
pub fn oom(_: core::alloc::Layout) -> ! {
|
||||
core::intrinsics::abort();
|
||||
}
|
||||
|
||||
#[cfg(all(not(feature = "std"), feature = "runtime-benchmarks"))]
|
||||
#[no_mangle]
|
||||
pub extern "C" fn validate_block(params: *const u8, len: usize) -> u64 {
|
||||
type Block = generic::Block<generic::Header<u32, traits::BlakeTwo256>, OpaqueExtrinsic>;
|
||||
let params = unsafe { alloc::slice::from_raw_parts(params, len) };
|
||||
proceed_storage_access::<Block>(params);
|
||||
1
|
||||
}
|
||||
@@ -0,0 +1,42 @@
|
||||
[package]
|
||||
description = "Endpoint to expose Prometheus metrics"
|
||||
name = "bizinikiwi-prometheus-endpoint"
|
||||
version = "0.17.0"
|
||||
license = "Apache-2.0"
|
||||
authors.workspace = true
|
||||
edition.workspace = true
|
||||
homepage.workspace = true
|
||||
repository.workspace = true
|
||||
readme = "README.md"
|
||||
|
||||
[lints]
|
||||
workspace = true
|
||||
|
||||
[package.metadata.docs.rs]
|
||||
targets = ["x86_64-unknown-linux-gnu"]
|
||||
|
||||
[dependencies]
|
||||
http-body-util = { workspace = true }
|
||||
hyper = { features = ["http1", "server"], workspace = true }
|
||||
hyper-util = { features = [
|
||||
"server-auto",
|
||||
"server-graceful",
|
||||
"tokio",
|
||||
], workspace = true }
|
||||
log = { workspace = true, default-features = true }
|
||||
prometheus = { workspace = true }
|
||||
thiserror = { workspace = true }
|
||||
tokio = { features = [
|
||||
"net",
|
||||
"parking_lot",
|
||||
], workspace = true, default-features = true }
|
||||
|
||||
[dev-dependencies]
|
||||
hyper-util = { features = [
|
||||
"client-legacy",
|
||||
"tokio",
|
||||
], workspace = true, default-features = true }
|
||||
tokio = { features = [
|
||||
"macros",
|
||||
"rt-multi-thread",
|
||||
], workspace = true, default-features = true }
|
||||
@@ -0,0 +1,20 @@
|
||||
# Bizinikiwi Prometheus Exporter
|
||||
|
||||
## Introduction
|
||||
|
||||
[Prometheus](https://prometheus.io/) is one of the most widely used monitoring tools for managing highly available
|
||||
services supported by [Cloud Native Computing Foundation](https://www.cncf.io/). By providing Prometheus metrics in
|
||||
Bizinikiwi, node operators can easily adopt widely used display/alert tools such as [Grafana](https://grafana.com/) and
|
||||
[Alertmanager](https://prometheus.io/docs/alerting/alertmanager/). Easy access to such monitoring tools will benefit
|
||||
teyrchain developers/operators and validators to have much higher availability of their services.
|
||||
|
||||
Metrics will be served under `/metrics` on TCP port 9615 by default.
|
||||
|
||||
## Quick Start
|
||||
|
||||
1. From the root of the repository start Bizinikiwi `cargo run --release`.
|
||||
|
||||
2. In another terminal run `curl localhost:9615/metrics` to retrieve the metrics.
|
||||
|
||||
To learn how to configure Prometheus see the Prometheus [Getting
|
||||
Started](https://prometheus.io/docs/prometheus/latest/getting_started/) guide.
|
||||
@@ -0,0 +1,173 @@
|
||||
// This file is part of Bizinikiwi.
|
||||
|
||||
// Copyright (C) Parity Technologies (UK) Ltd.
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
mod sourced;
|
||||
|
||||
use hyper::{http::StatusCode, Request, Response};
|
||||
use prometheus::{core::Collector, Encoder, TextEncoder};
|
||||
use std::net::SocketAddr;
|
||||
|
||||
pub use prometheus::{
|
||||
self,
|
||||
core::{
|
||||
AtomicF64 as F64, AtomicI64 as I64, AtomicU64 as U64, GenericCounter as Counter,
|
||||
GenericCounterVec as CounterVec, GenericGauge as Gauge, GenericGaugeVec as GaugeVec,
|
||||
},
|
||||
exponential_buckets, histogram_opts, linear_buckets, Error as PrometheusError, Histogram,
|
||||
HistogramOpts, HistogramVec, Opts, Registry,
|
||||
};
|
||||
pub use sourced::{MetricSource, SourcedCounter, SourcedGauge, SourcedMetric};
|
||||
|
||||
type Body = http_body_util::Full<hyper::body::Bytes>;
|
||||
|
||||
pub fn register<T: Clone + Collector + 'static>(
|
||||
metric: T,
|
||||
registry: &Registry,
|
||||
) -> Result<T, PrometheusError> {
|
||||
registry.register(Box::new(metric.clone()))?;
|
||||
Ok(metric)
|
||||
}
|
||||
|
||||
#[derive(Debug, thiserror::Error)]
|
||||
pub enum Error {
|
||||
/// Hyper internal error.
|
||||
#[error(transparent)]
|
||||
Hyper(#[from] hyper::Error),
|
||||
|
||||
/// Http request error.
|
||||
#[error(transparent)]
|
||||
Http(#[from] hyper::http::Error),
|
||||
|
||||
/// i/o error.
|
||||
#[error(transparent)]
|
||||
Io(#[from] std::io::Error),
|
||||
|
||||
#[error("Prometheus port {0} already in use.")]
|
||||
PortInUse(SocketAddr),
|
||||
}
|
||||
|
||||
async fn request_metrics(
|
||||
req: Request<hyper::body::Incoming>,
|
||||
registry: Registry,
|
||||
) -> Result<Response<Body>, Error> {
|
||||
if req.uri().path() == "/metrics" {
|
||||
let metric_families = registry.gather();
|
||||
let mut buffer = vec![];
|
||||
let encoder = TextEncoder::new();
|
||||
encoder.encode(&metric_families, &mut buffer).unwrap();
|
||||
|
||||
Response::builder()
|
||||
.status(StatusCode::OK)
|
||||
.header("Content-Type", encoder.format_type())
|
||||
.body(Body::from(buffer))
|
||||
.map_err(Error::Http)
|
||||
} else {
|
||||
Response::builder()
|
||||
.status(StatusCode::NOT_FOUND)
|
||||
.body(Body::from("Not found."))
|
||||
.map_err(Error::Http)
|
||||
}
|
||||
}
|
||||
|
||||
/// Initializes the metrics context, and starts an HTTP server
|
||||
/// to serve metrics.
|
||||
pub async fn init_prometheus(prometheus_addr: SocketAddr, registry: Registry) -> Result<(), Error> {
|
||||
let listener = tokio::net::TcpListener::bind(&prometheus_addr).await.map_err(|e| {
|
||||
log::error!(target: "prometheus", "Error binding to '{prometheus_addr:?}': {e:?}");
|
||||
Error::PortInUse(prometheus_addr)
|
||||
})?;
|
||||
|
||||
init_prometheus_with_listener(listener, registry).await
|
||||
}
|
||||
|
||||
/// Init prometheus using the given listener.
|
||||
async fn init_prometheus_with_listener(
|
||||
listener: tokio::net::TcpListener,
|
||||
registry: Registry,
|
||||
) -> Result<(), Error> {
|
||||
log::info!(target: "prometheus", "〽️ Prometheus exporter started at {}", listener.local_addr()?);
|
||||
|
||||
let server = hyper_util::server::conn::auto::Builder::new(hyper_util::rt::TokioExecutor::new());
|
||||
let graceful = hyper_util::server::graceful::GracefulShutdown::new();
|
||||
|
||||
loop {
|
||||
let io = match listener.accept().await {
|
||||
Ok((sock, _)) => hyper_util::rt::TokioIo::new(sock),
|
||||
Err(e) => {
|
||||
log::debug!(target: "prometheus", "Error accepting connection: {:?}", e);
|
||||
continue;
|
||||
},
|
||||
};
|
||||
|
||||
let registry = registry.clone();
|
||||
|
||||
let conn = server
|
||||
.serve_connection_with_upgrades(
|
||||
io,
|
||||
hyper::service::service_fn(move |req| request_metrics(req, registry.clone())),
|
||||
)
|
||||
.into_owned();
|
||||
let conn = graceful.watch(conn);
|
||||
|
||||
tokio::spawn(async move {
|
||||
if let Err(err) = conn.await {
|
||||
log::debug!(target: "prometheus", "connection error: {:?}", err);
|
||||
}
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use http_body_util::BodyExt;
|
||||
use hyper::Uri;
|
||||
use hyper_util::{client::legacy::Client, rt::TokioExecutor};
|
||||
|
||||
const METRIC_NAME: &str = "test_test_metric_name_test_test";
|
||||
|
||||
#[tokio::test]
|
||||
async fn prometheus_works() {
|
||||
let listener =
|
||||
tokio::net::TcpListener::bind("127.0.0.1:0").await.expect("Creates listener");
|
||||
|
||||
let local_addr = listener.local_addr().expect("Returns the local addr");
|
||||
|
||||
let registry = Registry::default();
|
||||
register(
|
||||
prometheus::Counter::new(METRIC_NAME, "yeah").expect("Creates test counter"),
|
||||
®istry,
|
||||
)
|
||||
.expect("Registers the test metric");
|
||||
|
||||
tokio::spawn(init_prometheus_with_listener(listener, registry));
|
||||
|
||||
let client = Client::builder(TokioExecutor::new()).build_http::<Body>();
|
||||
|
||||
let res = client
|
||||
.get(Uri::try_from(&format!("http://{}/metrics", local_addr)).expect("Parses URI"))
|
||||
.await
|
||||
.expect("Requests metrics");
|
||||
|
||||
assert!(res.status().is_success());
|
||||
|
||||
let buf = res.into_body().collect().await.expect("Failed to read HTTP body").to_bytes();
|
||||
let body = String::from_utf8(buf.to_vec()).expect("Converts body to String");
|
||||
|
||||
assert!(body.contains(&format!("{} 0", METRIC_NAME)));
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,157 @@
|
||||
// This file is part of Bizinikiwi.
|
||||
|
||||
// Copyright (C) Parity Technologies (UK) Ltd.
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
//! Metrics that are collected from existing sources.
|
||||
|
||||
use prometheus::{
|
||||
core::{Collector, Desc, Describer, Number, Opts},
|
||||
proto,
|
||||
};
|
||||
use std::{cmp::Ordering, marker::PhantomData};
|
||||
|
||||
/// A counter whose values are obtained from an existing source.
|
||||
///
|
||||
/// > **Note*: The counter values provided by the source `S`
|
||||
/// > must be monotonically increasing. Otherwise use a
|
||||
/// > [`SourcedGauge`] instead.
|
||||
pub type SourcedCounter<S> = SourcedMetric<Counter, S>;
|
||||
|
||||
/// A gauge whose values are obtained from an existing source.
|
||||
pub type SourcedGauge<S> = SourcedMetric<Gauge, S>;
|
||||
|
||||
/// The type of a sourced counter.
|
||||
#[derive(Copy, Clone)]
|
||||
pub enum Counter {}
|
||||
|
||||
/// The type of a sourced gauge.
|
||||
#[derive(Copy, Clone)]
|
||||
pub enum Gauge {}
|
||||
|
||||
/// A metric whose values are obtained from an existing source,
|
||||
/// instead of being independently recorded.
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct SourcedMetric<T, S> {
|
||||
source: S,
|
||||
desc: Desc,
|
||||
_type: PhantomData<T>,
|
||||
}
|
||||
|
||||
/// A source of values for a [`SourcedMetric`].
|
||||
pub trait MetricSource: Sync + Send + Clone {
|
||||
/// The type of the collected values.
|
||||
type N: Number;
|
||||
/// Collects the current values of the metrics from the source.
|
||||
fn collect(&self, set: impl FnMut(&[&str], Self::N));
|
||||
}
|
||||
|
||||
impl<T: SourcedType, S: MetricSource> SourcedMetric<T, S> {
|
||||
/// Creates a new metric that obtains its values from the given source.
|
||||
pub fn new(opts: &Opts, source: S) -> prometheus::Result<Self> {
|
||||
let desc = opts.describe()?;
|
||||
Ok(Self { source, desc, _type: PhantomData })
|
||||
}
|
||||
}
|
||||
|
||||
impl<T: SourcedType, S: MetricSource> Collector for SourcedMetric<T, S> {
|
||||
fn desc(&self) -> Vec<&Desc> {
|
||||
vec![&self.desc]
|
||||
}
|
||||
|
||||
fn collect(&self) -> Vec<proto::MetricFamily> {
|
||||
let mut counters = Vec::new();
|
||||
|
||||
self.source.collect(|label_values, value| {
|
||||
let mut m = proto::Metric::default();
|
||||
|
||||
match T::proto() {
|
||||
proto::MetricType::COUNTER => {
|
||||
let mut c = proto::Counter::default();
|
||||
c.set_value(value.into_f64());
|
||||
m.set_counter(c);
|
||||
},
|
||||
proto::MetricType::GAUGE => {
|
||||
let mut g = proto::Gauge::default();
|
||||
g.set_value(value.into_f64());
|
||||
m.set_gauge(g);
|
||||
},
|
||||
t => {
|
||||
log::error!("Unsupported sourced metric type: {:?}", t);
|
||||
},
|
||||
}
|
||||
|
||||
debug_assert_eq!(self.desc.variable_labels.len(), label_values.len());
|
||||
match self.desc.variable_labels.len().cmp(&label_values.len()) {
|
||||
Ordering::Greater => {
|
||||
log::warn!("Missing label values for sourced metric {}", self.desc.fq_name)
|
||||
},
|
||||
Ordering::Less => {
|
||||
log::warn!("Too many label values for sourced metric {}", self.desc.fq_name)
|
||||
},
|
||||
Ordering::Equal => {},
|
||||
}
|
||||
|
||||
m.set_label(
|
||||
self.desc
|
||||
.variable_labels
|
||||
.iter()
|
||||
.zip(label_values)
|
||||
.map(|(l_name, l_value)| {
|
||||
let mut l = proto::LabelPair::default();
|
||||
l.set_name(l_name.to_string());
|
||||
l.set_value(l_value.to_string());
|
||||
l
|
||||
})
|
||||
.chain(self.desc.const_label_pairs.iter().cloned())
|
||||
.collect::<Vec<_>>(),
|
||||
);
|
||||
|
||||
counters.push(m);
|
||||
});
|
||||
|
||||
let mut m = proto::MetricFamily::default();
|
||||
m.set_name(self.desc.fq_name.clone());
|
||||
m.set_help(self.desc.help.clone());
|
||||
m.set_field_type(T::proto());
|
||||
m.set_metric(counters);
|
||||
|
||||
vec![m]
|
||||
}
|
||||
}
|
||||
|
||||
/// Types of metrics that can obtain their values from an existing source.
|
||||
pub trait SourcedType: private::Sealed + Sync + Send {
|
||||
#[doc(hidden)]
|
||||
fn proto() -> proto::MetricType;
|
||||
}
|
||||
|
||||
impl SourcedType for Counter {
|
||||
fn proto() -> proto::MetricType {
|
||||
proto::MetricType::COUNTER
|
||||
}
|
||||
}
|
||||
|
||||
impl SourcedType for Gauge {
|
||||
fn proto() -> proto::MetricType {
|
||||
proto::MetricType::GAUGE
|
||||
}
|
||||
}
|
||||
|
||||
mod private {
|
||||
pub trait Sealed {}
|
||||
impl Sealed for super::Counter {}
|
||||
impl Sealed for super::Gauge {}
|
||||
}
|
||||
@@ -0,0 +1,70 @@
|
||||
[package]
|
||||
name = "bizinikiwi-wasm-builder"
|
||||
version = "17.0.0"
|
||||
authors.workspace = true
|
||||
description = "Utility for building WASM binaries"
|
||||
edition.workspace = true
|
||||
repository.workspace = true
|
||||
license = "Apache-2.0"
|
||||
homepage.workspace = true
|
||||
|
||||
[lints]
|
||||
workspace = true
|
||||
|
||||
[package.metadata.docs.rs]
|
||||
targets = ["x86_64-unknown-linux-gnu"]
|
||||
|
||||
[dependencies]
|
||||
build-helper = { workspace = true }
|
||||
cargo_metadata = { workspace = true }
|
||||
console = { workspace = true }
|
||||
filetime = { workspace = true }
|
||||
jobserver = { workspace = true }
|
||||
parity-wasm = { workspace = true }
|
||||
polkavm-linker = { workspace = true }
|
||||
pezsp-maybe-compressed-blob = { workspace = true, default-features = true }
|
||||
strum = { features = ["derive"], workspace = true, default-features = true }
|
||||
tempfile = { workspace = true }
|
||||
toml = { workspace = true }
|
||||
walkdir = { workspace = true }
|
||||
wasm-opt = { workspace = true }
|
||||
|
||||
# Dependencies required for the `metadata-hash` feature.
|
||||
array-bytes = { optional = true, workspace = true, default-features = true }
|
||||
codec = { optional = true, workspace = true, default-features = true }
|
||||
frame-metadata = { features = [
|
||||
"current",
|
||||
"unstable",
|
||||
], optional = true, workspace = true, default-features = true }
|
||||
merkleized-metadata = { optional = true, workspace = true }
|
||||
pezsc-executor = { optional = true, workspace = true, default-features = true }
|
||||
shlex = { workspace = true }
|
||||
pezsp-core = { optional = true, workspace = true, default-features = true }
|
||||
pezsp-io = { optional = true, workspace = true, default-features = true }
|
||||
pezsp-tracing = { optional = true, workspace = true, default-features = true }
|
||||
pezsp-version = { optional = true, workspace = true, default-features = true }
|
||||
|
||||
[features]
|
||||
# Enable support for generating the metadata hash.
|
||||
#
|
||||
# To generate the metadata hash the runtime is build once, executed to build the metadata and then
|
||||
# build a second time with the `RUNTIME_METADATA_HASH` environment variable set. The environment
|
||||
# variable then contains the hash and can be used inside the runtime.
|
||||
#
|
||||
# This pulls in quite a lot of dependencies and thus, is disabled by default.
|
||||
metadata-hash = [
|
||||
"array-bytes",
|
||||
"codec",
|
||||
"frame-metadata",
|
||||
"merkleized-metadata",
|
||||
"pezsc-executor",
|
||||
"pezsp-core",
|
||||
"pezsp-io",
|
||||
"pezsp-tracing",
|
||||
"pezsp-version",
|
||||
]
|
||||
runtime-benchmarks = [
|
||||
"pezsc-executor?/runtime-benchmarks",
|
||||
"pezsp-io?/runtime-benchmarks",
|
||||
"pezsp-version?/runtime-benchmarks",
|
||||
]
|
||||
@@ -0,0 +1,392 @@
|
||||
// This file is part of Bizinikiwi.
|
||||
|
||||
// Copyright (C) Parity Technologies (UK) Ltd.
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
use std::{
|
||||
env,
|
||||
path::{Path, PathBuf},
|
||||
process,
|
||||
};
|
||||
|
||||
use crate::RuntimeTarget;
|
||||
|
||||
/// Extra information when generating the `metadata-hash`.
|
||||
#[cfg(feature = "metadata-hash")]
|
||||
pub(crate) struct MetadataExtraInfo {
|
||||
pub decimals: u8,
|
||||
pub token_symbol: String,
|
||||
}
|
||||
|
||||
/// Returns the manifest dir from the `CARGO_MANIFEST_DIR` env.
|
||||
fn get_manifest_dir() -> PathBuf {
|
||||
env::var("CARGO_MANIFEST_DIR")
|
||||
.expect("`CARGO_MANIFEST_DIR` is always set for `build.rs` files; qed")
|
||||
.into()
|
||||
}
|
||||
|
||||
/// First step of the [`WasmBuilder`] to select the project to build.
|
||||
pub struct WasmBuilderSelectProject {
|
||||
/// This parameter just exists to make it impossible to construct
|
||||
/// this type outside of this crate.
|
||||
_ignore: (),
|
||||
}
|
||||
|
||||
impl WasmBuilderSelectProject {
|
||||
/// Use the current project as project for building the WASM binary.
|
||||
///
|
||||
/// # Panics
|
||||
///
|
||||
/// Panics if the `CARGO_MANIFEST_DIR` variable is not set. This variable
|
||||
/// is always set by `Cargo` in `build.rs` files.
|
||||
pub fn with_current_project(self) -> WasmBuilder {
|
||||
WasmBuilder {
|
||||
rust_flags: Vec::new(),
|
||||
file_name: None,
|
||||
project_cargo_toml: get_manifest_dir().join("Cargo.toml"),
|
||||
features_to_enable: Vec::new(),
|
||||
disable_runtime_version_section_check: false,
|
||||
export_heap_base: false,
|
||||
import_memory: false,
|
||||
#[cfg(feature = "metadata-hash")]
|
||||
enable_metadata_hash: None,
|
||||
}
|
||||
}
|
||||
|
||||
/// Use the given `path` as project for building the WASM binary.
|
||||
///
|
||||
/// Returns an error if the given `path` does not points to a `Cargo.toml`.
|
||||
pub fn with_project(self, path: impl Into<PathBuf>) -> Result<WasmBuilder, &'static str> {
|
||||
let path = path.into();
|
||||
|
||||
if path.ends_with("Cargo.toml") && path.exists() {
|
||||
Ok(WasmBuilder {
|
||||
rust_flags: Vec::new(),
|
||||
file_name: None,
|
||||
project_cargo_toml: path,
|
||||
features_to_enable: Vec::new(),
|
||||
disable_runtime_version_section_check: false,
|
||||
export_heap_base: false,
|
||||
import_memory: false,
|
||||
#[cfg(feature = "metadata-hash")]
|
||||
enable_metadata_hash: None,
|
||||
})
|
||||
} else {
|
||||
Err("Project path must point to the `Cargo.toml` of the project")
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// The builder for building a wasm binary.
|
||||
///
|
||||
/// The builder itself is separated into multiple structs to make the setup type safe.
|
||||
///
|
||||
/// Building a wasm binary:
|
||||
///
|
||||
/// 1. Call [`WasmBuilder::new`] to create a new builder.
|
||||
/// 2. Select the project to build using the methods of [`WasmBuilderSelectProject`].
|
||||
/// 3. Set additional `RUST_FLAGS` or a different name for the file containing the WASM code using
|
||||
/// methods of [`WasmBuilder`].
|
||||
/// 4. Build the WASM binary using [`Self::build`].
|
||||
pub struct WasmBuilder {
|
||||
/// Flags that should be appended to `RUST_FLAGS` env variable.
|
||||
rust_flags: Vec<String>,
|
||||
/// The name of the file that is being generated in `OUT_DIR`.
|
||||
///
|
||||
/// Defaults to `wasm_binary.rs`.
|
||||
file_name: Option<String>,
|
||||
/// The path to the `Cargo.toml` of the project that should be built
|
||||
/// for wasm.
|
||||
project_cargo_toml: PathBuf,
|
||||
/// Features that should be enabled when building the wasm binary.
|
||||
features_to_enable: Vec<String>,
|
||||
/// Should the builder not check that the `runtime_version` section exists in the wasm binary?
|
||||
disable_runtime_version_section_check: bool,
|
||||
|
||||
/// Whether `__heap_base` should be exported (WASM-only).
|
||||
export_heap_base: bool,
|
||||
/// Whether `--import-memory` should be added to the link args (WASM-only).
|
||||
import_memory: bool,
|
||||
|
||||
/// Whether to enable the metadata hash generation.
|
||||
#[cfg(feature = "metadata-hash")]
|
||||
enable_metadata_hash: Option<MetadataExtraInfo>,
|
||||
}
|
||||
|
||||
impl WasmBuilder {
|
||||
/// Create a new instance of the builder.
|
||||
pub fn new() -> WasmBuilderSelectProject {
|
||||
WasmBuilderSelectProject { _ignore: () }
|
||||
}
|
||||
|
||||
/// Build the WASM binary using the recommended default values.
|
||||
///
|
||||
/// This is the same as calling:
|
||||
/// ```no_run
|
||||
/// bizinikiwi_wasm_builder::WasmBuilder::new()
|
||||
/// .with_current_project()
|
||||
/// .import_memory()
|
||||
/// .export_heap_base()
|
||||
/// .build();
|
||||
/// ```
|
||||
pub fn build_using_defaults() {
|
||||
WasmBuilder::new()
|
||||
.with_current_project()
|
||||
.import_memory()
|
||||
.export_heap_base()
|
||||
.build();
|
||||
}
|
||||
|
||||
/// Init the wasm builder with the recommended default values.
|
||||
///
|
||||
/// In contrast to [`Self::build_using_defaults`] it does not build the WASM binary directly.
|
||||
///
|
||||
/// This is the same as calling:
|
||||
/// ```no_run
|
||||
/// bizinikiwi_wasm_builder::WasmBuilder::new()
|
||||
/// .with_current_project()
|
||||
/// .import_memory()
|
||||
/// .export_heap_base();
|
||||
/// ```
|
||||
pub fn init_with_defaults() -> Self {
|
||||
WasmBuilder::new().with_current_project().import_memory().export_heap_base()
|
||||
}
|
||||
|
||||
/// Enable exporting `__heap_base` as global variable in the WASM binary.
|
||||
///
|
||||
/// This adds `-C link-arg=--export=__heap_base` to `RUST_FLAGS`.
|
||||
pub fn export_heap_base(mut self) -> Self {
|
||||
self.export_heap_base = true;
|
||||
self
|
||||
}
|
||||
|
||||
/// Set the name of the file that will be generated in `OUT_DIR`.
|
||||
///
|
||||
/// This file needs to be included to get access to the build WASM binary.
|
||||
///
|
||||
/// If this function is not called, `file_name` defaults to `wasm_binary.rs`
|
||||
pub fn set_file_name(mut self, file_name: impl Into<String>) -> Self {
|
||||
self.file_name = Some(file_name.into());
|
||||
self
|
||||
}
|
||||
|
||||
/// Instruct the linker to import the memory into the WASM binary.
|
||||
///
|
||||
/// This adds `-C link-arg=--import-memory` to `RUST_FLAGS`.
|
||||
pub fn import_memory(mut self) -> Self {
|
||||
self.import_memory = true;
|
||||
self
|
||||
}
|
||||
|
||||
/// Append the given `flag` to `RUST_FLAGS`.
|
||||
///
|
||||
/// `flag` is appended as is, so it needs to be a valid flag.
|
||||
pub fn append_to_rust_flags(mut self, flag: impl Into<String>) -> Self {
|
||||
self.rust_flags.push(flag.into());
|
||||
self
|
||||
}
|
||||
|
||||
/// Enable the given feature when building the wasm binary.
|
||||
///
|
||||
/// `feature` needs to be a valid feature that is defined in the project `Cargo.toml`.
|
||||
pub fn enable_feature(mut self, feature: impl Into<String>) -> Self {
|
||||
self.features_to_enable.push(feature.into());
|
||||
self
|
||||
}
|
||||
|
||||
/// Enable generation of the metadata hash.
|
||||
///
|
||||
/// This will compile the runtime once, fetch the metadata, build the metadata hash and
|
||||
/// then compile again with the env `RUNTIME_METADATA_HASH` set. For more information
|
||||
/// about the metadata hash see [RFC78](https://polkadot-fellows.github.io/RFCs/approved/0078-merkleized-metadata.html).
|
||||
///
|
||||
/// - `token_symbol`: The symbol of the main native token of the chain.
|
||||
/// - `decimals`: The number of decimals of the main native token.
|
||||
#[cfg(feature = "metadata-hash")]
|
||||
pub fn enable_metadata_hash(mut self, token_symbol: impl Into<String>, decimals: u8) -> Self {
|
||||
self.enable_metadata_hash =
|
||||
Some(MetadataExtraInfo { token_symbol: token_symbol.into(), decimals });
|
||||
|
||||
self
|
||||
}
|
||||
|
||||
/// Disable the check for the `runtime_version` wasm section.
|
||||
///
|
||||
/// By default the `wasm-builder` will ensure that the `runtime_version` section will
|
||||
/// exists in the build wasm binary. This `runtime_version` section is used to get the
|
||||
/// `RuntimeVersion` without needing to call into the wasm binary. However, for some
|
||||
/// use cases (like tests) you may want to disable this check.
|
||||
pub fn disable_runtime_version_section_check(mut self) -> Self {
|
||||
self.disable_runtime_version_section_check = true;
|
||||
self
|
||||
}
|
||||
|
||||
/// Build the WASM binary.
|
||||
pub fn build(mut self) {
|
||||
let target = RuntimeTarget::new();
|
||||
|
||||
if target == RuntimeTarget::Wasm {
|
||||
if self.export_heap_base {
|
||||
self.rust_flags.push("-C link-arg=--export=__heap_base".into());
|
||||
}
|
||||
|
||||
if self.import_memory {
|
||||
self.rust_flags.push("-C link-arg=--import-memory".into());
|
||||
}
|
||||
}
|
||||
|
||||
let out_dir = PathBuf::from(env::var("OUT_DIR").expect("`OUT_DIR` is set by cargo!"));
|
||||
let file_path =
|
||||
out_dir.join(self.file_name.clone().unwrap_or_else(|| "wasm_binary.rs".into()));
|
||||
|
||||
if check_skip_build() {
|
||||
// If we skip the build, we still want to make sure to be called when an env variable
|
||||
// changes
|
||||
generate_rerun_if_changed_instructions();
|
||||
|
||||
provide_dummy_wasm_binary_if_not_exist(&file_path);
|
||||
|
||||
return;
|
||||
}
|
||||
|
||||
build_project(
|
||||
target,
|
||||
file_path,
|
||||
self.project_cargo_toml,
|
||||
self.rust_flags.join(" "),
|
||||
self.features_to_enable,
|
||||
self.file_name,
|
||||
!self.disable_runtime_version_section_check,
|
||||
#[cfg(feature = "metadata-hash")]
|
||||
self.enable_metadata_hash,
|
||||
);
|
||||
|
||||
// As last step we need to generate our `rerun-if-changed` stuff. If a build fails, we don't
|
||||
// want to spam the output!
|
||||
generate_rerun_if_changed_instructions();
|
||||
}
|
||||
}
|
||||
|
||||
/// Generate the name of the skip build environment variable for the current crate.
|
||||
fn generate_crate_skip_build_env_name() -> String {
|
||||
format!(
|
||||
"SKIP_{}_WASM_BUILD",
|
||||
env::var("CARGO_PKG_NAME")
|
||||
.expect("Package name is set")
|
||||
.to_uppercase()
|
||||
.replace('-', "_"),
|
||||
)
|
||||
}
|
||||
|
||||
/// Checks if the build of the WASM binary should be skipped.
|
||||
fn check_skip_build() -> bool {
|
||||
env::var(crate::SKIP_BUILD_ENV).is_ok() ||
|
||||
env::var(generate_crate_skip_build_env_name()).is_ok() ||
|
||||
// If we are running in docs.rs, let's skip building.
|
||||
// https://docs.rs/about/builds#detecting-docsrs
|
||||
env::var("DOCS_RS").is_ok()
|
||||
}
|
||||
|
||||
/// Provide a dummy WASM binary if there doesn't exist one.
|
||||
fn provide_dummy_wasm_binary_if_not_exist(file_path: &Path) {
|
||||
if !file_path.exists() {
|
||||
crate::write_file_if_changed(
|
||||
file_path,
|
||||
"pub const WASM_BINARY_PATH: Option<&str> = None;\
|
||||
pub const WASM_BINARY: Option<&[u8]> = None;\
|
||||
pub const WASM_BINARY_BLOATY: Option<&[u8]> = None;",
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
/// Generate the `rerun-if-changed` instructions for cargo to make sure that the WASM binary is
|
||||
/// rebuilt when needed.
|
||||
fn generate_rerun_if_changed_instructions() {
|
||||
// Make sure that the `build.rs` is called again if one of the following env variables changes.
|
||||
println!("cargo:rerun-if-env-changed={}", crate::SKIP_BUILD_ENV);
|
||||
println!("cargo:rerun-if-env-changed={}", crate::FORCE_WASM_BUILD_ENV);
|
||||
println!("cargo:rerun-if-env-changed={}", generate_crate_skip_build_env_name());
|
||||
}
|
||||
|
||||
/// Build the currently built project as wasm binary.
|
||||
///
|
||||
/// The current project is determined by using the `CARGO_MANIFEST_DIR` environment variable.
|
||||
///
|
||||
/// `file_name` - The name + path of the file being generated. The file contains the
|
||||
/// constant `WASM_BINARY`, which contains the built wasm binary.
|
||||
///
|
||||
/// `project_cargo_toml` - The path to the `Cargo.toml` of the project that should be built.
|
||||
///
|
||||
/// `default_rustflags` - Default `RUSTFLAGS` that will always be set for the build.
|
||||
///
|
||||
/// `features_to_enable` - Features that should be enabled for the project.
|
||||
///
|
||||
/// `wasm_binary_name` - The optional wasm binary name that is extended with
|
||||
/// `.compact.compressed.wasm`. If `None`, the project name will be used.
|
||||
///
|
||||
/// `check_for_runtime_version_section` - Should the wasm binary be checked for the
|
||||
/// `runtime_version` section?
|
||||
fn build_project(
|
||||
target: RuntimeTarget,
|
||||
file_name: PathBuf,
|
||||
project_cargo_toml: PathBuf,
|
||||
default_rustflags: String,
|
||||
features_to_enable: Vec<String>,
|
||||
wasm_binary_name: Option<String>,
|
||||
check_for_runtime_version_section: bool,
|
||||
#[cfg(feature = "metadata-hash")] enable_metadata_hash: Option<MetadataExtraInfo>,
|
||||
) {
|
||||
// Init jobserver as soon as possible
|
||||
crate::wasm_project::get_jobserver();
|
||||
let cargo_cmd = match crate::prerequisites::check(target) {
|
||||
Ok(cmd) => cmd,
|
||||
Err(err_msg) => {
|
||||
eprintln!("{err_msg}");
|
||||
process::exit(1);
|
||||
},
|
||||
};
|
||||
|
||||
let (wasm_binary, bloaty) = crate::wasm_project::create_and_compile(
|
||||
target,
|
||||
&project_cargo_toml,
|
||||
&default_rustflags,
|
||||
cargo_cmd,
|
||||
features_to_enable,
|
||||
wasm_binary_name,
|
||||
check_for_runtime_version_section,
|
||||
#[cfg(feature = "metadata-hash")]
|
||||
enable_metadata_hash,
|
||||
);
|
||||
|
||||
let (wasm_binary, wasm_binary_bloaty) = if let Some(wasm_binary) = wasm_binary {
|
||||
(wasm_binary.wasm_binary_path_escaped(), bloaty.bloaty_path_escaped())
|
||||
} else {
|
||||
(bloaty.bloaty_path_escaped(), bloaty.bloaty_path_escaped())
|
||||
};
|
||||
|
||||
crate::write_file_if_changed(
|
||||
file_name,
|
||||
format!(
|
||||
r#"
|
||||
pub const WASM_BINARY_PATH: Option<&str> = Some("{wasm_binary_path}");
|
||||
pub const WASM_BINARY: Option<&[u8]> = Some(include_bytes!("{wasm_binary}"));
|
||||
pub const WASM_BINARY_BLOATY: Option<&[u8]> = Some(include_bytes!("{wasm_binary_bloaty}"));
|
||||
"#,
|
||||
wasm_binary_path = wasm_binary,
|
||||
wasm_binary = wasm_binary,
|
||||
wasm_binary_bloaty = wasm_binary_bloaty,
|
||||
),
|
||||
);
|
||||
}
|
||||
@@ -0,0 +1,480 @@
|
||||
// This file is part of Bizinikiwi.
|
||||
|
||||
// Copyright (C) Parity Technologies (UK) Ltd.
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
//! # Wasm builder is a utility for building a project as a Wasm binary
|
||||
//!
|
||||
//! The Wasm builder is a tool that integrates the process of building the WASM binary of your
|
||||
//! project into the main `cargo` build process.
|
||||
//!
|
||||
//! ## Project setup
|
||||
//!
|
||||
//! A project that should be compiled as a Wasm binary needs to:
|
||||
//!
|
||||
//! 1. Add a `build.rs` file.
|
||||
//! 2. Add `wasm-builder` as dependency into `build-dependencies`.
|
||||
//!
|
||||
//! The `build.rs` file needs to contain the following code:
|
||||
//!
|
||||
//! ```no_run
|
||||
//! use bizinikiwi_wasm_builder::WasmBuilder;
|
||||
//!
|
||||
//! fn main() {
|
||||
//! // Builds the WASM binary using the recommended defaults.
|
||||
//! // If you need more control, you can call `new` or `init_with_defaults`.
|
||||
//! WasmBuilder::build_using_defaults();
|
||||
//! }
|
||||
//! ```
|
||||
//!
|
||||
//! As the final step, you need to add the following to your project:
|
||||
//!
|
||||
//! ```ignore
|
||||
//! include!(concat!(env!("OUT_DIR"), "/wasm_binary.rs"));
|
||||
//! ```
|
||||
//!
|
||||
//! This will include the generated Wasm binary as two constants `WASM_BINARY` and
|
||||
//! `WASM_BINARY_BLOATY`. The former is a compact Wasm binary and the latter is the Wasm binary as
|
||||
//! being generated by the compiler. Both variables have `Option<&'static [u8]>` as type.
|
||||
//! Additionally it will create the `WASM_BINARY_PATH` which is the path to the WASM blob on the
|
||||
//! filesystem.
|
||||
//!
|
||||
//! ### Feature
|
||||
//!
|
||||
//! Wasm builder supports to enable cargo features while building the Wasm binary. By default it
|
||||
//! will enable all features in the wasm build that are enabled for the native build except the
|
||||
//! `default` and `std` features. Besides that, wasm builder supports the special `runtime-wasm`
|
||||
//! feature. This `runtime-wasm` feature will be enabled by the wasm builder when it compiles the
|
||||
//! Wasm binary. If this feature is not present, it will not be enabled.
|
||||
//!
|
||||
//! ## Environment variables
|
||||
//!
|
||||
//! By using environment variables, you can configure which Wasm binaries are built and how:
|
||||
//!
|
||||
//! - `BIZINIKIWI_RUNTIME_TARGET` - Sets the target for building runtime. Supported values are `wasm`
|
||||
//! or `riscv` (experimental, do not use it in production!). By default the target is equal to
|
||||
//! `wasm`.
|
||||
//! - `SKIP_WASM_BUILD` - Skips building any Wasm binary. This is useful when only native should be
|
||||
//! recompiled. If this is the first run and there doesn't exist a Wasm binary, this will set both
|
||||
//! variables to `None`.
|
||||
//! - `WASM_BUILD_TYPE` - Sets the build type for building Wasm binaries. Supported values are
|
||||
//! `release` or `debug`. By default the build type is equal to the build type used by the main
|
||||
//! build.
|
||||
//! - `FORCE_WASM_BUILD` - Can be set to force a Wasm build. On subsequent calls the value of the
|
||||
//! variable needs to change. As wasm-builder instructs `cargo` to watch for file changes this
|
||||
//! environment variable should only be required in certain circumstances.
|
||||
//! - `WASM_BUILD_RUSTFLAGS` - Extend `RUSTFLAGS` given to `cargo build` while building the wasm
|
||||
//! binary.
|
||||
//! - `WASM_BUILD_NO_COLOR` - Disable color output of the wasm build.
|
||||
//! - `WASM_TARGET_DIRECTORY` - Will copy any build Wasm binary to the given directory. The path
|
||||
//! needs to be absolute.
|
||||
//! - `WASM_BUILD_TOOLCHAIN` - The toolchain that should be used to build the Wasm binaries. The
|
||||
//! format needs to be the same as used by cargo, e.g. `nightly-2024-12-26`.
|
||||
//! - `WASM_BUILD_WORKSPACE_HINT` - Hint the workspace that is being built. This is normally not
|
||||
//! required as we walk up from the target directory until we find a `Cargo.toml`. If the target
|
||||
//! directory is changed for the build, this environment variable can be used to point to the
|
||||
//! actual workspace.
|
||||
//! - `WASM_BUILD_STD` - Sets whether the Rust's standard library crates (`core` and `alloc`) will
|
||||
//! also be built. This is necessary to make sure the standard library crates only use the exact
|
||||
//! WASM feature set that our executor supports. Enabled by default for RISC-V target and WASM
|
||||
//! target (but only if Rust < 1.84). Disabled by default for WASM target and Rust >= 1.84.
|
||||
//! - `WASM_BUILD_CARGO_ARGS` - This can take a string as space separated list of `cargo` arguments.
|
||||
//! It was added specifically for the use case of enabling JSON diagnostic messages during the
|
||||
//! build phase, to be used by IDEs that parse them, but it might be useful for other cases too.
|
||||
//! - `CARGO_NET_OFFLINE` - If `true`, `--offline` will be passed to all processes launched to
|
||||
//! prevent network access. Useful in offline environments.
|
||||
//!
|
||||
//! Each project can be skipped individually by using the environment variable
|
||||
//! `SKIP_PROJECT_NAME_WASM_BUILD`. Where `PROJECT_NAME` needs to be replaced by the name of the
|
||||
//! cargo project, e.g. `kitchensink-runtime` will be `NODE_RUNTIME`.
|
||||
//!
|
||||
//! ## Prerequisites:
|
||||
//!
|
||||
//! Wasm builder requires the following prerequisites for building the Wasm binary:
|
||||
//! - Rust >= 1.68 and Rust < 1.84:
|
||||
//! - `wasm32-unknown-unknown` target
|
||||
//! - `rust-src` component
|
||||
//! - Rust >= 1.84:
|
||||
//! - `wasm32v1-none` target
|
||||
//!
|
||||
//! If a specific Rust is installed with `rustup`, it is important that the WASM
|
||||
//! target is installed as well. For example if installing the Rust from
|
||||
//! 26.12.2024 using `rustup install nightly-2024-12-26`, the WASM target
|
||||
//! (`wasm32-unknown-unknown` or `wasm32v1-none`) needs to be installed as well
|
||||
//! `rustup target add wasm32-unknown-unknown --toolchain nightly-2024-12-26`.
|
||||
//! To install the `rust-src` component, use `rustup component add rust-src
|
||||
//! --toolchain nightly-2024-12-26`.
|
||||
|
||||
use prerequisites::DummyCrate;
|
||||
use std::{
|
||||
env, fs,
|
||||
io::BufRead,
|
||||
path::{Path, PathBuf},
|
||||
process::Command,
|
||||
};
|
||||
use version::Version;
|
||||
|
||||
mod builder;
|
||||
#[cfg(feature = "metadata-hash")]
|
||||
mod metadata_hash;
|
||||
mod prerequisites;
|
||||
mod version;
|
||||
mod wasm_project;
|
||||
|
||||
pub use builder::{WasmBuilder, WasmBuilderSelectProject};
|
||||
|
||||
/// Environment variable that tells us to skip building the wasm binary.
|
||||
const SKIP_BUILD_ENV: &str = "SKIP_WASM_BUILD";
|
||||
|
||||
/// Environment variable that tells us whether we should avoid network requests
|
||||
const OFFLINE: &str = "CARGO_NET_OFFLINE";
|
||||
|
||||
/// Environment variable to force a certain build type when building the wasm binary.
|
||||
/// Expects "debug", "release" or "production" as value.
|
||||
///
|
||||
/// When unset the WASM binary uses the same build type as the main cargo build with
|
||||
/// the exception of a debug build: In this case the wasm build defaults to `release` in
|
||||
/// order to avoid a slowdown when not explicitly requested.
|
||||
const WASM_BUILD_TYPE_ENV: &str = "WASM_BUILD_TYPE";
|
||||
|
||||
/// Environment variable to extend the `RUSTFLAGS` variable given to the wasm build.
|
||||
const WASM_BUILD_RUSTFLAGS_ENV: &str = "WASM_BUILD_RUSTFLAGS";
|
||||
|
||||
/// Environment variable to set the target directory to copy the final wasm binary.
|
||||
///
|
||||
/// The directory needs to be an absolute path.
|
||||
const WASM_TARGET_DIRECTORY: &str = "WASM_TARGET_DIRECTORY";
|
||||
|
||||
/// Environment variable to disable color output of the wasm build.
|
||||
const WASM_BUILD_NO_COLOR: &str = "WASM_BUILD_NO_COLOR";
|
||||
|
||||
/// Environment variable to set the toolchain used to compile the wasm binary.
|
||||
const WASM_BUILD_TOOLCHAIN: &str = "WASM_BUILD_TOOLCHAIN";
|
||||
|
||||
/// Environment variable that makes sure the WASM build is triggered.
|
||||
const FORCE_WASM_BUILD_ENV: &str = "FORCE_WASM_BUILD";
|
||||
|
||||
/// Environment variable that hints the workspace we are building.
|
||||
const WASM_BUILD_WORKSPACE_HINT: &str = "WASM_BUILD_WORKSPACE_HINT";
|
||||
|
||||
/// Environment variable to set whether we'll build `core`/`alloc`.
|
||||
const WASM_BUILD_STD: &str = "WASM_BUILD_STD";
|
||||
|
||||
/// Environment variable to set additional cargo arguments that might be useful
|
||||
/// during the build phase.
|
||||
const WASM_BUILD_CARGO_ARGS: &str = "WASM_BUILD_CARGO_ARGS";
|
||||
|
||||
/// The target to use for the runtime. Valid values are `wasm` (default) or `riscv`.
|
||||
const RUNTIME_TARGET: &str = "BIZINIKIWI_RUNTIME_TARGET";
|
||||
|
||||
/// Write to the given `file` if the `content` is different.
|
||||
fn write_file_if_changed(file: impl AsRef<Path>, content: impl AsRef<str>) {
|
||||
if fs::read_to_string(file.as_ref()).ok().as_deref() != Some(content.as_ref()) {
|
||||
fs::write(file.as_ref(), content.as_ref())
|
||||
.unwrap_or_else(|_| panic!("Writing `{}` can not fail!", file.as_ref().display()));
|
||||
}
|
||||
}
|
||||
|
||||
/// Copy `src` to `dst` if the `dst` does not exist or is different.
|
||||
fn copy_file_if_changed(src: PathBuf, dst: PathBuf) {
|
||||
let src_file = fs::read_to_string(&src).ok();
|
||||
let dst_file = fs::read_to_string(&dst).ok();
|
||||
|
||||
if src_file != dst_file {
|
||||
fs::copy(&src, &dst).unwrap_or_else(|_| {
|
||||
panic!("Copying `{}` to `{}` can not fail; qed", src.display(), dst.display())
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
/// Get a cargo command that should be used to invoke the compilation.
|
||||
fn get_cargo_command(target: RuntimeTarget) -> CargoCommand {
|
||||
let env_cargo =
|
||||
CargoCommand::new(&env::var("CARGO").expect("`CARGO` env variable is always set by cargo"));
|
||||
let default_cargo = CargoCommand::new("cargo");
|
||||
let wasm_toolchain = env::var(WASM_BUILD_TOOLCHAIN).ok();
|
||||
|
||||
// First check if the user requested a specific toolchain
|
||||
if let Some(cmd) =
|
||||
wasm_toolchain.map(|t| CargoCommand::new_with_args("rustup", &["run", &t, "cargo"]))
|
||||
{
|
||||
cmd
|
||||
} else if env_cargo.supports_bizinikiwi_runtime_env(target) {
|
||||
env_cargo
|
||||
} else if default_cargo.supports_bizinikiwi_runtime_env(target) {
|
||||
default_cargo
|
||||
} else {
|
||||
// If no command before provided us with a cargo that supports our Bizinikiwi wasm env, we
|
||||
// try to search one with rustup. If that fails as well, we return the default cargo and let
|
||||
// the perquisites check fail.
|
||||
get_rustup_command(target).unwrap_or(default_cargo)
|
||||
}
|
||||
}
|
||||
|
||||
/// Get the newest rustup command that supports compiling a runtime.
|
||||
///
|
||||
/// Stable versions are always favored over nightly versions even if the nightly versions are
|
||||
/// newer.
|
||||
fn get_rustup_command(target: RuntimeTarget) -> Option<CargoCommand> {
|
||||
let output = Command::new("rustup").args(&["toolchain", "list"]).output().ok()?.stdout;
|
||||
let lines = output.as_slice().lines();
|
||||
|
||||
let mut versions = Vec::new();
|
||||
for line in lines.filter_map(|l| l.ok()) {
|
||||
// Split by a space to get rid of e.g. " (default)" at the end.
|
||||
let rustup_version = line.split(" ").next().unwrap();
|
||||
let cmd = CargoCommand::new_with_args("rustup", &["run", &rustup_version, "cargo"]);
|
||||
|
||||
if !cmd.supports_bizinikiwi_runtime_env(target) {
|
||||
continue;
|
||||
}
|
||||
|
||||
let Some(cargo_version) = cmd.version() else { continue };
|
||||
|
||||
versions.push((cargo_version, rustup_version.to_string()));
|
||||
}
|
||||
|
||||
// Sort by the parsed version to get the latest version (greatest version) at the end of the
|
||||
// vec.
|
||||
versions.sort_by_key(|v| v.0);
|
||||
let version = &versions.last()?.1;
|
||||
|
||||
Some(CargoCommand::new_with_args("rustup", &["run", &version, "cargo"]))
|
||||
}
|
||||
|
||||
/// Wraps a specific command which represents a cargo invocation.
|
||||
#[derive(Debug, Clone)]
|
||||
struct CargoCommand {
|
||||
program: String,
|
||||
args: Vec<String>,
|
||||
version: Option<Version>,
|
||||
}
|
||||
|
||||
impl CargoCommand {
|
||||
fn new(program: &str) -> Self {
|
||||
let version = Self::extract_version(program, &[]);
|
||||
|
||||
CargoCommand { program: program.into(), args: Vec::new(), version }
|
||||
}
|
||||
|
||||
fn new_with_args(program: &str, args: &[&str]) -> Self {
|
||||
let version = Self::extract_version(program, args);
|
||||
|
||||
CargoCommand {
|
||||
program: program.into(),
|
||||
args: args.iter().map(ToString::to_string).collect(),
|
||||
version,
|
||||
}
|
||||
}
|
||||
|
||||
fn command(&self) -> Command {
|
||||
let mut cmd = Command::new(&self.program);
|
||||
cmd.args(&self.args);
|
||||
cmd
|
||||
}
|
||||
|
||||
fn extract_version(program: &str, args: &[&str]) -> Option<Version> {
|
||||
let version = Command::new(program)
|
||||
.args(args)
|
||||
.arg("--version")
|
||||
.output()
|
||||
.ok()
|
||||
.and_then(|o| String::from_utf8(o.stdout).ok())?;
|
||||
|
||||
Version::extract(&version)
|
||||
}
|
||||
|
||||
/// Returns the version of this cargo command or `None` if it failed to extract the version.
|
||||
fn version(&self) -> Option<Version> {
|
||||
self.version
|
||||
}
|
||||
|
||||
/// Returns whether this version of the toolchain supports nightly features.
|
||||
fn supports_nightly_features(&self) -> bool {
|
||||
self.version.map_or(false, |version| version.is_nightly) ||
|
||||
env::var("RUSTC_BOOTSTRAP").is_ok()
|
||||
}
|
||||
|
||||
/// Check if the supplied cargo command supports our runtime environment.
|
||||
fn supports_bizinikiwi_runtime_env(&self, target: RuntimeTarget) -> bool {
|
||||
match target {
|
||||
RuntimeTarget::Wasm => self.supports_bizinikiwi_runtime_env_wasm(),
|
||||
RuntimeTarget::Riscv => true,
|
||||
}
|
||||
}
|
||||
|
||||
/// Check if the supplied cargo command supports our Bizinikiwi wasm environment.
|
||||
///
|
||||
/// This means that either the cargo version is at minimum 1.68.0 or this is a nightly cargo.
|
||||
///
|
||||
/// Assumes that cargo version matches the rustc version.
|
||||
fn supports_bizinikiwi_runtime_env_wasm(&self) -> bool {
|
||||
// `RUSTC_BOOTSTRAP` tells a stable compiler to behave like a nightly. So, when this env
|
||||
// variable is set, we can assume that whatever rust compiler we have, it is a nightly
|
||||
// compiler. For "more" information, see:
|
||||
// https://github.com/rust-lang/rust/blob/fa0f7d0080d8e7e9eb20aa9cbf8013f96c81287f/src/libsyntax/feature_gate/check.rs#L891
|
||||
if env::var("RUSTC_BOOTSTRAP").is_ok() {
|
||||
return true;
|
||||
}
|
||||
|
||||
let Some(version) = self.version() else { return false };
|
||||
|
||||
// Check if major and minor are greater or equal than 1.68 or this is a nightly.
|
||||
version.major > 1 || (version.major == 1 && version.minor >= 68) || version.is_nightly
|
||||
}
|
||||
|
||||
/// Returns whether this version of the toolchain supports the `wasm32v1-none` target.
|
||||
fn supports_wasm32v1_none_target(&self) -> bool {
|
||||
self.version.map_or(false, |version| {
|
||||
// Check if major and minor are greater or equal than 1.84.
|
||||
version.major > 1 || (version.major == 1 && version.minor >= 84)
|
||||
})
|
||||
}
|
||||
|
||||
/// Returns whether the `wasm32v1-none` target is installed in this version of the toolchain.
|
||||
fn is_wasm32v1_none_target_installed(&self) -> bool {
|
||||
let dummy_crate = DummyCrate::new(self, RuntimeTarget::Wasm, true);
|
||||
dummy_crate.is_target_installed("wasm32v1-none").unwrap_or(false)
|
||||
}
|
||||
|
||||
/// Returns whether the `wasm32v1-none` target is available in this version of the toolchain.
|
||||
fn is_wasm32v1_none_target_available(&self) -> bool {
|
||||
// Check if major and minor are greater or equal than 1.84 and that the `wasm32v1-none`
|
||||
// target is installed for this toolchain.
|
||||
self.supports_wasm32v1_none_target() && self.is_wasm32v1_none_target_installed()
|
||||
}
|
||||
}
|
||||
|
||||
/// Wraps a [`CargoCommand`] and the version of `rustc` the cargo command uses.
|
||||
#[derive(Clone)]
|
||||
struct CargoCommandVersioned {
|
||||
command: CargoCommand,
|
||||
version: String,
|
||||
}
|
||||
|
||||
impl CargoCommandVersioned {
|
||||
fn new(command: CargoCommand, version: String) -> Self {
|
||||
Self { command, version }
|
||||
}
|
||||
|
||||
/// Returns the `rustc` version.
|
||||
fn rustc_version(&self) -> &str {
|
||||
&self.version
|
||||
}
|
||||
}
|
||||
|
||||
impl std::ops::Deref for CargoCommandVersioned {
|
||||
type Target = CargoCommand;
|
||||
|
||||
fn deref(&self) -> &CargoCommand {
|
||||
&self.command
|
||||
}
|
||||
}
|
||||
|
||||
/// Returns `true` when color output is enabled.
|
||||
fn color_output_enabled() -> bool {
|
||||
env::var(crate::WASM_BUILD_NO_COLOR).is_err()
|
||||
}
|
||||
|
||||
/// Fetches a boolean environment variable. Will exit the process if the value is invalid.
|
||||
fn get_bool_environment_variable(name: &str) -> Option<bool> {
|
||||
let value = env::var_os(name)?;
|
||||
|
||||
// We're comparing `OsString`s here so we can't use a `match`.
|
||||
if value == "1" {
|
||||
Some(true)
|
||||
} else if value == "0" {
|
||||
Some(false)
|
||||
} else {
|
||||
build_helper::warning!(
|
||||
"the '{name}' environment variable has an invalid value; it must be either '1' or '0'",
|
||||
);
|
||||
std::process::exit(1);
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Copy, Clone, PartialEq, Eq)]
|
||||
enum RuntimeTarget {
|
||||
Wasm,
|
||||
Riscv,
|
||||
}
|
||||
|
||||
impl RuntimeTarget {
|
||||
/// Creates a new instance.
|
||||
fn new() -> Self {
|
||||
let Some(value) = env::var_os(RUNTIME_TARGET) else {
|
||||
return Self::Wasm;
|
||||
};
|
||||
|
||||
if value == "wasm" {
|
||||
Self::Wasm
|
||||
} else if value == "riscv" {
|
||||
Self::Riscv
|
||||
} else {
|
||||
build_helper::warning!(
|
||||
"RUNTIME_TARGET environment variable must be set to either \"wasm\" or \"riscv\""
|
||||
);
|
||||
std::process::exit(1);
|
||||
}
|
||||
}
|
||||
|
||||
/// Figures out the target parameter value for rustc.
|
||||
fn rustc_target(self, cargo_command: &CargoCommand) -> String {
|
||||
match self {
|
||||
RuntimeTarget::Wasm =>
|
||||
if cargo_command.is_wasm32v1_none_target_available() {
|
||||
"wasm32v1-none".into()
|
||||
} else {
|
||||
"wasm32-unknown-unknown".into()
|
||||
},
|
||||
RuntimeTarget::Riscv => {
|
||||
let path = polkavm_linker::target_json_32_path().expect("riscv not found");
|
||||
path.into_os_string().into_string().unwrap()
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
/// Figures out the target directory name used by cargo.
|
||||
fn rustc_target_dir(self, cargo_command: &CargoCommand) -> &'static str {
|
||||
match self {
|
||||
RuntimeTarget::Wasm =>
|
||||
if cargo_command.is_wasm32v1_none_target_available() {
|
||||
"wasm32v1-none".into()
|
||||
} else {
|
||||
"wasm32-unknown-unknown".into()
|
||||
},
|
||||
RuntimeTarget::Riscv => "riscv32emac-unknown-none-polkavm",
|
||||
}
|
||||
}
|
||||
|
||||
/// Figures out the build-std argument.
|
||||
fn rustc_target_build_std(self, cargo_command: &CargoCommand) -> Option<&'static str> {
|
||||
if !crate::get_bool_environment_variable(crate::WASM_BUILD_STD).unwrap_or_else(
|
||||
|| match self {
|
||||
RuntimeTarget::Wasm => !cargo_command.is_wasm32v1_none_target_available(),
|
||||
RuntimeTarget::Riscv => true,
|
||||
},
|
||||
) {
|
||||
return None;
|
||||
}
|
||||
|
||||
// This is a nightly-only flag.
|
||||
|
||||
// We only build `core` and `alloc` crates since wasm-builder disables `std` featue for
|
||||
// runtime. Thus the runtime is `#![no_std]` crate.
|
||||
|
||||
Some("build-std=core,alloc")
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,132 @@
|
||||
// This file is part of Bizinikiwi.
|
||||
|
||||
// Copyright (C) Parity Technologies (UK) Ltd.
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
use crate::builder::MetadataExtraInfo;
|
||||
use codec::{Decode, Encode};
|
||||
use frame_metadata::{RuntimeMetadata, RuntimeMetadataPrefixed};
|
||||
use merkleized_metadata::{generate_metadata_digest, ExtraInfo};
|
||||
use pezsc_executor::WasmExecutor;
|
||||
use pezsp_core::traits::{CallContext, CodeExecutor, RuntimeCode, WrappedRuntimeCode};
|
||||
use std::path::Path;
|
||||
|
||||
/// The host functions that we provide when calling into the wasm file.
|
||||
///
|
||||
/// Any other host function will return an error.
|
||||
type HostFunctions = (
|
||||
// The allocator functions.
|
||||
pezsp_io::allocator::HostFunctions,
|
||||
// Logging is good to have for debugging issues.
|
||||
pezsp_io::logging::HostFunctions,
|
||||
// Give access to the "state", actually the state will be empty, but some chains put constants
|
||||
// into the state and this would panic at metadata generation. Thus, we give them an empty
|
||||
// state to not panic.
|
||||
pezsp_io::storage::HostFunctions,
|
||||
// The hashing functions.
|
||||
pezsp_io::hashing::HostFunctions,
|
||||
);
|
||||
|
||||
/// Generate the metadata hash.
|
||||
///
|
||||
/// The metadata hash is generated as specced in
|
||||
/// [RFC78](https://polkadot-fellows.github.io/RFCs/approved/0078-merkleized-metadata.html).
|
||||
///
|
||||
/// Returns the metadata hash.
|
||||
pub fn generate_metadata_hash(wasm: &Path, extra_info: MetadataExtraInfo) -> [u8; 32] {
|
||||
pezsp_tracing::try_init_simple();
|
||||
|
||||
let wasm = std::fs::read(wasm).expect("Wasm file was just created and should be readable.");
|
||||
|
||||
let executor = WasmExecutor::<HostFunctions>::builder()
|
||||
.with_allow_missing_host_functions(true)
|
||||
.build();
|
||||
|
||||
let runtime_code = RuntimeCode {
|
||||
code_fetcher: &WrappedRuntimeCode(wasm.into()),
|
||||
heap_pages: None,
|
||||
// The hash is only used for caching and thus, not that important for our use case here.
|
||||
hash: vec![1, 2, 3],
|
||||
};
|
||||
|
||||
let metadata = executor
|
||||
.call(
|
||||
&mut pezsp_io::TestExternalities::default().ext(),
|
||||
&runtime_code,
|
||||
"Metadata_metadata_at_version",
|
||||
&15u32.encode(),
|
||||
CallContext::Offchain,
|
||||
)
|
||||
.0
|
||||
.expect("`Metadata::metadata_at_version` should exist.");
|
||||
|
||||
let metadata = Option::<Vec<u8>>::decode(&mut &metadata[..])
|
||||
.ok()
|
||||
.flatten()
|
||||
.expect("Metadata V15 support is required.");
|
||||
|
||||
let metadata = RuntimeMetadataPrefixed::decode(&mut &metadata[..])
|
||||
.expect("Invalid encoded metadata?")
|
||||
.1;
|
||||
|
||||
let runtime_version = executor
|
||||
.call(
|
||||
&mut pezsp_io::TestExternalities::default().ext(),
|
||||
&runtime_code,
|
||||
"Core_version",
|
||||
&[],
|
||||
CallContext::Offchain,
|
||||
)
|
||||
.0
|
||||
.expect("`Core_version` should exist.");
|
||||
let runtime_version = pezsp_version::RuntimeVersion::decode(&mut &runtime_version[..])
|
||||
.expect("Invalid `RuntimeVersion` encoding");
|
||||
|
||||
let base58_prefix = extract_ss58_prefix(&metadata);
|
||||
|
||||
let extra_info = ExtraInfo {
|
||||
spec_version: runtime_version.spec_version,
|
||||
spec_name: runtime_version.spec_name.into(),
|
||||
base58_prefix,
|
||||
decimals: extra_info.decimals,
|
||||
token_symbol: extra_info.token_symbol,
|
||||
};
|
||||
|
||||
generate_metadata_digest(&metadata, extra_info)
|
||||
.expect("Failed to generate the metadata digest")
|
||||
.hash()
|
||||
}
|
||||
|
||||
/// Extract the `SS58` from the constants in the given `metadata`.
|
||||
fn extract_ss58_prefix(metadata: &RuntimeMetadata) -> u16 {
|
||||
let RuntimeMetadata::V15(ref metadata) = metadata else {
|
||||
panic!("Metadata version 15 required")
|
||||
};
|
||||
|
||||
let system = metadata
|
||||
.pallets
|
||||
.iter()
|
||||
.find(|p| p.name == "System")
|
||||
.expect("Each FRAME runtime has the `System` pallet; qed");
|
||||
|
||||
system
|
||||
.constants
|
||||
.iter()
|
||||
.find_map(|c| {
|
||||
(c.name == "SS58Prefix")
|
||||
.then(|| u16::decode(&mut &c.value[..]).expect("SS58 is an `u16`; qed"))
|
||||
})
|
||||
.expect("`SS58PREFIX` exists in the `System` constants; qed")
|
||||
}
|
||||
@@ -0,0 +1,279 @@
|
||||
// This file is part of Bizinikiwi.
|
||||
|
||||
// Copyright (C) Parity Technologies (UK) Ltd.
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
use crate::{write_file_if_changed, CargoCommand, CargoCommandVersioned, RuntimeTarget};
|
||||
|
||||
use console::style;
|
||||
use std::{
|
||||
fs,
|
||||
path::{Path, PathBuf},
|
||||
process::Command,
|
||||
};
|
||||
|
||||
use tempfile::tempdir;
|
||||
|
||||
/// Colorizes an error message, if color output is enabled.
|
||||
fn colorize_error_message(message: &str) -> String {
|
||||
if super::color_output_enabled() {
|
||||
style(message).red().bold().to_string()
|
||||
} else {
|
||||
message.into()
|
||||
}
|
||||
}
|
||||
|
||||
/// Colorizes an auxiliary message, if color output is enabled.
|
||||
fn colorize_aux_message(message: &str) -> String {
|
||||
if super::color_output_enabled() {
|
||||
style(message).yellow().bold().to_string()
|
||||
} else {
|
||||
message.into()
|
||||
}
|
||||
}
|
||||
|
||||
/// Checks that all prerequisites are installed.
|
||||
///
|
||||
/// Returns the versioned cargo command on success.
|
||||
pub(crate) fn check(target: RuntimeTarget) -> Result<CargoCommandVersioned, String> {
|
||||
let cargo_command = crate::get_cargo_command(target);
|
||||
match target {
|
||||
RuntimeTarget::Wasm => {
|
||||
if !cargo_command.supports_bizinikiwi_runtime_env(target) {
|
||||
return Err(colorize_error_message(
|
||||
"Cannot compile a WASM runtime: no compatible Rust compiler found!\n\
|
||||
Install at least Rust 1.68.0 or a recent nightly version.",
|
||||
));
|
||||
}
|
||||
|
||||
check_wasm_toolchain_installed(cargo_command)
|
||||
},
|
||||
RuntimeTarget::Riscv => {
|
||||
if !cargo_command.supports_bizinikiwi_runtime_env(target) {
|
||||
return Err(colorize_error_message(
|
||||
"Cannot compile a RISC-V runtime: no compatible Rust compiler found!\n\
|
||||
Install a toolchain from here and try again: https://github.com/paritytech/rustc-rv32e-toolchain/",
|
||||
));
|
||||
}
|
||||
|
||||
let dummy_crate = DummyCrate::new(&cargo_command, target, false);
|
||||
let version = dummy_crate.get_rustc_version();
|
||||
Ok(CargoCommandVersioned::new(cargo_command, version))
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) struct DummyCrate<'a> {
|
||||
cargo_command: &'a CargoCommand,
|
||||
temp: tempfile::TempDir,
|
||||
manifest_path: PathBuf,
|
||||
target: RuntimeTarget,
|
||||
ignore_target: bool,
|
||||
}
|
||||
|
||||
impl<'a> DummyCrate<'a> {
|
||||
/// Creates a minimal dummy crate.
|
||||
pub(crate) fn new(
|
||||
cargo_command: &'a CargoCommand,
|
||||
target: RuntimeTarget,
|
||||
ignore_target: bool,
|
||||
) -> Self {
|
||||
let temp = tempdir().expect("Creating temp dir does not fail; qed");
|
||||
let project_dir = temp.path();
|
||||
fs::create_dir_all(project_dir.join("src")).expect("Creating src dir does not fail; qed");
|
||||
|
||||
let manifest_path = project_dir.join("Cargo.toml");
|
||||
match target {
|
||||
RuntimeTarget::Wasm => {
|
||||
write_file_if_changed(
|
||||
&manifest_path,
|
||||
r#"
|
||||
[package]
|
||||
name = "dummy-crate"
|
||||
version = "1.0.0"
|
||||
edition = "2021"
|
||||
|
||||
[lib]
|
||||
crate-type = ["cdylib"]
|
||||
|
||||
[workspace]
|
||||
"#,
|
||||
);
|
||||
|
||||
write_file_if_changed(
|
||||
project_dir.join("src/lib.rs"),
|
||||
r#"
|
||||
#![no_std]
|
||||
|
||||
#[panic_handler]
|
||||
fn panic(_: &core::panic::PanicInfo<'_>) -> ! {
|
||||
loop {}
|
||||
}
|
||||
"#,
|
||||
);
|
||||
},
|
||||
RuntimeTarget::Riscv => {
|
||||
write_file_if_changed(
|
||||
&manifest_path,
|
||||
r#"
|
||||
[package]
|
||||
name = "dummy-crate"
|
||||
version = "1.0.0"
|
||||
edition = "2021"
|
||||
|
||||
[workspace]
|
||||
"#,
|
||||
);
|
||||
|
||||
write_file_if_changed(
|
||||
project_dir.join("src/main.rs"),
|
||||
"#![allow(missing_docs)] fn main() {}",
|
||||
);
|
||||
},
|
||||
}
|
||||
|
||||
DummyCrate { cargo_command, temp, manifest_path, target, ignore_target }
|
||||
}
|
||||
|
||||
fn prepare_command(&self, subcommand: &str) -> Command {
|
||||
let mut cmd = self.cargo_command.command();
|
||||
// Chdir to temp to avoid including project's .cargo/config.toml
|
||||
// by accident - it can happen in some CI environments.
|
||||
cmd.current_dir(&self.temp);
|
||||
cmd.arg(subcommand);
|
||||
if !self.ignore_target {
|
||||
cmd.arg(format!("--target={}", self.target.rustc_target(self.cargo_command)));
|
||||
}
|
||||
cmd.args(&["--manifest-path", &self.manifest_path.display().to_string()]);
|
||||
|
||||
if super::color_output_enabled() {
|
||||
cmd.arg("--color=always");
|
||||
}
|
||||
|
||||
// manually set the `CARGO_TARGET_DIR` to prevent a cargo deadlock
|
||||
let target_dir = self.temp.path().join("target").display().to_string();
|
||||
cmd.env("CARGO_TARGET_DIR", &target_dir);
|
||||
|
||||
// Make sure the host's flags aren't used here, e.g. if an alternative linker is specified
|
||||
// in the RUSTFLAGS then the check we do here will break unless we clear these.
|
||||
cmd.env_remove("CARGO_ENCODED_RUSTFLAGS");
|
||||
cmd.env_remove("RUSTFLAGS");
|
||||
// Make sure if we're called from within a `build.rs` the host toolchain won't override a
|
||||
// rustup toolchain we've picked.
|
||||
cmd.env_remove("RUSTC");
|
||||
cmd
|
||||
}
|
||||
|
||||
fn get_rustc_version(&self) -> String {
|
||||
let mut run_cmd = self.prepare_command("rustc");
|
||||
run_cmd.args(&["-q", "--", "--version"]);
|
||||
run_cmd
|
||||
.output()
|
||||
.ok()
|
||||
.and_then(|o| String::from_utf8(o.stdout).ok())
|
||||
.unwrap_or_else(|| "unknown rustc version".into())
|
||||
}
|
||||
|
||||
fn get_sysroot(&self) -> Option<String> {
|
||||
let mut sysroot_cmd = self.prepare_command("rustc");
|
||||
sysroot_cmd.args(&["-q", "--", "--print", "sysroot"]);
|
||||
sysroot_cmd.output().ok().and_then(|o| String::from_utf8(o.stdout).ok())
|
||||
}
|
||||
|
||||
fn get_toolchain(&self) -> Option<String> {
|
||||
let sysroot = self.get_sysroot()?;
|
||||
Path::new(sysroot.trim())
|
||||
.file_name()
|
||||
.and_then(|s| s.to_str())
|
||||
.map(|s| s.to_string())
|
||||
}
|
||||
|
||||
pub(crate) fn is_target_installed(&self, target: &str) -> Option<bool> {
|
||||
let sysroot = self.get_sysroot()?;
|
||||
let target_libdir_path =
|
||||
Path::new(sysroot.trim()).join("lib").join("rustlib").join(target).join("lib");
|
||||
Some(target_libdir_path.exists())
|
||||
}
|
||||
|
||||
fn try_build(&self) -> Result<(), Option<String>> {
|
||||
let Ok(result) = self.prepare_command("build").output() else { return Err(None) };
|
||||
if !result.status.success() {
|
||||
return Err(Some(String::from_utf8_lossy(&result.stderr).into()));
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
fn check_wasm_toolchain_installed(
|
||||
cargo_command: CargoCommand,
|
||||
) -> Result<CargoCommandVersioned, String> {
|
||||
let target = RuntimeTarget::Wasm;
|
||||
let rustc_target = target.rustc_target(&cargo_command);
|
||||
|
||||
let dummy_crate = DummyCrate::new(&cargo_command, target, false);
|
||||
let toolchain = dummy_crate.get_toolchain().unwrap_or("<unknown>".to_string());
|
||||
|
||||
if let Err(error) = dummy_crate.try_build() {
|
||||
let basic_error_message = colorize_error_message(
|
||||
&format!("Rust WASM target for toolchain {toolchain} is not properly installed; please install it!")
|
||||
);
|
||||
return match error {
|
||||
None => Err(basic_error_message),
|
||||
Some(error) if error.contains(&format!("the `{rustc_target}` target may not be installed")) => {
|
||||
Err(colorize_error_message(&format!("Cannot compile the WASM runtime: the `{rustc_target}` target is not installed!\n\
|
||||
You can install it with `rustup target add {rustc_target} --toolchain {toolchain}` if you're using `rustup`.")))
|
||||
},
|
||||
// Apparently this can happen when we're running on a non Tier 1 platform.
|
||||
Some(ref error) if error.contains("linker `rust-lld` not found") =>
|
||||
Err(colorize_error_message("Cannot compile the WASM runtime: `rust-lld` not found!")),
|
||||
Some(error) => Err(format!(
|
||||
"{}\n\n{}\n{}\n{}{}\n",
|
||||
basic_error_message,
|
||||
colorize_aux_message("Further error information:"),
|
||||
colorize_aux_message(&"-".repeat(60)),
|
||||
error,
|
||||
colorize_aux_message(&"-".repeat(60)),
|
||||
))
|
||||
};
|
||||
}
|
||||
|
||||
let version = dummy_crate.get_rustc_version();
|
||||
|
||||
let target = RuntimeTarget::new();
|
||||
assert!(target == RuntimeTarget::Wasm);
|
||||
if target.rustc_target_build_std(&cargo_command).is_some() {
|
||||
if let Some(sysroot) = dummy_crate.get_sysroot() {
|
||||
let src_path =
|
||||
Path::new(sysroot.trim()).join("lib").join("rustlib").join("src").join("rust");
|
||||
if !src_path.exists() {
|
||||
let toolchain = dummy_crate.get_toolchain().unwrap_or("<toolchain>".to_string());
|
||||
return Err(colorize_error_message(
|
||||
&format!("Cannot compile the WASM runtime: no standard library sources found at {}!\n\
|
||||
You can install them with `rustup component add rust-src --toolchain {toolchain}` if you're using `rustup`.", src_path.display()),
|
||||
));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if cargo_command.supports_wasm32v1_none_target() &&
|
||||
!cargo_command.is_wasm32v1_none_target_installed()
|
||||
{
|
||||
build_helper::warning!("You are building WASM runtime using `wasm32-unknown-unknown` target, although Rust >= 1.84 supports `wasm32v1-none` target!");
|
||||
build_helper::warning!("You can install it with `rustup target add wasm32v1-none --toolchain {toolchain}` if you're using `rustup`.");
|
||||
build_helper::warning!("After installing `wasm32v1-none` target, you must rebuild WASM runtime from scratch, use `cargo clean` before building.");
|
||||
}
|
||||
|
||||
Ok(CargoCommandVersioned::new(cargo_command, version))
|
||||
}
|
||||
@@ -0,0 +1,232 @@
|
||||
// This file is part of Bizinikiwi.
|
||||
|
||||
// Copyright (C) Parity Technologies (UK) Ltd.
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
use std::cmp::Ordering;
|
||||
|
||||
/// The version of rustc/cargo.
|
||||
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
|
||||
pub struct Version {
|
||||
pub major: u32,
|
||||
pub minor: u32,
|
||||
pub patch: u32,
|
||||
pub is_nightly: bool,
|
||||
pub year: Option<u32>,
|
||||
pub month: Option<u32>,
|
||||
pub day: Option<u32>,
|
||||
}
|
||||
|
||||
impl Version {
|
||||
/// Returns if `self` is a stable version.
|
||||
pub fn is_stable(&self) -> bool {
|
||||
!self.is_nightly
|
||||
}
|
||||
|
||||
/// Return if `self` is a nightly version.
|
||||
pub fn is_nightly(&self) -> bool {
|
||||
self.is_nightly
|
||||
}
|
||||
|
||||
/// Extract from the given `version` string.
|
||||
pub fn extract(version: &str) -> Option<Self> {
|
||||
let mut is_nightly = false;
|
||||
let version_parts = version
|
||||
.trim()
|
||||
.split(" ")
|
||||
.nth(1)?
|
||||
.split(".")
|
||||
.filter_map(|v| {
|
||||
if let Some(rest) = v.strip_suffix("-nightly") {
|
||||
is_nightly = true;
|
||||
rest.parse().ok()
|
||||
} else {
|
||||
v.parse().ok()
|
||||
}
|
||||
})
|
||||
.collect::<Vec<u32>>();
|
||||
|
||||
if version_parts.len() != 3 {
|
||||
return None;
|
||||
}
|
||||
|
||||
let date_parts = version
|
||||
.split(" ")
|
||||
.nth(3)
|
||||
.map(|date| {
|
||||
date.split("-")
|
||||
.filter_map(|v| v.trim().strip_suffix(")").unwrap_or(v).parse().ok())
|
||||
.collect::<Vec<u32>>()
|
||||
})
|
||||
.unwrap_or_default();
|
||||
|
||||
Some(Version {
|
||||
major: version_parts[0],
|
||||
minor: version_parts[1],
|
||||
patch: version_parts[2],
|
||||
is_nightly,
|
||||
year: date_parts.get(0).copied(),
|
||||
month: date_parts.get(1).copied(),
|
||||
day: date_parts.get(2).copied(),
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
/// Ordering is done in the following way:
|
||||
///
|
||||
/// 1. `stable` > `nightly`
|
||||
/// 2. Then compare major, minor and patch.
|
||||
/// 3. Last compare the date.
|
||||
impl Ord for Version {
|
||||
fn cmp(&self, other: &Self) -> Ordering {
|
||||
if self == other {
|
||||
return Ordering::Equal;
|
||||
}
|
||||
|
||||
// Ensure that `stable > nightly`
|
||||
if self.is_stable() && other.is_nightly() {
|
||||
return Ordering::Greater;
|
||||
} else if self.is_nightly() && other.is_stable() {
|
||||
return Ordering::Less;
|
||||
}
|
||||
|
||||
let to_compare = [
|
||||
(Some(self.major), Some(other.major)),
|
||||
(Some(self.minor), Some(other.minor)),
|
||||
(Some(self.patch), Some(other.patch)),
|
||||
(self.year, other.year),
|
||||
(self.month, other.month),
|
||||
(self.day, other.day),
|
||||
];
|
||||
|
||||
to_compare
|
||||
.iter()
|
||||
.find_map(|(l, r)| if l != r { l.partial_cmp(&r) } else { None })
|
||||
// We already checked this right at the beginning, so we should never return here
|
||||
// `Equal`.
|
||||
.unwrap_or(Ordering::Equal)
|
||||
}
|
||||
}
|
||||
|
||||
impl PartialOrd for Version {
|
||||
fn partial_cmp(&self, other: &Self) -> Option<Ordering> {
|
||||
Some(self.cmp(other))
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn version_compare_and_extract_works() {
|
||||
let version_1_66_0 = Version::extract("cargo 1.66.0 (d65d197ad 2022-11-15)").unwrap();
|
||||
let version_1_66_1 = Version::extract("cargo 1.66.1 (d65d197ad 2022-11-15)").unwrap();
|
||||
let version_1_66_0_nightly =
|
||||
Version::extract("cargo 1.66.0-nightly (d65d197ad 2022-10-15)").unwrap();
|
||||
let version_1_66_0_nightly_older_date =
|
||||
Version::extract("cargo 1.66.0-nightly (d65d197ad 2022-10-14)").unwrap();
|
||||
let version_1_65_0 = Version::extract("cargo 1.65.0 (d65d197ad 2022-10-15)").unwrap();
|
||||
let version_1_65_0_older_date =
|
||||
Version::extract("cargo 1.65.0 (d65d197ad 2022-10-14)").unwrap();
|
||||
|
||||
assert!(version_1_66_1 > version_1_66_0);
|
||||
assert!(version_1_66_1 > version_1_65_0);
|
||||
assert!(version_1_66_1 > version_1_66_0_nightly);
|
||||
assert!(version_1_66_1 > version_1_66_0_nightly_older_date);
|
||||
assert!(version_1_66_1 > version_1_65_0_older_date);
|
||||
|
||||
assert!(version_1_66_0 > version_1_65_0);
|
||||
assert!(version_1_66_0 > version_1_66_0_nightly);
|
||||
assert!(version_1_66_0 > version_1_66_0_nightly_older_date);
|
||||
assert!(version_1_66_0 > version_1_65_0_older_date);
|
||||
|
||||
assert!(version_1_65_0 > version_1_66_0_nightly);
|
||||
assert!(version_1_65_0 > version_1_66_0_nightly_older_date);
|
||||
assert!(version_1_65_0 > version_1_65_0_older_date);
|
||||
|
||||
let mut versions = vec![
|
||||
version_1_66_0,
|
||||
version_1_66_0_nightly,
|
||||
version_1_66_0_nightly_older_date,
|
||||
version_1_65_0_older_date,
|
||||
version_1_65_0,
|
||||
version_1_66_1,
|
||||
];
|
||||
versions.sort_by(|a, b| b.cmp(a));
|
||||
|
||||
let expected_versions_order = vec![
|
||||
version_1_66_1,
|
||||
version_1_66_0,
|
||||
version_1_65_0,
|
||||
version_1_65_0_older_date,
|
||||
version_1_66_0_nightly,
|
||||
version_1_66_0_nightly_older_date,
|
||||
];
|
||||
assert_eq!(expected_versions_order, versions);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn parse_with_newline() {
|
||||
let version_1_66_0 = Version::extract("cargo 1.66.0 (d65d197ad 2022-11-15)\n").unwrap();
|
||||
assert_eq!(
|
||||
Version {
|
||||
major: 1,
|
||||
minor: 66,
|
||||
patch: 0,
|
||||
is_nightly: false,
|
||||
year: Some(2022),
|
||||
month: Some(11),
|
||||
day: Some(15),
|
||||
},
|
||||
version_1_66_0,
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn version_without_hash_and_date() {
|
||||
// Apparently there are installations that print without the hash and date.
|
||||
let version_1_69_0 = Version::extract("cargo 1.69.0-nightly").unwrap();
|
||||
assert_eq!(
|
||||
Version {
|
||||
major: 1,
|
||||
minor: 69,
|
||||
patch: 0,
|
||||
is_nightly: true,
|
||||
year: None,
|
||||
month: None,
|
||||
day: None,
|
||||
},
|
||||
version_1_69_0,
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn parse_rustc_version() {
|
||||
let version = Version::extract("rustc 1.73.0 (cc66ad468 2023-10-03)").unwrap();
|
||||
assert_eq!(
|
||||
version,
|
||||
Version {
|
||||
major: 1,
|
||||
minor: 73,
|
||||
patch: 0,
|
||||
is_nightly: false,
|
||||
year: Some(2023),
|
||||
month: Some(10),
|
||||
day: Some(03),
|
||||
}
|
||||
);
|
||||
}
|
||||
}
|
||||
File diff suppressed because it is too large
Load Diff
Reference in New Issue
Block a user