Create Script to Run All Benchmarks (#11493)

* Create run_all_benchmarks.sh

* Update run_all_benchmarks.sh

* Update run_all_benchmarks.sh

* Review fixes

Signed-off-by: Oliver Tale-Yazdi <oliver.tale-yazdi@parity.io>

* Update scripts/run_all_benchmarks.sh

Co-authored-by: Bastian Köcher <bkchr@users.noreply.github.com>

* typo

Signed-off-by: Oliver Tale-Yazdi <oliver.tale-yazdi@parity.io>

* add default for $1

* Typo

Signed-off-by: Oliver Tale-Yazdi <oliver.tale-yazdi@parity.io>

* Update run_all_benchmarks.sh

* new weights on benchmarking machine

* prefer `--chain=dev`

* fix compile

* fix command

* fmt

* dont use square brackets

* Extend doc

Signed-off-by: Oliver Tale-Yazdi <oliver.tale-yazdi@parity.io>

* Remove +nightly

Signed-off-by: Oliver Tale-Yazdi <oliver.tale-yazdi@parity.io>

* Add error file an run execute everything optimistically

Signed-off-by: Oliver Tale-Yazdi <oliver.tale-yazdi@parity.io>

Co-authored-by: Oliver Tale-Yazdi <oliver.tale-yazdi@parity.io>
Co-authored-by: Bastian Köcher <bkchr@users.noreply.github.com>
This commit is contained in:
Shawn Tabrizi
2022-05-23 12:56:42 -04:00
committed by GitHub
parent 8d3f692e0c
commit d9f33dc7e9
43 changed files with 2317 additions and 2169 deletions
@@ -15,32 +15,61 @@
// See the License for the specific language governing permissions and
// limitations under the License.
pub mod constants {
use frame_support::{
parameter_types,
weights::{constants, Weight},
};
//! THIS FILE WAS AUTO-GENERATED USING THE SUBSTRATE BENCHMARK CLI VERSION 4.0.0-dev
//! DATE: 2022-05-23 (Y/M/D)
//!
//! SHORT-NAME: `block`, LONG-NAME: `BlockExecution`, RUNTIME: `Development`
//! WARMUPS: `10`, REPEAT: `100`
//! WEIGHT-PATH: `./frame/support/src/weights/`
//! WEIGHT-METRIC: `Average`, WEIGHT-MUL: `1`, WEIGHT-ADD: `0`
parameter_types! {
/// Importing a block with 0 Extrinsics.
pub const BlockExecutionWeight: Weight = 5_000_000 * constants::WEIGHT_PER_NANOS;
}
// Executed Command:
// ./target/production/substrate
// benchmark
// overhead
// --dev
// --execution=wasm
// --wasm-execution=compiled
// --weight-path=./frame/support/src/weights/
// --warmup=10
// --repeat=100
#[cfg(test)]
mod test_weights {
use frame_support::weights::constants;
use frame_support::{
parameter_types,
weights::{constants::WEIGHT_PER_NANOS, Weight},
};
/// Checks that the weight exists and is sane.
// NOTE: If this test fails but you are sure that the generated values are fine,
// you can delete it.
#[test]
fn sane() {
let w = super::constants::BlockExecutionWeight::get();
parameter_types! {
/// Time to execute an empty block.
/// Calculated by multiplying the *Average* with `1` and adding `0`.
///
/// Stats nanoseconds:
/// Min, Max: 5_295_788, 5_473_440
/// Average: 5_343_308
/// Median: 5_323_240
/// Std-Dev: 38368.68
///
/// Percentiles nanoseconds:
/// 99th: 5_470_141
/// 95th: 5_418_269
/// 75th: 5_355_579
pub const BlockExecutionWeight: Weight = 5_343_308 * WEIGHT_PER_NANOS;
}
// At least 100 µs.
assert!(w >= 100 * constants::WEIGHT_PER_MICROS, "Weight should be at least 100 µs.");
// At most 50 ms.
assert!(w <= 50 * constants::WEIGHT_PER_MILLIS, "Weight should be at most 50 ms.");
}
#[cfg(test)]
mod test_weights {
use frame_support::weights::constants;
/// Checks that the weight exists and is sane.
// NOTE: If this test fails but you are sure that the generated values are fine,
// you can delete it.
#[test]
fn sane() {
let w = super::BlockExecutionWeight::get();
// At least 100 µs.
assert!(w >= 100 * constants::WEIGHT_PER_MICROS, "Weight should be at least 100 µs.");
// At most 50 ms.
assert!(w <= 50 * constants::WEIGHT_PER_MILLIS, "Weight should be at most 50 ms.");
}
}
@@ -15,32 +15,61 @@
// See the License for the specific language governing permissions and
// limitations under the License.
pub mod constants {
use frame_support::{
parameter_types,
weights::{constants, Weight},
};
//! THIS FILE WAS AUTO-GENERATED USING THE SUBSTRATE BENCHMARK CLI VERSION 4.0.0-dev
//! DATE: 2022-05-23 (Y/M/D)
//!
//! SHORT-NAME: `extrinsic`, LONG-NAME: `ExtrinsicBase`, RUNTIME: `Development`
//! WARMUPS: `10`, REPEAT: `100`
//! WEIGHT-PATH: `./frame/support/src/weights/`
//! WEIGHT-METRIC: `Average`, WEIGHT-MUL: `1`, WEIGHT-ADD: `0`
parameter_types! {
/// Executing a NO-OP `System::remarks` Extrinsic.
pub const ExtrinsicBaseWeight: Weight = 125_000 * constants::WEIGHT_PER_NANOS;
}
// Executed Command:
// ./target/production/substrate
// benchmark
// overhead
// --dev
// --execution=wasm
// --wasm-execution=compiled
// --weight-path=./frame/support/src/weights/
// --warmup=10
// --repeat=100
#[cfg(test)]
mod test_weights {
use frame_support::weights::constants;
use frame_support::{
parameter_types,
weights::{constants::WEIGHT_PER_NANOS, Weight},
};
/// Checks that the weight exists and is sane.
// NOTE: If this test fails but you are sure that the generated values are fine,
// you can delete it.
#[test]
fn sane() {
let w = super::constants::ExtrinsicBaseWeight::get();
parameter_types! {
/// Time to execute a NO-OP extrinsic, for example `System::remark`.
/// Calculated by multiplying the *Average* with `1` and adding `0`.
///
/// Stats nanoseconds:
/// Min, Max: 85_423, 86_142
/// Average: 85_795
/// Median: 85_790
/// Std-Dev: 162.37
///
/// Percentiles nanoseconds:
/// 99th: 86_115
/// 95th: 86_069
/// 75th: 85_937
pub const ExtrinsicBaseWeight: Weight = 85_795 * WEIGHT_PER_NANOS;
}
// At least 10 µs.
assert!(w >= 10 * constants::WEIGHT_PER_MICROS, "Weight should be at least 10 µs.");
// At most 1 ms.
assert!(w <= constants::WEIGHT_PER_MILLIS, "Weight should be at most 1 ms.");
}
#[cfg(test)]
mod test_weights {
use frame_support::weights::constants;
/// Checks that the weight exists and is sane.
// NOTE: If this test fails but you are sure that the generated values are fine,
// you can delete it.
#[test]
fn sane() {
let w = super::ExtrinsicBaseWeight::get();
// At least 10 µs.
assert!(w >= 10 * constants::WEIGHT_PER_MICROS, "Weight should be at least 10 µs.");
// At most 1 ms.
assert!(w <= constants::WEIGHT_PER_MILLIS, "Weight should be at most 1 ms.");
}
}