switch to pallet_revive runtime (#40)

Signed-off-by: xermicus <cyrill@parity.io>

crates/benchmarks/BENCHMARKS.md

@@ -0,0 +1,73 @@
+# Benchmarks
+
+## Table of Contents
+
+- [Benchmark Results](#benchmark-results)
+    - [Baseline](#baseline)
+    - [OddPorduct](#oddporduct)
+    - [TriangleNumber](#trianglenumber)
+    - [FibonacciRecursive](#fibonaccirecursive)
+    - [FibonacciIterative](#fibonacciiterative)
+    - [FibonacciBinet](#fibonaccibinet)
+    - [SHA1](#sha1)
+
+## Benchmark Results
+
+### Baseline
+
+|         | `EVM`                   | `PVMInterpreter`                 |
+|:--------|:------------------------|:-------------------------------- |
+| **`0`** | `5.97 us` (✅ **1.00x**) | `27.04 us` (❌ *4.53x slower*)    |
+
+### OddPorduct
+
+|              | `EVM`                     | `PVMInterpreter`                 |
+|:-------------|:--------------------------|:-------------------------------- |
+| **`10000`**  | `4.26 ms` (✅ **1.00x**)   | `2.88 ms` (✅ **1.48x faster**)   |
+| **`100000`** | `42.37 ms` (✅ **1.00x**)  | `28.35 ms` (✅ **1.49x faster**)  |
+| **`300000`** | `127.88 ms` (✅ **1.00x**) | `88.43 ms` (✅ **1.45x faster**)  |
+
+### TriangleNumber
+
+|              | `EVM`                     | `PVMInterpreter`                 |
+|:-------------|:--------------------------|:-------------------------------- |
+| **`10000`**  | `2.85 ms` (✅ **1.00x**)   | `2.37 ms` (✅ **1.20x faster**)   |
+| **`100000`** | `27.85 ms` (✅ **1.00x**)  | `23.01 ms` (✅ **1.21x faster**)  |
+| **`360000`** | `103.01 ms` (✅ **1.00x**) | `83.66 ms` (✅ **1.23x faster**)  |
+
+### FibonacciRecursive
+
+|          | `EVM`                     | `PVMInterpreter`                  |
+|:---------|:--------------------------|:--------------------------------- |
+| **`12`** | `195.19 us` (✅ **1.00x**) | `333.53 us` (❌ *1.71x slower*)    |
+| **`16`** | `1.22 ms` (✅ **1.00x**)   | `1.97 ms` (❌ *1.62x slower*)      |
+| **`20`** | `8.14 ms` (✅ **1.00x**)   | `13.20 ms` (❌ *1.62x slower*)     |
+| **`24`** | `55.09 ms` (✅ **1.00x**)  | `88.56 ms` (❌ *1.61x slower*)     |
+
+### FibonacciIterative
+
+|           | `EVM`                    | `PVMInterpreter`                  |
+|:----------|:-------------------------|:--------------------------------- |
+| **`64`**  | `33.39 us` (✅ **1.00x**) | `86.02 us` (❌ *2.58x slower*)     |
+| **`128`** | `52.91 us` (✅ **1.00x**) | `126.38 us` (❌ *2.39x slower*)    |
+| **`256`** | `82.33 us` (✅ **1.00x**) | `208.74 us` (❌ *2.54x slower*)    |
+
+### FibonacciBinet
+
+|           | `EVM`                    | `PVMInterpreter`                  |
+|:----------|:-------------------------|:--------------------------------- |
+| **`64`**  | `32.29 us` (✅ **1.00x**) | `161.75 us` (❌ *5.01x slower*)    |
+| **`128`** | `36.02 us` (✅ **1.00x**) | `172.59 us` (❌ *4.79x slower*)    |
+| **`256`** | `41.21 us` (✅ **1.00x**) | `185.30 us` (❌ *4.50x slower*)    |
+
+### SHA1
+
+|           | `EVM`                     | `PVMInterpreter`                  |
+|:----------|:--------------------------|:--------------------------------- |
+| **`1`**   | `160.17 us` (✅ **1.00x**) | `403.46 us` (❌ *2.52x slower*)    |
+| **`64`**  | `286.69 us` (✅ **1.00x**) | `479.79 us` (❌ *1.67x slower*)    |
+| **`512`** | `1.18 ms` (✅ **1.00x**)   | `1.37 ms` (❌ *1.16x slower*)      |
+
+---
+Made with [criterion-table](https://github.com/nu11ptr/criterion-table)
+

crates/benchmarks/Cargo.toml

@@ -9,17 +9,16 @@ description = "revive compiler benchmarks"
 
 [features]
 default = ["bench-pvm-interpreter"]
-bench-pvm-interpreter = []
-bench-pvm = []
+bench-pvm-interpreter = ["revive-runner"]
 bench-evm = ["revive-differential"]
-bench-extensive = []
 
 [dependencies]
 hex = { workspace = true }
-polkavm = { workspace = true }
+alloy-primitives = { workspace = true }
+
 revive-integration = { workspace = true }
 revive-differential = { workspace = true, optional = true }
-alloy-primitives = { workspace = true }
+revive-runner = { workspace = true, optional = true }
 
 [dev-dependencies]
 criterion = { workspace = true }
@@ -27,7 +26,3 @@ criterion = { workspace = true }
 [[bench]]
 name = "execute"
 harness = false
-
-[[bench]]
-name = "prepare"
-harness = false

crates/benchmarks/benches/execute.rs

@@ -1,104 +1,42 @@
+#![cfg(any(feature = "bench-pvm-interpreter", feature = "bench-evm"))]
+
 use alloy_primitives::U256;
 use criterion::{
-    criterion_group, criterion_main, measurement::Measurement, BenchmarkGroup, BenchmarkId,
-    Criterion,
+    criterion_group, criterion_main,
+    measurement::{Measurement, WallTime},
+    BenchmarkGroup, BenchmarkId, Criterion,
 };
 use revive_integration::cases::Contract;
 
-fn bench<P, L, I, M>(mut group: BenchmarkGroup<'_, M>, parameters: &[P], labels: &[L], contract: I)
-where
+fn bench<P, L, I>(
+    mut group: BenchmarkGroup<'_, WallTime>,
+    parameters: &[P],
+    labels: &[L],
+    contract: I,
+) where
     P: Clone,
     L: std::fmt::Display,
     I: Fn(P) -> Contract,
-    M: Measurement,
 {
     assert_eq!(parameters.len(), labels.len());
 
+    group.sample_size(10);
+
     for (p, l) in parameters.iter().zip(labels.iter()) {
+        let contract = contract(p.clone());
+
         #[cfg(feature = "bench-evm")]
-        {
-            let contract = contract(p.clone());
-            group.bench_with_input(BenchmarkId::new("EVM", l), p, move |b, _| {
-                b.iter(|| {
-                    revive_differential::execute(revive_differential::prepare(
-                        contract.evm_runtime.clone(),
-                        contract.calldata.clone(),
-                    ));
-                });
-            });
-        }
+        group.bench_with_input(BenchmarkId::new("EVM", l), p, |b, _| {
+            let code = &contract.evm_runtime;
+            let input = &contract.calldata;
+            b.iter_custom(|iters| revive_benchmarks::measure_evm(code, input, iters));
+        });
 
-        #[cfg(not(feature = "bench-extensive"))]
-        {
-            #[cfg(all(feature = "bench-pvm-interpreter", not(feature = "bench-extensive")))]
-            {
-                let contract = contract(p.clone());
-                let (transaction, mut instance, export) = revive_benchmarks::prepare_pvm(
-                    &contract.pvm_runtime,
-                    contract.calldata,
-                    polkavm::BackendKind::Interpreter,
-                );
-                group.bench_with_input(BenchmarkId::new("PVMInterpreter", l), p, |b, _| {
-                    b.iter(|| {
-                        let _ = transaction.clone().call_on(&mut instance, export);
-                    });
-                });
-            }
-
-            #[cfg(all(feature = "bench-pvm", not(feature = "bench-extensive")))]
-            {
-                let contract = contract(p.clone());
-                let (transaction, mut instance, export) = revive_benchmarks::prepare_pvm(
-                    &contract.pvm_runtime,
-                    contract.calldata,
-                    polkavm::BackendKind::Compiler,
-                );
-                group.bench_with_input(BenchmarkId::new("PVM", l), p, |b, _| {
-                    b.iter(|| {
-                        let _ = transaction.clone().call_on(&mut instance, export);
-                    });
-                });
-            }
-        }
-        #[cfg(feature = "bench-extensive")]
-        {
-            use revive_benchmarks::instantiate_engine;
-            use revive_integration::mock_runtime::{instantiate_module, recompile_code, State};
-
-            #[cfg(feature = "bench-pvm-interpreter")]
-            {
-                let contract = contract(p.clone());
-                let engine = instantiate_engine(polkavm::BackendKind::Interpreter);
-                let module = recompile_code(&contract.pvm_runtime, &engine);
-                let transaction = State::default()
-                    .transaction()
-                    .with_default_account(&contract.pvm_runtime)
-                    .calldata(contract.calldata);
-                group.bench_with_input(BenchmarkId::new("PVMInterpreter", l), p, |b, _| {
-                    b.iter(|| {
-                        let (mut instance, export) = instantiate_module(&module, &engine);
-                        let _ = transaction.clone().call_on(&mut instance, export);
-                    });
-                });
-            }
-
-            #[cfg(feature = "bench-pvm")]
-            {
-                let contract = contract(p.clone());
-                let engine = instantiate_engine(polkavm::BackendKind::Compiler);
-                let module = recompile_code(&contract.pvm_runtime, &engine);
-                let transaction = State::default()
-                    .transaction()
-                    .with_default_account(&contract.pvm_runtime)
-                    .calldata(contract.calldata);
-                group.bench_with_input(BenchmarkId::new("PVM", l), p, |b, _| {
-                    b.iter(|| {
-                        let (mut instance, export) = instantiate_module(&module, &engine);
-                        let _ = transaction.clone().call_on(&mut instance, export);
-                    });
-                });
-            }
-        }
+        #[cfg(feature = "bench-pvm-interpreter")]
+        group.bench_with_input(BenchmarkId::new("PVMInterpreter", l), p, |b, _| {
+            let specs = revive_benchmarks::create_specs(&contract);
+            b.iter_custom(|iters| revive_benchmarks::measure_pvm(&specs, iters));
+        });
     }
 
     group.finish();
@@ -108,14 +46,6 @@ fn group<'error, M>(c: &'error mut Criterion<M>, group_name: &str) -> BenchmarkG
 where
     M: Measurement,
 {
-    #[cfg(feature = "bench-extensive")]
-    {
-        let mut group = c.benchmark_group(group_name);
-        group.sample_size(10);
-        group
-    }
-
-    #[cfg(not(feature = "bench-extensive"))]
     return c.benchmark_group(group_name);
 }
 
@@ -128,33 +58,21 @@ fn bench_baseline(c: &mut Criterion) {
 
 fn bench_odd_product(c: &mut Criterion) {
     let group = group(c, "OddPorduct");
-    #[cfg(feature = "bench-extensive")]
-    let parameters = &[300000, 1200000, 12000000, 180000000, 720000000];
-    #[cfg(not(feature = "bench-extensive"))]
-    let parameters = &[10_000, 100_000];
+    let parameters = &[10_000, 100_000, 300000];
 
     bench(group, parameters, parameters, Contract::odd_product);
 }
 
 fn bench_triangle_number(c: &mut Criterion) {
     let group = group(c, "TriangleNumber");
-    #[cfg(feature = "bench-extensive")]
-    let parameters = &[360000, 1440000, 14400000, 216000000, 864000000];
-    #[cfg(not(feature = "bench-extensive"))]
-    let parameters = &[10_000, 100_000];
+    let parameters = &[10_000, 100_000, 360000];
 
     bench(group, parameters, parameters, Contract::triangle_number);
 }
 
 fn bench_fibonacci_recurisve(c: &mut Criterion) {
     let group = group(c, "FibonacciRecursive");
-    #[cfg(feature = "bench-extensive")]
-    let parameters = [24, 27, 31, 36, 39]
-        .iter()
-        .map(|p| U256::from(*p))
-        .collect::<Vec<_>>();
-    #[cfg(not(feature = "bench-extensive"))]
-    let parameters = [12, 16, 20]
+    let parameters = [12, 16, 20, 24]
         .iter()
         .map(|p| U256::from(*p))
         .collect::<Vec<_>>();
@@ -164,12 +82,6 @@ fn bench_fibonacci_recurisve(c: &mut Criterion) {
 
 fn bench_fibonacci_iterative(c: &mut Criterion) {
     let group = group(c, "FibonacciIterative");
-    #[cfg(feature = "bench-extensive")]
-    let parameters = [256, 162500, 650000, 6500000, 100000000, 400000000]
-        .iter()
-        .map(|p| U256::from(*p))
-        .collect::<Vec<_>>();
-    #[cfg(not(feature = "bench-extensive"))]
     let parameters = [64, 128, 256]
         .iter()
         .map(|p| U256::from(*p))
@@ -193,7 +105,9 @@ fn bench_sha1(c: &mut Criterion) {
     let parameters = &[vec![0xff], vec![0xff; 64], vec![0xff; 512]];
     let labels = parameters.iter().map(|p| p.len()).collect::<Vec<_>>();
 
-    bench(group, parameters, &labels, Contract::sha1);
+    bench(group, parameters, &labels, |input| {
+        Contract::sha1(input.into())
+    });
 }
 
 criterion_group!(

crates/benchmarks/benches/prepare.rs

@@ -1,171 +0,0 @@
-use alloy_primitives::U256;
-use criterion::{criterion_group, criterion_main, BenchmarkId, Criterion};
-
-use revive_integration::cases::Contract;
-
-fn bench(
-    c: &mut Criterion,
-    group_name: &str,
-    #[cfg(feature = "bench-evm")] evm_runtime: Vec<u8>,
-    #[cfg(any(feature = "bench-pvm-interpreter", feature = "bench-pvm"))] pvm_runtime: Vec<u8>,
-) {
-    let mut group = c.benchmark_group(group_name);
-    let code_size = 0;
-
-    #[cfg(feature = "bench-evm")]
-    group.bench_with_input(
-        BenchmarkId::new("Evm", code_size),
-        &evm_runtime,
-        |b, code| b.iter(|| revive_differential::prepare(code.clone(), Vec::new())),
-    );
-
-    #[cfg(feature = "bench-pvm-interpreter")]
-    {
-        let engine = revive_benchmarks::instantiate_engine(polkavm::BackendKind::Interpreter);
-        group.bench_with_input(
-            BenchmarkId::new("PVMInterpreterCompile", code_size),
-            &(&pvm_runtime, engine),
-            |b, (code, engine)| {
-                b.iter(|| {
-                    revive_integration::mock_runtime::recompile_code(code, engine);
-                });
-            },
-        );
-    }
-
-    #[cfg(feature = "bench-pvm-interpreter")]
-    {
-        let engine = revive_benchmarks::instantiate_engine(polkavm::BackendKind::Interpreter);
-        let module = revive_integration::mock_runtime::recompile_code(&pvm_runtime, &engine);
-        group.bench_with_input(
-            BenchmarkId::new("PVMInterpreterInstantiate", code_size),
-            &(module, engine),
-            |b, (module, engine)| {
-                b.iter(|| {
-                    revive_integration::mock_runtime::instantiate_module(module, engine);
-                });
-            },
-        );
-    }
-
-    #[cfg(feature = "bench-pvm")]
-    {
-        let engine = revive_benchmarks::instantiate_engine(polkavm::BackendKind::Compiler);
-        group.bench_with_input(
-            BenchmarkId::new("PVMCompile", code_size),
-            &(&pvm_runtime, engine),
-            |b, (code, engine)| {
-                b.iter(|| {
-                    revive_integration::mock_runtime::recompile_code(code, engine);
-                });
-            },
-        );
-    }
-
-    #[cfg(feature = "bench-pvm")]
-    {
-        let engine = revive_benchmarks::instantiate_engine(polkavm::BackendKind::Compiler);
-        let module = revive_integration::mock_runtime::recompile_code(&pvm_runtime, &engine);
-        group.bench_with_input(
-            BenchmarkId::new("PVMInstantiate", code_size),
-            &(module, engine),
-            |b, (module, engine)| {
-                b.iter(|| {
-                    revive_integration::mock_runtime::instantiate_module(module, engine);
-                });
-            },
-        );
-    }
-
-    group.finish();
-}
-
-fn bench_baseline(c: &mut Criterion) {
-    bench(
-        c,
-        "PrepareBaseline",
-        #[cfg(feature = "bench-evm")]
-        Contract::baseline().evm_runtime,
-        #[cfg(any(feature = "bench-pvm-interpreter", feature = "bench-pvm"))]
-        Contract::baseline().pvm_runtime,
-    );
-}
-
-fn bench_odd_product(c: &mut Criterion) {
-    bench(
-        c,
-        "PrepareOddProduct",
-        #[cfg(feature = "bench-evm")]
-        Contract::odd_product(0).evm_runtime,
-        #[cfg(any(feature = "bench-pvm-interpreter", feature = "bench-pvm"))]
-        Contract::baseline().pvm_runtime,
-    );
-}
-
-fn bench_triangle_number(c: &mut Criterion) {
-    bench(
-        c,
-        "PrepareTriangleNumber",
-        #[cfg(feature = "bench-evm")]
-        Contract::triangle_number(0).evm_runtime,
-        #[cfg(any(feature = "bench-pvm-interpreter", feature = "bench-pvm"))]
-        Contract::triangle_number(0).pvm_runtime,
-    );
-}
-
-fn bench_fibonacci_recursive(c: &mut Criterion) {
-    bench(
-        c,
-        "PrepareFibonacciRecursive",
-        #[cfg(feature = "bench-evm")]
-        Contract::fib_recursive(U256::ZERO).evm_runtime,
-        #[cfg(any(feature = "bench-pvm-interpreter", feature = "bench-pvm"))]
-        Contract::fib_recursive(U256::ZERO).pvm_runtime,
-    );
-}
-
-fn bench_fibonacci_iterative(c: &mut Criterion) {
-    bench(
-        c,
-        "PrepareFibonacciIterative",
-        #[cfg(feature = "bench-evm")]
-        Contract::fib_iterative(U256::ZERO).evm_runtime,
-        #[cfg(any(feature = "bench-pvm-interpreter", feature = "bench-pvm"))]
-        Contract::fib_iterative(U256::ZERO).pvm_runtime,
-    );
-}
-
-fn bench_fibonacci_binet(c: &mut Criterion) {
-    bench(
-        c,
-        "PrepareFibonacciBinet",
-        #[cfg(feature = "bench-evm")]
-        Contract::fib_binet(U256::ZERO).evm_runtime,
-        #[cfg(any(feature = "bench-pvm-interpreter", feature = "bench-pvm"))]
-        Contract::fib_binet(U256::ZERO).pvm_runtime,
-    );
-}
-
-fn bench_sha1(c: &mut Criterion) {
-    bench(
-        c,
-        "PrepareSHA1",
-        #[cfg(feature = "bench-evm")]
-        Contract::sha1(Default::default()).evm_runtime,
-        #[cfg(any(feature = "bench-pvm-interpreter", feature = "bench-pvm"))]
-        Contract::sha1(Default::default()).pvm_runtime,
-    );
-}
-
-criterion_group!(
-    name = prepare;
-    config = Criterion::default();
-    targets = bench_baseline,
-    bench_odd_product,
-    bench_triangle_number,
-    bench_fibonacci_recursive,
-    bench_fibonacci_iterative,
-    bench_fibonacci_binet,
-    bench_sha1
-);
-criterion_main!(prepare);

crates/benchmarks/src/lib.rs

@@ -1,28 +1,71 @@
-use polkavm::{BackendKind, Config, Engine, ExportIndex, Instance, SandboxKind};
-use revive_integration::mock_runtime::{self, TransactionBuilder};
-use revive_integration::mock_runtime::{State, Transaction};
-
-pub fn prepare_pvm(
-    code: &[u8],
-    input: Vec<u8>,
-    backend: BackendKind,
-) -> (TransactionBuilder, Instance<Transaction>, ExportIndex) {
-    let mut config = Config::new();
-    config.set_backend(Some(backend));
-    config.set_sandbox(Some(SandboxKind::Linux));
-
-    let (instance, export_index) = mock_runtime::prepare(code, Some(config));
-    let transaction = State::default()
-        .transaction()
-        .with_default_account(code)
-        .calldata(input);
-
-    (transaction, instance, export_index)
+#[cfg(feature = "bench-pvm-interpreter")]
+pub fn create_specs(contract: &revive_integration::cases::Contract) -> revive_runner::Specs {
+    use revive_runner::*;
+    use SpecsAction::*;
+    Specs {
+        differential: false,
+        actions: vec![
+            Instantiate {
+                code: Code::Bytes(contract.pvm_runtime.to_vec()),
+                origin: TestAccountId::Alice,
+                data: Default::default(),
+                value: Default::default(),
+                gas_limit: Default::default(),
+                storage_deposit_limit: Default::default(),
+                salt: Default::default(),
+            },
+            Call {
+                origin: TestAccountId::Alice,
+                dest: TestAccountId::Instantiated(0),
+                data: contract.calldata.to_vec(),
+                value: Default::default(),
+                gas_limit: Default::default(),
+                storage_deposit_limit: Default::default(),
+            },
+        ],
+        ..Default::default()
+    }
 }
 
-pub fn instantiate_engine(backend: BackendKind) -> Engine {
-    let mut config = Config::new();
-    config.set_backend(Some(backend));
-    config.set_sandbox(Some(SandboxKind::Linux));
-    mock_runtime::setup(Some(config))
+#[cfg(feature = "bench-pvm-interpreter")]
+pub fn measure_pvm(specs: &revive_runner::Specs, iters: u64) -> std::time::Duration {
+    use revive_runner::*;
+    let mut total_time = std::time::Duration::default();
+
+    for _ in 0..iters {
+        let results = specs.clone().run();
+
+        let CallResult::Exec { result, wall_time } =
+            results.get(1).expect("contract should have been called")
+        else {
+            panic!("expected a execution result");
+        };
+        let ret = result.result.as_ref().unwrap();
+        assert!(!ret.did_revert());
+
+        total_time += *wall_time;
+    }
+
+    total_time
+}
+
+#[cfg(feature = "bench-evm")]
+pub fn measure_evm(code: &[u8], input: &[u8], iters: u64) -> std::time::Duration {
+    let mut total_time = std::time::Duration::default();
+
+    let code = hex::encode(code);
+
+    for _ in 0..iters {
+        let log = revive_differential::Evm::default()
+            .code_blob(code.as_bytes().to_vec())
+            .input(input.to_vec().into())
+            .genesis_path("/tmp/genesis.json".into())
+            .bench(true)
+            .run();
+        assert!(log.output.run_success(), "evm run failed: {log:?}");
+
+        total_time += log.execution_time().unwrap();
+    }
+
+    total_time
 }