Integrate benchmarks and differential tests against an EVM interpreter (#7)

crates/benchmarks/Cargo.toml

@@ -0,0 +1,32 @@
+[package]
+name = "revive-benchmarks"
+version = "0.1.0"
+edition = "2021"
+authors = [
+    "Cyrill Leutwiler <cyrill@parity.io>",
+]
+
+[features]
+default = ["bench-pvm-interpreter"]
+bench-pvm-interpreter = []
+bench-pvm = []
+bench-evm = ["revive-differential"]
+bench-extensive = []
+
+[dependencies]
+hex = { workspace = true }
+polkavm = { workspace = true }
+revive-integration = { path = "../integration" }
+revive-differential = { path = "../differential", optional = true }
+alloy-primitives = { workspace = true }
+
+[dev-dependencies]
+criterion = { workspace = true }
+
+[[bench]]
+name = "execute"
+harness = false
+
+[[bench]]
+name = "prepare"
+harness = false

crates/benchmarks/benches/execute.rs

@@ -0,0 +1,185 @@
+#[cfg(feature = "bench-extensive")]
+use std::time::Duration;
+
+use criterion::{
+    criterion_group, criterion_main, measurement::Measurement, BenchmarkGroup, BenchmarkId,
+    Criterion,
+};
+#[cfg(any(feature = "bench-pvm-interpreter", feature = "bench-pvm"))]
+use polkavm::BackendKind;
+
+use revive_benchmarks::prepare_pvm;
+use revive_integration::cases::Contract;
+
+fn bench<P, L, I, M>(mut group: BenchmarkGroup<'_, M>, parameters: &[P], labels: &[L], contract: I)
+where
+    P: Clone,
+    L: std::fmt::Display,
+    I: Fn(P) -> Contract,
+    M: Measurement,
+{
+    assert_eq!(parameters.len(), labels.len());
+
+    for (p, l) in parameters.iter().zip(labels.iter()) {
+        #[cfg(feature = "bench-evm")]
+        {
+            let contract = contract(p.clone());
+            let vm = revive_differential::prepare(contract.evm_runtime, contract.calldata);
+            group.bench_with_input(BenchmarkId::new("EVM", l), p, move |b, _| {
+                b.iter(|| {
+                    revive_differential::execute(vm.clone());
+                });
+            });
+        }
+
+        #[cfg(feature = "bench-pvm-interpreter")]
+        {
+            let contract = contract(p.clone());
+            let (state, mut instance, export) = prepare_pvm(
+                &contract.pvm_runtime,
+                &contract.calldata,
+                BackendKind::Interpreter,
+            );
+            group.bench_with_input(BenchmarkId::new("PVMInterpreter", l), p, |b, _| {
+                b.iter(|| {
+                    revive_integration::mock_runtime::call(state.clone(), &mut instance, export);
+                });
+            });
+        }
+
+        #[cfg(feature = "bench-pvm")]
+        {
+            let contract = contract(p.clone());
+            let (state, mut instance, export) = prepare_pvm(
+                &contract.pvm_runtime,
+                &contract.calldata,
+                BackendKind::Compiler,
+            );
+            group.bench_with_input(BenchmarkId::new("PVM", l), p, |b, _| {
+                b.iter(|| {
+                    revive_integration::mock_runtime::call(state.clone(), &mut instance, export);
+                });
+            });
+        }
+    }
+
+    group.finish();
+}
+
+#[cfg(feature = "bench-extensive")]
+fn group_extensive<'error, M>(
+    c: &'error mut Criterion<M>,
+    group_name: &str,
+) -> BenchmarkGroup<'error, M>
+where
+    M: Measurement,
+{
+    let mut group = c.benchmark_group(group_name);
+    group
+        .sample_size(10)
+        .measurement_time(Duration::from_secs(60));
+    group
+}
+
+fn bench_baseline(c: &mut Criterion) {
+    let parameters = &[0u8];
+
+    bench(
+        c.benchmark_group("Baseline"),
+        parameters,
+        parameters,
+        |_| Contract::baseline(),
+    );
+}
+
+fn bench_odd_product(c: &mut Criterion) {
+    #[cfg(feature = "bench-extensive")]
+    let group = group_extensive(c, "OddProduct");
+    #[cfg(not(feature = "bench-extensive"))]
+    let group = c.benchmark_group("OddProduct");
+
+    #[cfg(feature = "bench-extensive")]
+    let parameters = &[2_000_000i32, 4_000_000, 8_000_000, 120_000_000];
+    #[cfg(not(feature = "bench-extensive"))]
+    let parameters = &[10_000, 100_000];
+
+    bench(group, parameters, parameters, Contract::odd_product);
+}
+
+fn bench_triangle_number(c: &mut Criterion) {
+    #[cfg(feature = "bench-extensive")]
+    let group = group_extensive(c, "TriangleNumber");
+    #[cfg(not(feature = "bench-extensive"))]
+    let group = c.benchmark_group("TriangleNumber");
+
+    #[cfg(feature = "bench-extensive")]
+    let parameters = &[3_000_000i64, 6_000_000, 12_000_000, 180_000_000];
+    #[cfg(not(feature = "bench-extensive"))]
+    let parameters = &[10_000, 100_000];
+
+    bench(group, parameters, parameters, Contract::triangle_number);
+}
+
+fn bench_fibonacci_recurisve(c: &mut Criterion) {
+    #[cfg(not(feature = "bench-extensive"))]
+    let group = c.benchmark_group("FibonacciRecursive");
+    #[cfg(feature = "bench-extensive")]
+    let group = group_extensive(c, "FibonacciRecursive");
+
+    #[cfg(feature = "bench-extensive")]
+    let parameters = &[26, 30, 34, 38];
+    #[cfg(not(feature = "bench-extensive"))]
+    let parameters = &[12, 16, 20];
+
+    bench(group, parameters, parameters, Contract::fib_recursive);
+}
+
+fn bench_fibonacci_iterative(c: &mut Criterion) {
+    #[cfg(not(feature = "bench-extensive"))]
+    let group = c.benchmark_group("FibonacciIterative");
+    #[cfg(feature = "bench-extensive")]
+    let group = group_extensive(c, "FibonacciIterative");
+
+    #[cfg(feature = "bench-extensive")]
+    let parameters = &[256, 100000, 1000000, 100000000];
+    #[cfg(not(feature = "bench-extensive"))]
+    let parameters = &[64, 128, 256];
+
+    bench(group, parameters, parameters, Contract::fib_iterative);
+}
+
+fn bench_fibonacci_binet(c: &mut Criterion) {
+    let parameters = &[64, 128, 256];
+
+    bench(
+        c.benchmark_group("FibonacciBinet"),
+        parameters,
+        parameters,
+        Contract::fib_binet,
+    );
+}
+
+fn bench_sha1(c: &mut Criterion) {
+    let parameters = &[vec![0xff], vec![0xff; 64], vec![0xff; 512]];
+    let labels = parameters.iter().map(|p| p.len()).collect::<Vec<_>>();
+
+    bench(
+        c.benchmark_group("SHA1"),
+        parameters,
+        &labels,
+        Contract::sha1,
+    );
+}
+
+criterion_group!(
+    name = execute;
+    config = Criterion::default();
+    targets = bench_baseline,
+    bench_odd_product,
+    bench_triangle_number,
+    bench_fibonacci_recurisve,
+    bench_fibonacci_iterative,
+    bench_fibonacci_binet,
+    bench_sha1
+);
+criterion_main!(execute);

crates/benchmarks/benches/prepare.rs

@@ -0,0 +1,172 @@
+use criterion::{criterion_group, criterion_main, BenchmarkId, Criterion};
+use polkavm::BackendKind;
+
+use revive_benchmarks::instantiate_engine;
+use revive_integration::cases::Contract;
+
+fn bench(
+    c: &mut Criterion,
+    group_name: &str,
+    #[cfg(feature = "bench-evm")] evm_runtime: Vec<u8>,
+    #[cfg(any(feature = "bench-pvm-interpreter", feature = "bench-pvm"))] pvm_runtime: Vec<u8>,
+) {
+    let mut group = c.benchmark_group(group_name);
+    let code_size = 0;
+
+    #[cfg(feature = "bench-evm")]
+    group.bench_with_input(
+        BenchmarkId::new("Evm", code_size),
+        &evm_runtime,
+        |b, code| b.iter(|| revive_differential::prepare(code.clone(), Vec::new())),
+    );
+
+    #[cfg(feature = "bench-pvm-interpreter")]
+    {
+        let engine = instantiate_engine(BackendKind::Interpreter);
+        group.bench_with_input(
+            BenchmarkId::new("PVMInterpreterCompile", code_size),
+            &(&pvm_runtime, engine),
+            |b, (code, engine)| {
+                b.iter(|| {
+                    revive_integration::mock_runtime::recompile_code(code, engine);
+                });
+            },
+        );
+    }
+
+    #[cfg(feature = "bench-pvm-interpreter")]
+    {
+        let engine = instantiate_engine(BackendKind::Interpreter);
+        let module = revive_integration::mock_runtime::recompile_code(&pvm_runtime, &engine);
+        group.bench_with_input(
+            BenchmarkId::new("PVMInterpreterInstantiate", code_size),
+            &(module, engine),
+            |b, (module, engine)| {
+                b.iter(|| {
+                    revive_integration::mock_runtime::instantiate_module(module, engine);
+                });
+            },
+        );
+    }
+
+    #[cfg(feature = "bench-pvm")]
+    {
+        let engine = instantiate_engine(BackendKind::Compiler);
+        group.bench_with_input(
+            BenchmarkId::new("PVMCompile", code_size),
+            &(&pvm_runtime, engine),
+            |b, (code, engine)| {
+                b.iter(|| {
+                    revive_integration::mock_runtime::recompile_code(code, engine);
+                });
+            },
+        );
+    }
+
+    #[cfg(feature = "bench-pvm")]
+    {
+        let engine = instantiate_engine(BackendKind::Compiler);
+        let module = revive_integration::mock_runtime::recompile_code(&pvm_runtime, &engine);
+        group.bench_with_input(
+            BenchmarkId::new("PVMInstantiate", code_size),
+            &(module, engine),
+            |b, (module, engine)| {
+                b.iter(|| {
+                    revive_integration::mock_runtime::instantiate_module(module, engine);
+                });
+            },
+        );
+    }
+
+    group.finish();
+}
+
+fn bench_baseline(c: &mut Criterion) {
+    bench(
+        c,
+        "PrepareBaseline",
+        #[cfg(feature = "bench-evm")]
+        Contract::baseline().evm_runtime,
+        #[cfg(any(feature = "bench-pvm-interpreter", feature = "bench-pvm"))]
+        Contract::baseline().pvm_runtime,
+    );
+}
+
+fn bench_odd_product(c: &mut Criterion) {
+    bench(
+        c,
+        "PrepareOddProduct",
+        #[cfg(feature = "bench-evm")]
+        Contract::odd_product(0).evm_runtime,
+        #[cfg(any(feature = "bench-pvm-interpreter", feature = "bench-pvm"))]
+        Contract::baseline().pvm_runtime,
+    );
+}
+
+fn bench_triangle_number(c: &mut Criterion) {
+    bench(
+        c,
+        "PrepareTriangleNumber",
+        #[cfg(feature = "bench-evm")]
+        Contract::triangle_number(0).evm_runtime,
+        #[cfg(any(feature = "bench-pvm-interpreter", feature = "bench-pvm"))]
+        Contract::triangle_number(0).pvm_runtime,
+    );
+}
+
+fn bench_fibonacci_recursive(c: &mut Criterion) {
+    bench(
+        c,
+        "PrepareFibonacciRecursive",
+        #[cfg(feature = "bench-evm")]
+        Contract::fib_recursive(0).evm_runtime,
+        #[cfg(any(feature = "bench-pvm-interpreter", feature = "bench-pvm"))]
+        Contract::fib_recursive(0).pvm_runtime,
+    );
+}
+
+fn bench_fibonacci_iterative(c: &mut Criterion) {
+    bench(
+        c,
+        "PrepareFibonacciIterative",
+        #[cfg(feature = "bench-evm")]
+        Contract::fib_iterative(0).evm_runtime,
+        #[cfg(any(feature = "bench-pvm-interpreter", feature = "bench-pvm"))]
+        Contract::fib_iterative(0).pvm_runtime,
+    );
+}
+
+fn bench_fibonacci_binet(c: &mut Criterion) {
+    bench(
+        c,
+        "PrepareFibonacciBinet",
+        #[cfg(feature = "bench-evm")]
+        Contract::fib_binet(0).evm_runtime,
+        #[cfg(any(feature = "bench-pvm-interpreter", feature = "bench-pvm"))]
+        Contract::fib_binet(0).pvm_runtime,
+    );
+}
+
+fn bench_sha1(c: &mut Criterion) {
+    bench(
+        c,
+        "PrepareSHA1",
+        #[cfg(feature = "bench-evm")]
+        Contract::sha1(Default::default()).evm_runtime,
+        #[cfg(any(feature = "bench-pvm-interpreter", feature = "bench-pvm"))]
+        Contract::sha1(Default::default()).pvm_runtime,
+    );
+}
+
+criterion_group!(
+    name = prepare;
+    config = Criterion::default();
+    targets = bench_baseline,
+    bench_odd_product,
+    bench_triangle_number,
+    bench_fibonacci_recursive,
+    bench_fibonacci_iterative,
+    bench_fibonacci_binet,
+    bench_sha1
+);
+criterion_main!(prepare);

crates/benchmarks/src/lib.rs

@@ -0,0 +1,24 @@
+use polkavm::{BackendKind, Config, Engine, ExportIndex, Instance, SandboxKind};
+use revive_integration::mock_runtime;
+use revive_integration::mock_runtime::State;
+
+pub fn prepare_pvm(
+    code: &[u8],
+    input: &[u8],
+    backend: BackendKind,
+) -> (State, Instance<State>, ExportIndex) {
+    let mut config = Config::new();
+    config.set_backend(Some(backend));
+    config.set_sandbox(Some(SandboxKind::Linux));
+
+    let (instance, export_index) = mock_runtime::prepare(code, Some(config));
+
+    (State::new(input.to_vec()), instance, export_index)
+}
+
+pub fn instantiate_engine(backend: BackendKind) -> Engine {
+    let mut config = Config::new();
+    config.set_backend(Some(backend));
+    config.set_sandbox(Some(SandboxKind::Linux));
+    mock_runtime::setup(Some(config))
+}