Add benchmarks for wasmi builtin metering (#56)

* Upgraded to newest wasmi
* Refactored benchmarks
* Two new benchmark strategies (`no_metering` and `wasmi_builtin`)

We can now benchmark the execution of modules using our two
instrumentation strategies in addition to no metering (as a baseline)
and wasmi's builtin metering.

We can learn from the following results (ran on my M1) that the builtin
metering decisively outperforms the instrumentation on every single
fixture.

cc @Robbepop 

```
coremark/no_metering                    [15.586 s 15.588 s 15.589 s]
coremark/wasmi_builtin                  [16.403 s 16.414 s 16.434 s]
coremark/host_function                  [18.245 s 18.248 s 18.252 s]
coremark/mutable_global                 [20.476 s 20.486 s 20.505 s]

recursive_ok/no_metering                [111.32 µs 111.33 µs 111.34 µs]
recursive_ok/wasmi_builtin              [138.64 µs 138.65 µs 138.66 µs]
recursive_ok/host_function              [495.55 µs 495.64 µs 495.78 µs]
recursive_ok/mutable_global             [514.07 µs 514.09 µs 514.11 µs]

fibonacci_recursive/no_metering         [3.9098 µs 3.9102 µs 3.9108 µs]
fibonacci_recursive/wasmi_builtin       [4.3242 µs 4.3246 µs 4.3250 µs]
fibonacci_recursive/host_function       [12.913 µs 12.914 µs 12.915 µs]
fibonacci_recursive/mutable_global      [13.202 µs 13.208 µs 13.212 µs]
              
factorial_recursive/no_metering         [530.72 ns 530.84 ns 530.91 ns]
factorial_recursive/wasmi_builtin       [619.17 ns 619.30 ns 619.44 ns]
factorial_recursive/host_function       [1.7656 µs 1.7657 µs 1.7659 µs]
factorial_recursive/mutable_global      [1.8783 µs 1.8786 µs 1.8788 µs]

count_until/no_metering                 [1.2422 ms 1.2423 ms 1.2424 ms]
count_until/wasmi_builtin               [1.3976 ms 1.3978 ms 1.3981 ms]
count_until/host_function               [4.8074 ms 4.8106 ms 4.8125 ms]
count_until/mutable_global              [5.9161 ms 5.9169 ms 5.9182 ms]

memory_vec_add/no_metering              [4.1630 ms 4.1638 ms 4.1648 ms]
memory_vec_add/wasmi_builtin            [4.3913 ms 4.3925 ms 4.3930 ms]
memory_vec_add/host_function            [8.2925 ms 8.2949 ms 8.2967 ms]
memory_vec_add/mutable_global           [9.1124 ms 9.1152 ms 9.1163 ms]

wasm_kernel::tiny_keccak/no_metering    [613.21 µs 613.42 µs 613.58 µs]
wasm_kernel::tiny_keccak/wasmi_builtin  [617.04 µs 617.46 µs 617.81 µs]
wasm_kernel::tiny_keccak/host_function  [817.24 µs 817.44 µs 817.89 µs]
wasm_kernel::tiny_keccak/mutable_global [873.42 µs 873.90 µs 874.65 µs]

global_bump/no_metering                 [1.4597 ms 1.4598 ms 1.4600 ms]
global_bump/wasmi_builtin               [1.6151 ms 1.6152 ms 1.6153 ms]
global_bump/host_function               [5.5393 ms 5.5418 ms 5.5435 ms]
global_bump/mutable_global              [6.9446 ms 6.9454 ms 6.9461 ms]
```

Cargo.toml

@@ -12,9 +12,14 @@ repository = "https://github.com/paritytech/wasm-instrument"
 include = ["src/**/*", "LICENSE-*", "README.md"]
 
 [[bench]]
-name = "benches"
+name = "instrumentation"
 harness = false
-path = "benches/benches.rs"
+path = "benches/instrumentation.rs"
+
+[[bench]]
+name = "execution"
+harness = false
+path = "benches/execution.rs"
 
 [profile.bench]
 lto = "fat"
@@ -32,7 +37,7 @@ rand = "0.8"
 wat = "1"
 wasmparser = "0.101"
 wasmprinter = "0.2"
-wasmi = "0.22"
+wasmi = "0.28"
 
 [features]
 default = ["std"]

benches/benches.rs

@@ -1,574 +0,0 @@
-use criterion::{
-	criterion_group, criterion_main, measurement::Measurement, Bencher, BenchmarkGroup, Criterion,
-	Throughput,
-};
-use std::{
-	fs::{read, read_dir},
-	path::PathBuf,
-	slice,
-};
-use wasm_instrument::{
-	gas_metering::{self, host_function, mutable_global, Backend, ConstantCostRules},
-	inject_stack_limiter,
-	parity_wasm::{deserialize_buffer, elements::Module, serialize},
-};
-
-fn fixture_dir() -> PathBuf {
-	let mut path = PathBuf::from(env!("CARGO_MANIFEST_DIR"));
-	path.push("benches");
-	path.push("fixtures");
-	path.push("wasm");
-	path
-}
-
-fn any_fixture<F, M>(group: &mut BenchmarkGroup<M>, f: F)
-where
-	F: Fn(Module),
-	M: Measurement,
-{
-	for entry in read_dir(fixture_dir()).unwrap() {
-		let entry = entry.unwrap();
-		let bytes = read(entry.path()).unwrap();
-		group.throughput(Throughput::Bytes(bytes.len().try_into().unwrap()));
-		group.bench_with_input(entry.file_name().to_str().unwrap(), &bytes, |bench, input| {
-			bench.iter(|| f(deserialize_buffer(input).unwrap()))
-		});
-	}
-}
-
-fn gas_metering(c: &mut Criterion) {
-	let mut group = c.benchmark_group("Gas Metering");
-	any_fixture(&mut group, |module| {
-		gas_metering::inject(
-			module,
-			host_function::Injector::new("env", "gas"),
-			&ConstantCostRules::default(),
-		)
-		.unwrap();
-	});
-}
-
-fn stack_height_limiter(c: &mut Criterion) {
-	let mut group = c.benchmark_group("Stack Height Limiter");
-	any_fixture(&mut group, |module| {
-		inject_stack_limiter(module, 128).unwrap();
-	});
-}
-
-use std::time::{Duration, SystemTime, UNIX_EPOCH};
-use wasmi::{
-	self,
-	core::{Pages, Value, F32},
-	Caller, Config, Engine, Extern, Func, Instance, Linker, Memory, StackLimits, Store,
-};
-fn prepare_module<P: Backend>(backend: P, input: &[u8]) -> (wasmi::Module, Store<u64>) {
-	let module = deserialize_buffer(input).unwrap();
-	let instrumented_module =
-		gas_metering::inject(module, backend, &ConstantCostRules::default()).unwrap();
-	let input = serialize(instrumented_module).unwrap();
-	// Prepare wasmi
-	let engine = Engine::new(&bench_config());
-	let module = wasmi::Module::new(&engine, &mut &input[..]).unwrap();
-	// Init host state with maximum gas_left
-	let store = Store::new(&engine, u64::MAX);
-
-	(module, store)
-}
-
-fn add_gas_host_func(linker: &mut Linker<u64>, store: &mut Store<u64>) {
-	// Create gas host function
-	let host_gas = Func::wrap(store, |mut caller: Caller<'_, u64>, param: u64| {
-		*caller.host_data_mut() -= param;
-	});
-	// Link the gas host function
-	linker.define("env", "gas", host_gas).unwrap();
-}
-
-fn add_gas_left_global(instance: &Instance, mut store: Store<u64>) -> Store<u64> {
-	instance
-		.get_export(&mut store, "gas_left")
-		.and_then(Extern::into_global)
-		.unwrap()
-		.set(&mut store, Value::I64(-1i64)) // the same as u64::MAX
-		.unwrap();
-	store
-}
-
-fn gas_metered_coremark(c: &mut Criterion) {
-	let mut group = c.benchmark_group("coremark, instrumented");
-	// Benchmark host_function::Injector
-	let wasm_filename = "coremark_minimal.wasm";
-	let bytes = read(fixture_dir().join(wasm_filename)).unwrap();
-	group.bench_function("with host_function::Injector", |bench| {
-		let backend = host_function::Injector::new("env", "gas");
-		let (module, mut store) = prepare_module(backend, &bytes);
-		// Link the host functions with the imported ones
-		let mut linker = <Linker<u64>>::new();
-		add_gas_host_func(&mut linker, &mut store);
-		// Create clock_ms host function.
-		let host_clock_ms = Func::wrap(&mut store, || {
-			SystemTime::now().duration_since(UNIX_EPOCH).unwrap().as_millis() as u64
-		});
-		// Link the time measurer for the coremark wasm
-		linker.define("env", "clock_ms", host_clock_ms).unwrap();
-
-		let instance = linker.instantiate(&mut store, &module).unwrap().start(&mut store).unwrap();
-
-		bench.iter(|| {
-			let run = instance
-				.get_export(&mut store, "run")
-				.and_then(Extern::into_func)
-				.unwrap()
-				.typed::<(), F32>(&mut store)
-				.unwrap();
-			// Call the wasm!
-			run.call(&mut store, ()).unwrap();
-		})
-	});
-
-	group.bench_function("with mutable_global::Injector", |bench| {
-		let backend = mutable_global::Injector::new("gas_left");
-		let (module, mut store) = prepare_module(backend, &bytes);
-		// Add the gas_left mutable global
-		let mut linker = <Linker<u64>>::new();
-		// Create clock_ms host function.
-		let host_clock_ms = Func::wrap(&mut store, || {
-			SystemTime::now().duration_since(UNIX_EPOCH).unwrap().as_millis() as u64
-		});
-		// Link the time measurer for the coremark wasm
-		linker.define("env", "clock_ms", host_clock_ms).unwrap();
-
-		let instance = linker.instantiate(&mut store, &module).unwrap().start(&mut store).unwrap();
-		let mut store = add_gas_left_global(&instance, store);
-
-		bench.iter(|| {
-			let run = instance
-				.get_export(&mut store, "run")
-				.and_then(Extern::into_func)
-				.unwrap()
-				.typed::<(), F32>(&mut store)
-				.unwrap();
-			// Call the wasm!
-			run.call(&mut store, ()).unwrap();
-		})
-	});
-}
-
-/// Converts the `.wat` encoded `bytes` into `.wasm` encoded bytes.
-pub fn wat2wasm(bytes: &[u8]) -> Vec<u8> {
-	wat::parse_bytes(bytes).unwrap().into_owned()
-}
-
-/// Returns a [`Config`] useful for benchmarking.
-fn bench_config() -> Config {
-	let mut config = Config::default();
-	config.set_stack_limits(StackLimits::new(1024, 1024 * 1024, 64 * 1024).unwrap());
-	config
-}
-
-fn gas_metered_recursive_ok(c: &mut Criterion) {
-	let mut group = c.benchmark_group("recursive_ok, instrumented");
-	const RECURSIVE_DEPTH: i32 = 8000;
-	let wasm_bytes = wat2wasm(include_bytes!("fixtures/wat/recursive_ok.wat"));
-
-	group.bench_function("with host_function::Injector", |bench| {
-		let backend = host_function::Injector::new("env", "gas");
-		let (module, mut store) = prepare_module(backend, &wasm_bytes);
-		// Link the host function with the imported one
-		let mut linker = <Linker<u64>>::new();
-		add_gas_host_func(&mut linker, &mut store);
-		let instance = linker.instantiate(&mut store, &module).unwrap().start(&mut store).unwrap();
-
-		let bench_call = instance.get_export(&store, "call").and_then(Extern::into_func).unwrap();
-		let mut result = [Value::I32(0)];
-
-		bench.iter(|| {
-			bench_call
-				.call(&mut store, &[Value::I32(RECURSIVE_DEPTH)], &mut result)
-				.unwrap();
-			assert_eq!(result, [Value::I32(0)]);
-		})
-	});
-
-	group.bench_function("with mutable_global::Injector", |bench| {
-		let backend = mutable_global::Injector::new("gas_left");
-		let (module, mut store) = prepare_module(backend, &wasm_bytes);
-		// Add the gas_left mutable global
-		let linker = <Linker<u64>>::new();
-		let instance = linker.instantiate(&mut store, &module).unwrap().start(&mut store).unwrap();
-		let mut store = add_gas_left_global(&instance, store);
-
-		let bench_call = instance.get_export(&store, "call").and_then(Extern::into_func).unwrap();
-		let mut result = [Value::I32(0)];
-
-		bench.iter(|| {
-			bench_call
-				.call(&mut store, &[Value::I32(RECURSIVE_DEPTH)], &mut result)
-				.unwrap();
-			assert_eq!(result, [Value::I32(0)]);
-		})
-	});
-}
-
-fn gas_metered_fibonacci_recursive(c: &mut Criterion) {
-	let mut group = c.benchmark_group("fibonacci_recursive, instrumented");
-	const FIBONACCI_REC_N: i64 = 10;
-	let wasm_bytes = wat2wasm(include_bytes!("fixtures/wat/fibonacci.wat"));
-
-	group.bench_function("with host_function::Injector", |bench| {
-		let backend = host_function::Injector::new("env", "gas");
-		let (module, mut store) = prepare_module(backend, &wasm_bytes);
-		// Link the host function with the imported one
-		let mut linker = <Linker<u64>>::new();
-		add_gas_host_func(&mut linker, &mut store);
-		let instance = linker.instantiate(&mut store, &module).unwrap().start(&mut store).unwrap();
-
-		let bench_call = instance
-			.get_export(&store, "fib_recursive")
-			.and_then(Extern::into_func)
-			.unwrap();
-		let mut result = [Value::I32(0)];
-
-		bench.iter(|| {
-			bench_call
-				.call(&mut store, &[Value::I64(FIBONACCI_REC_N)], &mut result)
-				.unwrap();
-		});
-	});
-
-	group.bench_function("with mutable_global::Injector", |bench| {
-		let backend = mutable_global::Injector::new("gas_left");
-		let (module, mut store) = prepare_module(backend, &wasm_bytes);
-
-		// Add the gas_left mutable global
-		let linker = <Linker<u64>>::new();
-		let instance = linker.instantiate(&mut store, &module).unwrap().start(&mut store).unwrap();
-		let mut store = add_gas_left_global(&instance, store);
-
-		let bench_call = instance
-			.get_export(&store, "fib_recursive")
-			.and_then(Extern::into_func)
-			.unwrap();
-		let mut result = [Value::I32(0)];
-
-		bench.iter(|| {
-			bench_call
-				.call(&mut store, &[Value::I64(FIBONACCI_REC_N)], &mut result)
-				.unwrap();
-		});
-	});
-}
-
-fn gas_metered_fac_recursive(c: &mut Criterion) {
-	let mut group = c.benchmark_group("factorial_recursive, instrumented");
-	let wasm_bytes = wat2wasm(include_bytes!("fixtures/wat/factorial.wat"));
-
-	group.bench_function("with host_function::Injector", |b| {
-		let backend = host_function::Injector::new("env", "gas");
-		let (module, mut store) = prepare_module(backend, &wasm_bytes);
-		// Link the host function with the imported one
-		let mut linker = <Linker<u64>>::new();
-		add_gas_host_func(&mut linker, &mut store);
-		let instance = linker.instantiate(&mut store, &module).unwrap().start(&mut store).unwrap();
-		let fac = instance
-			.get_export(&store, "recursive_factorial")
-			.and_then(Extern::into_func)
-			.unwrap();
-		let mut result = [Value::I64(0)];
-
-		b.iter(|| {
-			fac.call(&mut store, &[Value::I64(25)], &mut result).unwrap();
-			assert_eq!(result, [Value::I64(7034535277573963776)]);
-		})
-	});
-
-	group.bench_function("with mutable_global::Injector", |b| {
-		let backend = mutable_global::Injector::new("gas_left");
-		let (module, mut store) = prepare_module(backend, &wasm_bytes);
-
-		// Add the gas_left mutable global
-		let linker = <Linker<u64>>::new();
-		let instance = linker.instantiate(&mut store, &module).unwrap().start(&mut store).unwrap();
-		let mut store = add_gas_left_global(&instance, store);
-		let fac = instance
-			.get_export(&store, "recursive_factorial")
-			.and_then(Extern::into_func)
-			.unwrap();
-		let mut result = [Value::I64(0)];
-
-		b.iter(|| {
-			fac.call(&mut store, &[Value::I64(25)], &mut result).unwrap();
-			assert_eq!(result, [Value::I64(7034535277573963776)]);
-		})
-	});
-}
-
-fn gas_metered_count_until(c: &mut Criterion) {
-	const COUNT_UNTIL: i32 = 100_000;
-	let mut group = c.benchmark_group("count_until, instrumented");
-	let wasm_bytes = wat2wasm(include_bytes!("fixtures/wat/count_until.wat"));
-
-	group.bench_function("with host_function::Injector", |b| {
-		let backend = host_function::Injector::new("env", "gas");
-		let (module, mut store) = prepare_module(backend, &wasm_bytes);
-		// Link the host function with the imported one
-		let mut linker = <Linker<u64>>::new();
-		add_gas_host_func(&mut linker, &mut store);
-		let instance = linker.instantiate(&mut store, &module).unwrap().start(&mut store).unwrap();
-		let count_until =
-			instance.get_export(&store, "count_until").and_then(Extern::into_func).unwrap();
-		let mut result = [Value::I32(0)];
-
-		b.iter(|| {
-			count_until.call(&mut store, &[Value::I32(COUNT_UNTIL)], &mut result).unwrap();
-			assert_eq!(result, [Value::I32(COUNT_UNTIL)]);
-		})
-	});
-
-	group.bench_function("with mutable_global::Injector", |b| {
-		let backend = mutable_global::Injector::new("gas_left");
-		let (module, mut store) = prepare_module(backend, &wasm_bytes);
-
-		// Add the gas_left mutable global
-		let linker = <Linker<u64>>::new();
-		let instance = linker.instantiate(&mut store, &module).unwrap().start(&mut store).unwrap();
-		let mut store = add_gas_left_global(&instance, store);
-		let count_until =
-			instance.get_export(&store, "count_until").and_then(Extern::into_func).unwrap();
-		let mut result = [Value::I32(0)];
-
-		b.iter(|| {
-			count_until.call(&mut store, &[Value::I32(COUNT_UNTIL)], &mut result).unwrap();
-			assert_eq!(result, [Value::I32(COUNT_UNTIL)]);
-		})
-	});
-}
-
-fn gas_metered_vec_add(c: &mut Criterion) {
-	fn test_for<A, B>(
-		b: &mut Bencher,
-		vec_add: Func,
-		mut store: &mut Store<u64>,
-		mem: Memory,
-		len: usize,
-		vec_a: A,
-		vec_b: B,
-	) where
-		A: IntoIterator<Item = i32>,
-		B: IntoIterator<Item = i32>,
-	{
-		use core::mem::size_of;
-
-		let ptr_result = 10;
-		let len_result = len * size_of::<i64>();
-		let ptr_a = ptr_result + len_result;
-		let len_a = len * size_of::<i32>();
-		let ptr_b = ptr_a + len_a;
-
-		// Reset `result` buffer to zeros:
-		mem.data_mut(&mut store)[ptr_result..ptr_result + (len * size_of::<i32>())].fill(0);
-		// Initialize `a` buffer:
-		for (n, a) in vec_a.into_iter().take(len).enumerate() {
-			mem.write(&mut store, ptr_a + (n * size_of::<i32>()), &a.to_le_bytes()).unwrap();
-		}
-		// Initialize `b` buffer:
-		for (n, b) in vec_b.into_iter().take(len).enumerate() {
-			mem.write(&mut store, ptr_b + (n * size_of::<i32>()), &b.to_le_bytes()).unwrap();
-		}
-
-		// Prepare parameters and all Wasm `vec_add`:
-		let params = [
-			Value::I32(ptr_result as i32),
-			Value::I32(ptr_a as i32),
-			Value::I32(ptr_b as i32),
-			Value::I32(len as i32),
-		];
-		b.iter(|| {
-			vec_add.call(&mut store, &params, &mut []).unwrap();
-		});
-
-		// Validate the result buffer:
-		for n in 0..len {
-			let mut buffer4 = [0x00; 4];
-			let mut buffer8 = [0x00; 8];
-			let a = {
-				mem.read(&store, ptr_a + (n * size_of::<i32>()), &mut buffer4).unwrap();
-				i32::from_le_bytes(buffer4)
-			};
-			let b = {
-				mem.read(&store, ptr_b + (n * size_of::<i32>()), &mut buffer4).unwrap();
-				i32::from_le_bytes(buffer4)
-			};
-			let actual_result = {
-				mem.read(&store, ptr_result + (n * size_of::<i64>()), &mut buffer8).unwrap();
-				i64::from_le_bytes(buffer8)
-			};
-			let expected_result = (a as i64) + (b as i64);
-			assert_eq!(
-				expected_result, actual_result,
-				"given a = {a} and b = {b}, results diverge at index {n}"
-			);
-		}
-	}
-
-	let mut group = c.benchmark_group("memory_vec_add, instrumented");
-	let wasm_bytes = wat2wasm(include_bytes!("fixtures/wat/memory-vec-add.wat"));
-	const LEN: usize = 100_000;
-
-	group.bench_function("with host_function::Injector", |b| {
-		let backend = host_function::Injector::new("env", "gas");
-		let (module, mut store) = prepare_module(backend, &wasm_bytes);
-		// Link the host function with the imported one
-		let mut linker = <Linker<u64>>::new();
-		add_gas_host_func(&mut linker, &mut store);
-		let instance = linker.instantiate(&mut store, &module).unwrap().start(&mut store).unwrap();
-		let vec_add = instance.get_export(&store, "vec_add").and_then(Extern::into_func).unwrap();
-		let mem = instance.get_export(&store, "mem").and_then(Extern::into_memory).unwrap();
-		mem.grow(&mut store, Pages::new(25).unwrap()).unwrap();
-		test_for(
-			b,
-			vec_add,
-			&mut store,
-			mem,
-			LEN,
-			(0..LEN).map(|i| (i * i) as i32),
-			(0..LEN).map(|i| (i * 10) as i32),
-		)
-	});
-
-	group.bench_function("with mutable_global::Injector", |b| {
-		let backend = mutable_global::Injector::new("gas_left");
-		let (module, mut store) = prepare_module(backend, &wasm_bytes);
-		// Add the gas_left mutable global
-		let linker = <Linker<u64>>::new();
-		let instance = linker.instantiate(&mut store, &module).unwrap().start(&mut store).unwrap();
-		let mut store = add_gas_left_global(&instance, store);
-		let vec_add = instance.get_export(&store, "vec_add").and_then(Extern::into_func).unwrap();
-		let mem = instance.get_export(&store, "mem").and_then(Extern::into_memory).unwrap();
-		mem.grow(&mut store, Pages::new(25).unwrap()).unwrap();
-		test_for(
-			b,
-			vec_add,
-			&mut store,
-			mem,
-			LEN,
-			(0..LEN).map(|i| (i * i) as i32),
-			(0..LEN).map(|i| (i * 10) as i32),
-		)
-	});
-}
-
-fn gas_metered_tiny_keccak(c: &mut Criterion) {
-	let mut group = c.benchmark_group("wasm_kernel::tiny_keccak, instrumented");
-	let wasm_filename = "wasm_kernel.wasm";
-	let wasm_bytes = read(fixture_dir().join(wasm_filename)).unwrap();
-
-	group.bench_function("with host_function::Injector", |b| {
-		let backend = host_function::Injector::new("env", "gas");
-		let (module, mut store) = prepare_module(backend, &wasm_bytes);
-		// Link the host function with the imported one
-		let mut linker = <Linker<u64>>::new();
-		add_gas_host_func(&mut linker, &mut store);
-		let instance = linker.instantiate(&mut store, &module).unwrap().start(&mut store).unwrap();
-		let prepare = instance
-			.get_export(&store, "prepare_tiny_keccak")
-			.and_then(Extern::into_func)
-			.unwrap();
-		let keccak = instance
-			.get_export(&store, "bench_tiny_keccak")
-			.and_then(Extern::into_func)
-			.unwrap();
-		let mut test_data_ptr = Value::I32(0);
-		prepare.call(&mut store, &[], slice::from_mut(&mut test_data_ptr)).unwrap();
-		b.iter(|| {
-			keccak.call(&mut store, slice::from_ref(&test_data_ptr), &mut []).unwrap();
-		})
-	});
-
-	group.bench_function("with mutable_global::Injector", |b| {
-		let backend = mutable_global::Injector::new("gas_left");
-		let (module, mut store) = prepare_module(backend, &wasm_bytes);
-		// Add the gas_left mutable global
-		let linker = <Linker<u64>>::new();
-		let instance = linker.instantiate(&mut store, &module).unwrap().start(&mut store).unwrap();
-		let mut store = add_gas_left_global(&instance, store);
-		let prepare = instance
-			.get_export(&store, "prepare_tiny_keccak")
-			.and_then(Extern::into_func)
-			.unwrap();
-		let keccak = instance
-			.get_export(&store, "bench_tiny_keccak")
-			.and_then(Extern::into_func)
-			.unwrap();
-		let mut test_data_ptr = Value::I32(0);
-		prepare.call(&mut store, &[], slice::from_mut(&mut test_data_ptr)).unwrap();
-		b.iter(|| {
-			keccak.call(&mut store, slice::from_ref(&test_data_ptr), &mut []).unwrap();
-		})
-	});
-}
-
-fn gas_metered_global_bump(c: &mut Criterion) {
-	const BUMP_AMOUNT: i32 = 100_000;
-	let mut group = c.benchmark_group("global_bump, instrumented");
-	let wasm_bytes = wat2wasm(include_bytes!("fixtures/wat/global_bump.wat"));
-
-	group.bench_function("with host_function::Injector", |b| {
-		let backend = host_function::Injector::new("env", "gas");
-		let (module, mut store) = prepare_module(backend, &wasm_bytes);
-		// Link the host function with the imported one
-		let mut linker = <Linker<u64>>::new();
-		add_gas_host_func(&mut linker, &mut store);
-		let instance = linker.instantiate(&mut store, &module).unwrap().start(&mut store).unwrap();
-		let bump = instance.get_export(&store, "bump").and_then(Extern::into_func).unwrap();
-		let mut result = [Value::I32(0)];
-
-		b.iter(|| {
-			bump.call(&mut store, &[Value::I32(BUMP_AMOUNT)], &mut result).unwrap();
-			assert_eq!(result, [Value::I32(BUMP_AMOUNT)]);
-		})
-	});
-
-	group.bench_function("with mutable_global::Injector", |b| {
-		let backend = mutable_global::Injector::new("gas_left");
-		let (module, mut store) = prepare_module(backend, &wasm_bytes);
-		// Add the gas_left mutable global
-		let linker = <Linker<u64>>::new();
-		let instance = linker.instantiate(&mut store, &module).unwrap().start(&mut store).unwrap();
-		let mut store = add_gas_left_global(&instance, store);
-		let bump = instance.get_export(&store, "bump").and_then(Extern::into_func).unwrap();
-		let mut result = [Value::I32(0)];
-
-		b.iter(|| {
-			bump.call(&mut store, &[Value::I32(BUMP_AMOUNT)], &mut result).unwrap();
-			assert_eq!(result, [Value::I32(BUMP_AMOUNT)]);
-		})
-	});
-}
-
-criterion_group!(benches, gas_metering, stack_height_limiter);
-criterion_group!(
-	name = coremark;
-	config = Criterion::default()
-		.sample_size(10)
-		.measurement_time(Duration::from_millis(275000))
-		.warm_up_time(Duration::from_millis(1000));
-	targets =
-		 gas_metered_coremark,
-);
-criterion_group!(
-	name = wasmi_fixtures;
-	config = Criterion::default()
-		.sample_size(10)
-		.measurement_time(Duration::from_millis(250000))
-	.warm_up_time(Duration::from_millis(1000));
-	targets =
-		gas_metered_recursive_ok,
-		gas_metered_fibonacci_recursive,
-		gas_metered_fac_recursive,
-		gas_metered_count_until,
-		gas_metered_vec_add,
-		gas_metered_tiny_keccak,
-		gas_metered_global_bump,
-);
-criterion_main!(coremark, wasmi_fixtures);

benches/execution.rs

@@ -0,0 +1,437 @@
+use criterion::{
+	criterion_group, criterion_main, measurement::Measurement, Bencher, BenchmarkGroup, Criterion,
+};
+use std::{
+	fs::read,
+	path::PathBuf,
+	time::{Duration, SystemTime, UNIX_EPOCH},
+};
+use wasm_instrument::{
+	gas_metering::{self, host_function, mutable_global, ConstantCostRules},
+	parity_wasm::{deserialize_buffer, elements::Module, serialize},
+};
+use wasmi::{
+	core::{Pages, TrapCode, F32},
+	Caller, Config, Engine, Instance, Linker, Memory, StackLimits, Store, TypedFunc, Value,
+};
+
+/// Describes a gas metering strategy we want to benchmark.
+///
+/// Most strategies just need a subset of these functions. Hence we added default
+/// implementations for all of them.
+trait MeteringStrategy {
+	/// The wasmi config we should be using for this strategy.
+	fn config() -> Config {
+		Config::default()
+	}
+
+	/// The strategy may or may not want to instrument the module.
+	fn instrument_module(module: Module) -> Module {
+		module
+	}
+
+	/// The strategy might need to define additional host functions.
+	fn define_host_funcs(_linker: &mut Linker<u64>) {}
+
+	/// The strategy might need to do some initialization of the wasm instance.
+	fn init_instance(_module: &mut BenchInstance) {}
+}
+
+/// Don't do any metering at all. This is helpful as a baseline.
+struct NoMetering;
+
+/// Use wasmi's builtin fuel metering.
+struct WasmiMetering;
+
+/// Instrument the module using [`host_function::Injector`].
+struct HostFunctionMetering;
+
+/// Instrument the module using [`mutable_global::Injector`].
+struct MutableGlobalMetering;
+
+impl MeteringStrategy for NoMetering {}
+
+impl MeteringStrategy for WasmiMetering {
+	fn config() -> Config {
+		let mut config = Config::default();
+		config.consume_fuel(true);
+		config
+	}
+
+	fn init_instance(module: &mut BenchInstance) {
+		module.store.add_fuel(u64::MAX).unwrap();
+	}
+}
+
+impl MeteringStrategy for HostFunctionMetering {
+	fn instrument_module(module: Module) -> Module {
+		let backend = host_function::Injector::new("env", "gas");
+		gas_metering::inject(module, backend, &ConstantCostRules::default()).unwrap()
+	}
+
+	fn define_host_funcs(linker: &mut Linker<u64>) {
+		// the instrumentation relies on the existing of this function
+		linker
+			.func_wrap("env", "gas", |mut caller: Caller<'_, u64>, amount_consumed: u64| {
+				let gas_remaining = caller.data_mut();
+				*gas_remaining =
+					gas_remaining.checked_sub(amount_consumed).ok_or(TrapCode::OutOfFuel)?;
+				Ok(())
+			})
+			.unwrap();
+	}
+}
+
+impl MeteringStrategy for MutableGlobalMetering {
+	fn instrument_module(module: Module) -> Module {
+		let backend = mutable_global::Injector::new("gas_left");
+		gas_metering::inject(module, backend, &ConstantCostRules::default()).unwrap()
+	}
+
+	fn init_instance(module: &mut BenchInstance) {
+		// the instrumentation relies on the host to initialize the global with the gas limit
+		// we just init to the maximum so it will never run out
+		module
+			.instance
+			.get_global(&mut module.store, "gas_left")
+			.unwrap()
+			.set(&mut module.store, Value::I64(-1i64)) // the same as u64::MAX
+			.unwrap();
+	}
+}
+
+/// A wasm instance ready to be benchmarked.
+struct BenchInstance {
+	store: Store<u64>,
+	instance: Instance,
+}
+
+impl BenchInstance {
+	/// Create a new instance for the supplied metering strategy.
+	///
+	/// `wasm`: The raw wasm module for the benchmark.
+	/// `define_host_func`: In here the caller can define additional host function.
+	fn new<S, H>(wasm: &[u8], define_host_funcs: &H) -> Self
+	where
+		S: MeteringStrategy,
+		H: Fn(&mut Linker<u64>),
+	{
+		let module = deserialize_buffer(wasm).unwrap();
+		let instrumented_module = S::instrument_module(module);
+		let input = serialize(instrumented_module).unwrap();
+		let mut config = S::config();
+		config.set_stack_limits(StackLimits::new(1024, 1024 * 1024, 64 * 1024).unwrap());
+		let engine = Engine::new(&config);
+		let module = wasmi::Module::new(&engine, &mut &input[..]).unwrap();
+		let mut linker = Linker::new(&engine);
+		S::define_host_funcs(&mut linker);
+		define_host_funcs(&mut linker);
+		// init host state with maximum gas_left (only used by host_function instrumentation)
+		let mut store = Store::new(&engine, u64::MAX);
+		let instance = linker.instantiate(&mut store, &module).unwrap().start(&mut store).unwrap();
+		let mut bench_module = Self { store, instance };
+		S::init_instance(&mut bench_module);
+		bench_module
+	}
+}
+
+/// Runs a benchmark for every strategy.
+///
+/// We require the closures to implement `Fn` as they are executed for every strategy and we
+/// don't want them to change in between.
+///
+/// `group`: The benchmark group within the benchmarks will be executed.
+/// `wasm`: The raw wasm module for the benchmark.
+/// `define_host_func`: In here the caller can define additional host function.
+/// `f`: In here the user should perform the benchmark. Will be executed for every strategy.
+fn for_strategies<M, H, F>(group: &mut BenchmarkGroup<M>, wasm: &[u8], define_host_funcs: H, f: F)
+where
+	M: Measurement,
+	H: Fn(&mut Linker<u64>),
+	F: Fn(&mut Bencher<M>, &mut BenchInstance),
+{
+	let mut module = BenchInstance::new::<NoMetering, _>(wasm, &define_host_funcs);
+	group.bench_function("no_metering", |bench| f(bench, &mut module));
+
+	let mut module = BenchInstance::new::<WasmiMetering, _>(wasm, &define_host_funcs);
+	group.bench_function("wasmi_builtin", |bench| f(bench, &mut module));
+
+	let mut module = BenchInstance::new::<HostFunctionMetering, _>(wasm, &define_host_funcs);
+	group.bench_function("host_function", |bench| f(bench, &mut module));
+
+	let mut module = BenchInstance::new::<MutableGlobalMetering, _>(wasm, &define_host_funcs);
+	group.bench_function("mutable_global", |bench| f(bench, &mut module));
+}
+
+/// Converts the `.wat` encoded `bytes` into `.wasm` encoded bytes.
+fn wat2wasm(bytes: &[u8]) -> Vec<u8> {
+	wat::parse_bytes(bytes).unwrap().into_owned()
+}
+
+fn fixture_dir() -> PathBuf {
+	let mut path = PathBuf::from(env!("CARGO_MANIFEST_DIR"));
+	path.push("benches");
+	path.push("fixtures");
+	path.push("wasm");
+	path
+}
+
+fn gas_metered_coremark(c: &mut Criterion) {
+	let mut group = c.benchmark_group("coremark");
+	// Benchmark host_function::Injector
+	let wasm_filename = "coremark_minimal.wasm";
+	let bytes = read(fixture_dir().join(wasm_filename)).unwrap();
+	let define_host_funcs = |linker: &mut Linker<u64>| {
+		linker
+			.func_wrap("env", "clock_ms", || {
+				SystemTime::now().duration_since(UNIX_EPOCH).unwrap().as_millis() as u64
+			})
+			.unwrap();
+	};
+	for_strategies(&mut group, &bytes, define_host_funcs, |bench, module| {
+		bench.iter(|| {
+			let run = module.instance.get_typed_func::<(), F32>(&mut module.store, "run").unwrap();
+			// Call the wasm!
+			run.call(&mut module.store, ()).unwrap();
+		})
+	});
+}
+
+fn gas_metered_recursive_ok(c: &mut Criterion) {
+	let mut group = c.benchmark_group("recursive_ok");
+	const RECURSIVE_DEPTH: i32 = 8000;
+	let wasm_bytes = wat2wasm(include_bytes!("fixtures/wat/recursive_ok.wat"));
+	for_strategies(
+		&mut group,
+		&wasm_bytes,
+		|_| {},
+		|bench, module| {
+			let bench_call =
+				module.instance.get_typed_func::<i32, i32>(&module.store, "call").unwrap();
+			bench.iter(|| {
+				let result = bench_call.call(&mut module.store, RECURSIVE_DEPTH).unwrap();
+				assert_eq!(result, 0);
+			})
+		},
+	);
+}
+
+fn gas_metered_fibonacci_recursive(c: &mut Criterion) {
+	let mut group = c.benchmark_group("fibonacci_recursive");
+	const FIBONACCI_REC_N: i64 = 10;
+	let wasm_bytes = wat2wasm(include_bytes!("fixtures/wat/fibonacci.wat"));
+	for_strategies(
+		&mut group,
+		&wasm_bytes,
+		|_| {},
+		|bench, module| {
+			let bench_call = module
+				.instance
+				.get_typed_func::<i64, i64>(&module.store, "fib_recursive")
+				.unwrap();
+			bench.iter(|| {
+				bench_call.call(&mut module.store, FIBONACCI_REC_N).unwrap();
+			});
+		},
+	);
+}
+
+fn gas_metered_fac_recursive(c: &mut Criterion) {
+	let mut group = c.benchmark_group("factorial_recursive");
+	let wasm_bytes = wat2wasm(include_bytes!("fixtures/wat/factorial.wat"));
+	for_strategies(
+		&mut group,
+		&wasm_bytes,
+		|_| {},
+		|bench, module| {
+			let fac = module
+				.instance
+				.get_typed_func::<i64, i64>(&module.store, "recursive_factorial")
+				.unwrap();
+			bench.iter(|| {
+				let result = fac.call(&mut module.store, 25).unwrap();
+				assert_eq!(result, 7034535277573963776);
+			})
+		},
+	);
+}
+
+fn gas_metered_count_until(c: &mut Criterion) {
+	const COUNT_UNTIL: i32 = 100_000;
+	let mut group = c.benchmark_group("count_until");
+	let wasm_bytes = wat2wasm(include_bytes!("fixtures/wat/count_until.wat"));
+	for_strategies(
+		&mut group,
+		&wasm_bytes,
+		|_| {},
+		|bench, module| {
+			let count_until = module
+				.instance
+				.get_typed_func::<i32, i32>(&module.store, "count_until")
+				.unwrap();
+			bench.iter(|| {
+				let result = count_until.call(&mut module.store, COUNT_UNTIL).unwrap();
+				assert_eq!(result, COUNT_UNTIL);
+			})
+		},
+	);
+}
+
+fn gas_metered_vec_add(c: &mut Criterion) {
+	fn test_for<A, B>(
+		b: &mut Bencher,
+		vec_add: TypedFunc<(i32, i32, i32, i32), ()>,
+		mut store: &mut Store<u64>,
+		mem: Memory,
+		len: usize,
+		vec_a: A,
+		vec_b: B,
+	) where
+		A: IntoIterator<Item = i32>,
+		B: IntoIterator<Item = i32>,
+	{
+		use core::mem::size_of;
+
+		let ptr_result = 10;
+		let len_result = len * size_of::<i64>();
+		let ptr_a = ptr_result + len_result;
+		let len_a = len * size_of::<i32>();
+		let ptr_b = ptr_a + len_a;
+
+		// Reset `result` buffer to zeros:
+		mem.data_mut(&mut store)[ptr_result..ptr_result + (len * size_of::<i32>())].fill(0);
+		// Initialize `a` buffer:
+		for (n, a) in vec_a.into_iter().take(len).enumerate() {
+			mem.write(&mut store, ptr_a + (n * size_of::<i32>()), &a.to_le_bytes()).unwrap();
+		}
+		// Initialize `b` buffer:
+		for (n, b) in vec_b.into_iter().take(len).enumerate() {
+			mem.write(&mut store, ptr_b + (n * size_of::<i32>()), &b.to_le_bytes()).unwrap();
+		}
+
+		// Prepare parameters and all Wasm `vec_add`:
+		let params = (ptr_result as i32, ptr_a as i32, ptr_b as i32, len as i32);
+		b.iter(|| {
+			vec_add.call(&mut store, params).unwrap();
+		});
+
+		// Validate the result buffer:
+		for n in 0..len {
+			let mut buffer4 = [0x00; 4];
+			let mut buffer8 = [0x00; 8];
+			let a = {
+				mem.read(&store, ptr_a + (n * size_of::<i32>()), &mut buffer4).unwrap();
+				i32::from_le_bytes(buffer4)
+			};
+			let b = {
+				mem.read(&store, ptr_b + (n * size_of::<i32>()), &mut buffer4).unwrap();
+				i32::from_le_bytes(buffer4)
+			};
+			let actual_result = {
+				mem.read(&store, ptr_result + (n * size_of::<i64>()), &mut buffer8).unwrap();
+				i64::from_le_bytes(buffer8)
+			};
+			let expected_result = (a as i64) + (b as i64);
+			assert_eq!(
+				expected_result, actual_result,
+				"given a = {a} and b = {b}, results diverge at index {n}"
+			);
+		}
+	}
+
+	let mut group = c.benchmark_group("memory_vec_add");
+	let wasm_bytes = wat2wasm(include_bytes!("fixtures/wat/memory-vec-add.wat"));
+	const LEN: usize = 100_000;
+
+	for_strategies(
+		&mut group,
+		&wasm_bytes,
+		|_| {},
+		|bench, module| {
+			let vec_add = module
+				.instance
+				.get_typed_func::<(i32, i32, i32, i32), ()>(&module.store, "vec_add")
+				.unwrap();
+			let mem = module.instance.get_memory(&module.store, "mem").unwrap();
+			mem.grow(&mut module.store, Pages::new(25).unwrap()).unwrap();
+			test_for(
+				bench,
+				vec_add,
+				&mut module.store,
+				mem,
+				LEN,
+				(0..LEN).map(|i| (i * i) as i32),
+				(0..LEN).map(|i| (i * 10) as i32),
+			)
+		},
+	);
+}
+
+fn gas_metered_tiny_keccak(c: &mut Criterion) {
+	let mut group = c.benchmark_group("wasm_kernel::tiny_keccak");
+	let wasm_filename = "wasm_kernel.wasm";
+	let wasm_bytes = read(fixture_dir().join(wasm_filename)).unwrap();
+	for_strategies(
+		&mut group,
+		&wasm_bytes,
+		|_| {},
+		|bench, module| {
+			let prepare = module
+				.instance
+				.get_typed_func::<(), i32>(&module.store, "prepare_tiny_keccak")
+				.unwrap();
+			let keccak = module
+				.instance
+				.get_typed_func::<i32, ()>(&module.store, "bench_tiny_keccak")
+				.unwrap();
+			let test_data_ptr = prepare.call(&mut module.store, ()).unwrap();
+			bench.iter(|| {
+				keccak.call(&mut module.store, test_data_ptr).unwrap();
+			})
+		},
+	);
+}
+
+fn gas_metered_global_bump(c: &mut Criterion) {
+	const BUMP_AMOUNT: i32 = 100_000;
+	let mut group = c.benchmark_group("global_bump");
+	let wasm_bytes = wat2wasm(include_bytes!("fixtures/wat/global_bump.wat"));
+	for_strategies(
+		&mut group,
+		&wasm_bytes,
+		|_| {},
+		|bench, module| {
+			let bump = module.instance.get_typed_func::<i32, i32>(&module.store, "bump").unwrap();
+			bench.iter(|| {
+				let result = bump.call(&mut module.store, BUMP_AMOUNT).unwrap();
+				assert_eq!(result, BUMP_AMOUNT);
+			})
+		},
+	);
+}
+
+criterion_group!(
+	name = coremark;
+	config = Criterion::default()
+		.sample_size(10)
+		.measurement_time(Duration::from_millis(275000))
+		.warm_up_time(Duration::from_millis(1000));
+	targets =
+		 gas_metered_coremark,
+);
+criterion_group!(
+	name = wasmi_fixtures;
+	config = Criterion::default()
+		.sample_size(10)
+		.measurement_time(Duration::from_millis(250000))
+	.warm_up_time(Duration::from_millis(1000));
+	targets =
+		gas_metered_recursive_ok,
+		gas_metered_fibonacci_recursive,
+		gas_metered_fac_recursive,
+		gas_metered_count_until,
+		gas_metered_vec_add,
+		gas_metered_tiny_keccak,
+		gas_metered_global_bump,
+);
+criterion_main!(coremark, wasmi_fixtures);

benches/instrumentation.rs

@@ -0,0 +1,58 @@
+use criterion::{
+	criterion_group, criterion_main, measurement::Measurement, BenchmarkGroup, Criterion,
+	Throughput,
+};
+use std::{
+	fs::{read, read_dir},
+	path::PathBuf,
+};
+use wasm_instrument::{
+	gas_metering::{self, host_function, ConstantCostRules},
+	inject_stack_limiter,
+	parity_wasm::{deserialize_buffer, elements::Module},
+};
+
+fn fixture_dir() -> PathBuf {
+	let mut path = PathBuf::from(env!("CARGO_MANIFEST_DIR"));
+	path.push("benches");
+	path.push("fixtures");
+	path.push("wasm");
+	path
+}
+
+fn for_fixtures<F, M>(group: &mut BenchmarkGroup<M>, f: F)
+where
+	F: Fn(Module),
+	M: Measurement,
+{
+	for entry in read_dir(fixture_dir()).unwrap() {
+		let entry = entry.unwrap();
+		let bytes = read(entry.path()).unwrap();
+		group.throughput(Throughput::Bytes(bytes.len().try_into().unwrap()));
+		group.bench_with_input(entry.file_name().to_str().unwrap(), &bytes, |bench, input| {
+			bench.iter(|| f(deserialize_buffer(input).unwrap()))
+		});
+	}
+}
+
+fn gas_metering(c: &mut Criterion) {
+	let mut group = c.benchmark_group("Gas Metering");
+	for_fixtures(&mut group, |module| {
+		gas_metering::inject(
+			module,
+			host_function::Injector::new("env", "gas"),
+			&ConstantCostRules::default(),
+		)
+		.unwrap();
+	});
+}
+
+fn stack_height_limiter(c: &mut Criterion) {
+	let mut group = c.benchmark_group("Stack Height Limiter");
+	for_fixtures(&mut group, |module| {
+		inject_stack_limiter(module, 128).unwrap();
+	});
+}
+
+criterion_group!(benches, gas_metering, stack_height_limiter);
+criterion_main!(benches);