Implement Yul to LLVM IR compilation benchmarks (#407)

# Description

Closes [#404](https://github.com/paritytech/revive/issues/404)

Adds compilation time benchmarks for:

* Parsing of Yul source code -> AST Object
* Lowering of AST Object -> LLVM IR (unoptimized)

The benchmarks can be run from the root via:

```sh
# Run all benchmarks in the revive-yul crate (parsing + lowering)
make bench-yul
```

HTML reports will be generated under `target/criterion`, and a summary
of the results at
[crates/yul/BENCHMARKS_PARSE_M4PRO.md](https://github.com/paritytech/revive/blob/lj/compilation-benchmarks-yul/crates/yul/BENCHMARKS_PARSE_M4PRO.md)
and
[crates/yul/BENCHMARKS_LOWER_M4PRO.md](https://github.com/paritytech/revive/blob/lj/compilation-benchmarks-yul/crates/yul/BENCHMARKS_LOWER_M4PRO.md)
(currently from running on a Mac M4 Pro).

---------

Co-authored-by: xermicus <cyrill@parity.io>

Cargo.lock

@@ -9208,11 +9208,14 @@ dependencies = [
 name = "revive-yul"
 version = "0.4.0"
 dependencies = [
+ "alloy-primitives",
  "anyhow",
+ "criterion",
  "inkwell",
  "num",
  "regex",
  "revive-common",
+ "revive-integration",
  "revive-llvm-context",
  "serde",
  "thiserror 2.0.17",

Makefile

@@ -14,6 +14,7 @@
 	test \
 	test-integration \
 	test-resolc \
+	test-yul \
 	test-workspace \
 	test-wasm \
 	test-llvm-builder \
@@ -21,6 +22,7 @@
 	bench-pvm \
 	bench-evm \
 	bench-resolc \
+	bench-yul \
 	clean
 
 install: install-bin install-npm
@@ -67,10 +69,13 @@ test-integration: install-bin
 	cargo test --package revive-integration
 
 test-resolc: install
-	cargo test --package resolc --benches
+	cargo test --package resolc --all-targets
+
+test-yul:
+	cargo test --package revive-yul --all-targets
 
 test-workspace: install
-	cargo test --workspace --exclude revive-llvm-builder
+	cargo test --workspace --all-targets --exclude revive-llvm-builder
 
 test-wasm: install-wasm
 	npm run test:wasm
@@ -95,6 +100,12 @@ bench-resolc: test-resolc
 	cargo criterion --package resolc --bench compile --message-format=json \
 	| criterion-table > crates/resolc/BENCHMARKS_M4PRO.md
 
+bench-yul: test-yul
+	cargo criterion --package revive-yul --bench parse --message-format=json \
+	| criterion-table > crates/yul/BENCHMARKS_PARSE_M4PRO.md
+	cargo criterion --package revive-yul --bench lower --message-format=json \
+	| criterion-table > crates/yul/BENCHMARKS_LOWER_M4PRO.md
+
 clean:
 	cargo clean ; \
 	revive-llvm clean ; \

crates/integration/src/cases.rs

@@ -10,6 +10,7 @@ pub struct Contract {
     pub evm_runtime: Vec<u8>,
     pub pvm_runtime: Vec<u8>,
     pub calldata: Vec<u8>,
+    pub yul: String,
 }
 
 impl Contract {
@@ -19,6 +20,7 @@ impl Contract {
             evm_runtime: compile_evm_bin_runtime(name, code),
             pvm_runtime: compile_blob(name, code),
             calldata,
+            yul: compile_to_yul(name, code, true),
         }
     }
 
@@ -28,6 +30,7 @@ impl Contract {
             evm_runtime: compile_evm_bin_runtime(name, code),
             pvm_runtime: compile_blob_with_options(name, code, true, OptimizerSettings::size()),
             calldata,
+            yul: compile_to_yul(name, code, true),
         }
     }
 }

crates/llvm-context/src/polkavm/context/mod.rs

@@ -256,6 +256,22 @@ impl<'ctx> Context<'ctx> {
         }
     }
 
+    /// Initializes a new dummy LLVM context.
+    ///
+    /// Omits the LLVM module initialization; use this only in tests and benchmarks.
+    pub fn new_dummy(
+        llvm: &'ctx inkwell::context::Context,
+        optimizer_settings: OptimizerSettings,
+    ) -> Self {
+        Self::new(
+            llvm,
+            llvm.create_module("dummy"),
+            Optimizer::new(optimizer_settings),
+            Default::default(),
+            Default::default(),
+        )
+    }
+
     /// Builds the LLVM IR module, returning the build artifacts.
     pub fn build(
         self,

crates/llvm-context/src/polkavm/context/tests.rs

@@ -1,33 +1,21 @@
 //! The LLVM IR generator context tests.
 
 use crate::optimizer::settings::Settings as OptimizerSettings;
-use crate::optimizer::Optimizer;
 use crate::polkavm::context::attribute::Attribute;
 use crate::polkavm::context::Context;
 use crate::PolkaVMTarget;
 
-pub fn create_context(
-    llvm: &inkwell::context::Context,
-    optimizer_settings: OptimizerSettings,
-) -> Context<'_> {
+/// Initializes the LLVM compiler backend.
+fn initialize_llvm() {
     crate::initialize_llvm(PolkaVMTarget::PVM, "resolc", Default::default());
-
-    let module = llvm.create_module("test");
-    let optimizer = Optimizer::new(optimizer_settings);
-
-    Context::new(
-        llvm,
-        module,
-        optimizer,
-        Default::default(),
-        Default::default(),
-    )
 }
 
 #[test]
 pub fn check_attribute_null_pointer_is_invalid() {
+    initialize_llvm();
+
     let llvm = inkwell::context::Context::create();
-    let mut context = create_context(&llvm, OptimizerSettings::cycles());
+    let mut context = Context::new_dummy(&llvm, OptimizerSettings::cycles());
 
     let function = context
         .add_function(
@@ -51,8 +39,10 @@ pub fn check_attribute_null_pointer_is_invalid() {
 
 #[test]
 pub fn check_attribute_optimize_for_size_mode_3() {
+    initialize_llvm();
+
     let llvm = inkwell::context::Context::create();
-    let mut context = create_context(&llvm, OptimizerSettings::cycles());
+    let mut context = Context::new_dummy(&llvm, OptimizerSettings::cycles());
 
     let function = context
         .add_function(
@@ -76,8 +66,10 @@ pub fn check_attribute_optimize_for_size_mode_3() {
 
 #[test]
 pub fn check_attribute_optimize_for_size_mode_z() {
+    initialize_llvm();
+
     let llvm = inkwell::context::Context::create();
-    let mut context = create_context(&llvm, OptimizerSettings::size());
+    let mut context = Context::new_dummy(&llvm, OptimizerSettings::size());
 
     let function = context
         .add_function(
@@ -101,8 +93,10 @@ pub fn check_attribute_optimize_for_size_mode_z() {
 
 #[test]
 pub fn check_attribute_min_size_mode_3() {
+    initialize_llvm();
+
     let llvm = inkwell::context::Context::create();
-    let mut context = create_context(&llvm, OptimizerSettings::cycles());
+    let mut context = Context::new_dummy(&llvm, OptimizerSettings::cycles());
 
     let function = context
         .add_function(
@@ -126,8 +120,10 @@ pub fn check_attribute_min_size_mode_3() {
 
 #[test]
 pub fn check_attribute_min_size_mode_z() {
+    initialize_llvm();
+
     let llvm = inkwell::context::Context::create();
-    let mut context = create_context(&llvm, OptimizerSettings::size());
+    let mut context = Context::new_dummy(&llvm, OptimizerSettings::size());
 
     let function = context
         .add_function(

crates/resolc/src/project/contract/ir/yul.rs

@@ -8,7 +8,7 @@ use serde::Serialize;
 
 use revive_yul::parser::statement::object::Object;
 
-/// he contract Yul source code.
+/// The contract Yul source code.
 #[derive(Debug, Serialize, Deserialize, Clone)]
 pub struct Yul {
     /// The Yul AST object.

crates/resolc/src/test_utils.rs

@@ -33,6 +33,7 @@ static PVM_BLOB_CACHE: Lazy<Mutex<HashMap<CachedBlob, Vec<u8>>>> = Lazy::new(Def
 static EVM_BLOB_CACHE: Lazy<Mutex<HashMap<CachedBlob, Vec<u8>>>> = Lazy::new(Default::default);
 static EVM_RUNTIME_BLOB_CACHE: Lazy<Mutex<HashMap<CachedBlob, Vec<u8>>>> =
     Lazy::new(Default::default);
+static YUL_IR_CACHE: Lazy<Mutex<HashMap<CachedBlob, String>>> = Lazy::new(Default::default);
 
 const DEBUG_CONFIG: revive_llvm_context::DebugConfig = DebugConfig::new(None, true);
 
@@ -475,3 +476,95 @@ fn compile_evm(
 
     blob
 }
+
+/// Compiles the Solidity source code into Yul IR and returns
+/// the Yul IR code of the contract named `contract_name`.
+pub fn compile_to_yul(
+    contract_name: &str,
+    source_code: &str,
+    solc_optimizer_enabled: bool,
+) -> String {
+    check_dependencies();
+
+    let optimizer = SolcStandardJsonInputSettingsOptimizer::new(
+        solc_optimizer_enabled,
+        Default::default(),
+        Default::default(),
+    );
+    let id = CachedBlob {
+        contract_name: contract_name.to_owned(),
+        solc_optimizer_enabled,
+        solidity: source_code.to_owned(),
+        opt: optimizer.mode.into(),
+    };
+
+    if let Some(yul) = YUL_IR_CACHE.lock().unwrap().get(&id) {
+        return yul.clone();
+    }
+
+    let file_name = "contract.sol";
+    let sources = BTreeMap::from([(
+        file_name.to_owned(),
+        SolcStandardJsonInputSource::from(source_code.to_owned()),
+    )]);
+    let mut input = SolcStandardJsonInput::try_from_solidity_sources(
+        None,
+        sources.clone(),
+        Default::default(),
+        Default::default(),
+        SolcStandardJsonInputSettingsSelection::new_required_for_tests(),
+        optimizer,
+        Default::default(),
+        Default::default(),
+        Default::default(),
+        Default::default(),
+        false,
+    )
+    .unwrap();
+
+    let solc = SolcCompiler::new(SolcCompiler::DEFAULT_EXECUTABLE_NAME.to_owned()).unwrap();
+    let output = solc
+        .standard_json(&mut input, &mut vec![], None, vec![], None)
+        .unwrap();
+    output.check_errors().unwrap();
+
+    let yul = output
+        .contracts
+        .get(file_name)
+        .unwrap_or_else(|| panic!("file `{file_name}` not found in solc output"))
+        .get(contract_name)
+        .unwrap_or_else(|| panic!("contract `{contract_name}` not found in solc output"))
+        .ir_optimized
+        .to_owned();
+
+    YUL_IR_CACHE.lock().unwrap().insert(id, yul.clone());
+
+    yul
+}
+
+#[cfg(test)]
+mod tests {
+    use std::fs::read_to_string;
+
+    use super::compile_to_yul;
+    use crate::cli_utils::SOLIDITY_DEPENDENCY_CONTRACT_PATH;
+
+    #[test]
+    fn compiles_to_yul() {
+        let contract_name = "Dependency";
+        let source_code = read_to_string(SOLIDITY_DEPENDENCY_CONTRACT_PATH).unwrap();
+        let yul = compile_to_yul(contract_name, &source_code, true);
+        assert!(
+            yul.contains(&format!("object \"{contract_name}")),
+            "the `{contract_name}` contract IR code should contain a Yul object"
+        );
+    }
+
+    #[test]
+    #[should_panic(expected = "contract `Nonexistent` not found in solc output")]
+    fn error_nonexistent_contract_in_yul() {
+        let contract_name = "Nonexistent";
+        let source_code = read_to_string(SOLIDITY_DEPENDENCY_CONTRACT_PATH).unwrap();
+        compile_to_yul(contract_name, &source_code, true);
+    }
+}

crates/yul/BENCHMARKS_LOWER_M4PRO.md

@@ -0,0 +1,46 @@
+# Benchmarks
+
+## Table of Contents
+
+- [Benchmark Results](#benchmark-results)
+    - [Baseline](#baseline)
+    - [ERC20](#erc20)
+    - [SHA1](#sha1)
+    - [Storage](#storage)
+    - [Transfer](#transfer)
+
+## Benchmark Results
+
+### Baseline
+
+|        | `lower`                    |
+|:-------|:-------------------------- |
+|        | `110.18 us` (✅ **1.00x**)  |
+
+### ERC20
+
+|        | `lower`                    |
+|:-------|:-------------------------- |
+|        | `623.57 us` (✅ **1.00x**)  |
+
+### SHA1
+
+|        | `lower`                    |
+|:-------|:-------------------------- |
+|        | `357.61 us` (✅ **1.00x**)  |
+
+### Storage
+
+|        | `lower`                    |
+|:-------|:-------------------------- |
+|        | `161.24 us` (✅ **1.00x**)  |
+
+### Transfer
+
+|        | `lower`                    |
+|:-------|:-------------------------- |
+|        | `162.35 us` (✅ **1.00x**)  |
+
+---
+Made with [criterion-table](https://github.com/nu11ptr/criterion-table)
+

crates/yul/BENCHMARKS_PARSE_M4PRO.md

@@ -0,0 +1,46 @@
+# Benchmarks
+
+## Table of Contents
+
+- [Benchmark Results](#benchmark-results)
+    - [Baseline](#baseline)
+    - [ERC20](#erc20)
+    - [SHA1](#sha1)
+    - [Storage](#storage)
+    - [Transfer](#transfer)
+
+## Benchmark Results
+
+### Baseline
+
+|        | `parse`                  |
+|:-------|:------------------------ |
+|        | `8.20 us` (✅ **1.00x**)  |
+
+### ERC20
+
+|        | `parse`                    |
+|:-------|:-------------------------- |
+|        | `155.02 us` (✅ **1.00x**)  |
+
+### SHA1
+
+|        | `parse`                   |
+|:-------|:------------------------- |
+|        | `74.76 us` (✅ **1.00x**)  |
+
+### Storage
+
+|        | `parse`                   |
+|:-------|:------------------------- |
+|        | `17.05 us` (✅ **1.00x**)  |
+
+### Transfer
+
+|        | `parse`                   |
+|:-------|:------------------------- |
+|        | `19.37 us` (✅ **1.00x**)  |
+
+---
+Made with [criterion-table](https://github.com/nu11ptr/criterion-table)
+

crates/yul/Cargo.toml

@@ -18,3 +18,16 @@ thiserror = { workspace = true }
 
 revive-common = { workspace = true }
 revive-llvm-context = { workspace = true }
+
+[dev-dependencies]
+alloy-primitives = { workspace = true }
+criterion = { workspace = true }
+revive-integration = { workspace = true }
+
+[[bench]]
+name = "parse"
+harness = false
+
+[[bench]]
+name = "lower"
+harness = false

crates/yul/benches/lower.rs

@@ -0,0 +1,102 @@
+use std::time::Duration;
+
+use alloy_primitives::U256;
+use criterion::{
+    criterion_group, criterion_main,
+    measurement::{Measurement, WallTime},
+    BatchSize, BenchmarkGroup, Criterion,
+};
+use inkwell::context::Context as InkwellContext;
+use revive_integration::cases::Contract;
+use revive_llvm_context::{
+    initialize_llvm, OptimizerSettings, PolkaVMContext, PolkaVMTarget, PolkaVMWriteLLVM,
+};
+use revive_yul::{lexer::Lexer, parser::statement::object::Object};
+
+/// The function under test lowers the Yul `Object` into unoptimized LLVM IR.
+fn lower(mut ast: Object, mut llvm_context: PolkaVMContext) {
+    ast.declare(&mut llvm_context)
+        .expect("the AST should be valid");
+    ast.into_llvm(&mut llvm_context)
+        .expect("the AST should lower to LLVM IR");
+}
+
+fn parse(source_code: &str) -> Object {
+    let mut lexer = Lexer::new(source_code.to_owned());
+    Object::parse(&mut lexer, None).expect("the Yul source should parse")
+}
+
+fn group<'error, M>(c: &'error mut Criterion<M>, group_name: &str) -> BenchmarkGroup<'error, M>
+where
+    M: Measurement,
+{
+    c.benchmark_group(group_name)
+}
+
+fn bench<F>(mut group: BenchmarkGroup<'_, WallTime>, contract: F)
+where
+    F: Fn() -> Contract,
+{
+    let ast = parse(&contract().yul);
+    let llvm = InkwellContext::create();
+    // The optimizer settings will not affect the benchmarks since we're
+    // not running the optimization passes.
+    let optimizer_settings = OptimizerSettings::none();
+
+    initialize_llvm(PolkaVMTarget::PVM, "resolc", Default::default());
+
+    group
+        .sample_size(90)
+        .measurement_time(Duration::from_secs(6));
+
+    group.bench_function("lower", |b| {
+        b.iter_batched(
+            || {
+                (
+                    ast.clone(),
+                    PolkaVMContext::new_dummy(&llvm, optimizer_settings.to_owned()),
+                )
+            },
+            |(ast, llvm_context)| lower(ast, llvm_context),
+            BatchSize::SmallInput,
+        );
+    });
+
+    group.finish();
+}
+
+fn bench_baseline(c: &mut Criterion) {
+    bench(group(c, "Baseline"), Contract::baseline);
+}
+
+fn bench_erc20(c: &mut Criterion) {
+    bench(group(c, "ERC20"), Contract::erc20);
+}
+
+fn bench_sha1(c: &mut Criterion) {
+    bench(group(c, "SHA1"), || Contract::sha1(vec![0xff].into()));
+}
+
+fn bench_storage(c: &mut Criterion) {
+    bench(group(c, "Storage"), || {
+        Contract::storage_transient(U256::from(0))
+    });
+}
+
+fn bench_transfer(c: &mut Criterion) {
+    bench(group(c, "Transfer"), || {
+        Contract::transfer_self(U256::from(0))
+    });
+}
+
+criterion_group!(
+    name = benches_lower;
+    config = Criterion::default();
+    targets =
+        bench_baseline,
+        bench_erc20,
+        bench_sha1,
+        bench_storage,
+        bench_transfer,
+);
+criterion_main!(benches_lower);

crates/yul/benches/parse.rs

@@ -0,0 +1,72 @@
+use alloy_primitives::U256;
+use criterion::{
+    criterion_group, criterion_main,
+    measurement::{Measurement, WallTime},
+    BenchmarkGroup, Criterion,
+};
+use revive_integration::cases::Contract;
+use revive_yul::{lexer::Lexer, parser::statement::object::Object};
+
+/// The function under test parses the Yul `source_code`.
+fn parse(source_code: &str) {
+    let mut lexer = Lexer::new(source_code.to_owned());
+    Object::parse(&mut lexer, None).expect("the Yul source should parse");
+}
+
+fn group<'error, M>(c: &'error mut Criterion<M>, group_name: &str) -> BenchmarkGroup<'error, M>
+where
+    M: Measurement,
+{
+    c.benchmark_group(group_name)
+}
+
+fn bench<F>(mut group: BenchmarkGroup<'_, WallTime>, contract: F)
+where
+    F: Fn() -> Contract,
+{
+    let source_code = contract().yul;
+
+    group.sample_size(200);
+
+    group.bench_function("parse", |b| {
+        b.iter(|| parse(&source_code));
+    });
+
+    group.finish();
+}
+
+fn bench_baseline(c: &mut Criterion) {
+    bench(group(c, "Baseline"), Contract::baseline);
+}
+
+fn bench_erc20(c: &mut Criterion) {
+    bench(group(c, "ERC20"), Contract::erc20);
+}
+
+fn bench_sha1(c: &mut Criterion) {
+    bench(group(c, "SHA1"), || Contract::sha1(vec![0xff].into()));
+}
+
+fn bench_storage(c: &mut Criterion) {
+    bench(group(c, "Storage"), || {
+        Contract::storage_transient(U256::from(0))
+    });
+}
+
+fn bench_transfer(c: &mut Criterion) {
+    bench(group(c, "Transfer"), || {
+        Contract::transfer_self(U256::from(0))
+    });
+}
+
+criterion_group!(
+    name = benches_parse;
+    config = Criterion::default();
+    targets =
+        bench_baseline,
+        bench_erc20,
+        bench_sha1,
+        bench_storage,
+        bench_transfer,
+);
+criterion_main!(benches_parse);