parsing complex tests works modulo the contract addresses in calldata

Signed-off-by: Cyrill Leutwiler <bigcyrill@hotmail.com>

crates/core/Cargo.toml

@@ -1,5 +1,5 @@
 [package]
-name = "revive-differential-testing-retester"
+name = "revive-differential-testing-core"
 version.workspace = true
 authors.workspace = true
 license.workspace = true
@@ -9,8 +9,13 @@ rust-version.workspace = true
 
 [[bin]]
 name = "retester"
-path = "src/main.rs"
+path = "src/bin/main.rs"
 
 [dependencies]
+anyhow = { workspace = true }
+clap = { workspace = true }
+log = { workspace = true }
+env_logger = { workspace = true }
 revive-differential-testing-format = { workspace = true }
+serde = { workspace = true, features = [ "derive" ] }
 serde_json = { workspace = true }

crates/core/src/arguments.rs

@@ -0,0 +1,17 @@
+use std::path::PathBuf;
+
+use clap::Parser;
+
+#[derive(Debug, Parser)]
+#[command(name = "The PolkaVM Solidity compiler", arg_required_else_help = true)]
+pub struct Arguments {
+    /// The path where the `resolc` executable to be tested is found at.
+    ///
+    /// By default it uses the `resolc` found in `$PATH`
+    #[arg(long = "resolc")]
+    pub resolc: Option<PathBuf>,
+
+    /// A list of test corpus JSON files to be tested.
+    #[arg(long = "corpus")]
+    pub corpus: Vec<PathBuf>,
+}

crates/core/src/bin/main.rs

@@ -0,0 +1,24 @@
+use std::collections::BTreeSet;
+
+use clap::Parser;
+
+use revive_differential_testing_core::arguments::Arguments;
+use revive_differential_testing_format::corpus::Corpus;
+
+fn main() -> anyhow::Result<()> {
+    env_logger::init();
+
+    let args = Arguments::try_parse()?;
+
+    for path in args.corpus.iter().collect::<BTreeSet<_>>() {
+        log::trace!("attempting corpus {path:?}");
+        let corpus = Corpus::try_from_path(path)?;
+        log::info!("found corpus: {corpus:?}");
+
+        let tests = corpus.enumerate_tests();
+
+        log::debug!("{tests:?}");
+    }
+
+    Ok(())
+}

crates/core/src/lib.rs

@@ -0,0 +1,6 @@
+//! The revive differential testing core library.
+//!
+//! This crate defines the testing configuration and
+//! provides a helper utilty to execute tests.
+
+pub mod arguments;

crates/format/Cargo.toml

@@ -14,6 +14,7 @@ alloy-serde = { workspace = true }
 alloy-primitives = { workspace = true }
 alloy-sol-types = { workspace = true }
 anyhow = { workspace = true }
+log = { workspace = true }
 semver = { workspace = true }
 serde = { workspace = true, features = [ "derive" ] }
 serde_json = { workspace = true }

crates/format/src/case.rs

@@ -1,4 +1,4 @@
-use serde::{Deserialize, de::Deserializer};
+use serde::Deserialize;
 
 use crate::{input::Input, mode::Mode};
 
@@ -8,5 +8,4 @@ pub struct Case {
     pub comment: Option<String>,
     pub modes: Option<Vec<Mode>>,
     pub inputs: Vec<Input>,
-    pub expected: Vec<String>,
 }

crates/format/src/corpus.rs

@@ -0,0 +1,67 @@
+use std::{
+    fs::File,
+    path::{Path, PathBuf},
+};
+
+use serde::Deserialize;
+
+use crate::metadata::Metadata;
+
+#[derive(Clone, Debug, Default, Deserialize, Eq, PartialEq)]
+pub struct Corpus {
+    pub name: String,
+    pub path: PathBuf,
+}
+
+impl Corpus {
+    /// Try to read and parse the corpus definition file at given `path`.
+    pub fn try_from_path(path: &Path) -> anyhow::Result<Self> {
+        let file = File::open(path)?;
+        Ok(serde_json::from_reader(file)?)
+    }
+
+    /// Scan the corpus base directory and return all tests found.
+    pub fn enumerate_tests(&self) -> Vec<Metadata> {
+        let mut tests = Vec::new();
+        collect_metadata(&self.path, &mut tests);
+        tests
+    }
+}
+
+/// Recursively walks `path` and parses any JSON or Solidity file into a test
+/// definition [Metadata].
+///
+/// Found tests are inserted into `tests`.
+///
+/// `path` is expected to be a directory.
+pub fn collect_metadata(path: &Path, tests: &mut Vec<Metadata>) {
+    let dir_entry = match std::fs::read_dir(path) {
+        Ok(dir_entry) => dir_entry,
+        Err(error) => {
+            log::error!("failed to read dir '{}': {error}", path.display());
+            return;
+        }
+    };
+
+    for entry in dir_entry {
+        let entry = match entry {
+            Ok(entry) => entry,
+            Err(error) => {
+                log::error!("error reading dir entry: {error}");
+                continue;
+            }
+        };
+
+        let path = entry.path();
+        if path.is_dir() {
+            collect_metadata(&path, tests);
+            continue;
+        }
+
+        if path.is_file() {
+            if let Some(metadata) = Metadata::try_from_file(&path) {
+                tests.push(metadata)
+            }
+        }
+    }
+}

crates/format/src/input.rs

@@ -41,7 +41,7 @@ where
         match calldata_string.parse::<U256>() {
             Ok(parsed) => result.extend_from_slice(&parsed.to_be_bytes::<32>()),
             Err(error) => {
-                return Err(serde::de::Error::custom(&format!(
+                return Err(serde::de::Error::custom(format!(
                     "parsing U256 {calldata_string} error: {error}"
                 )));
             }
@@ -67,7 +67,7 @@ where
             match Function::parse(&signature) {
                 Ok(function) => Method::Function(function.selector().0),
                 Err(error) => {
-                    return Err(serde::de::Error::custom(&format!(
+                    return Err(serde::de::Error::custom(format!(
                         "parsing function signature '{signature}' error: {error}"
                     )));
                 }

crates/format/src/lib.rs

@@ -1,6 +1,7 @@
 //! The revive differential tests case format.
 
 pub mod case;
+pub mod corpus;
 pub mod input;
 pub mod metadata;
 pub mod mode;

crates/format/src/metadata.rs

@@ -1,9 +1,16 @@
-use std::collections::BTreeMap;
+use std::{
+    collections::BTreeMap,
+    fs::File,
+    path::{Path, PathBuf},
+};
 
 use serde::Deserialize;
 
 use crate::{case::Case, mode::Mode};
 
+pub const METADATA_FILE_EXTENSION: &str = "json";
+pub const SOLIDITY_CASE_FILE_EXTENSION: &str = "sol";
+
 #[derive(Debug, Default, Deserialize, Clone, Eq, PartialEq)]
 pub struct Metadata {
     pub cases: Vec<Case>,
@@ -11,4 +18,64 @@ pub struct Metadata {
     pub libraries: Option<BTreeMap<String, BTreeMap<String, String>>>,
     pub ignore: Option<bool>,
     pub modes: Option<Vec<Mode>>,
+    pub path: Option<PathBuf>,
+    pub directory: Option<PathBuf>,
+}
+
+impl Metadata {
+    /// Try to parse the test metadata struct from the given file at `path`.
+    ///
+    /// Returns `None` if `path` didn't contain a test metadata or case definition.
+    ///
+    /// # Panics
+    /// Expects the supplied `path` to be a file.
+    pub fn try_from_file(path: &Path) -> Option<Self> {
+        assert!(path.is_file(), "not a file: {}", path.display());
+
+        let Some(file_extension) = path.extension() else {
+            log::debug!("skipping corpus file: {}", path.display());
+            return None;
+        };
+
+        if file_extension == METADATA_FILE_EXTENSION {
+            return Self::try_from_json(path);
+        }
+
+        if file_extension == SOLIDITY_CASE_FILE_EXTENSION {
+            return None;
+            //return Self::try_from_solidity(path);
+        }
+
+        log::debug!("ignoring invalid corpus file: {}", path.display());
+        None
+    }
+
+    fn try_from_json(path: &Path) -> Option<Self> {
+        let file = File::open(path)
+            .inspect_err(|error| {
+                log::error!(
+                    "opening JSON test metadata file '{}' error: {error}",
+                    path.display()
+                );
+            })
+            .ok()?;
+
+        match serde_json::from_reader::<_, Metadata>(file) {
+            Ok(mut metadata) => {
+                metadata.directory = Some(path.to_path_buf());
+                Some(metadata)
+            }
+            Err(error) => {
+                log::error!(
+                    "parsing JSON test metadata file '{}' error: {error}",
+                    path.display()
+                );
+                None
+            }
+        }
+    }
+
+    fn try_from_solidity(path: &Path) -> Option<Self> {
+        todo!()
+    }
 }

crates/format/src/mode.rs

@@ -38,13 +38,13 @@ impl SolcMode {
             _ => return None,
         }
 
-        while let Some(part) = parts.next() {
+        for part in parts {
             if let Ok(solc_version) = semver::VersionReq::parse(part) {
                 result.solc_version = Some(solc_version);
                 continue;
             }
-            if part.starts_with("-O") {
-                result.llvm_optimizer_settings.push(part[2..].to_string());
+            if let Some(level) = part.strip_prefix("-O") {
+                result.llvm_optimizer_settings.push(level.to_string());
                 continue;
             }
             panic!("the YUL mode string {mode_string} failed to parse, invalid part: {part}")

crates/retester/src/main.rs

@@ -1,9 +0,0 @@
-use revive_differential_testing_format::metadata::Metadata;
-
-fn main() {
-    let example_def = include_str!("../../../test.json");
-
-    let metadata: Metadata = serde_json::from_str(example_def).unwrap();
-
-    println!("{metadata:?}");
-}