Get rid of corpus files

Cargo.lock

@@ -5628,6 +5628,7 @@ dependencies = [
  "semver 1.0.26",
  "serde",
  "serde_json",
+ "serde_with",
  "strum",
  "temp-dir",
 ]
@@ -5668,6 +5669,7 @@ dependencies = [
  "alloy",
  "anyhow",
  "futures",
+ "itertools 0.14.0",
  "revive-common",
  "revive-dt-common",
  "schemars 1.0.4",

Cargo.toml

@@ -69,6 +69,7 @@ tracing-subscriber = { version = "0.3.19", default-features = false, features =
     "env-filter",
 ] }
 indexmap = { version = "2.10.0", default-features = false }
+itertools = { version = "0.14.0" }
 
 # revive compiler
 revive-solc-json-interface = { git = "https://github.com/paritytech/revive", rev = "3389865af7c3ff6f29a586d82157e8bc573c1a8e" }

crates/common/src/types/mod.rs

@@ -1,11 +1,13 @@
 mod identifiers;
 mod mode;
+mod parsed_test_specifier;
 mod private_key_allocator;
 mod round_robin_pool;
 mod version_or_requirement;
 
 pub use identifiers::*;
 pub use mode::*;
+pub use parsed_test_specifier::*;
 pub use private_key_allocator::*;
 pub use round_robin_pool::*;
 pub use version_or_requirement::*;

crates/common/src/types/parsed_test_specifier.rs

@@ -0,0 +1,133 @@
+use std::{fmt::Display, path::PathBuf, str::FromStr};
+
+use anyhow::{Context as _, bail};
+
+use crate::types::Mode;
+
+#[derive(Clone, Debug, PartialEq, Eq, Hash)]
+pub enum ParsedTestSpecifier {
+    /// All of the test cases in the file should be ran across all of the specified modes
+    FileOrDirectory {
+        /// The path of the metadata file containing the test cases.
+        metadata_or_directory_file_path: PathBuf,
+    },
+    /// Only a specific case within the metadata file should be ran across all of the modes in the
+    /// file.
+    Case {
+        /// The path of the metadata file containing the test cases.
+        metadata_file_path: PathBuf,
+
+        /// The index of the specific case to run.
+        case_idx: usize,
+    },
+    /// A specific case and a specific mode should be ran. This is the most specific out of all of
+    /// the specifier types.
+    CaseWithMode {
+        /// The path of the metadata file containing the test cases.
+        metadata_file_path: PathBuf,
+
+        /// The index of the specific case to run.
+        case_idx: usize,
+
+        /// The parsed mode that the test should be run in.
+        mode: Mode,
+    },
+}
+
+impl Display for ParsedTestSpecifier {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        match self {
+            ParsedTestSpecifier::FileOrDirectory {
+                metadata_or_directory_file_path,
+            } => {
+                write!(f, "{}", metadata_or_directory_file_path.display())
+            }
+            ParsedTestSpecifier::Case {
+                metadata_file_path,
+                case_idx,
+            } => {
+                write!(f, "{}::{}", metadata_file_path.display(), case_idx)
+            }
+            ParsedTestSpecifier::CaseWithMode {
+                metadata_file_path,
+                case_idx,
+                mode,
+            } => {
+                write!(
+                    f,
+                    "{}::{}::{}",
+                    metadata_file_path.display(),
+                    case_idx,
+                    mode
+                )
+            }
+        }
+    }
+}
+
+impl FromStr for ParsedTestSpecifier {
+    type Err = anyhow::Error;
+
+    fn from_str(s: &str) -> Result<Self, Self::Err> {
+        let mut split_iter = s.split("::");
+
+        let Some(path_string) = split_iter.next() else {
+            bail!("Could not find the path in the test specifier")
+        };
+        let path = PathBuf::from(path_string)
+            .canonicalize()
+            .context("Failed to canonicalize the path of the test")?;
+
+        let Some(case_idx_string) = split_iter.next() else {
+            return Ok(Self::FileOrDirectory {
+                metadata_or_directory_file_path: path,
+            });
+        };
+        let case_idx = usize::from_str(case_idx_string)
+            .context("Failed to parse the case idx of the test specifier from string")?;
+
+        // At this point the provided path must be a file.
+        if !path.is_file() {
+            bail!(
+                "Test specifier with a path and case idx must point to a file and not a directory"
+            )
+        }
+
+        let Some(mode_string) = split_iter.next() else {
+            return Ok(Self::Case {
+                metadata_file_path: path,
+                case_idx,
+            });
+        };
+        let mode = Mode::from_str(mode_string)
+            .context("Failed to parse the mode string in the parsed test specifier")?;
+
+        Ok(Self::CaseWithMode {
+            metadata_file_path: path,
+            case_idx,
+            mode,
+        })
+    }
+}
+
+impl From<ParsedTestSpecifier> for String {
+    fn from(value: ParsedTestSpecifier) -> Self {
+        value.to_string()
+    }
+}
+
+impl TryFrom<String> for ParsedTestSpecifier {
+    type Error = anyhow::Error;
+
+    fn try_from(value: String) -> Result<Self, Self::Error> {
+        value.parse()
+    }
+}
+
+impl TryFrom<&str> for ParsedTestSpecifier {
+    type Error = anyhow::Error;
+
+    fn try_from(value: &str) -> Result<Self, Self::Error> {
+        value.parse()
+    }
+}

crates/config/Cargo.toml

@@ -18,6 +18,7 @@ semver = { workspace = true }
 temp-dir = { workspace = true }
 serde = { workspace = true }
 serde_json = { workspace = true }
+serde_with = { workspace = true }
 strum = { workspace = true }
 
 [lints]

crates/config/src/lib.rs

@@ -17,7 +17,7 @@ use alloy::{
     signers::local::PrivateKeySigner,
 };
 use clap::{Parser, ValueEnum, ValueHint};
-use revive_dt_common::types::PlatformIdentifier;
+use revive_dt_common::types::{ParsedTestSpecifier, PlatformIdentifier};
 use semver::Version;
 use serde::{Deserialize, Serialize, Serializer};
 use strum::{AsRefStr, Display, EnumString, IntoStaticStr};
@@ -652,11 +652,24 @@ impl AsRef<WalletConfiguration> for ExportGenesisContext {
 }
 
 /// A set of configuration parameters for the corpus files to use for the execution.
+#[serde_with::serde_as]
 #[derive(Clone, Debug, Parser, Serialize, Deserialize)]
 pub struct CorpusConfiguration {
-    /// A list of test corpus JSON files to be tested.
+    /// A list of test specifiers for the tests that the tool should run.
-    #[arg(short = 'c', long = "corpus")]
+    ///
-    pub paths: Vec<PathBuf>,
+    /// Test specifiers follow the following format:
+    ///
+    /// - `{directory_path|metadata_file_path}`: A path to a metadata file where all of the cases
+    ///   live and should be run. Alternatively, it points to a directory instructing the framework
+    ///   to discover of the metadata files that live there an execute them.
+    /// - `{metadata_file_path}::{case_idx}`: The path to a metadata file and then a case idx
+    ///   separated by two colons. This specifies that only this specific test case within the
+    ///   metadata file should be executed.
+    /// - `{metadata_file_path}::{case_idx}::{mode}`: This is very similar to the above specifier
+    ///   with the exception that in this case the mode is specified and will be used in the test.
+    #[serde_as(as = "Vec<serde_with::DisplayFromStr>")]
+    #[arg(short = 't', long = "test")]
+    pub test_specifiers: Vec<ParsedTestSpecifier>,
 }
 
 /// A set of configuration parameters for Solc.

crates/core/src/differential_benchmarks/entry_point.rs

@@ -6,7 +6,10 @@ use anyhow::Context as _;
 use futures::{FutureExt, StreamExt};
 use revive_dt_common::types::PrivateKeyAllocator;
 use revive_dt_core::Platform;
-use revive_dt_format::steps::{Step, StepIdx, StepPath};
+use revive_dt_format::{
+    corpus::Corpus,
+    steps::{Step, StepIdx, StepPath},
+};
 use tokio::sync::Mutex;
 use tracing::{Instrument, error, info, info_span, instrument, warn};
 
@@ -15,7 +18,7 @@ use revive_dt_report::Reporter;
 
 use crate::{
     differential_benchmarks::{Driver, Watcher, WatcherEvent},
-    helpers::{CachedCompiler, NodePool, collect_metadata_files, create_test_definitions_stream},
+    helpers::{CachedCompiler, NodePool, create_test_definitions_stream},
 };
 
 /// Handles the differential testing executing it according to the information defined in the
@@ -39,9 +42,17 @@ pub async fn handle_differential_benchmarks(
     let full_context = Context::Benchmark(Box::new(context.clone()));
 
     // Discover all of the metadata files that are defined in the context.
-    let metadata_files = collect_metadata_files(&context)
+    let corpus = context
-        .context("Failed to collect metadata files for differential testing")?;
+        .corpus_configuration
-    info!(len = metadata_files.len(), "Discovered metadata files");
+        .test_specifiers
+        .clone()
+        .into_iter()
+        .try_fold(Corpus::default(), Corpus::with_test_specifier)
+        .context("Failed to parse the test corpus")?;
+    info!(
+        len = corpus.metadata_file_count(),
+        "Discovered metadata files"
+    );
 
     // Discover the list of platforms that the tests should run on based on the context.
     let platforms = context
@@ -84,7 +95,7 @@ pub async fn handle_differential_benchmarks(
     // Preparing test definitions for the execution.
     let test_definitions = create_test_definitions_stream(
         &full_context,
-        metadata_files.iter(),
+        &corpus,
         &platforms_and_nodes,
         None,
         reporter.clone(),

crates/core/src/differential_tests/entry_point.rs

@@ -12,6 +12,7 @@ use anyhow::Context as _;
 use futures::{FutureExt, StreamExt};
 use revive_dt_common::{cached_fs::read_to_string, types::PrivateKeyAllocator};
 use revive_dt_core::Platform;
+use revive_dt_format::corpus::Corpus;
 use tokio::sync::{Mutex, RwLock, Semaphore};
 use tracing::{Instrument, error, info, info_span, instrument};
 
@@ -20,7 +21,7 @@ use revive_dt_report::{Reporter, ReporterEvent, TestCaseStatus};
 
 use crate::{
     differential_tests::Driver,
-    helpers::{CachedCompiler, NodePool, collect_metadata_files, create_test_definitions_stream},
+    helpers::{CachedCompiler, NodePool, create_test_definitions_stream},
 };
 
 /// Handles the differential testing executing it according to the information defined in the
@@ -33,9 +34,17 @@ pub async fn handle_differential_tests(
     let reporter_clone = reporter.clone();
 
     // Discover all of the metadata files that are defined in the context.
-    let metadata_files = collect_metadata_files(&context)
+    let corpus = context
-        .context("Failed to collect metadata files for differential testing")?;
+        .corpus_configuration
-    info!(len = metadata_files.len(), "Discovered metadata files");
+        .test_specifiers
+        .clone()
+        .into_iter()
+        .try_fold(Corpus::default(), Corpus::with_test_specifier)
+        .context("Failed to parse the test corpus")?;
+    info!(
+        len = corpus.metadata_file_count(),
+        "Discovered metadata files"
+    );
 
     // Discover the list of platforms that the tests should run on based on the context.
     let platforms = context
@@ -83,7 +92,7 @@ pub async fn handle_differential_tests(
     let full_context = Context::Test(Box::new(context.clone()));
     let test_definitions = create_test_definitions_stream(
         &full_context,
-        metadata_files.iter(),
+        &corpus,
         &platforms_and_nodes,
         only_execute_failed_tests.as_ref(),
         reporter.clone(),

crates/core/src/helpers/metadata.rs

@@ -1,33 +0,0 @@
-use revive_dt_config::CorpusConfiguration;
-use revive_dt_format::{corpus::Corpus, metadata::MetadataFile};
-use tracing::{info, info_span, instrument};
-
-/// Given an object that implements [`AsRef<CorpusConfiguration>`], this function finds all of the
-/// corpus files and produces a map containing all of the [`MetadataFile`]s discovered.
-#[instrument(level = "debug", name = "Collecting Corpora", skip_all)]
-pub fn collect_metadata_files(
-    context: impl AsRef<CorpusConfiguration>,
-) -> anyhow::Result<Vec<MetadataFile>> {
-    let mut metadata_files = Vec::new();
-
-    let corpus_configuration = AsRef::<CorpusConfiguration>::as_ref(&context);
-    for path in &corpus_configuration.paths {
-        let span = info_span!("Processing corpus file", path = %path.display());
-        let _guard = span.enter();
-
-        let corpus = Corpus::try_from_path(path)?;
-        info!(
-            name = corpus.name(),
-            number_of_contained_paths = corpus.path_count(),
-            "Deserialized corpus file"
-        );
-        metadata_files.extend(corpus.enumerate_tests());
-    }
-
-    // There's a possibility that there are certain paths that all lead to the same metadata files
-    // and therefore it's important that we sort them and then deduplicate them.
-    metadata_files.sort_by(|a, b| a.metadata_file_path.cmp(&b.metadata_file_path));
-    metadata_files.dedup_by(|a, b| a.metadata_file_path == b.metadata_file_path);
-
-    Ok(metadata_files)
-}

crates/core/src/helpers/mod.rs

@@ -1,9 +1,7 @@
 mod cached_compiler;
-mod metadata;
 mod pool;
 mod test;
 
 pub use cached_compiler::*;
-pub use metadata::*;
 pub use pool::*;
 pub use test::*;

crates/core/src/helpers/test.rs

@@ -4,9 +4,9 @@ use std::{borrow::Cow, path::Path};
 
 use futures::{Stream, StreamExt, stream};
 use indexmap::{IndexMap, indexmap};
-use revive_dt_common::iterators::EitherIter;
+use revive_dt_common::types::PlatformIdentifier;
-use revive_dt_common::types::{ParsedMode, PlatformIdentifier};
 use revive_dt_config::Context;
+use revive_dt_format::corpus::Corpus;
 use serde_json::{Value, json};
 
 use revive_dt_compiler::Mode;
@@ -27,47 +27,28 @@ pub async fn create_test_definitions_stream<'a>(
     // This is only required for creating the compiler objects and is not used anywhere else in the
     // function.
     context: &Context,
-    metadata_files: impl IntoIterator<Item = &'a MetadataFile>,
+    corpus: &'a Corpus,
     platforms_and_nodes: &'a BTreeMap<PlatformIdentifier, (&dyn Platform, NodePool)>,
     only_execute_failed_tests: Option<&Report>,
     reporter: Reporter,
 ) -> impl Stream<Item = TestDefinition<'a>> {
     stream::iter(
-        metadata_files
+        corpus
-            .into_iter()
+            .cases_iterator()
-            // Flatten over the cases.
+            .map(move |(metadata_file, case_idx, case, mode)| {
-            .flat_map(|metadata_file| {
-                metadata_file
-                    .cases
-                    .iter()
-                    .enumerate()
-                    .map(move |(case_idx, case)| (metadata_file, case_idx, case))
-            })
-            // Flatten over the modes, prefer the case modes over the metadata file modes.
-            .flat_map(move |(metadata_file, case_idx, case)| {
                 let reporter = reporter.clone();
 
-                let modes = case.modes.as_ref().or(metadata_file.modes.as_ref());
+                (
-                let modes = match modes {
+                    metadata_file,
-                    Some(modes) => EitherIter::A(
+                    case_idx,
-                        ParsedMode::many_to_modes(modes.iter()).map(Cow::<'static, _>::Owned),
+                    case,
-                    ),
+                    mode.clone(),
-                    None => EitherIter::B(Mode::all().map(Cow::<'static, _>::Borrowed)),
+                    reporter.test_specific_reporter(Arc::new(TestSpecifier {
-                };
+                        solc_mode: mode.as_ref().clone(),
-
+                        metadata_file_path: metadata_file.metadata_file_path.clone(),
-                modes.into_iter().map(move |mode| {
+                        case_idx: CaseIdx::new(case_idx),
-                    (
+                    })),
-                        metadata_file,
+                )
-                        case_idx,
-                        case,
-                        mode.clone(),
-                        reporter.test_specific_reporter(Arc::new(TestSpecifier {
-                            solc_mode: mode.as_ref().clone(),
-                            metadata_file_path: metadata_file.metadata_file_path.clone(),
-                            case_idx: CaseIdx::new(case_idx),
-                        })),
-                    )
-                })
             })
             // Inform the reporter of each one of the test cases that were discovered which we expect to
             // run.

crates/format/Cargo.toml

@@ -21,6 +21,7 @@ schemars = { workspace = true }
 semver = { workspace = true }
 serde = { workspace = true, features = ["derive"] }
 serde_json = { workspace = true }
+itertools = { workspace = true }
 
 [dev-dependencies]
 tokio = { workspace = true }

crates/format/src/corpus.rs

@@ -1,131 +1,200 @@
 use std::{
-    fs::File,
+    borrow::Cow,
+    collections::HashMap,
     path::{Path, PathBuf},
 };
 
-use revive_dt_common::iterators::FilesWithExtensionIterator;
+use itertools::Itertools;
-use serde::{Deserialize, Serialize};
+use revive_dt_common::{
-use tracing::{debug, info};
+    iterators::{EitherIter, FilesWithExtensionIterator},
+    types::{Mode, ParsedMode, ParsedTestSpecifier},
+};
+use tracing::{debug, warn};
 
-use crate::metadata::{Metadata, MetadataFile};
+use crate::{
-use anyhow::Context as _;
+    case::{Case, CaseIdx},
+    metadata::{Metadata, MetadataFile},
+};
 
-#[derive(Clone, Debug, PartialEq, Eq, PartialOrd, Ord, Hash, Serialize, Deserialize)]
+#[derive(Default)]
-#[serde(untagged)]
+pub struct Corpus {
-pub enum Corpus {
+    test_specifiers: HashMap<ParsedTestSpecifier, Vec<PathBuf>>,
-    SinglePath { name: String, path: PathBuf },
+    metadata_files: HashMap<PathBuf, MetadataFile>,
-    MultiplePaths { name: String, paths: Vec<PathBuf> },
 }
 
 impl Corpus {
-    pub fn try_from_path(file_path: impl AsRef<Path>) -> anyhow::Result<Self> {
+    pub fn new() -> Self {
-        let mut corpus = File::open(file_path.as_ref())
+        Default::default()
-            .map_err(anyhow::Error::from)
-            .and_then(|file| serde_json::from_reader::<_, Corpus>(file).map_err(Into::into))
-            .with_context(|| {
-                format!(
-                    "Failed to open and deserialize corpus file at {}",
-                    file_path.as_ref().display()
-                )
-            })?;
-
-        let corpus_directory = file_path
-            .as_ref()
-            .canonicalize()
-            .context("Failed to canonicalize the path to the corpus file")?
-            .parent()
-            .context("Corpus file has no parent")?
-            .to_path_buf();
-
-        for path in corpus.paths_iter_mut() {
-            *path = corpus_directory.join(path.as_path())
-        }
-
-        Ok(corpus)
     }
 
-    pub fn enumerate_tests(&self) -> Vec<MetadataFile> {
+    pub fn with_test_specifier(
-        let mut tests = self
+        mut self,
-            .paths_iter()
+        test_specifier: ParsedTestSpecifier,
-            .flat_map(|root_path| {
+    ) -> anyhow::Result<Self> {
-                if !root_path.is_dir() {
+        match &test_specifier {
-                    Box::new(std::iter::once(root_path.to_path_buf()))
+            ParsedTestSpecifier::FileOrDirectory {
-                        as Box<dyn Iterator<Item = _>>
+                metadata_or_directory_file_path: metadata_file_path,
-                } else {
+            }
-                    Box::new(
+            | ParsedTestSpecifier::Case {
-                        FilesWithExtensionIterator::new(root_path)
+                metadata_file_path, ..
-                            .with_use_cached_fs(true)
+            }
-                            .with_allowed_extension("sol")
+            | ParsedTestSpecifier::CaseWithMode {
-                            .with_allowed_extension("json"),
+                metadata_file_path, ..
-                    )
+            } => {
+                let metadata_files = enumerate_metadata_files(metadata_file_path);
+                self.test_specifiers.insert(
+                    test_specifier,
+                    metadata_files
+                        .iter()
+                        .map(|metadata_file| metadata_file.metadata_file_path.clone())
+                        .collect(),
+                );
+                for metadata_file in metadata_files.into_iter() {
+                    self.metadata_files
+                        .insert(metadata_file.metadata_file_path.clone(), metadata_file);
                 }
-                .map(move |metadata_file_path| (root_path, metadata_file_path))
+            }
-            })
+        };
-            .filter_map(|(root_path, metadata_file_path)| {
+
-                Metadata::try_from_file(&metadata_file_path)
+        Ok(self)
-                    .or_else(|| {
+    }
-                        debug!(
+
-                            discovered_from = %root_path.display(),
+    pub fn cases_iterator(
-                            metadata_file_path = %metadata_file_path.display(),
+        &self,
-                            "Skipping file since it doesn't contain valid metadata"
+    ) -> impl Iterator<Item = (&'_ MetadataFile, CaseIdx, &'_ Case, Cow<'_, Mode>)> + '_ {
-                        );
+        let mut iterator = Box::new(std::iter::empty())
-                        None
+            as Box<dyn Iterator<Item = (&'_ MetadataFile, CaseIdx, &'_ Case, Cow<'_, Mode>)> + '_>;
-                    })
+
-                    .map(|metadata| MetadataFile {
+        for (test_specifier, metadata_file_paths) in self.test_specifiers.iter() {
-                        metadata_file_path,
+            for metadata_file_path in metadata_file_paths {
-                        corpus_file_path: root_path.to_path_buf(),
+                let metadata_file = self
-                        content: metadata,
+                    .metadata_files
-                    })
+                    .get(metadata_file_path)
-                    .inspect(|metadata_file| {
+                    .expect("Must succeed");
-                        debug!(
+
-                            metadata_file_path = %metadata_file.relative_path().display(),
+                match test_specifier {
-                            "Loaded metadata file"
+                    ParsedTestSpecifier::FileOrDirectory { .. } => {
+                        for (case_idx, case) in metadata_file.cases.iter().enumerate() {
+                            let case_idx = CaseIdx::new(case_idx);
+
+                            let modes = case.modes.as_ref().or(metadata_file.modes.as_ref());
+                            let modes = match modes {
+                                Some(modes) => EitherIter::A(
+                                    ParsedMode::many_to_modes(modes.iter())
+                                        .map(Cow::<'static, _>::Owned),
+                                ),
+                                None => EitherIter::B(Mode::all().map(Cow::<'static, _>::Borrowed)),
+                            };
+
+                            iterator = Box::new(
+                                iterator.chain(
+                                    modes
+                                        .into_iter()
+                                        .map(move |mode| (metadata_file, case_idx, case, mode)),
+                                ),
+                            )
+                        }
+                    }
+                    ParsedTestSpecifier::Case { case_idx, .. } => {
+                        let Some(case) = metadata_file.cases.get(*case_idx) else {
+                            warn!(
+                                test_specifier = %test_specifier,
+                                metadata_file_path = %metadata_file_path.display(),
+                                case_idx = case_idx,
+                                case_count = metadata_file.cases.len(),
+                                "Specified case not found in metadata file"
+                            );
+                            continue;
+                        };
+                        let case_idx = CaseIdx::new(*case_idx);
+
+                        let modes = case.modes.as_ref().or(metadata_file.modes.as_ref());
+                        let modes = match modes {
+                            Some(modes) => EitherIter::A(
+                                ParsedMode::many_to_modes(modes.iter())
+                                    .map(Cow::<'static, Mode>::Owned),
+                            ),
+                            None => EitherIter::B(Mode::all().map(Cow::<'static, _>::Borrowed)),
+                        };
+
+                        iterator = Box::new(
+                            iterator.chain(
+                                modes
+                                    .into_iter()
+                                    .map(move |mode| (metadata_file, case_idx, case, mode)),
+                            ),
                         )
-                    })
+                    }
-            })
+                    ParsedTestSpecifier::CaseWithMode { case_idx, mode, .. } => {
-            .collect::<Vec<_>>();
+                        let Some(case) = metadata_file.cases.get(*case_idx) else {
-        tests.sort_by(|a, b| a.metadata_file_path.cmp(&b.metadata_file_path));
+                            warn!(
-        tests.dedup_by(|a, b| a.metadata_file_path == b.metadata_file_path);
+                                test_specifier = %test_specifier,
-        info!(
+                                metadata_file_path = %metadata_file_path.display(),
-            len = tests.len(),
+                                case_idx = case_idx,
-            corpus_name = self.name(),
+                                case_count = metadata_file.cases.len(),
-            "Found tests in Corpus"
+                                "Specified case not found in metadata file"
-        );
+                            );
-        tests
+                            continue;
-    }
+                        };
+                        let case_idx = CaseIdx::new(*case_idx);
 
-    pub fn name(&self) -> &str {
+                        let mode = Cow::Borrowed(mode);
-        match self {
+                        iterator = Box::new(iterator.chain(std::iter::once((
-            Corpus::SinglePath { name, .. } | Corpus::MultiplePaths { name, .. } => name.as_str(),
+                            metadata_file,
-        }
+                            case_idx,
-    }
+                            case,
-
+                            mode,
-    pub fn paths_iter(&self) -> impl Iterator<Item = &Path> {
+                        ))))
-        match self {
+                    }
-            Corpus::SinglePath { path, .. } => {
+                }
-                Box::new(std::iter::once(path.as_path())) as Box<dyn Iterator<Item = _>>
-            }
-            Corpus::MultiplePaths { paths, .. } => {
-                Box::new(paths.iter().map(|path| path.as_path())) as Box<dyn Iterator<Item = _>>
             }
         }
+
+        iterator.unique_by(|item| (&item.0.metadata_file_path, item.1, item.3.clone()))
     }
 
-    pub fn paths_iter_mut(&mut self) -> impl Iterator<Item = &mut PathBuf> {
+    pub fn metadata_file_count(&self) -> usize {
-        match self {
+        self.metadata_files.len()
-            Corpus::SinglePath { path, .. } => {
-                Box::new(std::iter::once(path)) as Box<dyn Iterator<Item = _>>
-            }
-            Corpus::MultiplePaths { paths, .. } => {
-                Box::new(paths.iter_mut()) as Box<dyn Iterator<Item = _>>
-            }
-        }
-    }
-
-    pub fn path_count(&self) -> usize {
-        match self {
-            Corpus::SinglePath { .. } => 1,
-            Corpus::MultiplePaths { paths, .. } => paths.len(),
-        }
     }
 }
+
+fn enumerate_metadata_files(path: impl AsRef<Path>) -> Vec<MetadataFile> {
+    let root_path = path.as_ref();
+    let mut tests = if !root_path.is_dir() {
+        Box::new(std::iter::once(root_path.to_path_buf())) as Box<dyn Iterator<Item = _>>
+    } else {
+        Box::new(
+            FilesWithExtensionIterator::new(root_path)
+                .with_use_cached_fs(true)
+                .with_allowed_extension("sol")
+                .with_allowed_extension("json"),
+        )
+    }
+    .map(move |metadata_file_path| (root_path, metadata_file_path))
+    .filter_map(|(root_path, metadata_file_path)| {
+        Metadata::try_from_file(&metadata_file_path)
+            .or_else(|| {
+                debug!(
+                    discovered_from = %root_path.display(),
+                    metadata_file_path = %metadata_file_path.display(),
+                    "Skipping file since it doesn't contain valid metadata"
+                );
+                None
+            })
+            .map(|metadata| MetadataFile {
+                metadata_file_path,
+                corpus_file_path: root_path.to_path_buf(),
+                content: metadata,
+            })
+            .inspect(|metadata_file| {
+                debug!(
+                    metadata_file_path = %metadata_file.relative_path().display(),
+                    "Loaded metadata file"
+                )
+            })
+    })
+    .collect::<Vec<_>>();
+    tests.sort_by(|a, b| a.metadata_file_path.cmp(&b.metadata_file_path));
+    tests.dedup_by(|a, b| a.metadata_file_path == b.metadata_file_path);
+    tests
+}

crates/report/src/aggregator.rs

@@ -11,10 +11,10 @@ use std::{
 use alloy::primitives::Address;
 use anyhow::{Context as _, Result};
 use indexmap::IndexMap;
-use revive_dt_common::types::PlatformIdentifier;
+use revive_dt_common::types::{ParsedTestSpecifier, PlatformIdentifier};
 use revive_dt_compiler::{CompilerInput, CompilerOutput, Mode};
 use revive_dt_config::Context;
-use revive_dt_format::{case::CaseIdx, corpus::Corpus, metadata::ContractInstance};
+use revive_dt_format::{case::CaseIdx, metadata::ContractInstance};
 use semver::Version;
 use serde::{Deserialize, Serialize};
 use serde_with::{DisplayFromStr, serde_as};
@@ -67,7 +67,7 @@ impl ReportAggregator {
                 RunnerEvent::SubscribeToEvents(event) => {
                     self.handle_subscribe_to_events_event(*event);
                 }
-                RunnerEvent::CorpusFileDiscovery(event) => {
+                RunnerEvent::CorpusDiscovery(event) => {
                     self.handle_corpus_file_discovered_event(*event)
                 }
                 RunnerEvent::MetadataFileDiscovery(event) => {
@@ -152,8 +152,8 @@ impl ReportAggregator {
         let _ = event.tx.send(self.listener_tx.subscribe());
     }
 
-    fn handle_corpus_file_discovered_event(&mut self, event: CorpusFileDiscoveryEvent) {
+    fn handle_corpus_file_discovered_event(&mut self, event: CorpusDiscoveryEvent) {
-        self.report.corpora.push(event.corpus);
+        self.report.corpora.extend(event.test_specifiers);
     }
 
     fn handle_metadata_file_discovery_event(&mut self, event: MetadataFileDiscoveryEvent) {
@@ -420,7 +420,8 @@ pub struct Report {
     /// The context that the tool was started up with.
     pub context: Context,
     /// The list of corpus files that the tool found.
-    pub corpora: Vec<Corpus>,
+    #[serde_as(as = "Vec<DisplayFromStr>")]
+    pub corpora: Vec<ParsedTestSpecifier>,
     /// The list of metadata files that were found by the tool.
     pub metadata_files: BTreeSet<MetadataFilePath>,
     /// Information relating to each test case.

crates/report/src/runner_event.rs

@@ -6,10 +6,11 @@ use std::{collections::BTreeMap, path::PathBuf, sync::Arc};
 use alloy::primitives::Address;
 use anyhow::Context as _;
 use indexmap::IndexMap;
+use revive_dt_common::types::ParsedTestSpecifier;
 use revive_dt_common::types::PlatformIdentifier;
 use revive_dt_compiler::{CompilerInput, CompilerOutput};
+use revive_dt_format::metadata::ContractInstance;
 use revive_dt_format::metadata::Metadata;
-use revive_dt_format::{corpus::Corpus, metadata::ContractInstance};
 use semver::Version;
 use tokio::sync::{broadcast, oneshot};
 
@@ -481,9 +482,9 @@ define_event! {
             tx: oneshot::Sender<broadcast::Receiver<ReporterEvent>>
         },
         /// An event emitted by runners when they've discovered a corpus file.
-        CorpusFileDiscovery {
+        CorpusDiscovery {
             /// The contents of the corpus file.
-            corpus: Corpus
+            test_specifiers: Vec<ParsedTestSpecifier>
         },
         /// An event emitted by runners when they've discovered a metadata file.
         MetadataFileDiscovery {

run_tests.sh

@@ -1,7 +1,7 @@
 #!/bin/bash
 
 # Revive Differential Tests - Quick Start Script
-# This script clones the test repository, sets up the corpus file, and runs the tool
+# This script clones the test repository, and runs the tool
 
 set -e  # Exit on any error
 
@@ -14,7 +14,6 @@ NC='\033[0m' # No Color
 # Configuration
 TEST_REPO_URL="https://github.com/paritytech/resolc-compiler-tests"
 TEST_REPO_DIR="resolc-compiler-tests"
-CORPUS_FILE="./corpus.json"
 WORKDIR="workdir"
 
 # Optional positional argument: path to polkadot-sdk directory
@@ -68,21 +67,6 @@ else
     echo -e "${YELLOW}No polkadot-sdk path provided. Using binaries from $PATH.${NC}"
 fi
 
-# Create corpus file with absolute path resolved at runtime
-echo -e "${GREEN}Creating corpus file...${NC}"
-ABSOLUTE_PATH=$(realpath "$TEST_REPO_DIR/fixtures/solidity/")
-
-cat > "$CORPUS_FILE" << EOF
-{
-  "name": "MatterLabs Solidity Simple, Complex, and Semantic Tests",
-  "paths": [
-    "$(realpath "$TEST_REPO_DIR/fixtures/solidity")"
-  ]
-}
-EOF
-
-echo -e "${GREEN}Corpus file created: $CORPUS_FILE${NC}"
-
 # Create workdir if it doesn't exist
 mkdir -p "$WORKDIR"
 
@@ -94,7 +78,7 @@ echo ""
 cargo build --release;
 RUST_LOG="info,alloy_pubsub::service=error" ./target/release/retester test \
     --platform revive-dev-node-polkavm-resolc \
-    --corpus "$CORPUS_FILE" \
+    --test $(realpath "$TEST_REPO_DIR/fixtures/solidity") \
     --working-directory "$WORKDIR" \
     --concurrency.number-of-nodes 10 \
     --concurrency.number-of-threads 5 \