From bf35c084d4822dd96e314e308ebf6ebe65518c1d Mon Sep 17 00:00:00 2001 From: Omar Abdulla Date: Thu, 16 Oct 2025 01:09:07 +0300 Subject: [PATCH] Get rid of corpus files --- Cargo.lock | 2 + Cargo.toml | 1 + crates/common/src/types/mod.rs | 2 + .../common/src/types/parsed_test_specifier.rs | 133 ++++++++ crates/config/Cargo.toml | 1 + crates/config/src/lib.rs | 21 +- .../differential_benchmarks/entry_point.rs | 23 +- .../src/differential_tests/entry_point.rs | 19 +- crates/core/src/helpers/metadata.rs | 33 -- crates/core/src/helpers/mod.rs | 2 - crates/core/src/helpers/test.rs | 53 +--- crates/format/Cargo.toml | 1 + crates/format/src/corpus.rs | 291 +++++++++++------- crates/report/src/aggregator.rs | 13 +- crates/report/src/runner_event.rs | 7 +- run_tests.sh | 20 +- 16 files changed, 398 insertions(+), 224 deletions(-) create mode 100644 crates/common/src/types/parsed_test_specifier.rs delete mode 100644 crates/core/src/helpers/metadata.rs diff --git a/Cargo.lock b/Cargo.lock index 7bc7514..3253ee3 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -5628,6 +5628,7 @@ dependencies = [ "semver 1.0.26", "serde", "serde_json", + "serde_with", "strum", "temp-dir", ] @@ -5668,6 +5669,7 @@ dependencies = [ "alloy", "anyhow", "futures", + "itertools 0.14.0", "revive-common", "revive-dt-common", "schemars 1.0.4", diff --git a/Cargo.toml b/Cargo.toml index 907844a..60178b6 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -69,6 +69,7 @@ tracing-subscriber = { version = "0.3.19", default-features = false, features = "env-filter", ] } indexmap = { version = "2.10.0", default-features = false } +itertools = { version = "0.14.0" } # revive compiler revive-solc-json-interface = { git = "https://github.com/paritytech/revive", rev = "3389865af7c3ff6f29a586d82157e8bc573c1a8e" } diff --git a/crates/common/src/types/mod.rs b/crates/common/src/types/mod.rs index 8019524..4d52d08 100644 --- a/crates/common/src/types/mod.rs +++ b/crates/common/src/types/mod.rs @@ -1,11 +1,13 @@ mod identifiers; mod mode; +mod parsed_test_specifier; mod private_key_allocator; mod round_robin_pool; mod version_or_requirement; pub use identifiers::*; pub use mode::*; +pub use parsed_test_specifier::*; pub use private_key_allocator::*; pub use round_robin_pool::*; pub use version_or_requirement::*; diff --git a/crates/common/src/types/parsed_test_specifier.rs b/crates/common/src/types/parsed_test_specifier.rs new file mode 100644 index 0000000..3460ae5 --- /dev/null +++ b/crates/common/src/types/parsed_test_specifier.rs @@ -0,0 +1,133 @@ +use std::{fmt::Display, path::PathBuf, str::FromStr}; + +use anyhow::{Context as _, bail}; + +use crate::types::Mode; + +#[derive(Clone, Debug, PartialEq, Eq, Hash)] +pub enum ParsedTestSpecifier { + /// All of the test cases in the file should be ran across all of the specified modes + FileOrDirectory { + /// The path of the metadata file containing the test cases. + metadata_or_directory_file_path: PathBuf, + }, + /// Only a specific case within the metadata file should be ran across all of the modes in the + /// file. + Case { + /// The path of the metadata file containing the test cases. + metadata_file_path: PathBuf, + + /// The index of the specific case to run. + case_idx: usize, + }, + /// A specific case and a specific mode should be ran. This is the most specific out of all of + /// the specifier types. + CaseWithMode { + /// The path of the metadata file containing the test cases. + metadata_file_path: PathBuf, + + /// The index of the specific case to run. + case_idx: usize, + + /// The parsed mode that the test should be run in. + mode: Mode, + }, +} + +impl Display for ParsedTestSpecifier { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + ParsedTestSpecifier::FileOrDirectory { + metadata_or_directory_file_path, + } => { + write!(f, "{}", metadata_or_directory_file_path.display()) + } + ParsedTestSpecifier::Case { + metadata_file_path, + case_idx, + } => { + write!(f, "{}::{}", metadata_file_path.display(), case_idx) + } + ParsedTestSpecifier::CaseWithMode { + metadata_file_path, + case_idx, + mode, + } => { + write!( + f, + "{}::{}::{}", + metadata_file_path.display(), + case_idx, + mode + ) + } + } + } +} + +impl FromStr for ParsedTestSpecifier { + type Err = anyhow::Error; + + fn from_str(s: &str) -> Result { + let mut split_iter = s.split("::"); + + let Some(path_string) = split_iter.next() else { + bail!("Could not find the path in the test specifier") + }; + let path = PathBuf::from(path_string) + .canonicalize() + .context("Failed to canonicalize the path of the test")?; + + let Some(case_idx_string) = split_iter.next() else { + return Ok(Self::FileOrDirectory { + metadata_or_directory_file_path: path, + }); + }; + let case_idx = usize::from_str(case_idx_string) + .context("Failed to parse the case idx of the test specifier from string")?; + + // At this point the provided path must be a file. + if !path.is_file() { + bail!( + "Test specifier with a path and case idx must point to a file and not a directory" + ) + } + + let Some(mode_string) = split_iter.next() else { + return Ok(Self::Case { + metadata_file_path: path, + case_idx, + }); + }; + let mode = Mode::from_str(mode_string) + .context("Failed to parse the mode string in the parsed test specifier")?; + + Ok(Self::CaseWithMode { + metadata_file_path: path, + case_idx, + mode, + }) + } +} + +impl From for String { + fn from(value: ParsedTestSpecifier) -> Self { + value.to_string() + } +} + +impl TryFrom for ParsedTestSpecifier { + type Error = anyhow::Error; + + fn try_from(value: String) -> Result { + value.parse() + } +} + +impl TryFrom<&str> for ParsedTestSpecifier { + type Error = anyhow::Error; + + fn try_from(value: &str) -> Result { + value.parse() + } +} diff --git a/crates/config/Cargo.toml b/crates/config/Cargo.toml index 1862e74..ad9e6dd 100644 --- a/crates/config/Cargo.toml +++ b/crates/config/Cargo.toml @@ -18,6 +18,7 @@ semver = { workspace = true } temp-dir = { workspace = true } serde = { workspace = true } serde_json = { workspace = true } +serde_with = { workspace = true } strum = { workspace = true } [lints] diff --git a/crates/config/src/lib.rs b/crates/config/src/lib.rs index 01b2064..4baf4e7 100644 --- a/crates/config/src/lib.rs +++ b/crates/config/src/lib.rs @@ -17,7 +17,7 @@ use alloy::{ signers::local::PrivateKeySigner, }; use clap::{Parser, ValueEnum, ValueHint}; -use revive_dt_common::types::PlatformIdentifier; +use revive_dt_common::types::{ParsedTestSpecifier, PlatformIdentifier}; use semver::Version; use serde::{Deserialize, Serialize, Serializer}; use strum::{AsRefStr, Display, EnumString, IntoStaticStr}; @@ -652,11 +652,24 @@ impl AsRef for ExportGenesisContext { } /// A set of configuration parameters for the corpus files to use for the execution. +#[serde_with::serde_as] #[derive(Clone, Debug, Parser, Serialize, Deserialize)] pub struct CorpusConfiguration { - /// A list of test corpus JSON files to be tested. - #[arg(short = 'c', long = "corpus")] - pub paths: Vec, + /// A list of test specifiers for the tests that the tool should run. + /// + /// Test specifiers follow the following format: + /// + /// - `{directory_path|metadata_file_path}`: A path to a metadata file where all of the cases + /// live and should be run. Alternatively, it points to a directory instructing the framework + /// to discover of the metadata files that live there an execute them. + /// - `{metadata_file_path}::{case_idx}`: The path to a metadata file and then a case idx + /// separated by two colons. This specifies that only this specific test case within the + /// metadata file should be executed. + /// - `{metadata_file_path}::{case_idx}::{mode}`: This is very similar to the above specifier + /// with the exception that in this case the mode is specified and will be used in the test. + #[serde_as(as = "Vec")] + #[arg(short = 't', long = "test")] + pub test_specifiers: Vec, } /// A set of configuration parameters for Solc. diff --git a/crates/core/src/differential_benchmarks/entry_point.rs b/crates/core/src/differential_benchmarks/entry_point.rs index d813519..16c5c21 100644 --- a/crates/core/src/differential_benchmarks/entry_point.rs +++ b/crates/core/src/differential_benchmarks/entry_point.rs @@ -6,7 +6,10 @@ use anyhow::Context as _; use futures::{FutureExt, StreamExt}; use revive_dt_common::types::PrivateKeyAllocator; use revive_dt_core::Platform; -use revive_dt_format::steps::{Step, StepIdx, StepPath}; +use revive_dt_format::{ + corpus::Corpus, + steps::{Step, StepIdx, StepPath}, +}; use tokio::sync::Mutex; use tracing::{Instrument, error, info, info_span, instrument, warn}; @@ -15,7 +18,7 @@ use revive_dt_report::Reporter; use crate::{ differential_benchmarks::{Driver, Watcher, WatcherEvent}, - helpers::{CachedCompiler, NodePool, collect_metadata_files, create_test_definitions_stream}, + helpers::{CachedCompiler, NodePool, create_test_definitions_stream}, }; /// Handles the differential testing executing it according to the information defined in the @@ -39,9 +42,17 @@ pub async fn handle_differential_benchmarks( let full_context = Context::Benchmark(Box::new(context.clone())); // Discover all of the metadata files that are defined in the context. - let metadata_files = collect_metadata_files(&context) - .context("Failed to collect metadata files for differential testing")?; - info!(len = metadata_files.len(), "Discovered metadata files"); + let corpus = context + .corpus_configuration + .test_specifiers + .clone() + .into_iter() + .try_fold(Corpus::default(), Corpus::with_test_specifier) + .context("Failed to parse the test corpus")?; + info!( + len = corpus.metadata_file_count(), + "Discovered metadata files" + ); // Discover the list of platforms that the tests should run on based on the context. let platforms = context @@ -84,7 +95,7 @@ pub async fn handle_differential_benchmarks( // Preparing test definitions for the execution. let test_definitions = create_test_definitions_stream( &full_context, - metadata_files.iter(), + &corpus, &platforms_and_nodes, None, reporter.clone(), diff --git a/crates/core/src/differential_tests/entry_point.rs b/crates/core/src/differential_tests/entry_point.rs index 5c7f5dc..5ca42d5 100644 --- a/crates/core/src/differential_tests/entry_point.rs +++ b/crates/core/src/differential_tests/entry_point.rs @@ -12,6 +12,7 @@ use anyhow::Context as _; use futures::{FutureExt, StreamExt}; use revive_dt_common::{cached_fs::read_to_string, types::PrivateKeyAllocator}; use revive_dt_core::Platform; +use revive_dt_format::corpus::Corpus; use tokio::sync::{Mutex, RwLock, Semaphore}; use tracing::{Instrument, error, info, info_span, instrument}; @@ -20,7 +21,7 @@ use revive_dt_report::{Reporter, ReporterEvent, TestCaseStatus}; use crate::{ differential_tests::Driver, - helpers::{CachedCompiler, NodePool, collect_metadata_files, create_test_definitions_stream}, + helpers::{CachedCompiler, NodePool, create_test_definitions_stream}, }; /// Handles the differential testing executing it according to the information defined in the @@ -33,9 +34,17 @@ pub async fn handle_differential_tests( let reporter_clone = reporter.clone(); // Discover all of the metadata files that are defined in the context. - let metadata_files = collect_metadata_files(&context) - .context("Failed to collect metadata files for differential testing")?; - info!(len = metadata_files.len(), "Discovered metadata files"); + let corpus = context + .corpus_configuration + .test_specifiers + .clone() + .into_iter() + .try_fold(Corpus::default(), Corpus::with_test_specifier) + .context("Failed to parse the test corpus")?; + info!( + len = corpus.metadata_file_count(), + "Discovered metadata files" + ); // Discover the list of platforms that the tests should run on based on the context. let platforms = context @@ -83,7 +92,7 @@ pub async fn handle_differential_tests( let full_context = Context::Test(Box::new(context.clone())); let test_definitions = create_test_definitions_stream( &full_context, - metadata_files.iter(), + &corpus, &platforms_and_nodes, only_execute_failed_tests.as_ref(), reporter.clone(), diff --git a/crates/core/src/helpers/metadata.rs b/crates/core/src/helpers/metadata.rs deleted file mode 100644 index 60f351b..0000000 --- a/crates/core/src/helpers/metadata.rs +++ /dev/null @@ -1,33 +0,0 @@ -use revive_dt_config::CorpusConfiguration; -use revive_dt_format::{corpus::Corpus, metadata::MetadataFile}; -use tracing::{info, info_span, instrument}; - -/// Given an object that implements [`AsRef`], this function finds all of the -/// corpus files and produces a map containing all of the [`MetadataFile`]s discovered. -#[instrument(level = "debug", name = "Collecting Corpora", skip_all)] -pub fn collect_metadata_files( - context: impl AsRef, -) -> anyhow::Result> { - let mut metadata_files = Vec::new(); - - let corpus_configuration = AsRef::::as_ref(&context); - for path in &corpus_configuration.paths { - let span = info_span!("Processing corpus file", path = %path.display()); - let _guard = span.enter(); - - let corpus = Corpus::try_from_path(path)?; - info!( - name = corpus.name(), - number_of_contained_paths = corpus.path_count(), - "Deserialized corpus file" - ); - metadata_files.extend(corpus.enumerate_tests()); - } - - // There's a possibility that there are certain paths that all lead to the same metadata files - // and therefore it's important that we sort them and then deduplicate them. - metadata_files.sort_by(|a, b| a.metadata_file_path.cmp(&b.metadata_file_path)); - metadata_files.dedup_by(|a, b| a.metadata_file_path == b.metadata_file_path); - - Ok(metadata_files) -} diff --git a/crates/core/src/helpers/mod.rs b/crates/core/src/helpers/mod.rs index eba4f67..d2948af 100644 --- a/crates/core/src/helpers/mod.rs +++ b/crates/core/src/helpers/mod.rs @@ -1,9 +1,7 @@ mod cached_compiler; -mod metadata; mod pool; mod test; pub use cached_compiler::*; -pub use metadata::*; pub use pool::*; pub use test::*; diff --git a/crates/core/src/helpers/test.rs b/crates/core/src/helpers/test.rs index 14d238a..6571c6b 100644 --- a/crates/core/src/helpers/test.rs +++ b/crates/core/src/helpers/test.rs @@ -4,9 +4,9 @@ use std::{borrow::Cow, path::Path}; use futures::{Stream, StreamExt, stream}; use indexmap::{IndexMap, indexmap}; -use revive_dt_common::iterators::EitherIter; -use revive_dt_common::types::{ParsedMode, PlatformIdentifier}; +use revive_dt_common::types::PlatformIdentifier; use revive_dt_config::Context; +use revive_dt_format::corpus::Corpus; use serde_json::{Value, json}; use revive_dt_compiler::Mode; @@ -27,47 +27,28 @@ pub async fn create_test_definitions_stream<'a>( // This is only required for creating the compiler objects and is not used anywhere else in the // function. context: &Context, - metadata_files: impl IntoIterator, + corpus: &'a Corpus, platforms_and_nodes: &'a BTreeMap, only_execute_failed_tests: Option<&Report>, reporter: Reporter, ) -> impl Stream> { stream::iter( - metadata_files - .into_iter() - // Flatten over the cases. - .flat_map(|metadata_file| { - metadata_file - .cases - .iter() - .enumerate() - .map(move |(case_idx, case)| (metadata_file, case_idx, case)) - }) - // Flatten over the modes, prefer the case modes over the metadata file modes. - .flat_map(move |(metadata_file, case_idx, case)| { + corpus + .cases_iterator() + .map(move |(metadata_file, case_idx, case, mode)| { let reporter = reporter.clone(); - let modes = case.modes.as_ref().or(metadata_file.modes.as_ref()); - let modes = match modes { - Some(modes) => EitherIter::A( - ParsedMode::many_to_modes(modes.iter()).map(Cow::<'static, _>::Owned), - ), - None => EitherIter::B(Mode::all().map(Cow::<'static, _>::Borrowed)), - }; - - modes.into_iter().map(move |mode| { - ( - metadata_file, - case_idx, - case, - mode.clone(), - reporter.test_specific_reporter(Arc::new(TestSpecifier { - solc_mode: mode.as_ref().clone(), - metadata_file_path: metadata_file.metadata_file_path.clone(), - case_idx: CaseIdx::new(case_idx), - })), - ) - }) + ( + metadata_file, + case_idx, + case, + mode.clone(), + reporter.test_specific_reporter(Arc::new(TestSpecifier { + solc_mode: mode.as_ref().clone(), + metadata_file_path: metadata_file.metadata_file_path.clone(), + case_idx: CaseIdx::new(case_idx), + })), + ) }) // Inform the reporter of each one of the test cases that were discovered which we expect to // run. diff --git a/crates/format/Cargo.toml b/crates/format/Cargo.toml index afc09a1..b5f1cde 100644 --- a/crates/format/Cargo.toml +++ b/crates/format/Cargo.toml @@ -21,6 +21,7 @@ schemars = { workspace = true } semver = { workspace = true } serde = { workspace = true, features = ["derive"] } serde_json = { workspace = true } +itertools = { workspace = true } [dev-dependencies] tokio = { workspace = true } diff --git a/crates/format/src/corpus.rs b/crates/format/src/corpus.rs index 62e81f6..83cc847 100644 --- a/crates/format/src/corpus.rs +++ b/crates/format/src/corpus.rs @@ -1,131 +1,200 @@ use std::{ - fs::File, + borrow::Cow, + collections::HashMap, path::{Path, PathBuf}, }; -use revive_dt_common::iterators::FilesWithExtensionIterator; -use serde::{Deserialize, Serialize}; -use tracing::{debug, info}; +use itertools::Itertools; +use revive_dt_common::{ + iterators::{EitherIter, FilesWithExtensionIterator}, + types::{Mode, ParsedMode, ParsedTestSpecifier}, +}; +use tracing::{debug, warn}; -use crate::metadata::{Metadata, MetadataFile}; -use anyhow::Context as _; +use crate::{ + case::{Case, CaseIdx}, + metadata::{Metadata, MetadataFile}, +}; -#[derive(Clone, Debug, PartialEq, Eq, PartialOrd, Ord, Hash, Serialize, Deserialize)] -#[serde(untagged)] -pub enum Corpus { - SinglePath { name: String, path: PathBuf }, - MultiplePaths { name: String, paths: Vec }, +#[derive(Default)] +pub struct Corpus { + test_specifiers: HashMap>, + metadata_files: HashMap, } impl Corpus { - pub fn try_from_path(file_path: impl AsRef) -> anyhow::Result { - let mut corpus = File::open(file_path.as_ref()) - .map_err(anyhow::Error::from) - .and_then(|file| serde_json::from_reader::<_, Corpus>(file).map_err(Into::into)) - .with_context(|| { - format!( - "Failed to open and deserialize corpus file at {}", - file_path.as_ref().display() - ) - })?; - - let corpus_directory = file_path - .as_ref() - .canonicalize() - .context("Failed to canonicalize the path to the corpus file")? - .parent() - .context("Corpus file has no parent")? - .to_path_buf(); - - for path in corpus.paths_iter_mut() { - *path = corpus_directory.join(path.as_path()) - } - - Ok(corpus) + pub fn new() -> Self { + Default::default() } - pub fn enumerate_tests(&self) -> Vec { - let mut tests = self - .paths_iter() - .flat_map(|root_path| { - if !root_path.is_dir() { - Box::new(std::iter::once(root_path.to_path_buf())) - as Box> - } else { - Box::new( - FilesWithExtensionIterator::new(root_path) - .with_use_cached_fs(true) - .with_allowed_extension("sol") - .with_allowed_extension("json"), - ) + pub fn with_test_specifier( + mut self, + test_specifier: ParsedTestSpecifier, + ) -> anyhow::Result { + match &test_specifier { + ParsedTestSpecifier::FileOrDirectory { + metadata_or_directory_file_path: metadata_file_path, + } + | ParsedTestSpecifier::Case { + metadata_file_path, .. + } + | ParsedTestSpecifier::CaseWithMode { + metadata_file_path, .. + } => { + let metadata_files = enumerate_metadata_files(metadata_file_path); + self.test_specifiers.insert( + test_specifier, + metadata_files + .iter() + .map(|metadata_file| metadata_file.metadata_file_path.clone()) + .collect(), + ); + for metadata_file in metadata_files.into_iter() { + self.metadata_files + .insert(metadata_file.metadata_file_path.clone(), metadata_file); } - .map(move |metadata_file_path| (root_path, metadata_file_path)) - }) - .filter_map(|(root_path, metadata_file_path)| { - Metadata::try_from_file(&metadata_file_path) - .or_else(|| { - debug!( - discovered_from = %root_path.display(), - metadata_file_path = %metadata_file_path.display(), - "Skipping file since it doesn't contain valid metadata" - ); - None - }) - .map(|metadata| MetadataFile { - metadata_file_path, - corpus_file_path: root_path.to_path_buf(), - content: metadata, - }) - .inspect(|metadata_file| { - debug!( - metadata_file_path = %metadata_file.relative_path().display(), - "Loaded metadata file" + } + }; + + Ok(self) + } + + pub fn cases_iterator( + &self, + ) -> impl Iterator)> + '_ { + let mut iterator = Box::new(std::iter::empty()) + as Box)> + '_>; + + for (test_specifier, metadata_file_paths) in self.test_specifiers.iter() { + for metadata_file_path in metadata_file_paths { + let metadata_file = self + .metadata_files + .get(metadata_file_path) + .expect("Must succeed"); + + match test_specifier { + ParsedTestSpecifier::FileOrDirectory { .. } => { + for (case_idx, case) in metadata_file.cases.iter().enumerate() { + let case_idx = CaseIdx::new(case_idx); + + let modes = case.modes.as_ref().or(metadata_file.modes.as_ref()); + let modes = match modes { + Some(modes) => EitherIter::A( + ParsedMode::many_to_modes(modes.iter()) + .map(Cow::<'static, _>::Owned), + ), + None => EitherIter::B(Mode::all().map(Cow::<'static, _>::Borrowed)), + }; + + iterator = Box::new( + iterator.chain( + modes + .into_iter() + .map(move |mode| (metadata_file, case_idx, case, mode)), + ), + ) + } + } + ParsedTestSpecifier::Case { case_idx, .. } => { + let Some(case) = metadata_file.cases.get(*case_idx) else { + warn!( + test_specifier = %test_specifier, + metadata_file_path = %metadata_file_path.display(), + case_idx = case_idx, + case_count = metadata_file.cases.len(), + "Specified case not found in metadata file" + ); + continue; + }; + let case_idx = CaseIdx::new(*case_idx); + + let modes = case.modes.as_ref().or(metadata_file.modes.as_ref()); + let modes = match modes { + Some(modes) => EitherIter::A( + ParsedMode::many_to_modes(modes.iter()) + .map(Cow::<'static, Mode>::Owned), + ), + None => EitherIter::B(Mode::all().map(Cow::<'static, _>::Borrowed)), + }; + + iterator = Box::new( + iterator.chain( + modes + .into_iter() + .map(move |mode| (metadata_file, case_idx, case, mode)), + ), ) - }) - }) - .collect::>(); - tests.sort_by(|a, b| a.metadata_file_path.cmp(&b.metadata_file_path)); - tests.dedup_by(|a, b| a.metadata_file_path == b.metadata_file_path); - info!( - len = tests.len(), - corpus_name = self.name(), - "Found tests in Corpus" - ); - tests - } + } + ParsedTestSpecifier::CaseWithMode { case_idx, mode, .. } => { + let Some(case) = metadata_file.cases.get(*case_idx) else { + warn!( + test_specifier = %test_specifier, + metadata_file_path = %metadata_file_path.display(), + case_idx = case_idx, + case_count = metadata_file.cases.len(), + "Specified case not found in metadata file" + ); + continue; + }; + let case_idx = CaseIdx::new(*case_idx); - pub fn name(&self) -> &str { - match self { - Corpus::SinglePath { name, .. } | Corpus::MultiplePaths { name, .. } => name.as_str(), - } - } - - pub fn paths_iter(&self) -> impl Iterator { - match self { - Corpus::SinglePath { path, .. } => { - Box::new(std::iter::once(path.as_path())) as Box> - } - Corpus::MultiplePaths { paths, .. } => { - Box::new(paths.iter().map(|path| path.as_path())) as Box> + let mode = Cow::Borrowed(mode); + iterator = Box::new(iterator.chain(std::iter::once(( + metadata_file, + case_idx, + case, + mode, + )))) + } + } } } + + iterator.unique_by(|item| (&item.0.metadata_file_path, item.1, item.3.clone())) } - pub fn paths_iter_mut(&mut self) -> impl Iterator { - match self { - Corpus::SinglePath { path, .. } => { - Box::new(std::iter::once(path)) as Box> - } - Corpus::MultiplePaths { paths, .. } => { - Box::new(paths.iter_mut()) as Box> - } - } - } - - pub fn path_count(&self) -> usize { - match self { - Corpus::SinglePath { .. } => 1, - Corpus::MultiplePaths { paths, .. } => paths.len(), - } + pub fn metadata_file_count(&self) -> usize { + self.metadata_files.len() } } + +fn enumerate_metadata_files(path: impl AsRef) -> Vec { + let root_path = path.as_ref(); + let mut tests = if !root_path.is_dir() { + Box::new(std::iter::once(root_path.to_path_buf())) as Box> + } else { + Box::new( + FilesWithExtensionIterator::new(root_path) + .with_use_cached_fs(true) + .with_allowed_extension("sol") + .with_allowed_extension("json"), + ) + } + .map(move |metadata_file_path| (root_path, metadata_file_path)) + .filter_map(|(root_path, metadata_file_path)| { + Metadata::try_from_file(&metadata_file_path) + .or_else(|| { + debug!( + discovered_from = %root_path.display(), + metadata_file_path = %metadata_file_path.display(), + "Skipping file since it doesn't contain valid metadata" + ); + None + }) + .map(|metadata| MetadataFile { + metadata_file_path, + corpus_file_path: root_path.to_path_buf(), + content: metadata, + }) + .inspect(|metadata_file| { + debug!( + metadata_file_path = %metadata_file.relative_path().display(), + "Loaded metadata file" + ) + }) + }) + .collect::>(); + tests.sort_by(|a, b| a.metadata_file_path.cmp(&b.metadata_file_path)); + tests.dedup_by(|a, b| a.metadata_file_path == b.metadata_file_path); + tests +} diff --git a/crates/report/src/aggregator.rs b/crates/report/src/aggregator.rs index 9a1a4ad..69d9743 100644 --- a/crates/report/src/aggregator.rs +++ b/crates/report/src/aggregator.rs @@ -11,10 +11,10 @@ use std::{ use alloy::primitives::Address; use anyhow::{Context as _, Result}; use indexmap::IndexMap; -use revive_dt_common::types::PlatformIdentifier; +use revive_dt_common::types::{ParsedTestSpecifier, PlatformIdentifier}; use revive_dt_compiler::{CompilerInput, CompilerOutput, Mode}; use revive_dt_config::Context; -use revive_dt_format::{case::CaseIdx, corpus::Corpus, metadata::ContractInstance}; +use revive_dt_format::{case::CaseIdx, metadata::ContractInstance}; use semver::Version; use serde::{Deserialize, Serialize}; use serde_with::{DisplayFromStr, serde_as}; @@ -67,7 +67,7 @@ impl ReportAggregator { RunnerEvent::SubscribeToEvents(event) => { self.handle_subscribe_to_events_event(*event); } - RunnerEvent::CorpusFileDiscovery(event) => { + RunnerEvent::CorpusDiscovery(event) => { self.handle_corpus_file_discovered_event(*event) } RunnerEvent::MetadataFileDiscovery(event) => { @@ -152,8 +152,8 @@ impl ReportAggregator { let _ = event.tx.send(self.listener_tx.subscribe()); } - fn handle_corpus_file_discovered_event(&mut self, event: CorpusFileDiscoveryEvent) { - self.report.corpora.push(event.corpus); + fn handle_corpus_file_discovered_event(&mut self, event: CorpusDiscoveryEvent) { + self.report.corpora.extend(event.test_specifiers); } fn handle_metadata_file_discovery_event(&mut self, event: MetadataFileDiscoveryEvent) { @@ -420,7 +420,8 @@ pub struct Report { /// The context that the tool was started up with. pub context: Context, /// The list of corpus files that the tool found. - pub corpora: Vec, + #[serde_as(as = "Vec")] + pub corpora: Vec, /// The list of metadata files that were found by the tool. pub metadata_files: BTreeSet, /// Information relating to each test case. diff --git a/crates/report/src/runner_event.rs b/crates/report/src/runner_event.rs index f6d322f..9f3d62b 100644 --- a/crates/report/src/runner_event.rs +++ b/crates/report/src/runner_event.rs @@ -6,10 +6,11 @@ use std::{collections::BTreeMap, path::PathBuf, sync::Arc}; use alloy::primitives::Address; use anyhow::Context as _; use indexmap::IndexMap; +use revive_dt_common::types::ParsedTestSpecifier; use revive_dt_common::types::PlatformIdentifier; use revive_dt_compiler::{CompilerInput, CompilerOutput}; +use revive_dt_format::metadata::ContractInstance; use revive_dt_format::metadata::Metadata; -use revive_dt_format::{corpus::Corpus, metadata::ContractInstance}; use semver::Version; use tokio::sync::{broadcast, oneshot}; @@ -481,9 +482,9 @@ define_event! { tx: oneshot::Sender> }, /// An event emitted by runners when they've discovered a corpus file. - CorpusFileDiscovery { + CorpusDiscovery { /// The contents of the corpus file. - corpus: Corpus + test_specifiers: Vec }, /// An event emitted by runners when they've discovered a metadata file. MetadataFileDiscovery { diff --git a/run_tests.sh b/run_tests.sh index c0bd229..1295d86 100755 --- a/run_tests.sh +++ b/run_tests.sh @@ -1,7 +1,7 @@ #!/bin/bash # Revive Differential Tests - Quick Start Script -# This script clones the test repository, sets up the corpus file, and runs the tool +# This script clones the test repository, and runs the tool set -e # Exit on any error @@ -14,7 +14,6 @@ NC='\033[0m' # No Color # Configuration TEST_REPO_URL="https://github.com/paritytech/resolc-compiler-tests" TEST_REPO_DIR="resolc-compiler-tests" -CORPUS_FILE="./corpus.json" WORKDIR="workdir" # Optional positional argument: path to polkadot-sdk directory @@ -68,21 +67,6 @@ else echo -e "${YELLOW}No polkadot-sdk path provided. Using binaries from $PATH.${NC}" fi -# Create corpus file with absolute path resolved at runtime -echo -e "${GREEN}Creating corpus file...${NC}" -ABSOLUTE_PATH=$(realpath "$TEST_REPO_DIR/fixtures/solidity/") - -cat > "$CORPUS_FILE" << EOF -{ - "name": "MatterLabs Solidity Simple, Complex, and Semantic Tests", - "paths": [ - "$(realpath "$TEST_REPO_DIR/fixtures/solidity")" - ] -} -EOF - -echo -e "${GREEN}Corpus file created: $CORPUS_FILE${NC}" - # Create workdir if it doesn't exist mkdir -p "$WORKDIR" @@ -94,7 +78,7 @@ echo "" cargo build --release; RUST_LOG="info,alloy_pubsub::service=error" ./target/release/retester test \ --platform revive-dev-node-polkavm-resolc \ - --corpus "$CORPUS_FILE" \ + --test $(realpath "$TEST_REPO_DIR/fixtures/solidity") \ --working-directory "$WORKDIR" \ --concurrency.number-of-nodes 10 \ --concurrency.number-of-threads 5 \