diff --git a/Cargo.lock b/Cargo.lock index 0f56c36..77f248d 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -5573,6 +5573,7 @@ dependencies = [ "clap", "moka", "once_cell", + "regex", "schemars 1.0.4", "semver 1.0.26", "serde", @@ -5651,7 +5652,6 @@ dependencies = [ "alloy", "anyhow", "futures", - "regex", "revive-common", "revive-dt-common", "schemars 1.0.4", diff --git a/crates/common/Cargo.toml b/crates/common/Cargo.toml index 68b185f..f637162 100644 --- a/crates/common/Cargo.toml +++ b/crates/common/Cargo.toml @@ -14,6 +14,7 @@ anyhow = { workspace = true } clap = { workspace = true } moka = { workspace = true, features = ["sync"] } once_cell = { workspace = true } +regex = { workspace = true } semver = { workspace = true } serde = { workspace = true } schemars = { workspace = true } diff --git a/crates/common/src/types/mode.rs b/crates/common/src/types/mode.rs index 3e2d3a9..a3aff67 100644 --- a/crates/common/src/types/mode.rs +++ b/crates/common/src/types/mode.rs @@ -1,6 +1,11 @@ +use crate::iterators::EitherIter; use crate::types::VersionOrRequirement; +use anyhow::{Context as _, bail}; +use regex::Regex; +use schemars::JsonSchema; use semver::Version; use serde::{Deserialize, Serialize}; +use std::collections::HashSet; use std::fmt::Display; use std::str::FromStr; use std::sync::LazyLock; @@ -33,6 +38,19 @@ impl Display for Mode { } } +impl FromStr for Mode { + type Err = anyhow::Error; + + fn from_str(s: &str) -> Result { + let parsed_mode = ParsedMode::from_str(s)?; + let mut iter = parsed_mode.to_modes(); + let (Some(mode), None) = (iter.next(), iter.next()) else { + bail!("Failed to parse the mode") + }; + Ok(mode) + } +} + impl Mode { /// Return all of the available mode combinations. pub fn all() -> impl Iterator { @@ -171,3 +189,250 @@ impl ModeOptimizerSetting { !matches!(self, ModeOptimizerSetting::M0) } } + +/// This represents a mode that has been parsed from test metadata. +/// +/// Mode strings can take the following form (in pseudo-regex): +/// +/// ```text +/// [YEILV][+-]? (M[0123sz])? ? +/// ``` +/// +/// We can parse valid mode strings into [`ParsedMode`] using [`ParsedMode::from_str`]. +#[derive(Clone, Debug, PartialEq, Eq, Hash, Deserialize, Serialize, JsonSchema)] +#[serde(try_from = "String", into = "String")] +pub struct ParsedMode { + pub pipeline: Option, + pub optimize_flag: Option, + pub optimize_setting: Option, + pub version: Option, +} + +impl FromStr for ParsedMode { + type Err = anyhow::Error; + fn from_str(s: &str) -> Result { + static REGEX: LazyLock = LazyLock::new(|| { + Regex::new(r"(?x) + ^ + (?:(?P[YEILV])(?P[+-])?)? # Pipeline to use eg Y, E+, E- + \s* + (?PM[a-zA-Z0-9])? # Optimize setting eg M0, Ms, Mz + \s* + (?P[>=<^]*\d+(?:\.\d+)*)? # Optional semver version eg >=0.8.0, 0.7, <0.8 + $ + ").unwrap() + }); + + let Some(caps) = REGEX.captures(s) else { + anyhow::bail!("Cannot parse mode '{s}' from string"); + }; + + let pipeline = match caps.name("pipeline") { + Some(m) => Some( + ModePipeline::from_str(m.as_str()) + .context("Failed to parse mode pipeline from string")?, + ), + None => None, + }; + + let optimize_flag = caps.name("optimize_flag").map(|m| m.as_str() == "+"); + + let optimize_setting = match caps.name("optimize_setting") { + Some(m) => Some( + ModeOptimizerSetting::from_str(m.as_str()) + .context("Failed to parse optimizer setting from string")?, + ), + None => None, + }; + + let version = match caps.name("version") { + Some(m) => Some( + semver::VersionReq::parse(m.as_str()) + .map_err(|e| { + anyhow::anyhow!( + "Cannot parse the version requirement '{}': {e}", + m.as_str() + ) + }) + .context("Failed to parse semver requirement from mode string")?, + ), + None => None, + }; + + Ok(ParsedMode { + pipeline, + optimize_flag, + optimize_setting, + version, + }) + } +} + +impl Display for ParsedMode { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + let mut has_written = false; + + if let Some(pipeline) = self.pipeline { + pipeline.fmt(f)?; + if let Some(optimize_flag) = self.optimize_flag { + f.write_str(if optimize_flag { "+" } else { "-" })?; + } + has_written = true; + } + + if let Some(optimize_setting) = self.optimize_setting { + if has_written { + f.write_str(" ")?; + } + optimize_setting.fmt(f)?; + has_written = true; + } + + if let Some(version) = &self.version { + if has_written { + f.write_str(" ")?; + } + version.fmt(f)?; + } + + Ok(()) + } +} + +impl From for String { + fn from(parsed_mode: ParsedMode) -> Self { + parsed_mode.to_string() + } +} + +impl TryFrom for ParsedMode { + type Error = anyhow::Error; + fn try_from(value: String) -> Result { + ParsedMode::from_str(&value) + } +} + +impl ParsedMode { + /// This takes a [`ParsedMode`] and expands it into a list of [`Mode`]s that we should try. + pub fn to_modes(&self) -> impl Iterator { + let pipeline_iter = self.pipeline.as_ref().map_or_else( + || EitherIter::A(ModePipeline::test_cases()), + |p| EitherIter::B(std::iter::once(*p)), + ); + + let optimize_flag_setting = self.optimize_flag.map(|flag| { + if flag { + ModeOptimizerSetting::M3 + } else { + ModeOptimizerSetting::M0 + } + }); + + let optimize_flag_iter = match optimize_flag_setting { + Some(setting) => EitherIter::A(std::iter::once(setting)), + None => EitherIter::B(ModeOptimizerSetting::test_cases()), + }; + + let optimize_settings_iter = self.optimize_setting.as_ref().map_or_else( + || EitherIter::A(optimize_flag_iter), + |s| EitherIter::B(std::iter::once(*s)), + ); + + pipeline_iter.flat_map(move |pipeline| { + optimize_settings_iter + .clone() + .map(move |optimize_setting| Mode { + pipeline, + optimize_setting, + version: self.version.clone(), + }) + }) + } + + /// Return a set of [`Mode`]s that correspond to the given [`ParsedMode`]s. + /// This avoids any duplicate entries. + pub fn many_to_modes<'a>( + parsed: impl Iterator, + ) -> impl Iterator { + let modes: HashSet<_> = parsed.flat_map(|p| p.to_modes()).collect(); + modes.into_iter() + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_parsed_mode_from_str() { + let strings = vec![ + ("Mz", "Mz"), + ("Y", "Y"), + ("Y+", "Y+"), + ("Y-", "Y-"), + ("E", "E"), + ("E+", "E+"), + ("E-", "E-"), + ("Y M0", "Y M0"), + ("Y M1", "Y M1"), + ("Y M2", "Y M2"), + ("Y M3", "Y M3"), + ("Y Ms", "Y Ms"), + ("Y Mz", "Y Mz"), + ("E M0", "E M0"), + ("E M1", "E M1"), + ("E M2", "E M2"), + ("E M3", "E M3"), + ("E Ms", "E Ms"), + ("E Mz", "E Mz"), + // When stringifying semver again, 0.8.0 becomes ^0.8.0 (same meaning) + ("Y 0.8.0", "Y ^0.8.0"), + ("E+ 0.8.0", "E+ ^0.8.0"), + ("Y M3 >=0.8.0", "Y M3 >=0.8.0"), + ("E Mz <0.7.0", "E Mz <0.7.0"), + // We can parse +- _and_ M1/M2 but the latter takes priority. + ("Y+ M1 0.8.0", "Y+ M1 ^0.8.0"), + ("E- M2 0.7.0", "E- M2 ^0.7.0"), + // We don't see this in the wild but it is parsed. + ("<=0.8", "<=0.8"), + ]; + + for (actual, expected) in strings { + let parsed = ParsedMode::from_str(actual) + .unwrap_or_else(|_| panic!("Failed to parse mode string '{actual}'")); + assert_eq!( + expected, + parsed.to_string(), + "Mode string '{actual}' did not parse to '{expected}': got '{parsed}'" + ); + } + } + + #[test] + fn test_parsed_mode_to_test_modes() { + let strings = vec![ + ("Mz", vec!["Y Mz", "E Mz"]), + ("Y", vec!["Y M0", "Y M3"]), + ("E", vec!["E M0", "E M3"]), + ("Y+", vec!["Y M3"]), + ("Y-", vec!["Y M0"]), + ("Y <=0.8", vec!["Y M0 <=0.8", "Y M3 <=0.8"]), + ( + "<=0.8", + vec!["Y M0 <=0.8", "Y M3 <=0.8", "E M0 <=0.8", "E M3 <=0.8"], + ), + ]; + + for (actual, expected) in strings { + let parsed = ParsedMode::from_str(actual) + .unwrap_or_else(|_| panic!("Failed to parse mode string '{actual}'")); + let expected_set: HashSet<_> = expected.into_iter().map(|s| s.to_owned()).collect(); + let actual_set: HashSet<_> = parsed.to_modes().map(|m| m.to_string()).collect(); + + assert_eq!( + expected_set, actual_set, + "Mode string '{actual}' did not expand to '{expected_set:?}': got '{actual_set:?}'" + ); + } + } +} diff --git a/crates/config/src/lib.rs b/crates/config/src/lib.rs index 0e87b13..a40de8d 100644 --- a/crates/config/src/lib.rs +++ b/crates/config/src/lib.rs @@ -12,19 +12,18 @@ use std::{ use alloy::{ genesis::Genesis, - hex::ToHexExt, network::EthereumWallet, - primitives::{FixedBytes, U256}, + primitives::{B256, FixedBytes, U256}, signers::local::PrivateKeySigner, }; use clap::{Parser, ValueEnum, ValueHint}; use revive_dt_common::types::PlatformIdentifier; use semver::Version; -use serde::{Serialize, Serializer}; +use serde::{Deserialize, Serialize, Serializer}; use strum::{AsRefStr, Display, EnumString, IntoStaticStr}; use temp_dir::TempDir; -#[derive(Clone, Debug, Parser, Serialize)] +#[derive(Clone, Debug, Parser, Serialize, Deserialize)] #[command(name = "retester")] pub enum Context { /// Executes tests in the MatterLabs format differentially on multiple targets concurrently. @@ -200,7 +199,17 @@ impl AsRef for Context { } } -#[derive(Clone, Debug, Parser, Serialize)] +impl AsRef for Context { + fn as_ref(&self) -> &IgnoreSuccessConfiguration { + match self { + Self::Test(context) => context.as_ref().as_ref(), + Self::Benchmark(..) => unreachable!(), + Self::ExportJsonSchema => unreachable!(), + } + } +} + +#[derive(Clone, Debug, Parser, Serialize, Deserialize)] pub struct TestExecutionContext { /// The set of platforms that the differential tests should run on. #[arg( @@ -282,9 +291,13 @@ pub struct TestExecutionContext { /// Configuration parameters for the report. #[clap(flatten, next_help_heading = "Report Configuration")] pub report_configuration: ReportConfiguration, + + /// Configuration parameters for ignoring certain test cases based on the report + #[clap(flatten, next_help_heading = "Ignore Success Configuration")] + pub ignore_success_configuration: IgnoreSuccessConfiguration, } -#[derive(Clone, Debug, Parser, Serialize)] +#[derive(Clone, Debug, Parser, Serialize, Deserialize)] pub struct BenchmarkingContext { /// The working directory that the program will use for all of the temporary artifacts needed at /// runtime. @@ -461,6 +474,12 @@ impl AsRef for TestExecutionContext { } } +impl AsRef for TestExecutionContext { + fn as_ref(&self) -> &IgnoreSuccessConfiguration { + &self.ignore_success_configuration + } +} + impl Default for BenchmarkingContext { fn default() -> Self { Self::parse_from(["execution-context"]) @@ -552,7 +571,7 @@ impl AsRef for BenchmarkingContext { } /// A set of configuration parameters for the corpus files to use for the execution. -#[derive(Clone, Debug, Parser, Serialize)] +#[derive(Clone, Debug, Parser, Serialize, Deserialize)] pub struct CorpusConfiguration { /// A list of test corpus JSON files to be tested. #[arg(short = 'c', long = "corpus")] @@ -560,7 +579,7 @@ pub struct CorpusConfiguration { } /// A set of configuration parameters for Solc. -#[derive(Clone, Debug, Parser, Serialize)] +#[derive(Clone, Debug, Parser, Serialize, Deserialize)] pub struct SolcConfiguration { /// Specifies the default version of the Solc compiler that should be used if there is no /// override specified by one of the test cases. @@ -569,7 +588,7 @@ pub struct SolcConfiguration { } /// A set of configuration parameters for Resolc. -#[derive(Clone, Debug, Parser, Serialize)] +#[derive(Clone, Debug, Parser, Serialize, Deserialize)] pub struct ResolcConfiguration { /// Specifies the path of the resolc compiler to be used by the tool. /// @@ -580,7 +599,7 @@ pub struct ResolcConfiguration { } /// A set of configuration parameters for Polkadot Parachain. -#[derive(Clone, Debug, Parser, Serialize)] +#[derive(Clone, Debug, Parser, Serialize, Deserialize)] pub struct PolkadotParachainConfiguration { /// Specifies the path of the polkadot-parachain node to be used by the tool. /// @@ -604,7 +623,7 @@ pub struct PolkadotParachainConfiguration { } /// A set of configuration parameters for Geth. -#[derive(Clone, Debug, Parser, Serialize)] +#[derive(Clone, Debug, Parser, Serialize, Deserialize)] pub struct GethConfiguration { /// Specifies the path of the geth node to be used by the tool. /// @@ -624,7 +643,7 @@ pub struct GethConfiguration { } /// A set of configuration parameters for kurtosis. -#[derive(Clone, Debug, Parser, Serialize)] +#[derive(Clone, Debug, Parser, Serialize, Deserialize)] pub struct KurtosisConfiguration { /// Specifies the path of the kurtosis node to be used by the tool. /// @@ -639,7 +658,7 @@ pub struct KurtosisConfiguration { } /// A set of configuration parameters for Kitchensink. -#[derive(Clone, Debug, Parser, Serialize)] +#[derive(Clone, Debug, Parser, Serialize, Deserialize)] pub struct KitchensinkConfiguration { /// Specifies the path of the kitchensink node to be used by the tool. /// @@ -663,7 +682,7 @@ pub struct KitchensinkConfiguration { } /// A set of configuration parameters for the revive dev node. -#[derive(Clone, Debug, Parser, Serialize)] +#[derive(Clone, Debug, Parser, Serialize, Deserialize)] pub struct ReviveDevNodeConfiguration { /// Specifies the path of the revive dev node to be used by the tool. /// @@ -695,7 +714,7 @@ pub struct ReviveDevNodeConfiguration { } /// A set of configuration parameters for the ETH RPC. -#[derive(Clone, Debug, Parser, Serialize)] +#[derive(Clone, Debug, Parser, Serialize, Deserialize)] pub struct EthRpcConfiguration { /// Specifies the path of the ETH RPC to be used by the tool. /// @@ -715,7 +734,7 @@ pub struct EthRpcConfiguration { } /// A set of configuration parameters for the genesis. -#[derive(Clone, Debug, Default, Parser, Serialize)] +#[derive(Clone, Debug, Default, Parser, Serialize, Deserialize)] pub struct GenesisConfiguration { /// Specifies the path of the genesis file to use for the nodes that are started. /// @@ -753,15 +772,14 @@ impl GenesisConfiguration { } /// A set of configuration parameters for the wallet. -#[derive(Clone, Debug, Parser, Serialize)] +#[derive(Clone, Debug, Parser, Serialize, Deserialize)] pub struct WalletConfiguration { /// The private key of the default signer. #[clap( long = "wallet.default-private-key", default_value = "0x4f3edf983ac636a65a842ce7c78d9aa706d3b113bce9c46f30d7d21715b23b1d" )] - #[serde(serialize_with = "serialize_private_key")] - default_key: PrivateKeySigner, + default_key: B256, /// This argument controls which private keys the nodes should have access to and be added to /// its wallet signers. With a value of N, private keys (0, N] will be added to the signer set @@ -779,7 +797,8 @@ impl WalletConfiguration { pub fn wallet(&self) -> Arc { self.wallet .get_or_init(|| { - let mut wallet = EthereumWallet::new(self.default_key.clone()); + let mut wallet = + EthereumWallet::new(PrivateKeySigner::from_bytes(&self.default_key).unwrap()); for signer in (1..=self.additional_keys) .map(|id| U256::from(id)) .map(|id| id.to_be_bytes::<32>()) @@ -797,15 +816,8 @@ impl WalletConfiguration { } } -fn serialize_private_key(value: &PrivateKeySigner, serializer: S) -> Result -where - S: Serializer, -{ - value.to_bytes().encode_hex().serialize(serializer) -} - /// A set of configuration for concurrency. -#[derive(Clone, Debug, Parser, Serialize)] +#[derive(Clone, Debug, Parser, Serialize, Deserialize)] pub struct ConcurrencyConfiguration { /// Determines the amount of nodes that will be spawned for each chain. #[clap(long = "concurrency.number-of-nodes", default_value_t = 5)] @@ -843,14 +855,14 @@ impl ConcurrencyConfiguration { } } -#[derive(Clone, Debug, Parser, Serialize)] +#[derive(Clone, Debug, Parser, Serialize, Deserialize)] pub struct CompilationConfiguration { /// Controls if the compilation cache should be invalidated or not. #[arg(long = "compilation.invalidate-cache")] pub invalidate_compilation_cache: bool, } -#[derive(Clone, Debug, Parser, Serialize)] +#[derive(Clone, Debug, Parser, Serialize, Deserialize)] pub struct ReportConfiguration { /// Controls if the compiler input is included in the final report. #[clap(long = "report.include-compiler-input")] @@ -861,6 +873,13 @@ pub struct ReportConfiguration { pub include_compiler_output: bool, } +#[derive(Clone, Debug, Parser, Serialize, Deserialize)] +pub struct IgnoreSuccessConfiguration { + /// The path of the report generated by the tool to use to ignore the cases that succeeded. + #[clap(long = "ignore-success.report-path")] + pub path: Option, +} + /// Represents the working directory that the program uses. #[derive(Debug, Clone)] pub enum WorkingDirectoryConfiguration { @@ -870,6 +889,24 @@ pub enum WorkingDirectoryConfiguration { Path(PathBuf), } +impl Serialize for WorkingDirectoryConfiguration { + fn serialize(&self, serializer: S) -> Result + where + S: Serializer, + { + self.as_path().serialize(serializer) + } +} + +impl<'a> Deserialize<'a> for WorkingDirectoryConfiguration { + fn deserialize(deserializer: D) -> Result + where + D: serde::Deserializer<'a>, + { + PathBuf::deserialize(deserializer).map(Self::Path) + } +} + impl WorkingDirectoryConfiguration { pub fn as_path(&self) -> &Path { self.as_ref() @@ -919,15 +956,6 @@ impl Display for WorkingDirectoryConfiguration { } } -impl Serialize for WorkingDirectoryConfiguration { - fn serialize(&self, serializer: S) -> Result - where - S: serde::Serializer, - { - self.as_path().serialize(serializer) - } -} - fn parse_duration(s: &str) -> anyhow::Result { u64::from_str(s) .map(Duration::from_millis) diff --git a/crates/core/src/differential_benchmarks/entry_point.rs b/crates/core/src/differential_benchmarks/entry_point.rs index fd80aa3..d813519 100644 --- a/crates/core/src/differential_benchmarks/entry_point.rs +++ b/crates/core/src/differential_benchmarks/entry_point.rs @@ -86,6 +86,7 @@ pub async fn handle_differential_benchmarks( &full_context, metadata_files.iter(), &platforms_and_nodes, + None, reporter.clone(), ) .await diff --git a/crates/core/src/differential_tests/entry_point.rs b/crates/core/src/differential_tests/entry_point.rs index 20fcfee..b9cd9a7 100644 --- a/crates/core/src/differential_tests/entry_point.rs +++ b/crates/core/src/differential_tests/entry_point.rs @@ -10,7 +10,7 @@ use std::{ use ansi_term::{ANSIStrings, Color}; use anyhow::Context as _; use futures::{FutureExt, StreamExt}; -use revive_dt_common::types::PrivateKeyAllocator; +use revive_dt_common::{cached_fs::read_to_string, types::PrivateKeyAllocator}; use revive_dt_core::Platform; use tokio::sync::{Mutex, RwLock, Semaphore}; use tracing::{Instrument, error, info, info_span, instrument}; @@ -72,11 +72,20 @@ pub async fn handle_differential_tests( info!("Spawned the platform nodes"); // Preparing test definitions. + let only_execute_failed_tests = match context.ignore_success_configuration.path.as_ref() { + Some(path) => { + let report = read_to_string(path) + .context("Failed to read the report file to ignore the succeeding test cases")?; + Some(serde_json::from_str(&report).context("Failed to deserialize report")?) + } + None => None, + }; let full_context = Context::Test(Box::new(context.clone())); let test_definitions = create_test_definitions_stream( &full_context, metadata_files.iter(), &platforms_and_nodes, + only_execute_failed_tests.as_ref(), reporter.clone(), ) .await diff --git a/crates/core/src/helpers/test.rs b/crates/core/src/helpers/test.rs index ec16b91..14d238a 100644 --- a/crates/core/src/helpers/test.rs +++ b/crates/core/src/helpers/test.rs @@ -5,9 +5,8 @@ use std::{borrow::Cow, path::Path}; use futures::{Stream, StreamExt, stream}; use indexmap::{IndexMap, indexmap}; use revive_dt_common::iterators::EitherIter; -use revive_dt_common::types::PlatformIdentifier; +use revive_dt_common::types::{ParsedMode, PlatformIdentifier}; use revive_dt_config::Context; -use revive_dt_format::mode::ParsedMode; use serde_json::{Value, json}; use revive_dt_compiler::Mode; @@ -17,7 +16,7 @@ use revive_dt_format::{ metadata::MetadataFile, }; use revive_dt_node_interaction::EthereumNode; -use revive_dt_report::{ExecutionSpecificReporter, Reporter}; +use revive_dt_report::{ExecutionSpecificReporter, Report, Reporter, TestCaseStatus}; use revive_dt_report::{TestSpecificReporter, TestSpecifier}; use tracing::{debug, error, info}; @@ -30,6 +29,7 @@ pub async fn create_test_definitions_stream<'a>( context: &Context, metadata_files: impl IntoIterator, platforms_and_nodes: &'a BTreeMap, + only_execute_failed_tests: Option<&Report>, reporter: Reporter, ) -> impl Stream> { stream::iter( @@ -140,7 +140,7 @@ pub async fn create_test_definitions_stream<'a>( ) // Filter out the test cases which are incompatible or that can't run in the current setup. .filter_map(move |test| async move { - match test.check_compatibility() { + match test.check_compatibility(only_execute_failed_tests) { Ok(()) => Some(test), Err((reason, additional_information)) => { debug!( @@ -200,12 +200,16 @@ pub struct TestDefinition<'a> { impl<'a> TestDefinition<'a> { /// Checks if this test can be ran with the current configuration. - pub fn check_compatibility(&self) -> TestCheckFunctionResult { + pub fn check_compatibility( + &self, + only_execute_failed_tests: Option<&Report>, + ) -> TestCheckFunctionResult { self.check_metadata_file_ignored()?; self.check_case_file_ignored()?; self.check_target_compatibility()?; self.check_evm_version_compatibility()?; self.check_compiler_compatibility()?; + self.check_ignore_succeeded(only_execute_failed_tests)?; Ok(()) } @@ -313,6 +317,36 @@ impl<'a> TestDefinition<'a> { )) } } + + /// Checks if the test case should be executed or not based on the passed report and whether the + /// user has instructed the tool to ignore the already succeeding test cases. + fn check_ignore_succeeded( + &self, + only_execute_failed_tests: Option<&Report>, + ) -> TestCheckFunctionResult { + let Some(report) = only_execute_failed_tests else { + return Ok(()); + }; + + let test_case_status = report + .test_case_information + .get(&(self.metadata_file_path.to_path_buf().into())) + .and_then(|obj| obj.get(&self.mode)) + .and_then(|obj| obj.get(&self.case_idx)) + .and_then(|obj| obj.status.as_ref()); + + match test_case_status { + Some(TestCaseStatus::Failed { .. }) => Ok(()), + Some(TestCaseStatus::Ignored { .. }) => Err(( + "Ignored since it was ignored in a previous run", + indexmap! {}, + )), + Some(TestCaseStatus::Succeeded { .. }) => { + Err(("Ignored since it succeeded in a prior run", indexmap! {})) + } + None => Ok(()), + } + } } pub struct TestPlatformInformation<'a> { diff --git a/crates/format/Cargo.toml b/crates/format/Cargo.toml index d1e5050..afc09a1 100644 --- a/crates/format/Cargo.toml +++ b/crates/format/Cargo.toml @@ -16,7 +16,6 @@ revive-common = { workspace = true } alloy = { workspace = true } anyhow = { workspace = true } futures = { workspace = true } -regex = { workspace = true } tracing = { workspace = true } schemars = { workspace = true } semver = { workspace = true } diff --git a/crates/format/src/case.rs b/crates/format/src/case.rs index 7c48279..44639dd 100644 --- a/crates/format/src/case.rs +++ b/crates/format/src/case.rs @@ -1,9 +1,12 @@ use schemars::JsonSchema; use serde::{Deserialize, Serialize}; -use revive_dt_common::{macros::define_wrapper_type, types::Mode}; +use revive_dt_common::{ + macros::define_wrapper_type, + types::{Mode, ParsedMode}, +}; -use crate::{mode::ParsedMode, steps::*}; +use crate::steps::*; #[derive(Debug, Default, Serialize, Deserialize, Clone, Eq, PartialEq, JsonSchema)] pub struct Case { diff --git a/crates/format/src/lib.rs b/crates/format/src/lib.rs index 7d250d9..ed2426a 100644 --- a/crates/format/src/lib.rs +++ b/crates/format/src/lib.rs @@ -3,6 +3,5 @@ pub mod case; pub mod corpus; pub mod metadata; -pub mod mode; pub mod steps; pub mod traits; diff --git a/crates/format/src/metadata.rs b/crates/format/src/metadata.rs index 7632866..173e288 100644 --- a/crates/format/src/metadata.rs +++ b/crates/format/src/metadata.rs @@ -16,11 +16,11 @@ use revive_dt_common::{ cached_fs::read_to_string, iterators::FilesWithExtensionIterator, macros::define_wrapper_type, - types::{Mode, VmIdentifier}, + types::{Mode, ParsedMode, VmIdentifier}, }; use tracing::error; -use crate::{case::Case, mode::ParsedMode}; +use crate::case::Case; pub const METADATA_FILE_EXTENSION: &str = "json"; pub const SOLIDITY_CASE_FILE_EXTENSION: &str = "sol"; diff --git a/crates/format/src/mode.rs b/crates/format/src/mode.rs deleted file mode 100644 index bf6ae03..0000000 --- a/crates/format/src/mode.rs +++ /dev/null @@ -1,257 +0,0 @@ -use anyhow::Context as _; -use regex::Regex; -use revive_dt_common::iterators::EitherIter; -use revive_dt_common::types::{Mode, ModeOptimizerSetting, ModePipeline}; -use schemars::JsonSchema; -use serde::{Deserialize, Serialize}; -use std::collections::HashSet; -use std::fmt::Display; -use std::str::FromStr; -use std::sync::LazyLock; - -/// This represents a mode that has been parsed from test metadata. -/// -/// Mode strings can take the following form (in pseudo-regex): -/// -/// ```text -/// [YEILV][+-]? (M[0123sz])? ? -/// ``` -/// -/// We can parse valid mode strings into [`ParsedMode`] using [`ParsedMode::from_str`]. -#[derive(Clone, Debug, PartialEq, Eq, Hash, Deserialize, Serialize, JsonSchema)] -#[serde(try_from = "String", into = "String")] -pub struct ParsedMode { - pub pipeline: Option, - pub optimize_flag: Option, - pub optimize_setting: Option, - pub version: Option, -} - -impl FromStr for ParsedMode { - type Err = anyhow::Error; - fn from_str(s: &str) -> Result { - static REGEX: LazyLock = LazyLock::new(|| { - Regex::new(r"(?x) - ^ - (?:(?P[YEILV])(?P[+-])?)? # Pipeline to use eg Y, E+, E- - \s* - (?PM[a-zA-Z0-9])? # Optimize setting eg M0, Ms, Mz - \s* - (?P[>=<]*\d+(?:\.\d+)*)? # Optional semver version eg >=0.8.0, 0.7, <0.8 - $ - ").unwrap() - }); - - let Some(caps) = REGEX.captures(s) else { - anyhow::bail!("Cannot parse mode '{s}' from string"); - }; - - let pipeline = match caps.name("pipeline") { - Some(m) => Some( - ModePipeline::from_str(m.as_str()) - .context("Failed to parse mode pipeline from string")?, - ), - None => None, - }; - - let optimize_flag = caps.name("optimize_flag").map(|m| m.as_str() == "+"); - - let optimize_setting = match caps.name("optimize_setting") { - Some(m) => Some( - ModeOptimizerSetting::from_str(m.as_str()) - .context("Failed to parse optimizer setting from string")?, - ), - None => None, - }; - - let version = match caps.name("version") { - Some(m) => Some( - semver::VersionReq::parse(m.as_str()) - .map_err(|e| { - anyhow::anyhow!( - "Cannot parse the version requirement '{}': {e}", - m.as_str() - ) - }) - .context("Failed to parse semver requirement from mode string")?, - ), - None => None, - }; - - Ok(ParsedMode { - pipeline, - optimize_flag, - optimize_setting, - version, - }) - } -} - -impl Display for ParsedMode { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - let mut has_written = false; - - if let Some(pipeline) = self.pipeline { - pipeline.fmt(f)?; - if let Some(optimize_flag) = self.optimize_flag { - f.write_str(if optimize_flag { "+" } else { "-" })?; - } - has_written = true; - } - - if let Some(optimize_setting) = self.optimize_setting { - if has_written { - f.write_str(" ")?; - } - optimize_setting.fmt(f)?; - has_written = true; - } - - if let Some(version) = &self.version { - if has_written { - f.write_str(" ")?; - } - version.fmt(f)?; - } - - Ok(()) - } -} - -impl From for String { - fn from(parsed_mode: ParsedMode) -> Self { - parsed_mode.to_string() - } -} - -impl TryFrom for ParsedMode { - type Error = anyhow::Error; - fn try_from(value: String) -> Result { - ParsedMode::from_str(&value) - } -} - -impl ParsedMode { - /// This takes a [`ParsedMode`] and expands it into a list of [`Mode`]s that we should try. - pub fn to_modes(&self) -> impl Iterator { - let pipeline_iter = self.pipeline.as_ref().map_or_else( - || EitherIter::A(ModePipeline::test_cases()), - |p| EitherIter::B(std::iter::once(*p)), - ); - - let optimize_flag_setting = self.optimize_flag.map(|flag| { - if flag { - ModeOptimizerSetting::M3 - } else { - ModeOptimizerSetting::M0 - } - }); - - let optimize_flag_iter = match optimize_flag_setting { - Some(setting) => EitherIter::A(std::iter::once(setting)), - None => EitherIter::B(ModeOptimizerSetting::test_cases()), - }; - - let optimize_settings_iter = self.optimize_setting.as_ref().map_or_else( - || EitherIter::A(optimize_flag_iter), - |s| EitherIter::B(std::iter::once(*s)), - ); - - pipeline_iter.flat_map(move |pipeline| { - optimize_settings_iter - .clone() - .map(move |optimize_setting| Mode { - pipeline, - optimize_setting, - version: self.version.clone(), - }) - }) - } - - /// Return a set of [`Mode`]s that correspond to the given [`ParsedMode`]s. - /// This avoids any duplicate entries. - pub fn many_to_modes<'a>( - parsed: impl Iterator, - ) -> impl Iterator { - let modes: HashSet<_> = parsed.flat_map(|p| p.to_modes()).collect(); - modes.into_iter() - } -} - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn test_parsed_mode_from_str() { - let strings = vec![ - ("Mz", "Mz"), - ("Y", "Y"), - ("Y+", "Y+"), - ("Y-", "Y-"), - ("E", "E"), - ("E+", "E+"), - ("E-", "E-"), - ("Y M0", "Y M0"), - ("Y M1", "Y M1"), - ("Y M2", "Y M2"), - ("Y M3", "Y M3"), - ("Y Ms", "Y Ms"), - ("Y Mz", "Y Mz"), - ("E M0", "E M0"), - ("E M1", "E M1"), - ("E M2", "E M2"), - ("E M3", "E M3"), - ("E Ms", "E Ms"), - ("E Mz", "E Mz"), - // When stringifying semver again, 0.8.0 becomes ^0.8.0 (same meaning) - ("Y 0.8.0", "Y ^0.8.0"), - ("E+ 0.8.0", "E+ ^0.8.0"), - ("Y M3 >=0.8.0", "Y M3 >=0.8.0"), - ("E Mz <0.7.0", "E Mz <0.7.0"), - // We can parse +- _and_ M1/M2 but the latter takes priority. - ("Y+ M1 0.8.0", "Y+ M1 ^0.8.0"), - ("E- M2 0.7.0", "E- M2 ^0.7.0"), - // We don't see this in the wild but it is parsed. - ("<=0.8", "<=0.8"), - ]; - - for (actual, expected) in strings { - let parsed = ParsedMode::from_str(actual) - .unwrap_or_else(|_| panic!("Failed to parse mode string '{actual}'")); - assert_eq!( - expected, - parsed.to_string(), - "Mode string '{actual}' did not parse to '{expected}': got '{parsed}'" - ); - } - } - - #[test] - fn test_parsed_mode_to_test_modes() { - let strings = vec![ - ("Mz", vec!["Y Mz", "E Mz"]), - ("Y", vec!["Y M0", "Y M3"]), - ("E", vec!["E M0", "E M3"]), - ("Y+", vec!["Y M3"]), - ("Y-", vec!["Y M0"]), - ("Y <=0.8", vec!["Y M0 <=0.8", "Y M3 <=0.8"]), - ( - "<=0.8", - vec!["Y M0 <=0.8", "Y M3 <=0.8", "E M0 <=0.8", "E M3 <=0.8"], - ), - ]; - - for (actual, expected) in strings { - let parsed = ParsedMode::from_str(actual) - .unwrap_or_else(|_| panic!("Failed to parse mode string '{actual}'")); - let expected_set: HashSet<_> = expected.into_iter().map(|s| s.to_owned()).collect(); - let actual_set: HashSet<_> = parsed.to_modes().map(|m| m.to_string()).collect(); - - assert_eq!( - expected_set, actual_set, - "Mode string '{actual}' did not expand to '{expected_set:?}': got '{actual_set:?}'" - ); - } - } -} diff --git a/crates/report/src/aggregator.rs b/crates/report/src/aggregator.rs index bb235eb..9a1a4ad 100644 --- a/crates/report/src/aggregator.rs +++ b/crates/report/src/aggregator.rs @@ -16,7 +16,7 @@ use revive_dt_compiler::{CompilerInput, CompilerOutput, Mode}; use revive_dt_config::Context; use revive_dt_format::{case::CaseIdx, corpus::Corpus, metadata::ContractInstance}; use semver::Version; -use serde::Serialize; +use serde::{Deserialize, Serialize}; use serde_with::{DisplayFromStr, serde_as}; use tokio::sync::{ broadcast::{Sender, channel}, @@ -415,7 +415,7 @@ impl ReportAggregator { } #[serde_as] -#[derive(Clone, Debug, Serialize)] +#[derive(Clone, Debug, Serialize, Deserialize)] pub struct Report { /// The context that the tool was started up with. pub context: Context, @@ -440,7 +440,7 @@ impl Report { } } -#[derive(Clone, Debug, Serialize, Default)] +#[derive(Clone, Debug, Serialize, Deserialize, Default)] pub struct TestCaseReport { /// Information on the status of the test case and whether it succeeded, failed, or was ignored. #[serde(skip_serializing_if = "Option::is_none")] @@ -451,7 +451,7 @@ pub struct TestCaseReport { /// Information related to the status of the test. Could be that the test succeeded, failed, or that /// it was ignored. -#[derive(Clone, Debug, Serialize)] +#[derive(Clone, Debug, Serialize, Deserialize)] #[serde(tag = "status")] pub enum TestCaseStatus { /// The test case succeeded. @@ -475,7 +475,7 @@ pub enum TestCaseStatus { } /// Information related to the platform node that's being used to execute the step. -#[derive(Clone, Debug, Serialize)] +#[derive(Clone, Debug, Serialize, Deserialize)] pub struct TestCaseNodeInformation { /// The ID of the node that this case is being executed on. pub id: usize, @@ -486,7 +486,7 @@ pub struct TestCaseNodeInformation { } /// Execution information tied to the platform. -#[derive(Clone, Debug, Default, Serialize)] +#[derive(Clone, Debug, Default, Serialize, Deserialize)] pub struct ExecutionInformation { /// Information related to the node assigned to this test case. #[serde(skip_serializing_if = "Option::is_none")] @@ -506,7 +506,7 @@ pub struct ExecutionInformation { } /// Information related to compilation -#[derive(Clone, Debug, Serialize)] +#[derive(Clone, Debug, Serialize, Deserialize)] #[serde(tag = "status")] pub enum CompilationStatus { /// The compilation was successful.