Merge branch 'main' into feature/ignore-tests-that-already-succeeded

Cargo.lock

@@ -5573,6 +5573,7 @@ dependencies = [
  "clap",
  "moka",
  "once_cell",
+ "regex",
  "schemars 1.0.4",
  "semver 1.0.26",
  "serde",
@@ -5651,7 +5652,6 @@ dependencies = [
  "alloy",
  "anyhow",
  "futures",
- "regex",
  "revive-common",
  "revive-dt-common",
  "schemars 1.0.4",

crates/common/Cargo.toml

@@ -14,6 +14,7 @@ anyhow = { workspace = true }
 clap = { workspace = true }
 moka = { workspace = true, features = ["sync"] }
 once_cell = { workspace = true }
+regex = { workspace = true }
 semver = { workspace = true }
 serde = { workspace = true }
 schemars = { workspace = true }

crates/common/src/types/mode.rs

@@ -1,6 +1,11 @@
+use crate::iterators::EitherIter;
 use crate::types::VersionOrRequirement;
+use anyhow::{Context as _, bail};
+use regex::Regex;
+use schemars::JsonSchema;
 use semver::Version;
 use serde::{Deserialize, Serialize};
+use std::collections::HashSet;
 use std::fmt::Display;
 use std::str::FromStr;
 use std::sync::LazyLock;
@@ -33,6 +38,19 @@ impl Display for Mode {
     }
 }
 
+impl FromStr for Mode {
+    type Err = anyhow::Error;
+
+    fn from_str(s: &str) -> Result<Self, Self::Err> {
+        let parsed_mode = ParsedMode::from_str(s)?;
+        let mut iter = parsed_mode.to_modes();
+        let (Some(mode), None) = (iter.next(), iter.next()) else {
+            bail!("Failed to parse the mode")
+        };
+        Ok(mode)
+    }
+}
+
 impl Mode {
     /// Return all of the available mode combinations.
     pub fn all() -> impl Iterator<Item = &'static Mode> {
@@ -171,3 +189,250 @@ impl ModeOptimizerSetting {
         !matches!(self, ModeOptimizerSetting::M0)
     }
 }
+
+/// This represents a mode that has been parsed from test metadata.
+///
+/// Mode strings can take the following form (in pseudo-regex):
+///
+/// ```text
+/// [YEILV][+-]? (M[0123sz])? <semver>?
+/// ```
+///
+/// We can parse valid mode strings into [`ParsedMode`] using [`ParsedMode::from_str`].
+#[derive(Clone, Debug, PartialEq, Eq, Hash, Deserialize, Serialize, JsonSchema)]
+#[serde(try_from = "String", into = "String")]
+pub struct ParsedMode {
+    pub pipeline: Option<ModePipeline>,
+    pub optimize_flag: Option<bool>,
+    pub optimize_setting: Option<ModeOptimizerSetting>,
+    pub version: Option<semver::VersionReq>,
+}
+
+impl FromStr for ParsedMode {
+    type Err = anyhow::Error;
+    fn from_str(s: &str) -> Result<Self, Self::Err> {
+        static REGEX: LazyLock<Regex> = LazyLock::new(|| {
+            Regex::new(r"(?x)
+                ^
+                (?:(?P<pipeline>[YEILV])(?P<optimize_flag>[+-])?)? # Pipeline to use eg Y, E+, E-
+                \s*
+                (?P<optimize_setting>M[a-zA-Z0-9])?                # Optimize setting eg M0, Ms, Mz
+                \s*
+                (?P<version>[>=<^]*\d+(?:\.\d+)*)?                  # Optional semver version eg >=0.8.0, 0.7, <0.8
+                $
+            ").unwrap()
+        });
+
+        let Some(caps) = REGEX.captures(s) else {
+            anyhow::bail!("Cannot parse mode '{s}' from string");
+        };
+
+        let pipeline = match caps.name("pipeline") {
+            Some(m) => Some(
+                ModePipeline::from_str(m.as_str())
+                    .context("Failed to parse mode pipeline from string")?,
+            ),
+            None => None,
+        };
+
+        let optimize_flag = caps.name("optimize_flag").map(|m| m.as_str() == "+");
+
+        let optimize_setting = match caps.name("optimize_setting") {
+            Some(m) => Some(
+                ModeOptimizerSetting::from_str(m.as_str())
+                    .context("Failed to parse optimizer setting from string")?,
+            ),
+            None => None,
+        };
+
+        let version = match caps.name("version") {
+            Some(m) => Some(
+                semver::VersionReq::parse(m.as_str())
+                    .map_err(|e| {
+                        anyhow::anyhow!(
+                            "Cannot parse the version requirement '{}': {e}",
+                            m.as_str()
+                        )
+                    })
+                    .context("Failed to parse semver requirement from mode string")?,
+            ),
+            None => None,
+        };
+
+        Ok(ParsedMode {
+            pipeline,
+            optimize_flag,
+            optimize_setting,
+            version,
+        })
+    }
+}
+
+impl Display for ParsedMode {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        let mut has_written = false;
+
+        if let Some(pipeline) = self.pipeline {
+            pipeline.fmt(f)?;
+            if let Some(optimize_flag) = self.optimize_flag {
+                f.write_str(if optimize_flag { "+" } else { "-" })?;
+            }
+            has_written = true;
+        }
+
+        if let Some(optimize_setting) = self.optimize_setting {
+            if has_written {
+                f.write_str(" ")?;
+            }
+            optimize_setting.fmt(f)?;
+            has_written = true;
+        }
+
+        if let Some(version) = &self.version {
+            if has_written {
+                f.write_str(" ")?;
+            }
+            version.fmt(f)?;
+        }
+
+        Ok(())
+    }
+}
+
+impl From<ParsedMode> for String {
+    fn from(parsed_mode: ParsedMode) -> Self {
+        parsed_mode.to_string()
+    }
+}
+
+impl TryFrom<String> for ParsedMode {
+    type Error = anyhow::Error;
+    fn try_from(value: String) -> Result<Self, Self::Error> {
+        ParsedMode::from_str(&value)
+    }
+}
+
+impl ParsedMode {
+    /// This takes a [`ParsedMode`] and expands it into a list of [`Mode`]s that we should try.
+    pub fn to_modes(&self) -> impl Iterator<Item = Mode> {
+        let pipeline_iter = self.pipeline.as_ref().map_or_else(
+            || EitherIter::A(ModePipeline::test_cases()),
+            |p| EitherIter::B(std::iter::once(*p)),
+        );
+
+        let optimize_flag_setting = self.optimize_flag.map(|flag| {
+            if flag {
+                ModeOptimizerSetting::M3
+            } else {
+                ModeOptimizerSetting::M0
+            }
+        });
+
+        let optimize_flag_iter = match optimize_flag_setting {
+            Some(setting) => EitherIter::A(std::iter::once(setting)),
+            None => EitherIter::B(ModeOptimizerSetting::test_cases()),
+        };
+
+        let optimize_settings_iter = self.optimize_setting.as_ref().map_or_else(
+            || EitherIter::A(optimize_flag_iter),
+            |s| EitherIter::B(std::iter::once(*s)),
+        );
+
+        pipeline_iter.flat_map(move |pipeline| {
+            optimize_settings_iter
+                .clone()
+                .map(move |optimize_setting| Mode {
+                    pipeline,
+                    optimize_setting,
+                    version: self.version.clone(),
+                })
+        })
+    }
+
+    /// Return a set of [`Mode`]s that correspond to the given [`ParsedMode`]s.
+    /// This avoids any duplicate entries.
+    pub fn many_to_modes<'a>(
+        parsed: impl Iterator<Item = &'a ParsedMode>,
+    ) -> impl Iterator<Item = Mode> {
+        let modes: HashSet<_> = parsed.flat_map(|p| p.to_modes()).collect();
+        modes.into_iter()
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn test_parsed_mode_from_str() {
+        let strings = vec![
+            ("Mz", "Mz"),
+            ("Y", "Y"),
+            ("Y+", "Y+"),
+            ("Y-", "Y-"),
+            ("E", "E"),
+            ("E+", "E+"),
+            ("E-", "E-"),
+            ("Y M0", "Y M0"),
+            ("Y M1", "Y M1"),
+            ("Y M2", "Y M2"),
+            ("Y M3", "Y M3"),
+            ("Y Ms", "Y Ms"),
+            ("Y Mz", "Y Mz"),
+            ("E M0", "E M0"),
+            ("E M1", "E M1"),
+            ("E M2", "E M2"),
+            ("E M3", "E M3"),
+            ("E Ms", "E Ms"),
+            ("E Mz", "E Mz"),
+            // When stringifying semver again, 0.8.0 becomes ^0.8.0 (same meaning)
+            ("Y 0.8.0", "Y ^0.8.0"),
+            ("E+ 0.8.0", "E+ ^0.8.0"),
+            ("Y M3 >=0.8.0", "Y M3 >=0.8.0"),
+            ("E Mz <0.7.0", "E Mz <0.7.0"),
+            // We can parse +- _and_ M1/M2 but the latter takes priority.
+            ("Y+ M1 0.8.0", "Y+ M1 ^0.8.0"),
+            ("E- M2 0.7.0", "E- M2 ^0.7.0"),
+            // We don't see this in the wild but it is parsed.
+            ("<=0.8", "<=0.8"),
+        ];
+
+        for (actual, expected) in strings {
+            let parsed = ParsedMode::from_str(actual)
+                .unwrap_or_else(|_| panic!("Failed to parse mode string '{actual}'"));
+            assert_eq!(
+                expected,
+                parsed.to_string(),
+                "Mode string '{actual}' did not parse to '{expected}': got '{parsed}'"
+            );
+        }
+    }
+
+    #[test]
+    fn test_parsed_mode_to_test_modes() {
+        let strings = vec![
+            ("Mz", vec!["Y Mz", "E Mz"]),
+            ("Y", vec!["Y M0", "Y M3"]),
+            ("E", vec!["E M0", "E M3"]),
+            ("Y+", vec!["Y M3"]),
+            ("Y-", vec!["Y M0"]),
+            ("Y <=0.8", vec!["Y M0 <=0.8", "Y M3 <=0.8"]),
+            (
+                "<=0.8",
+                vec!["Y M0 <=0.8", "Y M3 <=0.8", "E M0 <=0.8", "E M3 <=0.8"],
+            ),
+        ];
+
+        for (actual, expected) in strings {
+            let parsed = ParsedMode::from_str(actual)
+                .unwrap_or_else(|_| panic!("Failed to parse mode string '{actual}'"));
+            let expected_set: HashSet<_> = expected.into_iter().map(|s| s.to_owned()).collect();
+            let actual_set: HashSet<_> = parsed.to_modes().map(|m| m.to_string()).collect();
+
+            assert_eq!(
+                expected_set, actual_set,
+                "Mode string '{actual}' did not expand to '{expected_set:?}': got '{actual_set:?}'"
+            );
+        }
+    }
+}

crates/config/src/lib.rs

@@ -12,19 +12,18 @@ use std::{
 
 use alloy::{
     genesis::Genesis,
-    hex::ToHexExt,
     network::EthereumWallet,
-    primitives::{FixedBytes, U256},
+    primitives::{B256, FixedBytes, U256},
     signers::local::PrivateKeySigner,
 };
 use clap::{Parser, ValueEnum, ValueHint};
 use revive_dt_common::types::PlatformIdentifier;
 use semver::Version;
-use serde::{Serialize, Serializer};
+use serde::{Deserialize, Serialize, Serializer};
 use strum::{AsRefStr, Display, EnumString, IntoStaticStr};
 use temp_dir::TempDir;
 
-#[derive(Clone, Debug, Parser, Serialize)]
+#[derive(Clone, Debug, Parser, Serialize, Deserialize)]
 #[command(name = "retester")]
 pub enum Context {
     /// Executes tests in the MatterLabs format differentially on multiple targets concurrently.
@@ -200,7 +199,17 @@ impl AsRef<ReportConfiguration> for Context {
     }
 }
 
-#[derive(Clone, Debug, Parser, Serialize)]
+impl AsRef<IgnoreSuccessConfiguration> for Context {
+    fn as_ref(&self) -> &IgnoreSuccessConfiguration {
+        match self {
+            Self::Test(context) => context.as_ref().as_ref(),
+            Self::Benchmark(..) => unreachable!(),
+            Self::ExportJsonSchema => unreachable!(),
+        }
+    }
+}
+
+#[derive(Clone, Debug, Parser, Serialize, Deserialize)]
 pub struct TestExecutionContext {
     /// The set of platforms that the differential tests should run on.
     #[arg(
@@ -282,9 +291,13 @@ pub struct TestExecutionContext {
     /// Configuration parameters for the report.
     #[clap(flatten, next_help_heading = "Report Configuration")]
     pub report_configuration: ReportConfiguration,
+
+    /// Configuration parameters for ignoring certain test cases based on the report
+    #[clap(flatten, next_help_heading = "Ignore Success Configuration")]
+    pub ignore_success_configuration: IgnoreSuccessConfiguration,
 }
 
-#[derive(Clone, Debug, Parser, Serialize)]
+#[derive(Clone, Debug, Parser, Serialize, Deserialize)]
 pub struct BenchmarkingContext {
     /// The working directory that the program will use for all of the temporary artifacts needed at
     /// runtime.
@@ -461,6 +474,12 @@ impl AsRef<ReportConfiguration> for TestExecutionContext {
     }
 }
 
+impl AsRef<IgnoreSuccessConfiguration> for TestExecutionContext {
+    fn as_ref(&self) -> &IgnoreSuccessConfiguration {
+        &self.ignore_success_configuration
+    }
+}
+
 impl Default for BenchmarkingContext {
     fn default() -> Self {
         Self::parse_from(["execution-context"])
@@ -552,7 +571,7 @@ impl AsRef<ReportConfiguration> for BenchmarkingContext {
 }
 
 /// A set of configuration parameters for the corpus files to use for the execution.
-#[derive(Clone, Debug, Parser, Serialize)]
+#[derive(Clone, Debug, Parser, Serialize, Deserialize)]
 pub struct CorpusConfiguration {
     /// A list of test corpus JSON files to be tested.
     #[arg(short = 'c', long = "corpus")]
@@ -560,7 +579,7 @@ pub struct CorpusConfiguration {
 }
 
 /// A set of configuration parameters for Solc.
-#[derive(Clone, Debug, Parser, Serialize)]
+#[derive(Clone, Debug, Parser, Serialize, Deserialize)]
 pub struct SolcConfiguration {
     /// Specifies the default version of the Solc compiler that should be used if there is no
     /// override specified by one of the test cases.
@@ -569,7 +588,7 @@ pub struct SolcConfiguration {
 }
 
 /// A set of configuration parameters for Resolc.
-#[derive(Clone, Debug, Parser, Serialize)]
+#[derive(Clone, Debug, Parser, Serialize, Deserialize)]
 pub struct ResolcConfiguration {
     /// Specifies the path of the resolc compiler to be used by the tool.
     ///
@@ -580,7 +599,7 @@ pub struct ResolcConfiguration {
 }
 
 /// A set of configuration parameters for Polkadot Parachain.
-#[derive(Clone, Debug, Parser, Serialize)]
+#[derive(Clone, Debug, Parser, Serialize, Deserialize)]
 pub struct PolkadotParachainConfiguration {
     /// Specifies the path of the polkadot-parachain node to be used by the tool.
     ///
@@ -604,7 +623,7 @@ pub struct PolkadotParachainConfiguration {
 }
 
 /// A set of configuration parameters for Geth.
-#[derive(Clone, Debug, Parser, Serialize)]
+#[derive(Clone, Debug, Parser, Serialize, Deserialize)]
 pub struct GethConfiguration {
     /// Specifies the path of the geth node to be used by the tool.
     ///
@@ -624,7 +643,7 @@ pub struct GethConfiguration {
 }
 
 /// A set of configuration parameters for kurtosis.
-#[derive(Clone, Debug, Parser, Serialize)]
+#[derive(Clone, Debug, Parser, Serialize, Deserialize)]
 pub struct KurtosisConfiguration {
     /// Specifies the path of the kurtosis node to be used by the tool.
     ///
@@ -639,7 +658,7 @@ pub struct KurtosisConfiguration {
 }
 
 /// A set of configuration parameters for Kitchensink.
-#[derive(Clone, Debug, Parser, Serialize)]
+#[derive(Clone, Debug, Parser, Serialize, Deserialize)]
 pub struct KitchensinkConfiguration {
     /// Specifies the path of the kitchensink node to be used by the tool.
     ///
@@ -663,7 +682,7 @@ pub struct KitchensinkConfiguration {
 }
 
 /// A set of configuration parameters for the revive dev node.
-#[derive(Clone, Debug, Parser, Serialize)]
+#[derive(Clone, Debug, Parser, Serialize, Deserialize)]
 pub struct ReviveDevNodeConfiguration {
     /// Specifies the path of the revive dev node to be used by the tool.
     ///
@@ -695,7 +714,7 @@ pub struct ReviveDevNodeConfiguration {
 }
 
 /// A set of configuration parameters for the ETH RPC.
-#[derive(Clone, Debug, Parser, Serialize)]
+#[derive(Clone, Debug, Parser, Serialize, Deserialize)]
 pub struct EthRpcConfiguration {
     /// Specifies the path of the ETH RPC to be used by the tool.
     ///
@@ -715,7 +734,7 @@ pub struct EthRpcConfiguration {
 }
 
 /// A set of configuration parameters for the genesis.
-#[derive(Clone, Debug, Default, Parser, Serialize)]
+#[derive(Clone, Debug, Default, Parser, Serialize, Deserialize)]
 pub struct GenesisConfiguration {
     /// Specifies the path of the genesis file to use for the nodes that are started.
     ///
@@ -753,15 +772,14 @@ impl GenesisConfiguration {
 }
 
 /// A set of configuration parameters for the wallet.
-#[derive(Clone, Debug, Parser, Serialize)]
+#[derive(Clone, Debug, Parser, Serialize, Deserialize)]
 pub struct WalletConfiguration {
     /// The private key of the default signer.
     #[clap(
         long = "wallet.default-private-key",
         default_value = "0x4f3edf983ac636a65a842ce7c78d9aa706d3b113bce9c46f30d7d21715b23b1d"
     )]
-    #[serde(serialize_with = "serialize_private_key")]
+    default_key: B256,
-    default_key: PrivateKeySigner,
 
     /// This argument controls which private keys the nodes should have access to and be added to
     /// its wallet signers. With a value of N, private keys (0, N] will be added to the signer set
@@ -779,7 +797,8 @@ impl WalletConfiguration {
     pub fn wallet(&self) -> Arc<EthereumWallet> {
         self.wallet
             .get_or_init(|| {
-                let mut wallet = EthereumWallet::new(self.default_key.clone());
+                let mut wallet =
+                    EthereumWallet::new(PrivateKeySigner::from_bytes(&self.default_key).unwrap());
                 for signer in (1..=self.additional_keys)
                     .map(|id| U256::from(id))
                     .map(|id| id.to_be_bytes::<32>())
@@ -797,15 +816,8 @@ impl WalletConfiguration {
     }
 }
 
-fn serialize_private_key<S>(value: &PrivateKeySigner, serializer: S) -> Result<S::Ok, S::Error>
-where
-    S: Serializer,
-{
-    value.to_bytes().encode_hex().serialize(serializer)
-}
-
 /// A set of configuration for concurrency.
-#[derive(Clone, Debug, Parser, Serialize)]
+#[derive(Clone, Debug, Parser, Serialize, Deserialize)]
 pub struct ConcurrencyConfiguration {
     /// Determines the amount of nodes that will be spawned for each chain.
     #[clap(long = "concurrency.number-of-nodes", default_value_t = 5)]
@@ -843,14 +855,14 @@ impl ConcurrencyConfiguration {
     }
 }
 
-#[derive(Clone, Debug, Parser, Serialize)]
+#[derive(Clone, Debug, Parser, Serialize, Deserialize)]
 pub struct CompilationConfiguration {
     /// Controls if the compilation cache should be invalidated or not.
     #[arg(long = "compilation.invalidate-cache")]
     pub invalidate_compilation_cache: bool,
 }
 
-#[derive(Clone, Debug, Parser, Serialize)]
+#[derive(Clone, Debug, Parser, Serialize, Deserialize)]
 pub struct ReportConfiguration {
     /// Controls if the compiler input is included in the final report.
     #[clap(long = "report.include-compiler-input")]
@@ -861,6 +873,13 @@ pub struct ReportConfiguration {
     pub include_compiler_output: bool,
 }
 
+#[derive(Clone, Debug, Parser, Serialize, Deserialize)]
+pub struct IgnoreSuccessConfiguration {
+    /// The path of the report generated by the tool to use to ignore the cases that succeeded.
+    #[clap(long = "ignore-success.report-path")]
+    pub path: Option<PathBuf>,
+}
+
 /// Represents the working directory that the program uses.
 #[derive(Debug, Clone)]
 pub enum WorkingDirectoryConfiguration {
@@ -870,6 +889,24 @@ pub enum WorkingDirectoryConfiguration {
     Path(PathBuf),
 }
 
+impl Serialize for WorkingDirectoryConfiguration {
+    fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>
+    where
+        S: Serializer,
+    {
+        self.as_path().serialize(serializer)
+    }
+}
+
+impl<'a> Deserialize<'a> for WorkingDirectoryConfiguration {
+    fn deserialize<D>(deserializer: D) -> Result<Self, D::Error>
+    where
+        D: serde::Deserializer<'a>,
+    {
+        PathBuf::deserialize(deserializer).map(Self::Path)
+    }
+}
+
 impl WorkingDirectoryConfiguration {
     pub fn as_path(&self) -> &Path {
         self.as_ref()
@@ -919,15 +956,6 @@ impl Display for WorkingDirectoryConfiguration {
     }
 }
 
-impl Serialize for WorkingDirectoryConfiguration {
-    fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>
-    where
-        S: serde::Serializer,
-    {
-        self.as_path().serialize(serializer)
-    }
-}
-
 fn parse_duration(s: &str) -> anyhow::Result<Duration> {
     u64::from_str(s)
         .map(Duration::from_millis)

crates/core/src/differential_benchmarks/entry_point.rs

@@ -86,6 +86,7 @@ pub async fn handle_differential_benchmarks(
         &full_context,
         metadata_files.iter(),
         &platforms_and_nodes,
+        None,
         reporter.clone(),
     )
     .await

crates/core/src/differential_tests/entry_point.rs

@@ -10,7 +10,7 @@ use std::{
 use ansi_term::{ANSIStrings, Color};
 use anyhow::Context as _;
 use futures::{FutureExt, StreamExt};
-use revive_dt_common::types::PrivateKeyAllocator;
+use revive_dt_common::{cached_fs::read_to_string, types::PrivateKeyAllocator};
 use revive_dt_core::Platform;
 use tokio::sync::{Mutex, RwLock, Semaphore};
 use tracing::{Instrument, error, info, info_span, instrument};
@@ -72,11 +72,20 @@ pub async fn handle_differential_tests(
     info!("Spawned the platform nodes");
 
     // Preparing test definitions.
+    let only_execute_failed_tests = match context.ignore_success_configuration.path.as_ref() {
+        Some(path) => {
+            let report = read_to_string(path)
+                .context("Failed to read the report file to ignore the succeeding test cases")?;
+            Some(serde_json::from_str(&report).context("Failed to deserialize report")?)
+        }
+        None => None,
+    };
     let full_context = Context::Test(Box::new(context.clone()));
     let test_definitions = create_test_definitions_stream(
         &full_context,
         metadata_files.iter(),
         &platforms_and_nodes,
+        only_execute_failed_tests.as_ref(),
         reporter.clone(),
     )
     .await

crates/core/src/helpers/test.rs

@@ -5,9 +5,8 @@ use std::{borrow::Cow, path::Path};
 use futures::{Stream, StreamExt, stream};
 use indexmap::{IndexMap, indexmap};
 use revive_dt_common::iterators::EitherIter;
-use revive_dt_common::types::PlatformIdentifier;
+use revive_dt_common::types::{ParsedMode, PlatformIdentifier};
 use revive_dt_config::Context;
-use revive_dt_format::mode::ParsedMode;
 use serde_json::{Value, json};
 
 use revive_dt_compiler::Mode;
@@ -17,7 +16,7 @@ use revive_dt_format::{
     metadata::MetadataFile,
 };
 use revive_dt_node_interaction::EthereumNode;
-use revive_dt_report::{ExecutionSpecificReporter, Reporter};
+use revive_dt_report::{ExecutionSpecificReporter, Report, Reporter, TestCaseStatus};
 use revive_dt_report::{TestSpecificReporter, TestSpecifier};
 use tracing::{debug, error, info};
 
@@ -30,6 +29,7 @@ pub async fn create_test_definitions_stream<'a>(
     context: &Context,
     metadata_files: impl IntoIterator<Item = &'a MetadataFile>,
     platforms_and_nodes: &'a BTreeMap<PlatformIdentifier, (&dyn Platform, NodePool)>,
+    only_execute_failed_tests: Option<&Report>,
     reporter: Reporter,
 ) -> impl Stream<Item = TestDefinition<'a>> {
     stream::iter(
@@ -140,7 +140,7 @@ pub async fn create_test_definitions_stream<'a>(
     )
     // Filter out the test cases which are incompatible or that can't run in the current setup.
     .filter_map(move |test| async move {
-        match test.check_compatibility() {
+        match test.check_compatibility(only_execute_failed_tests) {
             Ok(()) => Some(test),
             Err((reason, additional_information)) => {
                 debug!(
@@ -200,12 +200,16 @@ pub struct TestDefinition<'a> {
 
 impl<'a> TestDefinition<'a> {
     /// Checks if this test can be ran with the current configuration.
-    pub fn check_compatibility(&self) -> TestCheckFunctionResult {
+    pub fn check_compatibility(
+        &self,
+        only_execute_failed_tests: Option<&Report>,
+    ) -> TestCheckFunctionResult {
         self.check_metadata_file_ignored()?;
         self.check_case_file_ignored()?;
         self.check_target_compatibility()?;
         self.check_evm_version_compatibility()?;
         self.check_compiler_compatibility()?;
+        self.check_ignore_succeeded(only_execute_failed_tests)?;
         Ok(())
     }
 
@@ -313,6 +317,36 @@ impl<'a> TestDefinition<'a> {
             ))
         }
     }
+
+    /// Checks if the test case should be executed or not based on the passed report and whether the
+    /// user has instructed the tool to ignore the already succeeding test cases.
+    fn check_ignore_succeeded(
+        &self,
+        only_execute_failed_tests: Option<&Report>,
+    ) -> TestCheckFunctionResult {
+        let Some(report) = only_execute_failed_tests else {
+            return Ok(());
+        };
+
+        let test_case_status = report
+            .test_case_information
+            .get(&(self.metadata_file_path.to_path_buf().into()))
+            .and_then(|obj| obj.get(&self.mode))
+            .and_then(|obj| obj.get(&self.case_idx))
+            .and_then(|obj| obj.status.as_ref());
+
+        match test_case_status {
+            Some(TestCaseStatus::Failed { .. }) => Ok(()),
+            Some(TestCaseStatus::Ignored { .. }) => Err((
+                "Ignored since it was ignored in a previous run",
+                indexmap! {},
+            )),
+            Some(TestCaseStatus::Succeeded { .. }) => {
+                Err(("Ignored since it succeeded in a prior run", indexmap! {}))
+            }
+            None => Ok(()),
+        }
+    }
 }
 
 pub struct TestPlatformInformation<'a> {

crates/format/Cargo.toml

@@ -16,7 +16,6 @@ revive-common = { workspace = true }
 alloy = { workspace = true }
 anyhow = { workspace = true }
 futures = { workspace = true }
-regex = { workspace = true }
 tracing = { workspace = true }
 schemars = { workspace = true }
 semver = { workspace = true }

crates/format/src/case.rs

@@ -1,9 +1,12 @@
 use schemars::JsonSchema;
 use serde::{Deserialize, Serialize};
 
-use revive_dt_common::{macros::define_wrapper_type, types::Mode};
+use revive_dt_common::{
+    macros::define_wrapper_type,
+    types::{Mode, ParsedMode},
+};
 
-use crate::{mode::ParsedMode, steps::*};
+use crate::steps::*;
 
 #[derive(Debug, Default, Serialize, Deserialize, Clone, Eq, PartialEq, JsonSchema)]
 pub struct Case {

crates/format/src/lib.rs

@@ -3,6 +3,5 @@
 pub mod case;
 pub mod corpus;
 pub mod metadata;
-pub mod mode;
 pub mod steps;
 pub mod traits;

crates/format/src/metadata.rs

@@ -16,11 +16,11 @@ use revive_dt_common::{
     cached_fs::read_to_string,
     iterators::FilesWithExtensionIterator,
     macros::define_wrapper_type,
-    types::{Mode, VmIdentifier},
+    types::{Mode, ParsedMode, VmIdentifier},
 };
 use tracing::error;
 
-use crate::{case::Case, mode::ParsedMode};
+use crate::case::Case;
 
 pub const METADATA_FILE_EXTENSION: &str = "json";
 pub const SOLIDITY_CASE_FILE_EXTENSION: &str = "sol";

crates/format/src/mode.rs

@@ -1,257 +0,0 @@
-use anyhow::Context as _;
-use regex::Regex;
-use revive_dt_common::iterators::EitherIter;
-use revive_dt_common::types::{Mode, ModeOptimizerSetting, ModePipeline};
-use schemars::JsonSchema;
-use serde::{Deserialize, Serialize};
-use std::collections::HashSet;
-use std::fmt::Display;
-use std::str::FromStr;
-use std::sync::LazyLock;
-
-/// This represents a mode that has been parsed from test metadata.
-///
-/// Mode strings can take the following form (in pseudo-regex):
-///
-/// ```text
-/// [YEILV][+-]? (M[0123sz])? <semver>?
-/// ```
-///
-/// We can parse valid mode strings into [`ParsedMode`] using [`ParsedMode::from_str`].
-#[derive(Clone, Debug, PartialEq, Eq, Hash, Deserialize, Serialize, JsonSchema)]
-#[serde(try_from = "String", into = "String")]
-pub struct ParsedMode {
-    pub pipeline: Option<ModePipeline>,
-    pub optimize_flag: Option<bool>,
-    pub optimize_setting: Option<ModeOptimizerSetting>,
-    pub version: Option<semver::VersionReq>,
-}
-
-impl FromStr for ParsedMode {
-    type Err = anyhow::Error;
-    fn from_str(s: &str) -> Result<Self, Self::Err> {
-        static REGEX: LazyLock<Regex> = LazyLock::new(|| {
-            Regex::new(r"(?x)
-                ^
-                (?:(?P<pipeline>[YEILV])(?P<optimize_flag>[+-])?)? # Pipeline to use eg Y, E+, E-
-                \s*
-                (?P<optimize_setting>M[a-zA-Z0-9])?                # Optimize setting eg M0, Ms, Mz
-                \s*
-                (?P<version>[>=<]*\d+(?:\.\d+)*)?                  # Optional semver version eg >=0.8.0, 0.7, <0.8
-                $
-            ").unwrap()
-        });
-
-        let Some(caps) = REGEX.captures(s) else {
-            anyhow::bail!("Cannot parse mode '{s}' from string");
-        };
-
-        let pipeline = match caps.name("pipeline") {
-            Some(m) => Some(
-                ModePipeline::from_str(m.as_str())
-                    .context("Failed to parse mode pipeline from string")?,
-            ),
-            None => None,
-        };
-
-        let optimize_flag = caps.name("optimize_flag").map(|m| m.as_str() == "+");
-
-        let optimize_setting = match caps.name("optimize_setting") {
-            Some(m) => Some(
-                ModeOptimizerSetting::from_str(m.as_str())
-                    .context("Failed to parse optimizer setting from string")?,
-            ),
-            None => None,
-        };
-
-        let version = match caps.name("version") {
-            Some(m) => Some(
-                semver::VersionReq::parse(m.as_str())
-                    .map_err(|e| {
-                        anyhow::anyhow!(
-                            "Cannot parse the version requirement '{}': {e}",
-                            m.as_str()
-                        )
-                    })
-                    .context("Failed to parse semver requirement from mode string")?,
-            ),
-            None => None,
-        };
-
-        Ok(ParsedMode {
-            pipeline,
-            optimize_flag,
-            optimize_setting,
-            version,
-        })
-    }
-}
-
-impl Display for ParsedMode {
-    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
-        let mut has_written = false;
-
-        if let Some(pipeline) = self.pipeline {
-            pipeline.fmt(f)?;
-            if let Some(optimize_flag) = self.optimize_flag {
-                f.write_str(if optimize_flag { "+" } else { "-" })?;
-            }
-            has_written = true;
-        }
-
-        if let Some(optimize_setting) = self.optimize_setting {
-            if has_written {
-                f.write_str(" ")?;
-            }
-            optimize_setting.fmt(f)?;
-            has_written = true;
-        }
-
-        if let Some(version) = &self.version {
-            if has_written {
-                f.write_str(" ")?;
-            }
-            version.fmt(f)?;
-        }
-
-        Ok(())
-    }
-}
-
-impl From<ParsedMode> for String {
-    fn from(parsed_mode: ParsedMode) -> Self {
-        parsed_mode.to_string()
-    }
-}
-
-impl TryFrom<String> for ParsedMode {
-    type Error = anyhow::Error;
-    fn try_from(value: String) -> Result<Self, Self::Error> {
-        ParsedMode::from_str(&value)
-    }
-}
-
-impl ParsedMode {
-    /// This takes a [`ParsedMode`] and expands it into a list of [`Mode`]s that we should try.
-    pub fn to_modes(&self) -> impl Iterator<Item = Mode> {
-        let pipeline_iter = self.pipeline.as_ref().map_or_else(
-            || EitherIter::A(ModePipeline::test_cases()),
-            |p| EitherIter::B(std::iter::once(*p)),
-        );
-
-        let optimize_flag_setting = self.optimize_flag.map(|flag| {
-            if flag {
-                ModeOptimizerSetting::M3
-            } else {
-                ModeOptimizerSetting::M0
-            }
-        });
-
-        let optimize_flag_iter = match optimize_flag_setting {
-            Some(setting) => EitherIter::A(std::iter::once(setting)),
-            None => EitherIter::B(ModeOptimizerSetting::test_cases()),
-        };
-
-        let optimize_settings_iter = self.optimize_setting.as_ref().map_or_else(
-            || EitherIter::A(optimize_flag_iter),
-            |s| EitherIter::B(std::iter::once(*s)),
-        );
-
-        pipeline_iter.flat_map(move |pipeline| {
-            optimize_settings_iter
-                .clone()
-                .map(move |optimize_setting| Mode {
-                    pipeline,
-                    optimize_setting,
-                    version: self.version.clone(),
-                })
-        })
-    }
-
-    /// Return a set of [`Mode`]s that correspond to the given [`ParsedMode`]s.
-    /// This avoids any duplicate entries.
-    pub fn many_to_modes<'a>(
-        parsed: impl Iterator<Item = &'a ParsedMode>,
-    ) -> impl Iterator<Item = Mode> {
-        let modes: HashSet<_> = parsed.flat_map(|p| p.to_modes()).collect();
-        modes.into_iter()
-    }
-}
-
-#[cfg(test)]
-mod tests {
-    use super::*;
-
-    #[test]
-    fn test_parsed_mode_from_str() {
-        let strings = vec![
-            ("Mz", "Mz"),
-            ("Y", "Y"),
-            ("Y+", "Y+"),
-            ("Y-", "Y-"),
-            ("E", "E"),
-            ("E+", "E+"),
-            ("E-", "E-"),
-            ("Y M0", "Y M0"),
-            ("Y M1", "Y M1"),
-            ("Y M2", "Y M2"),
-            ("Y M3", "Y M3"),
-            ("Y Ms", "Y Ms"),
-            ("Y Mz", "Y Mz"),
-            ("E M0", "E M0"),
-            ("E M1", "E M1"),
-            ("E M2", "E M2"),
-            ("E M3", "E M3"),
-            ("E Ms", "E Ms"),
-            ("E Mz", "E Mz"),
-            // When stringifying semver again, 0.8.0 becomes ^0.8.0 (same meaning)
-            ("Y 0.8.0", "Y ^0.8.0"),
-            ("E+ 0.8.0", "E+ ^0.8.0"),
-            ("Y M3 >=0.8.0", "Y M3 >=0.8.0"),
-            ("E Mz <0.7.0", "E Mz <0.7.0"),
-            // We can parse +- _and_ M1/M2 but the latter takes priority.
-            ("Y+ M1 0.8.0", "Y+ M1 ^0.8.0"),
-            ("E- M2 0.7.0", "E- M2 ^0.7.0"),
-            // We don't see this in the wild but it is parsed.
-            ("<=0.8", "<=0.8"),
-        ];
-
-        for (actual, expected) in strings {
-            let parsed = ParsedMode::from_str(actual)
-                .unwrap_or_else(|_| panic!("Failed to parse mode string '{actual}'"));
-            assert_eq!(
-                expected,
-                parsed.to_string(),
-                "Mode string '{actual}' did not parse to '{expected}': got '{parsed}'"
-            );
-        }
-    }
-
-    #[test]
-    fn test_parsed_mode_to_test_modes() {
-        let strings = vec![
-            ("Mz", vec!["Y Mz", "E Mz"]),
-            ("Y", vec!["Y M0", "Y M3"]),
-            ("E", vec!["E M0", "E M3"]),
-            ("Y+", vec!["Y M3"]),
-            ("Y-", vec!["Y M0"]),
-            ("Y <=0.8", vec!["Y M0 <=0.8", "Y M3 <=0.8"]),
-            (
-                "<=0.8",
-                vec!["Y M0 <=0.8", "Y M3 <=0.8", "E M0 <=0.8", "E M3 <=0.8"],
-            ),
-        ];
-
-        for (actual, expected) in strings {
-            let parsed = ParsedMode::from_str(actual)
-                .unwrap_or_else(|_| panic!("Failed to parse mode string '{actual}'"));
-            let expected_set: HashSet<_> = expected.into_iter().map(|s| s.to_owned()).collect();
-            let actual_set: HashSet<_> = parsed.to_modes().map(|m| m.to_string()).collect();
-
-            assert_eq!(
-                expected_set, actual_set,
-                "Mode string '{actual}' did not expand to '{expected_set:?}': got '{actual_set:?}'"
-            );
-        }
-    }
-}

crates/report/src/aggregator.rs

@@ -16,7 +16,7 @@ use revive_dt_compiler::{CompilerInput, CompilerOutput, Mode};
 use revive_dt_config::Context;
 use revive_dt_format::{case::CaseIdx, corpus::Corpus, metadata::ContractInstance};
 use semver::Version;
-use serde::Serialize;
+use serde::{Deserialize, Serialize};
 use serde_with::{DisplayFromStr, serde_as};
 use tokio::sync::{
     broadcast::{Sender, channel},
@@ -415,7 +415,7 @@ impl ReportAggregator {
 }
 
 #[serde_as]
-#[derive(Clone, Debug, Serialize)]
+#[derive(Clone, Debug, Serialize, Deserialize)]
 pub struct Report {
     /// The context that the tool was started up with.
     pub context: Context,
@@ -440,7 +440,7 @@ impl Report {
     }
 }
 
-#[derive(Clone, Debug, Serialize, Default)]
+#[derive(Clone, Debug, Serialize, Deserialize, Default)]
 pub struct TestCaseReport {
     /// Information on the status of the test case and whether it succeeded, failed, or was ignored.
     #[serde(skip_serializing_if = "Option::is_none")]
@@ -451,7 +451,7 @@ pub struct TestCaseReport {
 
 /// Information related to the status of the test. Could be that the test succeeded, failed, or that
 /// it was ignored.
-#[derive(Clone, Debug, Serialize)]
+#[derive(Clone, Debug, Serialize, Deserialize)]
 #[serde(tag = "status")]
 pub enum TestCaseStatus {
     /// The test case succeeded.
@@ -475,7 +475,7 @@ pub enum TestCaseStatus {
 }
 
 /// Information related to the platform node that's being used to execute the step.
-#[derive(Clone, Debug, Serialize)]
+#[derive(Clone, Debug, Serialize, Deserialize)]
 pub struct TestCaseNodeInformation {
     /// The ID of the node that this case is being executed on.
     pub id: usize,
@@ -486,7 +486,7 @@ pub struct TestCaseNodeInformation {
 }
 
 /// Execution information tied to the platform.
-#[derive(Clone, Debug, Default, Serialize)]
+#[derive(Clone, Debug, Default, Serialize, Deserialize)]
 pub struct ExecutionInformation {
     /// Information related to the node assigned to this test case.
     #[serde(skip_serializing_if = "Option::is_none")]
@@ -506,7 +506,7 @@ pub struct ExecutionInformation {
 }
 
 /// Information related to compilation
-#[derive(Clone, Debug, Serialize)]
+#[derive(Clone, Debug, Serialize, Deserialize)]
 #[serde(tag = "status")]
 pub enum CompilationStatus {
     /// The compilation was successful.