Add a report processing tool

Cargo.lock

@@ -5910,6 +5910,18 @@ dependencies = [
  "tracing",
 ]
 
+[[package]]
+name = "revive-dt-report-processor"
+version = "0.1.0"
+dependencies = [
+ "anyhow",
+ "clap",
+ "revive-dt-common",
+ "revive-dt-report",
+ "serde",
+ "serde_json",
+]
+
 [[package]]
 name = "revive-dt-solc-binaries"
 version = "0.1.0"

Cargo.toml

@@ -21,6 +21,7 @@ revive-dt-node-interaction = { version = "0.1.0", path = "crates/node-interactio
 revive-dt-node-pool = { version = "0.1.0", path = "crates/node-pool" }
 revive-dt-report = { version = "0.1.0", path = "crates/report" }
 revive-dt-solc-binaries = { version = "0.1.0", path = "crates/solc-binaries" }
+revive-dt-report-processor = { version = "0.1.0", path = "crates/report-processor" }
 
 alloy = { version = "1.4.1", features = ["full", "genesis", "json-rpc"] }
 ansi_term = "0.12.1"

crates/common/src/types/mode.rs

@@ -23,6 +23,18 @@ pub struct Mode {
     pub version: Option<semver::VersionReq>,
 }
 
+impl Ord for Mode {
+    fn cmp(&self, other: &Self) -> std::cmp::Ordering {
+        self.to_string().cmp(&other.to_string())
+    }
+}
+
+impl PartialOrd for Mode {
+    fn partial_cmp(&self, other: &Self) -> Option<std::cmp::Ordering> {
+        Some(self.cmp(other))
+    }
+}
+
 impl Display for Mode {
     fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
         self.pipeline.fmt(f)?;

crates/common/src/types/parsed_test_specifier.rs

@@ -1,10 +1,15 @@
-use std::{fmt::Display, path::PathBuf, str::FromStr};
+use std::{
+    fmt::Display,
+    path::{Path, PathBuf},
+    str::FromStr,
+};
 
 use anyhow::{Context as _, bail};
+use serde::{Deserialize, Serialize};
 
 use crate::types::Mode;
 
-#[derive(Clone, Debug, PartialEq, Eq, Hash)]
+#[derive(Clone, Debug, PartialEq, Eq, PartialOrd, Ord, Hash)]
 pub enum ParsedTestSpecifier {
     /// All of the test cases in the file should be ran across all of the specified modes
     FileOrDirectory {
@@ -34,6 +39,22 @@ pub enum ParsedTestSpecifier {
     },
 }
 
+impl ParsedTestSpecifier {
+    pub fn metadata_path(&self) -> &Path {
+        match self {
+            ParsedTestSpecifier::FileOrDirectory {
+                metadata_or_directory_file_path: metadata_file_path,
+            }
+            | ParsedTestSpecifier::Case {
+                metadata_file_path, ..
+            }
+            | ParsedTestSpecifier::CaseWithMode {
+                metadata_file_path, ..
+            } => metadata_file_path,
+        }
+    }
+}
+
 impl Display for ParsedTestSpecifier {
     fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
         match self {
@@ -131,3 +152,22 @@ impl TryFrom<&str> for ParsedTestSpecifier {
         value.parse()
     }
 }
+
+impl Serialize for ParsedTestSpecifier {
+    fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>
+    where
+        S: serde::Serializer,
+    {
+        self.to_string().serialize(serializer)
+    }
+}
+
+impl<'de> Deserialize<'de> for ParsedTestSpecifier {
+    fn deserialize<D>(deserializer: D) -> Result<Self, D::Error>
+    where
+        D: serde::Deserializer<'de>,
+    {
+        let string = String::deserialize(deserializer)?;
+        string.parse().map_err(serde::de::Error::custom)
+    }
+}

crates/report-processor/Cargo.toml

@@ -0,0 +1,25 @@
+[package]
+name = "revive-dt-report-processor"
+description = "revive differential testing report processor utility"
+version.workspace = true
+authors.workspace = true
+license.workspace = true
+edition.workspace = true
+repository.workspace = true
+rust-version.workspace = true
+
+[[bin]]
+name = "report-processor"
+path = "src/main.rs"
+
+[dependencies]
+revive-dt-report = { workspace = true }
+revive-dt-common = { workspace = true }
+
+anyhow = { workspace = true }
+clap = { workspace = true }
+serde = { workspace = true }
+serde_json = { workspace = true }
+
+[lints]
+workspace = true

crates/report-processor/src/main.rs

@@ -0,0 +1,301 @@
+use std::{
+    borrow::Cow,
+    collections::{BTreeMap, BTreeSet},
+    fmt::Display,
+    fs::{File, OpenOptions},
+    ops::{Deref, DerefMut},
+    path::{Path, PathBuf},
+    str::FromStr,
+};
+
+use anyhow::{Context as _, Error, Result, bail};
+use clap::Parser;
+use serde::{Deserialize, Serialize, de::DeserializeOwned};
+
+use revive_dt_common::types::{Mode, ParsedTestSpecifier};
+use revive_dt_report::{Report, TestCaseStatus};
+
+fn main() -> Result<()> {
+    let cli = Cli::try_parse().context("Failed to parse the CLI arguments")?;
+
+    match cli {
+        Cli::GenerateExpectationsFile {
+            report_path,
+            output_path: output_file,
+            remove_prefix,
+        } => {
+            let remove_prefix = remove_prefix
+                .into_iter()
+                .map(|path| path.canonicalize().context("Failed to canonicalize path"))
+                .collect::<Result<Vec<_>>>()?;
+
+            let expectations = report_path
+                .execution_information
+                .iter()
+                .flat_map(|(metadata_file_path, metadata_file_report)| {
+                    metadata_file_report
+                        .case_reports
+                        .iter()
+                        .map(move |(case_idx, case_report)| {
+                            (metadata_file_path, case_idx, case_report)
+                        })
+                })
+                .flat_map(|(metadata_file_path, case_idx, case_report)| {
+                    case_report.mode_execution_reports.iter().map(
+                        move |(mode, execution_report)| {
+                            (
+                                metadata_file_path,
+                                case_idx,
+                                mode,
+                                execution_report.status.as_ref(),
+                            )
+                        },
+                    )
+                })
+                .filter_map(|(metadata_file_path, case_idx, mode, status)| {
+                    status.map(|status| (metadata_file_path, case_idx, mode, status))
+                })
+                .map(|(metadata_file_path, case_idx, mode, status)| {
+                    (
+                        TestSpecifier {
+                            metadata_file_path: Cow::Borrowed(
+                                remove_prefix
+                                    .iter()
+                                    .filter_map(|prefix| {
+                                        metadata_file_path.as_inner().strip_prefix(prefix).ok()
+                                    })
+                                    .next()
+                                    .unwrap_or(metadata_file_path.as_inner()),
+                            ),
+                            case_idx: case_idx.into_inner(),
+                            mode: Cow::Borrowed(mode),
+                        },
+                        Cow::Borrowed(status),
+                    )
+                })
+                .collect::<Expectations>();
+
+            let output_file = OpenOptions::new()
+                .truncate(true)
+                .create(true)
+                .write(true)
+                .open(output_file)
+                .context("Failed to create the output file")?;
+            serde_json::to_writer_pretty(output_file, &expectations)
+                .context("Failed to write the expectations to file")?;
+        }
+        Cli::CompareExpectationFiles {
+            base_expectation_path,
+            other_expectation_path,
+        } => {
+            let keys = base_expectation_path
+                .keys()
+                .chain(other_expectation_path.keys())
+                .collect::<BTreeSet<_>>();
+
+            for key in keys {
+                let base_status = base_expectation_path.get(key).context(format!(
+                    "Entry not found in the base expectations: \"{}\"",
+                    key
+                ))?;
+                let other_status = other_expectation_path.get(key).context(format!(
+                    "Entry not found in the other expectations: \"{}\"",
+                    key
+                ))?;
+
+                if base_status != other_status {
+                    bail!(
+                        "Expectations for entry \"{}\" have changed. They were {:?} and now they are {:?}",
+                        key,
+                        base_status,
+                        other_status
+                    )
+                }
+            }
+        }
+    };
+
+    Ok(())
+}
+
+type Expectations<'a> = BTreeMap<TestSpecifier<'a>, Cow<'a, TestCaseStatus>>;
+
+/// A tool that's used to process the reports generated by the retester binary in various ways.
+#[derive(Clone, Debug, Parser)]
+#[command(name = "retester", term_width = 100)]
+pub enum Cli {
+    /// Generates an expectation file out of a given report.
+    GenerateExpectationsFile {
+        /// The path of the report's JSON file to generate the expectation's file for.
+        #[clap(long)]
+        report_path: JsonFile<Report>,
+
+        /// The path of the output file to generate.
+        ///
+        /// Note that we expect that:
+        /// 1. The provided path points to a JSON file.
+        /// 1. The ancestor's of the provided path already exist such that no directory creations
+        ///    are required.
+        #[clap(long)]
+        output_path: PathBuf,
+
+        /// Prefix paths to remove from the paths in the final expectations file.
+        #[clap(long)]
+        remove_prefix: Vec<PathBuf>,
+    },
+
+    /// Compares two expectation files to ensure that they match each other.
+    CompareExpectationFiles {
+        /// The path of the base expectation file.
+        #[clap(long)]
+        base_expectation_path: JsonFile<Expectations<'static>>,
+
+        /// The path of the other expectation file.
+        #[clap(long)]
+        other_expectation_path: JsonFile<Expectations<'static>>,
+    },
+}
+
+#[derive(Clone, Debug, PartialEq, Eq, PartialOrd, Ord, Hash)]
+pub struct JsonFile<T> {
+    path: PathBuf,
+    content: Box<T>,
+}
+
+impl<T> Deref for JsonFile<T> {
+    type Target = T;
+
+    fn deref(&self) -> &Self::Target {
+        &self.content
+    }
+}
+
+impl<T> DerefMut for JsonFile<T> {
+    fn deref_mut(&mut self) -> &mut Self::Target {
+        &mut self.content
+    }
+}
+
+impl<T> FromStr for JsonFile<T>
+where
+    T: DeserializeOwned,
+{
+    type Err = Error;
+
+    fn from_str(s: &str) -> Result<Self, Self::Err> {
+        let path = PathBuf::from(s);
+        let file = File::open(&path).context("Failed to open the file")?;
+        serde_json::from_reader(&file)
+            .map(|content| Self { path, content })
+            .context(format!(
+                "Failed to deserialize file's content as {}",
+                std::any::type_name::<T>()
+            ))
+    }
+}
+
+impl<T> Display for JsonFile<T> {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        Display::fmt(&self.path.display(), f)
+    }
+}
+
+impl<T> From<JsonFile<T>> for String {
+    fn from(value: JsonFile<T>) -> Self {
+        value.to_string()
+    }
+}
+
+#[derive(Clone, Debug, PartialEq, Eq, PartialOrd, Ord, Hash)]
+pub struct TestSpecifier<'a> {
+    pub metadata_file_path: Cow<'a, Path>,
+    pub case_idx: usize,
+    pub mode: Cow<'a, Mode>,
+}
+
+impl<'a> Display for TestSpecifier<'a> {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        write!(
+            f,
+            "{}::{}::{}",
+            self.metadata_file_path.display(),
+            self.case_idx,
+            self.mode
+        )
+    }
+}
+
+impl<'a> From<TestSpecifier<'a>> for ParsedTestSpecifier {
+    fn from(
+        TestSpecifier {
+            metadata_file_path,
+            case_idx,
+            mode,
+        }: TestSpecifier,
+    ) -> Self {
+        Self::CaseWithMode {
+            metadata_file_path: metadata_file_path.to_path_buf(),
+            case_idx,
+            mode: mode.into_owned(),
+        }
+    }
+}
+
+impl TryFrom<ParsedTestSpecifier> for TestSpecifier<'static> {
+    type Error = Error;
+
+    fn try_from(value: ParsedTestSpecifier) -> Result<Self> {
+        let ParsedTestSpecifier::CaseWithMode {
+            metadata_file_path,
+            case_idx,
+            mode,
+        } = value
+        else {
+            bail!("Expected a full test case specifier")
+        };
+        Ok(Self {
+            metadata_file_path: Cow::Owned(metadata_file_path),
+            case_idx,
+            mode: Cow::Owned(mode),
+        })
+    }
+}
+
+impl<'a> Serialize for TestSpecifier<'a> {
+    fn serialize<S>(&self, serializer: S) -> std::result::Result<S::Ok, S::Error>
+    where
+        S: serde::Serializer,
+    {
+        self.to_string().serialize(serializer)
+    }
+}
+
+impl<'d, 'a> Deserialize<'d> for TestSpecifier<'a> {
+    fn deserialize<D>(deserializer: D) -> std::result::Result<Self, D::Error>
+    where
+        D: serde::Deserializer<'d>,
+    {
+        let string = String::deserialize(deserializer)?;
+        let mut splitted = string.split("::");
+        let (Some(metadata_file_path), Some(case_idx), Some(mode), None) = (
+            splitted.next(),
+            splitted.next(),
+            splitted.next(),
+            splitted.next(),
+        ) else {
+            return Err(serde::de::Error::custom(
+                "Test specifier doesn't contain the components required",
+            ));
+        };
+        let metadata_file_path = PathBuf::from(metadata_file_path);
+        let case_idx = usize::from_str(case_idx)
+            .map_err(|_| serde::de::Error::custom("Case idx is not a usize"))?;
+        let mode = Mode::from_str(mode).map_err(|_| serde::de::Error::custom("Invalid mode"))?;
+
+        Ok(Self {
+            metadata_file_path: Cow::Owned(metadata_file_path),
+            case_idx,
+            mode: Cow::Owned(mode),
+        })
+    }
+}

crates/report/src/aggregator.rs

@@ -562,7 +562,7 @@ pub struct Report {
     /// The list of metadata files that were found by the tool.
     pub metadata_files: BTreeSet<MetadataFilePath>,
     /// Metrics from the execution.
-    #[serde(skip_serializing_if = "Option::is_none")]
+    #[serde(default, skip_serializing_if = "Option::is_none")]
     pub metrics: Option<Metrics>,
     /// Information relating to each test case.
     pub execution_information: BTreeMap<MetadataFilePath, MetadataFileReport>,
@@ -582,7 +582,7 @@ impl Report {
 #[derive(Clone, Debug, Serialize, Deserialize, Default)]
 pub struct MetadataFileReport {
     /// Metrics from the execution.
-    #[serde(skip_serializing_if = "Option::is_none")]
+    #[serde(default, skip_serializing_if = "Option::is_none")]
     pub metrics: Option<Metrics>,
     /// The report of each case keyed by the case idx.
     pub case_reports: BTreeMap<CaseIdx, CaseReport>,
@@ -592,7 +592,7 @@ pub struct MetadataFileReport {
 #[derive(Clone, Debug, Serialize, Deserialize, Default)]
 pub struct CaseReport {
     /// Metrics from the execution.
-    #[serde(skip_serializing_if = "Option::is_none")]
+    #[serde(default, skip_serializing_if = "Option::is_none")]
     pub metrics: Option<Metrics>,
     /// The [`ExecutionReport`] for each one of the [`Mode`]s.
     #[serde_as(as = "HashMap<DisplayFromStr, _>")]
@@ -602,31 +602,31 @@ pub struct CaseReport {
 #[derive(Clone, Debug, Serialize, Deserialize, Default)]
 pub struct ExecutionReport {
     /// Information on the status of the test case and whether it succeeded, failed, or was ignored.
-    #[serde(skip_serializing_if = "Option::is_none")]
+    #[serde(default, skip_serializing_if = "Option::is_none")]
     pub status: Option<TestCaseStatus>,
     /// Metrics from the execution.
-    #[serde(skip_serializing_if = "Option::is_none")]
+    #[serde(default, skip_serializing_if = "Option::is_none")]
     pub metrics: Option<Metrics>,
     /// Information related to the execution on one of the platforms.
-    #[serde(skip_serializing_if = "BTreeMap::is_empty")]
+    #[serde(default, skip_serializing_if = "BTreeMap::is_empty")]
     pub platform_execution: PlatformKeyedInformation<Option<ExecutionInformation>>,
     /// Information on the compiled contracts.
-    #[serde(skip_serializing_if = "BTreeMap::is_empty")]
+    #[serde(default, skip_serializing_if = "BTreeMap::is_empty")]
     pub compiled_contracts: BTreeMap<PathBuf, BTreeMap<String, ContractInformation>>,
     /// The addresses of the deployed contracts
-    #[serde(skip_serializing_if = "BTreeMap::is_empty")]
+    #[serde(default, skip_serializing_if = "BTreeMap::is_empty")]
     pub contract_addresses: BTreeMap<ContractInstance, PlatformKeyedInformation<Vec<Address>>>,
     /// Information on the mined blocks as part of this execution.
-    #[serde(skip_serializing_if = "BTreeMap::is_empty")]
+    #[serde(default, skip_serializing_if = "BTreeMap::is_empty")]
     pub mined_block_information: PlatformKeyedInformation<Vec<MinedBlockInformation>>,
     /// Information tracked for each step that was executed.
-    #[serde(skip_serializing_if = "BTreeMap::is_empty")]
+    #[serde(default, skip_serializing_if = "BTreeMap::is_empty")]
     pub steps: BTreeMap<StepPath, StepReport>,
 }
 
 /// Information related to the status of the test. Could be that the test succeeded, failed, or that
 /// it was ignored.
-#[derive(Clone, Debug, Serialize, Deserialize)]
+#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
 #[serde(tag = "status")]
 pub enum TestCaseStatus {
     /// The test case succeeded.
@@ -664,19 +664,19 @@ pub struct TestCaseNodeInformation {
 #[derive(Clone, Debug, Default, Serialize, Deserialize)]
 pub struct ExecutionInformation {
     /// Information related to the node assigned to this test case.
-    #[serde(skip_serializing_if = "Option::is_none")]
+    #[serde(default, skip_serializing_if = "Option::is_none")]
     pub node: Option<TestCaseNodeInformation>,
     /// Information on the pre-link compiled contracts.
-    #[serde(skip_serializing_if = "Option::is_none")]
+    #[serde(default, skip_serializing_if = "Option::is_none")]
     pub pre_link_compilation_status: Option<CompilationStatus>,
     /// Information on the post-link compiled contracts.
-    #[serde(skip_serializing_if = "Option::is_none")]
+    #[serde(default, skip_serializing_if = "Option::is_none")]
     pub post_link_compilation_status: Option<CompilationStatus>,
     /// Information on the deployed libraries.
-    #[serde(skip_serializing_if = "Option::is_none")]
+    #[serde(default, skip_serializing_if = "Option::is_none")]
     pub deployed_libraries: Option<BTreeMap<ContractInstance, Address>>,
     /// Information on the deployed contracts.
-    #[serde(skip_serializing_if = "Option::is_none")]
+    #[serde(default, skip_serializing_if = "Option::is_none")]
     pub deployed_contracts: Option<BTreeMap<ContractInstance, Address>>,
 }
 
@@ -695,11 +695,11 @@ pub enum CompilationStatus {
         /// The input provided to the compiler to compile the contracts. This is only included if
         /// the appropriate flag is set in the CLI context and if the contracts were not cached and
         /// the compiler was invoked.
-        #[serde(skip_serializing_if = "Option::is_none")]
+        #[serde(default, skip_serializing_if = "Option::is_none")]
         compiler_input: Option<CompilerInput>,
         /// The output of the compiler. This is only included if the appropriate flag is set in the
         /// CLI contexts.
-        #[serde(skip_serializing_if = "Option::is_none")]
+        #[serde(default, skip_serializing_if = "Option::is_none")]
         compiler_output: Option<CompilerOutput>,
     },
     /// The compilation failed.
@@ -707,15 +707,15 @@ pub enum CompilationStatus {
         /// The failure reason.
         reason: String,
         /// The version of the compiler used to compile the contracts.
-        #[serde(skip_serializing_if = "Option::is_none")]
+        #[serde(default, skip_serializing_if = "Option::is_none")]
         compiler_version: Option<Version>,
         /// The path of the compiler used to compile the contracts.
-        #[serde(skip_serializing_if = "Option::is_none")]
+        #[serde(default, skip_serializing_if = "Option::is_none")]
         compiler_path: Option<PathBuf>,
         /// The input provided to the compiler to compile the contracts. This is only included if
         /// the appropriate flag is set in the CLI context and if the contracts were not cached and
         /// the compiler was invoked.
-        #[serde(skip_serializing_if = "Option::is_none")]
+        #[serde(default, skip_serializing_if = "Option::is_none")]
         compiler_input: Option<CompilerInput>,
     },
 }
@@ -743,24 +743,24 @@ pub struct Metrics {
     pub gas_per_second: Metric<u64>,
     /* Block Fullness */
     pub gas_block_fullness: Metric<u64>,
-    #[serde(skip_serializing_if = "Option::is_none")]
+    #[serde(default, skip_serializing_if = "Option::is_none")]
     pub ref_time_block_fullness: Option<Metric<u64>>,
-    #[serde(skip_serializing_if = "Option::is_none")]
+    #[serde(default, skip_serializing_if = "Option::is_none")]
     pub proof_size_block_fullness: Option<Metric<u64>>,
 }
 
 /// The data that we store for a given metric (e.g., TPS).
 #[derive(Clone, Debug, Default, Serialize, Deserialize)]
 pub struct Metric<T> {
-    #[serde(skip_serializing_if = "Option::is_none")]
+    #[serde(default, skip_serializing_if = "Option::is_none")]
     pub minimum: Option<PlatformKeyedInformation<T>>,
-    #[serde(skip_serializing_if = "Option::is_none")]
+    #[serde(default, skip_serializing_if = "Option::is_none")]
     pub maximum: Option<PlatformKeyedInformation<T>>,
-    #[serde(skip_serializing_if = "Option::is_none")]
+    #[serde(default, skip_serializing_if = "Option::is_none")]
     pub mean: Option<PlatformKeyedInformation<T>>,
-    #[serde(skip_serializing_if = "Option::is_none")]
+    #[serde(default, skip_serializing_if = "Option::is_none")]
     pub median: Option<PlatformKeyedInformation<T>>,
-    #[serde(skip_serializing_if = "Option::is_none")]
+    #[serde(default, skip_serializing_if = "Option::is_none")]
     pub raw: Option<PlatformKeyedInformation<Vec<T>>>,
 }