Merge remote-tracking branch 'origin/main' into feature/output-formats

Cargo.lock

@@ -782,6 +782,15 @@ dependencies = [
  "libc",
 ]
 
+[[package]]
+name = "ansi_term"
+version = "0.12.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "d52a9bb7ec0cf484c551830a7ce27bd20d67eac647e1befb56b0be4ee39a55d2"
+dependencies = [
+ "winapi",
+]
+
 [[package]]
 name = "anstream"
 version = "0.6.18"
@@ -2328,7 +2337,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "8d162beedaa69905488a8da94f5ac3edb4dd4788b732fadb7bd120b2625c1976"
 dependencies = [
  "data-encoding",
- "syn 1.0.109",
+ "syn 2.0.101",
 ]
 
 [[package]]
@@ -5611,6 +5620,7 @@ name = "revive-dt-core"
 version = "0.1.0"
 dependencies = [
  "alloy",
+ "ansi_term",
  "anyhow",
  "bson",
  "cacache",
@@ -5943,7 +5953,7 @@ dependencies = [
  "security-framework 3.3.0",
  "security-framework-sys",
  "webpki-root-certs 0.26.11",
- "windows-sys 0.52.0",
+ "windows-sys 0.59.0",
 ]
 
 [[package]]

Cargo.toml

@@ -22,6 +22,7 @@ revive-dt-node-pool = { version = "0.1.0", path = "crates/node-pool" }
 revive-dt-report = { version = "0.1.0", path = "crates/report" }
 revive-dt-solc-binaries = { version = "0.1.0", path = "crates/solc-binaries" }
 
+ansi_term = "0.12.1"
 anyhow = "1.0"
 async-stream = { version = "0.3.6" }
 bson = { version = "2.15.0" }

crates/compiler/tests/assets/array_one_element/main.sol

@@ -7,7 +7,10 @@ pragma solidity >=0.6.9;
 import "./callable.sol";
 
 contract Main {
-    function main(uint[1] calldata p1, Callable callable) public returns(uint) {
+    function main(
+        uint[1] calldata p1,
+        Callable callable
+    ) public pure returns (uint) {
         return callable.f(p1);
     }
 }

crates/config/src/lib.rs

@@ -202,6 +202,18 @@ impl AsRef<ReportConfiguration> for Context {
 
 #[derive(Clone, Debug, Parser, Serialize)]
 pub struct TestExecutionContext {
+    /// The set of platforms that the differential tests should run on.
+    #[arg(
+        short = 'p',
+        long = "platform",
+        default_values = ["geth-evm-solc", "revive-dev-node-polkavm-resolc"]
+    )]
+    pub platforms: Vec<PlatformIdentifier>,
+
+    /// The output format to use for the tool's output.
+    #[arg(short, long, default_value_t = OutputFormat::CargoTestLike)]
+    pub output_format: OutputFormat,
+
     /// The working directory that the program will use for all of the temporary artifacts needed at
     /// runtime.
     ///
@@ -215,14 +227,6 @@ pub struct TestExecutionContext {
     )]
     pub working_directory: WorkingDirectoryConfiguration,
 
-    /// The set of platforms that the differential tests should run on.
-    #[arg(
-        short = 'p',
-        long = "platform",
-        default_values = ["geth-evm-solc", "revive-dev-node-polkavm-resolc"]
-    )]
-    pub platforms: Vec<PlatformIdentifier>,
-
     /// Configuration parameters for the corpus files to use.
     #[clap(flatten, next_help_heading = "Corpus Configuration")]
     pub corpus_configuration: CorpusConfiguration,
@@ -958,3 +962,29 @@ pub enum TestingPlatform {
     /// A polkadot/Substrate based network
     Zombienet,
 }
+
+/// The output format to use for the test execution output.
+#[derive(
+    Clone,
+    Copy,
+    Debug,
+    PartialEq,
+    Eq,
+    PartialOrd,
+    Ord,
+    Hash,
+    Serialize,
+    ValueEnum,
+    EnumString,
+    Display,
+    AsRefStr,
+    IntoStaticStr,
+)]
+#[strum(serialize_all = "kebab-case")]
+pub enum OutputFormat {
+    /// The legacy format that was used in the past for the output.
+    Legacy,
+
+    /// An output format that looks heavily resembles the output from `cargo test`.
+    CargoTestLike,
+}

crates/core/Cargo.toml

@@ -21,6 +21,7 @@ revive-dt-node = { workspace = true }
 revive-dt-node-interaction = { workspace = true }
 revive-dt-report = { workspace = true }
 
+ansi_term = { workspace = true }
 alloy = { workspace = true }
 anyhow = { workspace = true }
 bson = { workspace = true }

crates/core/src/differential_tests/entry_point.rs

@@ -7,6 +7,7 @@ use std::{
     time::{Duration, Instant},
 };
 
+use ansi_term::{ANSIStrings, Color};
 use anyhow::Context as _;
 use futures::{FutureExt, StreamExt};
 use revive_dt_common::types::PrivateKeyAllocator;
@@ -14,7 +15,7 @@ use revive_dt_core::Platform;
 use tokio::sync::{Mutex, RwLock, Semaphore};
 use tracing::{Instrument, error, info, info_span, instrument};
 
-use revive_dt_config::{Context, TestExecutionContext};
+use revive_dt_config::{Context, OutputFormat, TestExecutionContext};
 use revive_dt_report::{Reporter, ReporterEvent, TestCaseStatus};
 
 use crate::{
@@ -176,7 +177,7 @@ pub async fn handle_differential_tests(
             .report_completion_event()
             .expect("Can't fail")
     });
-    let cli_reporting_task = start_cli_reporting_task(reporter);
+    let cli_reporting_task = start_cli_reporting_task(context.output_format, reporter);
 
     tokio::task::spawn(async move {
         loop {
@@ -196,21 +197,15 @@ pub async fn handle_differential_tests(
 }
 
 #[allow(irrefutable_let_patterns, clippy::uninlined_format_args)]
-async fn start_cli_reporting_task(reporter: Reporter) {
+async fn start_cli_reporting_task(output_format: OutputFormat, reporter: Reporter) {
     let mut aggregator_events_rx = reporter.subscribe().await.expect("Can't fail");
     drop(reporter);
 
     let start = Instant::now();
 
-    const GREEN: &str = "\x1B[32m";
-    const RED: &str = "\x1B[31m";
-    const GREY: &str = "\x1B[90m";
-    const COLOR_RESET: &str = "\x1B[0m";
-    const BOLD: &str = "\x1B[1m";
-    const BOLD_RESET: &str = "\x1B[22m";
-
-    let mut number_of_successes = 0;
-    let mut number_of_failures = 0;
+    let mut global_success_count = 0;
+    let mut global_failure_count = 0;
+    let mut global_ignore_count = 0;
 
     let mut buf = BufWriter::new(stderr());
     while let Ok(event) = aggregator_events_rx.recv().await {
@@ -223,55 +218,125 @@ async fn start_cli_reporting_task(reporter: Reporter) {
             continue;
         };
 
-        let _ = writeln!(buf, "{} - {}", mode, metadata_file_path.display());
-        for (case_idx, case_status) in case_status.into_iter() {
-            let _ = write!(buf, "\tCase Index {case_idx:>3}: ");
-            let _ = match case_status {
-                TestCaseStatus::Succeeded { steps_executed } => {
-                    number_of_successes += 1;
-                    writeln!(
-                        buf,
-                        "{}{}Case Succeeded{} - Steps Executed: {}{}",
-                        GREEN, BOLD, BOLD_RESET, steps_executed, COLOR_RESET
-                    )
+        match output_format {
+            OutputFormat::Legacy => {
+                let _ = writeln!(buf, "{} - {}", mode, metadata_file_path.display());
+                for (case_idx, case_status) in case_status.into_iter() {
+                    let _ = write!(buf, "\tCase Index {case_idx:>3}: ");
+                    let _ = match case_status {
+                        TestCaseStatus::Succeeded { steps_executed } => {
+                            global_success_count += 1;
+                            writeln!(
+                                buf,
+                                "{}",
+                                ANSIStrings(&[
+                                    Color::Green.bold().paint("Case Succeeded"),
+                                    Color::Green
+                                        .paint(format!(" - Steps Executed: {steps_executed}")),
+                                ])
+                            )
+                        }
+                        TestCaseStatus::Failed { reason } => {
+                            global_failure_count += 1;
+                            writeln!(
+                                buf,
+                                "{}",
+                                ANSIStrings(&[
+                                    Color::Red.bold().paint("Case Failed"),
+                                    Color::Red.paint(format!(" - Reason: {}", reason.trim())),
+                                ])
+                            )
+                        }
+                        TestCaseStatus::Ignored { reason, .. } => {
+                            global_ignore_count += 1;
+                            writeln!(
+                                buf,
+                                "{}",
+                                ANSIStrings(&[
+                                    Color::Yellow.bold().paint("Case Ignored"),
+                                    Color::Yellow.paint(format!(" - Reason: {}", reason.trim())),
+                                ])
+                            )
+                        }
+                    };
                 }
-                TestCaseStatus::Failed { reason } => {
-                    number_of_failures += 1;
-                    writeln!(
-                        buf,
-                        "{}{}Case Failed{} - Reason: {}{}",
-                        RED,
-                        BOLD,
-                        BOLD_RESET,
-                        reason.trim(),
-                        COLOR_RESET,
-                    )
-                }
-                TestCaseStatus::Ignored { reason, .. } => writeln!(
+                let _ = writeln!(buf);
+            }
+            OutputFormat::CargoTestLike => {
+                writeln!(
                     buf,
-                    "{}{}Case Ignored{} - Reason: {}{}",
-                    GREY,
-                    BOLD,
-                    BOLD_RESET,
-                    reason.trim(),
-                    COLOR_RESET,
-                ),
-            };
+                    "\t{} {} - {}\n",
+                    Color::Green.paint("Running"),
+                    metadata_file_path.display(),
+                    mode
+                )
+                .unwrap();
+
+                let mut success_count = 0;
+                let mut failure_count = 0;
+                let mut ignored_count = 0;
+                writeln!(buf, "running {} tests", case_status.len()).unwrap();
+                for (case_idx, case_result) in case_status.iter() {
+                    let status = match case_result {
+                        TestCaseStatus::Succeeded { .. } => {
+                            success_count += 1;
+                            global_success_count += 1;
+                            Color::Green.paint("ok")
+                        }
+                        TestCaseStatus::Failed { reason } => {
+                            failure_count += 1;
+                            global_failure_count += 1;
+                            Color::Red.paint(format!("FAILED, {reason}"))
+                        }
+                        TestCaseStatus::Ignored { reason, .. } => {
+                            ignored_count += 1;
+                            global_ignore_count += 1;
+                            Color::Yellow.paint(format!("ignored, {reason:?}"))
+                        }
+                    };
+                    writeln!(buf, "test case_idx_{} ... {}", case_idx, status).unwrap();
+                }
+                writeln!(buf).unwrap();
+
+                let status = if failure_count > 0 {
+                    Color::Red.paint("FAILED")
+                } else {
+                    Color::Green.paint("ok")
+                };
+                writeln!(
+                    buf,
+                    "test result: {}. {} passed; {} failed; {} ignored",
+                    status, success_count, failure_count, ignored_count,
+                )
+                .unwrap();
+                writeln!(buf).unwrap()
+            }
         }
-        let _ = writeln!(buf);
     }
 
     // Summary at the end.
-    let _ = writeln!(
-        buf,
-        "{} cases: {}{}{} cases succeeded, {}{}{} cases failed in {} seconds",
-        number_of_successes + number_of_failures,
-        GREEN,
-        number_of_successes,
-        COLOR_RESET,
-        RED,
-        number_of_failures,
-        COLOR_RESET,
-        start.elapsed().as_secs()
-    );
+    match output_format {
+        OutputFormat::Legacy => {
+            writeln!(
+                buf,
+                "{} cases: {} cases succeeded, {} cases failed in {} seconds",
+                global_success_count + global_failure_count + global_ignore_count,
+                Color::Green.paint(global_success_count.to_string()),
+                Color::Red.paint(global_failure_count.to_string()),
+                start.elapsed().as_secs()
+            )
+            .unwrap();
+        }
+        OutputFormat::CargoTestLike => {
+            writeln!(
+                buf,
+                "run finished. {} passed; {} failed; {} ignored; finished in {}s",
+                global_success_count,
+                global_failure_count,
+                global_ignore_count,
+                start.elapsed().as_secs()
+            )
+            .unwrap();
+        }
+    }
 }

run_tests.sh

@@ -93,17 +93,16 @@ echo ""
 # Run the tool
 cargo build --release;
 RUST_LOG="info,alloy_pubsub::service=error" ./target/release/retester test \
-    --platform revive-dev-node-revm-solc \
+    --platform geth-evm-solc \
     --corpus "$CORPUS_FILE" \
     --working-directory "$WORKDIR" \
     --concurrency.number-of-nodes 10 \
     --concurrency.number-of-threads 5 \
-    --concurrency.number-of-concurrent-tasks 1000 \
+    --concurrency.ignore-concurrency-limit \
     --wallet.additional-keys 100000 \
     --kitchensink.path "$SUBSTRATE_NODE_BIN" \
     --revive-dev-node.path "$REVIVE_DEV_NODE_BIN" \
     --eth-rpc.path "$ETH_RPC_BIN" \
-    > logs.log \
-    2> output.log
+    > logs.log 
 
 echo -e "${GREEN}=== Test run completed! ===${NC}"