diff --git a/Cargo.lock b/Cargo.lock index d0e8908..0f56c36 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -782,6 +782,15 @@ dependencies = [ "libc", ] +[[package]] +name = "ansi_term" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d52a9bb7ec0cf484c551830a7ce27bd20d67eac647e1befb56b0be4ee39a55d2" +dependencies = [ + "winapi", +] + [[package]] name = "anstream" version = "0.6.18" @@ -2328,7 +2337,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8d162beedaa69905488a8da94f5ac3edb4dd4788b732fadb7bd120b2625c1976" dependencies = [ "data-encoding", - "syn 1.0.109", + "syn 2.0.101", ] [[package]] @@ -5611,6 +5620,7 @@ name = "revive-dt-core" version = "0.1.0" dependencies = [ "alloy", + "ansi_term", "anyhow", "bson", "cacache", @@ -5943,7 +5953,7 @@ dependencies = [ "security-framework 3.3.0", "security-framework-sys", "webpki-root-certs 0.26.11", - "windows-sys 0.52.0", + "windows-sys 0.59.0", ] [[package]] diff --git a/Cargo.toml b/Cargo.toml index 63eab61..d1c48b5 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -22,6 +22,7 @@ revive-dt-node-pool = { version = "0.1.0", path = "crates/node-pool" } revive-dt-report = { version = "0.1.0", path = "crates/report" } revive-dt-solc-binaries = { version = "0.1.0", path = "crates/solc-binaries" } +ansi_term = "0.12.1" anyhow = "1.0" async-stream = { version = "0.3.6" } bson = { version = "2.15.0" } diff --git a/crates/compiler/tests/assets/array_one_element/main.sol b/crates/compiler/tests/assets/array_one_element/main.sol index 74789c9..5882471 100644 --- a/crates/compiler/tests/assets/array_one_element/main.sol +++ b/crates/compiler/tests/assets/array_one_element/main.sol @@ -7,7 +7,10 @@ pragma solidity >=0.6.9; import "./callable.sol"; contract Main { - function main(uint[1] calldata p1, Callable callable) public returns(uint) { + function main( + uint[1] calldata p1, + Callable callable + ) public pure returns (uint) { return callable.f(p1); } } diff --git a/crates/config/src/lib.rs b/crates/config/src/lib.rs index 46b978c..0e87b13 100644 --- a/crates/config/src/lib.rs +++ b/crates/config/src/lib.rs @@ -202,6 +202,18 @@ impl AsRef for Context { #[derive(Clone, Debug, Parser, Serialize)] pub struct TestExecutionContext { + /// The set of platforms that the differential tests should run on. + #[arg( + short = 'p', + long = "platform", + default_values = ["geth-evm-solc", "revive-dev-node-polkavm-resolc"] + )] + pub platforms: Vec, + + /// The output format to use for the tool's output. + #[arg(short, long, default_value_t = OutputFormat::CargoTestLike)] + pub output_format: OutputFormat, + /// The working directory that the program will use for all of the temporary artifacts needed at /// runtime. /// @@ -215,14 +227,6 @@ pub struct TestExecutionContext { )] pub working_directory: WorkingDirectoryConfiguration, - /// The set of platforms that the differential tests should run on. - #[arg( - short = 'p', - long = "platform", - default_values = ["geth-evm-solc", "revive-dev-node-polkavm-resolc"] - )] - pub platforms: Vec, - /// Configuration parameters for the corpus files to use. #[clap(flatten, next_help_heading = "Corpus Configuration")] pub corpus_configuration: CorpusConfiguration, @@ -958,3 +962,29 @@ pub enum TestingPlatform { /// A polkadot/Substrate based network Zombienet, } + +/// The output format to use for the test execution output. +#[derive( + Clone, + Copy, + Debug, + PartialEq, + Eq, + PartialOrd, + Ord, + Hash, + Serialize, + ValueEnum, + EnumString, + Display, + AsRefStr, + IntoStaticStr, +)] +#[strum(serialize_all = "kebab-case")] +pub enum OutputFormat { + /// The legacy format that was used in the past for the output. + Legacy, + + /// An output format that looks heavily resembles the output from `cargo test`. + CargoTestLike, +} diff --git a/crates/core/Cargo.toml b/crates/core/Cargo.toml index 2c973a8..63c8275 100644 --- a/crates/core/Cargo.toml +++ b/crates/core/Cargo.toml @@ -21,6 +21,7 @@ revive-dt-node = { workspace = true } revive-dt-node-interaction = { workspace = true } revive-dt-report = { workspace = true } +ansi_term = { workspace = true } alloy = { workspace = true } anyhow = { workspace = true } bson = { workspace = true } diff --git a/crates/core/src/differential_tests/entry_point.rs b/crates/core/src/differential_tests/entry_point.rs index a2b6c53..20fcfee 100644 --- a/crates/core/src/differential_tests/entry_point.rs +++ b/crates/core/src/differential_tests/entry_point.rs @@ -7,6 +7,7 @@ use std::{ time::{Duration, Instant}, }; +use ansi_term::{ANSIStrings, Color}; use anyhow::Context as _; use futures::{FutureExt, StreamExt}; use revive_dt_common::types::PrivateKeyAllocator; @@ -14,7 +15,7 @@ use revive_dt_core::Platform; use tokio::sync::{Mutex, RwLock, Semaphore}; use tracing::{Instrument, error, info, info_span, instrument}; -use revive_dt_config::{Context, TestExecutionContext}; +use revive_dt_config::{Context, OutputFormat, TestExecutionContext}; use revive_dt_report::{Reporter, ReporterEvent, TestCaseStatus}; use crate::{ @@ -176,7 +177,7 @@ pub async fn handle_differential_tests( .report_completion_event() .expect("Can't fail") }); - let cli_reporting_task = start_cli_reporting_task(reporter); + let cli_reporting_task = start_cli_reporting_task(context.output_format, reporter); tokio::task::spawn(async move { loop { @@ -196,21 +197,15 @@ pub async fn handle_differential_tests( } #[allow(irrefutable_let_patterns, clippy::uninlined_format_args)] -async fn start_cli_reporting_task(reporter: Reporter) { +async fn start_cli_reporting_task(output_format: OutputFormat, reporter: Reporter) { let mut aggregator_events_rx = reporter.subscribe().await.expect("Can't fail"); drop(reporter); let start = Instant::now(); - const GREEN: &str = "\x1B[32m"; - const RED: &str = "\x1B[31m"; - const GREY: &str = "\x1B[90m"; - const COLOR_RESET: &str = "\x1B[0m"; - const BOLD: &str = "\x1B[1m"; - const BOLD_RESET: &str = "\x1B[22m"; - - let mut number_of_successes = 0; - let mut number_of_failures = 0; + let mut global_success_count = 0; + let mut global_failure_count = 0; + let mut global_ignore_count = 0; let mut buf = BufWriter::new(stderr()); while let Ok(event) = aggregator_events_rx.recv().await { @@ -223,55 +218,125 @@ async fn start_cli_reporting_task(reporter: Reporter) { continue; }; - let _ = writeln!(buf, "{} - {}", mode, metadata_file_path.display()); - for (case_idx, case_status) in case_status.into_iter() { - let _ = write!(buf, "\tCase Index {case_idx:>3}: "); - let _ = match case_status { - TestCaseStatus::Succeeded { steps_executed } => { - number_of_successes += 1; - writeln!( - buf, - "{}{}Case Succeeded{} - Steps Executed: {}{}", - GREEN, BOLD, BOLD_RESET, steps_executed, COLOR_RESET - ) + match output_format { + OutputFormat::Legacy => { + let _ = writeln!(buf, "{} - {}", mode, metadata_file_path.display()); + for (case_idx, case_status) in case_status.into_iter() { + let _ = write!(buf, "\tCase Index {case_idx:>3}: "); + let _ = match case_status { + TestCaseStatus::Succeeded { steps_executed } => { + global_success_count += 1; + writeln!( + buf, + "{}", + ANSIStrings(&[ + Color::Green.bold().paint("Case Succeeded"), + Color::Green + .paint(format!(" - Steps Executed: {steps_executed}")), + ]) + ) + } + TestCaseStatus::Failed { reason } => { + global_failure_count += 1; + writeln!( + buf, + "{}", + ANSIStrings(&[ + Color::Red.bold().paint("Case Failed"), + Color::Red.paint(format!(" - Reason: {}", reason.trim())), + ]) + ) + } + TestCaseStatus::Ignored { reason, .. } => { + global_ignore_count += 1; + writeln!( + buf, + "{}", + ANSIStrings(&[ + Color::Yellow.bold().paint("Case Ignored"), + Color::Yellow.paint(format!(" - Reason: {}", reason.trim())), + ]) + ) + } + }; } - TestCaseStatus::Failed { reason } => { - number_of_failures += 1; - writeln!( - buf, - "{}{}Case Failed{} - Reason: {}{}", - RED, - BOLD, - BOLD_RESET, - reason.trim(), - COLOR_RESET, - ) - } - TestCaseStatus::Ignored { reason, .. } => writeln!( + let _ = writeln!(buf); + } + OutputFormat::CargoTestLike => { + writeln!( buf, - "{}{}Case Ignored{} - Reason: {}{}", - GREY, - BOLD, - BOLD_RESET, - reason.trim(), - COLOR_RESET, - ), - }; + "\t{} {} - {}\n", + Color::Green.paint("Running"), + metadata_file_path.display(), + mode + ) + .unwrap(); + + let mut success_count = 0; + let mut failure_count = 0; + let mut ignored_count = 0; + writeln!(buf, "running {} tests", case_status.len()).unwrap(); + for (case_idx, case_result) in case_status.iter() { + let status = match case_result { + TestCaseStatus::Succeeded { .. } => { + success_count += 1; + global_success_count += 1; + Color::Green.paint("ok") + } + TestCaseStatus::Failed { reason } => { + failure_count += 1; + global_failure_count += 1; + Color::Red.paint(format!("FAILED, {reason}")) + } + TestCaseStatus::Ignored { reason, .. } => { + ignored_count += 1; + global_ignore_count += 1; + Color::Yellow.paint(format!("ignored, {reason:?}")) + } + }; + writeln!(buf, "test case_idx_{} ... {}", case_idx, status).unwrap(); + } + writeln!(buf).unwrap(); + + let status = if failure_count > 0 { + Color::Red.paint("FAILED") + } else { + Color::Green.paint("ok") + }; + writeln!( + buf, + "test result: {}. {} passed; {} failed; {} ignored", + status, success_count, failure_count, ignored_count, + ) + .unwrap(); + writeln!(buf).unwrap() + } } - let _ = writeln!(buf); } // Summary at the end. - let _ = writeln!( - buf, - "{} cases: {}{}{} cases succeeded, {}{}{} cases failed in {} seconds", - number_of_successes + number_of_failures, - GREEN, - number_of_successes, - COLOR_RESET, - RED, - number_of_failures, - COLOR_RESET, - start.elapsed().as_secs() - ); + match output_format { + OutputFormat::Legacy => { + writeln!( + buf, + "{} cases: {} cases succeeded, {} cases failed in {} seconds", + global_success_count + global_failure_count + global_ignore_count, + Color::Green.paint(global_success_count.to_string()), + Color::Red.paint(global_failure_count.to_string()), + start.elapsed().as_secs() + ) + .unwrap(); + } + OutputFormat::CargoTestLike => { + writeln!( + buf, + "run finished. {} passed; {} failed; {} ignored; finished in {}s", + global_success_count, + global_failure_count, + global_ignore_count, + start.elapsed().as_secs() + ) + .unwrap(); + } + } } diff --git a/run_tests.sh b/run_tests.sh index 8f17191..37817c6 100755 --- a/run_tests.sh +++ b/run_tests.sh @@ -93,17 +93,16 @@ echo "" # Run the tool cargo build --release; RUST_LOG="info,alloy_pubsub::service=error" ./target/release/retester test \ - --platform revive-dev-node-revm-solc \ + --platform geth-evm-solc \ --corpus "$CORPUS_FILE" \ --working-directory "$WORKDIR" \ --concurrency.number-of-nodes 10 \ --concurrency.number-of-threads 5 \ - --concurrency.number-of-concurrent-tasks 1000 \ + --concurrency.ignore-concurrency-limit \ --wallet.additional-keys 100000 \ --kitchensink.path "$SUBSTRATE_NODE_BIN" \ --revive-dev-node.path "$REVIVE_DEV_NODE_BIN" \ --eth-rpc.path "$ETH_RPC_BIN" \ - > logs.log \ - 2> output.log + > logs.log echo -e "${GREEN}=== Test run completed! ===${NC}" \ No newline at end of file