diff --git a/crates/common/src/macros/define_wrapper_type.rs b/crates/common/src/macros/define_wrapper_type.rs index 95e0de9..a5f1098 100644 --- a/crates/common/src/macros/define_wrapper_type.rs +++ b/crates/common/src/macros/define_wrapper_type.rs @@ -1,6 +1,7 @@ #[macro_export] macro_rules! impl_for_wrapper { (Display, $ident: ident) => { + #[automatically_derived] impl std::fmt::Display for $ident { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { std::fmt::Display::fmt(&self.0, f) @@ -8,6 +9,7 @@ macro_rules! impl_for_wrapper { } }; (FromStr, $ident: ident) => { + #[automatically_derived] impl std::str::FromStr for $ident { type Err = anyhow::Error; diff --git a/crates/core/src/main.rs b/crates/core/src/main.rs index 8ec81ad..e3c17ac 100644 --- a/crates/core/src/main.rs +++ b/crates/core/src/main.rs @@ -1,7 +1,7 @@ mod cached_compiler; use std::{ - collections::{BTreeMap, HashMap}, + collections::HashMap, io::{BufWriter, Write, stderr}, path::Path, sync::{Arc, LazyLock}, @@ -14,11 +14,10 @@ use alloy::{ }; use anyhow::Context; use clap::Parser; -use futures::{FutureExt, stream}; +use futures::stream; use futures::{Stream, StreamExt}; -use indexmap::IndexMap; use revive_dt_node_interaction::EthereumNode; -use revive_dt_report::{ReportAggregator, Reporter}; +use revive_dt_report::{ReportAggregator, Reporter, TestSpecificReporter, TestSpecifier}; use temp_dir::TempDir; use tokio::{join, sync::mpsc, try_join}; use tracing::{debug, info, info_span, instrument}; @@ -53,6 +52,7 @@ struct Test<'a> { mode: Mode, case_idx: CaseIdx, case: &'a Case, + reporter: TestSpecificReporter, } /// This represents the results that we gather from running test cases. @@ -183,16 +183,11 @@ where { let (report_tx, report_rx) = mpsc::unbounded_channel::<(Test<'_>, CaseResult)>(); - let tests = prepare_tests::(args, metadata_files); + let tests = prepare_tests::(args, metadata_files, reporter); let driver_task = start_driver_task::(args, tests, report_tx).await?; let status_reporter_task = start_reporter_task(report_rx); - drop(reporter); - let (_, _, rtn) = tokio::join!( - status_reporter_task.inspect(|_| info!("Status reporter completed")), - driver_task.inspect(|_| info!("Driver completed")), - report_aggregator_task.inspect(|_| info!("Report aggregator completed")) - ); + let (_, _, rtn) = tokio::join!(status_reporter_task, driver_task, report_aggregator_task,); rtn?; Ok(()) @@ -201,6 +196,7 @@ where fn prepare_tests<'a, L, F>( args: &Arguments, metadata_files: &'a [MetadataFile], + reporter: Reporter, ) -> impl Stream> where L: Platform, @@ -227,27 +223,25 @@ where .into_iter() .map(move |mode| (metadata_file, case_idx, case, mode)) }) - .fold( - IndexMap::<_, BTreeMap<_, Vec<_>>>::new(), - |mut map, (metadata_file, case_idx, case, mode)| { - let test = Test { - metadata: metadata_file, - metadata_file_path: metadata_file.metadata_file_path.as_path(), - mode: mode.clone(), - case_idx: CaseIdx::new(case_idx), - case, - }; - map.entry(mode) - .or_default() - .entry(test.case_idx) - .or_default() - .push(test); - map - }, - ) - .into_values() - .flatten() - .flat_map(|(_, value)| value.into_iter()) + .map(move |(metadata_file, case_idx, case, mode)| Test { + metadata: metadata_file, + metadata_file_path: metadata_file.metadata_file_path.as_path(), + mode: mode.clone(), + case_idx: CaseIdx::new(case_idx), + case, + reporter: reporter.test_specific_reporter(Arc::new(TestSpecifier { + solc_mode: mode.clone(), + metadata_file_path: metadata_file.metadata_file_path.clone(), + case_idx: CaseIdx::new(case_idx), + })), + }) + .inspect(|test| { + test.reporter + .report_test_case_discovery_event() + .expect("Can't fail") + }) + .collect::>() + .into_iter() // Filter the test out if the leader and follower do not support the target. .filter(|test| { let leader_support = @@ -262,7 +256,30 @@ where leader_support, follower_support, "Target is not supported, throwing metadata file out" - ) + ); + test + .reporter + .report_test_ignored_event( + "Either the leader or the follower do not support the target desired by the test", + HashMap::from_iter([ + ( + "test_desired_targets".to_string(), + serde_json::to_value(test.metadata.targets.as_ref()) + .expect("Can't fail") + ), + ( + "leader_support".to_string(), + serde_json::to_value(leader_support) + .expect("Can't fail") + ), + ( + "follower_support".to_string(), + serde_json::to_value(follower_support) + .expect("Can't fail") + ) + ]) + ) + .expect("Can't fail"); } is_allowed @@ -274,6 +291,13 @@ where file_path = %test.metadata.relative_path().display(), "Metadata file is ignored, throwing case out" ); + test + .reporter + .report_test_ignored_event( + "Metadata file is ignored, therefore all cases are ignored", + HashMap::new(), + ) + .expect("Can't fail"); false } else { true @@ -287,6 +311,13 @@ where case_idx = %test.case_idx, "Case is ignored, throwing case out" ); + test + .reporter + .report_test_ignored_event( + "Case is ignored", + HashMap::new(), + ) + .expect("Can't fail"); false } else { true @@ -309,6 +340,29 @@ where follower_compatibility, "EVM Version is incompatible, throwing case out" ); + test + .reporter + .report_test_ignored_event( + "EVM version is incompatible with either the leader or the follower", + HashMap::from_iter([ + ( + "test_desired_evm_version".to_string(), + serde_json::to_value(test.metadata.required_evm_version) + .expect("Can't fail") + ), + ( + "leader_compatibility".to_string(), + serde_json::to_value(leader_compatibility) + .expect("Can't fail") + ), + ( + "follower_compatibility".to_string(), + serde_json::to_value(follower_compatibility) + .expect("Can't fail") + ) + ]) + ) + .expect("Can't fail"); } is_allowed @@ -337,6 +391,24 @@ where follower_support, "Compilers do not support this, throwing case out" ); + test + .reporter + .report_test_ignored_event( + "Compilers do not support this mode either for the leader or for the follower.", + HashMap::from_iter([ + ( + "leader_support".to_string(), + serde_json::to_value(leader_support) + .expect("Can't fail") + ), + ( + "follower_support".to_string(), + serde_json::to_value(follower_support) + .expect("Can't fail") + ) + ]) + ) + .expect("Can't fail"); } is_allowed.then_some(test) diff --git a/crates/report/src/aggregator.rs b/crates/report/src/aggregator.rs index ef99cca..ea35b22 100644 --- a/crates/report/src/aggregator.rs +++ b/crates/report/src/aggregator.rs @@ -19,7 +19,7 @@ use tokio::sync::{ }; use crate::{ - SubscribeToEventsEvent, TestIgnoredEvent, + SubscribeToEventsEvent, TestCaseDiscoveryEvent, TestIgnoredEvent, TestSpecifier, common::MetadataFilePath, reporter_event::ReporterEvent, runner_event::{CorpusFileDiscoveryEvent, MetadataFileDiscoveryEvent, Reporter, RunnerEvent}, @@ -28,7 +28,7 @@ use crate::{ pub struct ReportAggregator { /* Internal Report State */ report: Report, - remaining_cases: HashMap>, + remaining_cases: HashMap>>, /* Channels */ runner_tx: Option>, runner_rx: UnboundedReceiver, @@ -60,7 +60,6 @@ impl ReportAggregator { async fn aggregate(mut self) -> Result<()> { while let Some(event) = self.runner_rx.recv().await { match event { - RunnerEvent::ExecutionCompleted(..) => break, RunnerEvent::SubscribeToEvents(event) => { self.handle_subscribe_to_events_event(*event); } @@ -70,6 +69,9 @@ impl ReportAggregator { RunnerEvent::MetadataFileDiscovery(event) => { self.handle_metadata_file_discovery_event(*event); } + RunnerEvent::TestCaseDiscovery(event) => { + self.handle_test_case_discovery(*event); + } RunnerEvent::TestIgnored(event) => { self.handle_test_ignored_event(*event); } @@ -104,21 +106,44 @@ impl ReportAggregator { fn handle_metadata_file_discovery_event(&mut self, event: MetadataFileDiscoveryEvent) { self.report.metadata_files.insert(event.path.clone()); - self.remaining_cases.insert( - event.path, - event - .metadata - .cases - .iter() - .enumerate() - .map(|(id, _)| id) - .map(CaseIdx::new) - .collect(), - ); } - fn handle_test_ignored_event(&mut self, _: TestIgnoredEvent) { - todo!() + fn handle_test_case_discovery(&mut self, event: TestCaseDiscoveryEvent) { + self.remaining_cases + .entry(event.test_specifier.metadata_file_path.clone().into()) + .or_default() + .entry(event.test_specifier.solc_mode.clone()) + .or_default() + .insert(event.test_specifier.case_idx); + } + + fn handle_test_ignored_event(&mut self, event: TestIgnoredEvent) { + // Remove this from the set of cases we're tracking + self.remaining_cases + .entry(event.test_specifier.metadata_file_path.clone().into()) + .or_default() + .entry(event.test_specifier.solc_mode.clone()) + .or_default() + .remove(&event.test_specifier.case_idx); + + // Add information on the fact that the case was ignored to the report. + let test_case_report = self.test_case_report(&event.test_specifier); + test_case_report.ignore = Some(TestCaseIgnoreInformation { + is_ignored: true, + reason: event.reason, + additional_fields: event.additional_fields, + }); + } + + fn test_case_report(&mut self, specifier: &TestSpecifier) -> &mut TestCaseReport { + self.report + .test_case_information + .entry(specifier.metadata_file_path.clone().into()) + .or_default() + .entry(specifier.solc_mode.clone()) + .or_default() + .entry(specifier.case_idx) + .or_default() } } @@ -148,5 +173,20 @@ impl Report { } } +#[derive(Clone, Debug, Serialize, Default)] +pub struct TestCaseReport { + /// Information related to the test case being ignored and why it's ignored. + ignore: Option, +} + +/// Information related to the test case being ignored and why it's ignored. #[derive(Clone, Debug, Serialize)] -pub struct TestCaseReport {} +pub struct TestCaseIgnoreInformation { + /// A boolean that defines if the test case is ignored or not. + pub is_ignored: bool, + /// The reason behind the test case being ignored. + pub reason: String, + /// Additional fields that describe more information on why the test case is ignored. + #[serde(flatten)] + pub additional_fields: HashMap, +} diff --git a/crates/report/src/lib.rs b/crates/report/src/lib.rs index dc66f46..7a6aa78 100644 --- a/crates/report/src/lib.rs +++ b/crates/report/src/lib.rs @@ -6,5 +6,6 @@ mod reporter_event; mod runner_event; pub use aggregator::*; +pub use common::*; pub use reporter_event::*; pub use runner_event::*; diff --git a/crates/report/src/runner_event.rs b/crates/report/src/runner_event.rs index 2d2e35a..81f3d81 100644 --- a/crates/report/src/runner_event.rs +++ b/crates/report/src/runner_event.rs @@ -1,11 +1,108 @@ //! The types associated with the events sent by the runner to the reporter. #![allow(dead_code)] +use std::{collections::HashMap, sync::Arc}; + use revive_dt_format::corpus::Corpus; use revive_dt_format::metadata::Metadata; use tokio::sync::{broadcast, oneshot}; -use crate::{ReporterEvent, common::MetadataFilePath}; +use crate::{ReporterEvent, TestSpecifier, common::MetadataFilePath}; + +macro_rules! __report_gen__emit_test_specific { + ( + $ident:ident, + $variant_ident:ident, + $skip_field:ident; + $( $bname:ident : $bty:ty, )* + ; + $( $aname:ident : $aty:ty, )* + ) => { + paste::paste! { + pub fn [< report_ $variant_ident:snake _event >]( + &self + $(, $bname: impl Into<$bty> )* + $(, $aname: impl Into<$aty> )* + ) -> anyhow::Result<()> { + self.report([< $variant_ident Event >] { + $skip_field: self.test_specifier.clone() + $(, $bname: $bname.into() )* + $(, $aname: $aname.into() )* + }) + } + } + }; +} + +macro_rules! __report_gen__emit_test_specific_by_parse { + ( + $ident:ident, + $variant_ident:ident, + $skip_field:ident; + $( $bname:ident : $bty:ty, )* ; $( $aname:ident : $aty:ty, )* + ) => { + __report_gen__emit_test_specific!( + $ident, $variant_ident, $skip_field; + $( $bname : $bty, )* ; $( $aname : $aty, )* + ); + }; +} + +macro_rules! __report_gen__scan_before { + ( + $ident:ident, $variant_ident:ident; + $( $before:ident : $bty:ty, )* + ; + test_specifier : $skip_ty:ty, + $( $after:ident : $aty:ty, )* + ; + ) => { + __report_gen__emit_test_specific_by_parse!( + $ident, $variant_ident, test_specifier; + $( $before : $bty, )* ; $( $after : $aty, )* + ); + }; + ( + $ident:ident, $variant_ident:ident; + $( $before:ident : $bty:ty, )* + ; + $name:ident : $ty:ty, $( $after:ident : $aty:ty, )* + ; + ) => { + __report_gen__scan_before!( + $ident, $variant_ident; + $( $before : $bty, )* $name : $ty, + ; + $( $after : $aty, )* + ; + ); + }; + ( + $ident:ident, $variant_ident:ident; + $( $before:ident : $bty:ty, )* + ; + ; + ) => {}; +} + +macro_rules! __report_gen_for_variant { + ( + $ident:ident, + $variant_ident:ident; + ) => {}; + ( + $ident:ident, + $variant_ident:ident; + $( $field_ident:ident : $field_ty:ty ),+ $(,)? + ) => { + __report_gen__scan_before!( + $ident, $variant_ident; + ; + $( $field_ident : $field_ty, )* + ; + ); + }; +} macro_rules! keep_if_doc { (#[doc = $doc:expr]) => { @@ -115,6 +212,7 @@ macro_rules! define_event { } /// A reporter that's tied to a specific test case. + #[derive(Clone, Debug)] pub struct [< $ident TestSpecificReporter >] { $vis reporter: [< $ident Reporter >], $vis test_specifier: std::sync::Arc, @@ -124,6 +222,10 @@ macro_rules! define_event { fn report(&self, event: impl Into<$ident>) -> anyhow::Result<()> { self.reporter.report(event) } + + $( + __report_gen_for_variant! { $ident, $variant_ident; $( $field_ident : $field_ty ),* } + )* } } }; @@ -150,13 +252,20 @@ define_event! { /// The content of the metadata file. metadata: Metadata }, + /// An event emitted by the runners when they discover a test case. + TestCaseDiscovery { + /// A specifier for the test that was discovered. + test_specifier: Arc, + }, /// An event emitted by the runners when a test case is ignored. TestIgnored { - + /// A specifier for the test that's been ignored. + test_specifier: Arc, + /// A reason for the test to be ignored. + reason: String, + /// Additional fields that describe more information on why the test was ignored. + additional_fields: HashMap }, - /// An event emitted by the runners when the execution is completed and the aggregator can - /// stop. - ExecutionCompleted {} } } @@ -170,3 +279,4 @@ impl RunnerEventReporter { } pub type Reporter = RunnerEventReporter; +pub type TestSpecificReporter = RunnerEventTestSpecificReporter;