Add reporting infra for reporting ignored tests

crates/common/src/macros/define_wrapper_type.rs

@@ -1,6 +1,7 @@
 #[macro_export]
 macro_rules! impl_for_wrapper {
     (Display, $ident: ident) => {
+        #[automatically_derived]
         impl std::fmt::Display for $ident {
             fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
                 std::fmt::Display::fmt(&self.0, f)
@@ -8,6 +9,7 @@ macro_rules! impl_for_wrapper {
         }
     };
     (FromStr, $ident: ident) => {
+        #[automatically_derived]
         impl std::str::FromStr for $ident {
             type Err = anyhow::Error;
 

crates/core/src/main.rs

@@ -1,7 +1,7 @@
 mod cached_compiler;
 
 use std::{
-    collections::{BTreeMap, HashMap},
+    collections::HashMap,
     io::{BufWriter, Write, stderr},
     path::Path,
     sync::{Arc, LazyLock},
@@ -14,11 +14,10 @@ use alloy::{
 };
 use anyhow::Context;
 use clap::Parser;
-use futures::{FutureExt, stream};
+use futures::stream;
 use futures::{Stream, StreamExt};
-use indexmap::IndexMap;
 use revive_dt_node_interaction::EthereumNode;
-use revive_dt_report::{ReportAggregator, Reporter};
+use revive_dt_report::{ReportAggregator, Reporter, TestSpecificReporter, TestSpecifier};
 use temp_dir::TempDir;
 use tokio::{join, sync::mpsc, try_join};
 use tracing::{debug, info, info_span, instrument};
@@ -53,6 +52,7 @@ struct Test<'a> {
     mode: Mode,
     case_idx: CaseIdx,
     case: &'a Case,
+    reporter: TestSpecificReporter,
 }
 
 /// This represents the results that we gather from running test cases.
@@ -183,16 +183,11 @@ where
 {
     let (report_tx, report_rx) = mpsc::unbounded_channel::<(Test<'_>, CaseResult)>();
 
-    let tests = prepare_tests::<L, F>(args, metadata_files);
+    let tests = prepare_tests::<L, F>(args, metadata_files, reporter);
     let driver_task = start_driver_task::<L, F>(args, tests, report_tx).await?;
     let status_reporter_task = start_reporter_task(report_rx);
 
-    drop(reporter);
+    let (_, _, rtn) = tokio::join!(status_reporter_task, driver_task, report_aggregator_task,);
-    let (_, _, rtn) = tokio::join!(
-        status_reporter_task.inspect(|_| info!("Status reporter completed")),
-        driver_task.inspect(|_| info!("Driver completed")),
-        report_aggregator_task.inspect(|_| info!("Report aggregator completed"))
-    );
     rtn?;
 
     Ok(())
@@ -201,6 +196,7 @@ where
 fn prepare_tests<'a, L, F>(
     args: &Arguments,
     metadata_files: &'a [MetadataFile],
+    reporter: Reporter,
 ) -> impl Stream<Item = Test<'a>>
 where
     L: Platform,
@@ -227,27 +223,25 @@ where
                 .into_iter()
                 .map(move |mode| (metadata_file, case_idx, case, mode))
         })
-        .fold(
+        .map(move |(metadata_file, case_idx, case, mode)| Test {
-            IndexMap::<_, BTreeMap<_, Vec<_>>>::new(),
+            metadata: metadata_file,
-            |mut map, (metadata_file, case_idx, case, mode)| {
+            metadata_file_path: metadata_file.metadata_file_path.as_path(),
-                let test = Test {
+            mode: mode.clone(),
-                    metadata: metadata_file,
+            case_idx: CaseIdx::new(case_idx),
-                    metadata_file_path: metadata_file.metadata_file_path.as_path(),
+            case,
-                    mode: mode.clone(),
+            reporter: reporter.test_specific_reporter(Arc::new(TestSpecifier {
-                    case_idx: CaseIdx::new(case_idx),
+                solc_mode: mode.clone(),
-                    case,
+                metadata_file_path: metadata_file.metadata_file_path.clone(),
-                };
+                case_idx: CaseIdx::new(case_idx),
-                map.entry(mode)
+            })),
-                    .or_default()
+        })
-                    .entry(test.case_idx)
+        .inspect(|test| {
-                    .or_default()
+            test.reporter
-                    .push(test);
+                .report_test_case_discovery_event()
-                map
+                .expect("Can't fail")
-            },
+        })
-        )
+        .collect::<Vec<_>>()
-        .into_values()
+        .into_iter()
-        .flatten()
-        .flat_map(|(_, value)| value.into_iter())
         // Filter the test out if the leader and follower do not support the target.
         .filter(|test| {
             let leader_support =
@@ -262,7 +256,30 @@ where
                     leader_support,
                     follower_support,
                     "Target is not supported, throwing metadata file out"
-                )
+                );
+                test
+                    .reporter
+                    .report_test_ignored_event(
+                        "Either the leader or the follower do not support the target desired by the test",
+                        HashMap::from_iter([
+                            (
+                                "test_desired_targets".to_string(),
+                                serde_json::to_value(test.metadata.targets.as_ref())
+                                    .expect("Can't fail")
+                            ),
+                            (
+                                "leader_support".to_string(),
+                                serde_json::to_value(leader_support)
+                                    .expect("Can't fail")
+                            ),
+                            (
+                                "follower_support".to_string(),
+                                serde_json::to_value(follower_support)
+                                    .expect("Can't fail")
+                            )
+                        ])
+                    )
+                    .expect("Can't fail");
             }
 
             is_allowed
@@ -274,6 +291,13 @@ where
                     file_path = %test.metadata.relative_path().display(),
                     "Metadata file is ignored, throwing case out"
                 );
+                test
+                    .reporter
+                    .report_test_ignored_event(
+                        "Metadata file is ignored, therefore all cases are ignored",
+                        HashMap::new(),
+                    )
+                    .expect("Can't fail");
                 false
             } else {
                 true
@@ -287,6 +311,13 @@ where
                     case_idx = %test.case_idx,
                     "Case is ignored, throwing case out"
                 );
+                test
+                    .reporter
+                    .report_test_ignored_event(
+                        "Case is ignored",
+                        HashMap::new(),
+                    )
+                    .expect("Can't fail");
                 false
             } else {
                 true
@@ -309,6 +340,29 @@ where
                         follower_compatibility,
                         "EVM Version is incompatible, throwing case out"
                     );
+                    test
+                        .reporter
+                        .report_test_ignored_event(
+                            "EVM version is incompatible with either the leader or the follower",
+                            HashMap::from_iter([
+                                (
+                                    "test_desired_evm_version".to_string(),
+                                    serde_json::to_value(test.metadata.required_evm_version)
+                                        .expect("Can't fail")
+                                ),
+                                (
+                                    "leader_compatibility".to_string(),
+                                    serde_json::to_value(leader_compatibility)
+                                        .expect("Can't fail")
+                                ),
+                                (
+                                    "follower_compatibility".to_string(),
+                                    serde_json::to_value(follower_compatibility)
+                                        .expect("Can't fail")
+                                )
+                            ])
+                        )
+                        .expect("Can't fail");
                 }
 
                 is_allowed
@@ -337,6 +391,24 @@ where
                     follower_support,
                     "Compilers do not support this, throwing case out"
                 );
+                test
+                    .reporter
+                    .report_test_ignored_event(
+                        "Compilers do not support this mode either for the leader or for the follower.",
+                        HashMap::from_iter([
+                            (
+                                "leader_support".to_string(),
+                                serde_json::to_value(leader_support)
+                                    .expect("Can't fail")
+                            ),
+                            (
+                                "follower_support".to_string(),
+                                serde_json::to_value(follower_support)
+                                    .expect("Can't fail")
+                            )
+                        ])
+                    )
+                    .expect("Can't fail");
             }
 
             is_allowed.then_some(test)

crates/report/src/aggregator.rs

@@ -19,7 +19,7 @@ use tokio::sync::{
 };
 
 use crate::{
-    SubscribeToEventsEvent, TestIgnoredEvent,
+    SubscribeToEventsEvent, TestCaseDiscoveryEvent, TestIgnoredEvent, TestSpecifier,
     common::MetadataFilePath,
     reporter_event::ReporterEvent,
     runner_event::{CorpusFileDiscoveryEvent, MetadataFileDiscoveryEvent, Reporter, RunnerEvent},
@@ -28,7 +28,7 @@ use crate::{
 pub struct ReportAggregator {
     /* Internal Report State */
     report: Report,
-    remaining_cases: HashMap<MetadataFilePath, HashSet<CaseIdx>>,
+    remaining_cases: HashMap<MetadataFilePath, HashMap<Mode, HashSet<CaseIdx>>>,
     /* Channels */
     runner_tx: Option<UnboundedSender<RunnerEvent>>,
     runner_rx: UnboundedReceiver<RunnerEvent>,
@@ -60,7 +60,6 @@ impl ReportAggregator {
     async fn aggregate(mut self) -> Result<()> {
         while let Some(event) = self.runner_rx.recv().await {
             match event {
-                RunnerEvent::ExecutionCompleted(..) => break,
                 RunnerEvent::SubscribeToEvents(event) => {
                     self.handle_subscribe_to_events_event(*event);
                 }
@@ -70,6 +69,9 @@ impl ReportAggregator {
                 RunnerEvent::MetadataFileDiscovery(event) => {
                     self.handle_metadata_file_discovery_event(*event);
                 }
+                RunnerEvent::TestCaseDiscovery(event) => {
+                    self.handle_test_case_discovery(*event);
+                }
                 RunnerEvent::TestIgnored(event) => {
                     self.handle_test_ignored_event(*event);
                 }
@@ -104,21 +106,44 @@ impl ReportAggregator {
 
     fn handle_metadata_file_discovery_event(&mut self, event: MetadataFileDiscoveryEvent) {
         self.report.metadata_files.insert(event.path.clone());
-        self.remaining_cases.insert(
-            event.path,
-            event
-                .metadata
-                .cases
-                .iter()
-                .enumerate()
-                .map(|(id, _)| id)
-                .map(CaseIdx::new)
-                .collect(),
-        );
     }
 
-    fn handle_test_ignored_event(&mut self, _: TestIgnoredEvent) {
+    fn handle_test_case_discovery(&mut self, event: TestCaseDiscoveryEvent) {
-        todo!()
+        self.remaining_cases
+            .entry(event.test_specifier.metadata_file_path.clone().into())
+            .or_default()
+            .entry(event.test_specifier.solc_mode.clone())
+            .or_default()
+            .insert(event.test_specifier.case_idx);
+    }
+
+    fn handle_test_ignored_event(&mut self, event: TestIgnoredEvent) {
+        // Remove this from the set of cases we're tracking
+        self.remaining_cases
+            .entry(event.test_specifier.metadata_file_path.clone().into())
+            .or_default()
+            .entry(event.test_specifier.solc_mode.clone())
+            .or_default()
+            .remove(&event.test_specifier.case_idx);
+
+        // Add information on the fact that the case was ignored to the report.
+        let test_case_report = self.test_case_report(&event.test_specifier);
+        test_case_report.ignore = Some(TestCaseIgnoreInformation {
+            is_ignored: true,
+            reason: event.reason,
+            additional_fields: event.additional_fields,
+        });
+    }
+
+    fn test_case_report(&mut self, specifier: &TestSpecifier) -> &mut TestCaseReport {
+        self.report
+            .test_case_information
+            .entry(specifier.metadata_file_path.clone().into())
+            .or_default()
+            .entry(specifier.solc_mode.clone())
+            .or_default()
+            .entry(specifier.case_idx)
+            .or_default()
     }
 }
 
@@ -148,5 +173,20 @@ impl Report {
     }
 }
 
+#[derive(Clone, Debug, Serialize, Default)]
+pub struct TestCaseReport {
+    /// Information related to the test case being ignored and why it's ignored.
+    ignore: Option<TestCaseIgnoreInformation>,
+}
+
+/// Information related to the test case being ignored and why it's ignored.
 #[derive(Clone, Debug, Serialize)]
-pub struct TestCaseReport {}
+pub struct TestCaseIgnoreInformation {
+    /// A boolean that defines if the test case is ignored or not.
+    pub is_ignored: bool,
+    /// The reason behind the test case being ignored.
+    pub reason: String,
+    /// Additional fields that describe more information on why the test case is ignored.
+    #[serde(flatten)]
+    pub additional_fields: HashMap<String, serde_json::Value>,
+}

crates/report/src/lib.rs

@@ -6,5 +6,6 @@ mod reporter_event;
 mod runner_event;
 
 pub use aggregator::*;
+pub use common::*;
 pub use reporter_event::*;
 pub use runner_event::*;

crates/report/src/runner_event.rs

@@ -1,11 +1,108 @@
 //! The types associated with the events sent by the runner to the reporter.
 #![allow(dead_code)]
 
+use std::{collections::HashMap, sync::Arc};
+
 use revive_dt_format::corpus::Corpus;
 use revive_dt_format::metadata::Metadata;
 use tokio::sync::{broadcast, oneshot};
 
-use crate::{ReporterEvent, common::MetadataFilePath};
+use crate::{ReporterEvent, TestSpecifier, common::MetadataFilePath};
+
+macro_rules! __report_gen__emit_test_specific {
+    (
+        $ident:ident,
+        $variant_ident:ident,
+        $skip_field:ident;
+        $( $bname:ident : $bty:ty, )*
+        ;
+        $( $aname:ident : $aty:ty, )*
+    ) => {
+        paste::paste! {
+            pub fn [< report_ $variant_ident:snake _event >](
+                &self
+                $(, $bname: impl Into<$bty> )*
+                $(, $aname: impl Into<$aty> )*
+            ) -> anyhow::Result<()> {
+                self.report([< $variant_ident Event >] {
+                    $skip_field: self.test_specifier.clone()
+                    $(, $bname: $bname.into() )*
+                    $(, $aname: $aname.into() )*
+                })
+            }
+        }
+    };
+}
+
+macro_rules! __report_gen__emit_test_specific_by_parse {
+    (
+        $ident:ident,
+        $variant_ident:ident,
+        $skip_field:ident;
+        $( $bname:ident : $bty:ty, )* ; $( $aname:ident : $aty:ty, )*
+    ) => {
+        __report_gen__emit_test_specific!(
+            $ident, $variant_ident, $skip_field;
+            $( $bname : $bty, )* ; $( $aname : $aty, )*
+        );
+    };
+}
+
+macro_rules! __report_gen__scan_before {
+    (
+        $ident:ident, $variant_ident:ident;
+        $( $before:ident : $bty:ty, )*
+        ;
+        test_specifier : $skip_ty:ty,
+        $( $after:ident : $aty:ty, )*
+        ;
+    ) => {
+        __report_gen__emit_test_specific_by_parse!(
+            $ident, $variant_ident, test_specifier;
+            $( $before : $bty, )* ; $( $after : $aty, )*
+        );
+    };
+    (
+        $ident:ident, $variant_ident:ident;
+        $( $before:ident : $bty:ty, )*
+        ;
+        $name:ident : $ty:ty, $( $after:ident : $aty:ty, )*
+        ;
+    ) => {
+        __report_gen__scan_before!(
+            $ident, $variant_ident;
+            $( $before : $bty, )* $name : $ty,
+            ;
+            $( $after : $aty, )*
+            ;
+        );
+    };
+    (
+        $ident:ident, $variant_ident:ident;
+        $( $before:ident : $bty:ty, )*
+        ;
+        ;
+    ) => {};
+}
+
+macro_rules! __report_gen_for_variant {
+    (
+        $ident:ident,
+        $variant_ident:ident;
+    ) => {};
+    (
+        $ident:ident,
+        $variant_ident:ident;
+        $( $field_ident:ident : $field_ty:ty ),+ $(,)?
+    ) => {
+        __report_gen__scan_before!(
+            $ident, $variant_ident;
+            ;
+            $( $field_ident : $field_ty, )*
+            ;
+        );
+    };
+}
 
 macro_rules! keep_if_doc {
     (#[doc = $doc:expr]) => {
@@ -115,6 +212,7 @@ macro_rules! define_event {
             }
 
             /// A reporter that's tied to a specific test case.
+            #[derive(Clone, Debug)]
             pub struct [< $ident TestSpecificReporter >] {
                 $vis reporter: [< $ident Reporter >],
                 $vis test_specifier: std::sync::Arc<crate::common::TestSpecifier>,
@@ -124,6 +222,10 @@ macro_rules! define_event {
                 fn report(&self, event: impl Into<$ident>) -> anyhow::Result<()> {
                     self.reporter.report(event)
                 }
+
+                $(
+                    __report_gen_for_variant! { $ident, $variant_ident; $( $field_ident : $field_ty ),* }
+                )*
             }
         }
     };
@@ -150,13 +252,20 @@ define_event! {
             /// The content of the metadata file.
             metadata: Metadata
         },
+        /// An event emitted by the runners when they discover a test case.
+        TestCaseDiscovery {
+            /// A specifier for the test that was discovered.
+            test_specifier: Arc<TestSpecifier>,
+        },
         /// An event emitted by the runners when a test case is ignored.
         TestIgnored {
-
+            /// A specifier for the test that's been ignored.
+            test_specifier: Arc<TestSpecifier>,
+            /// A reason for the test to be ignored.
+            reason: String,
+            /// Additional fields that describe more information on why the test was ignored.
+            additional_fields: HashMap<String, serde_json::Value>
         },
-        /// An event emitted by the runners when the execution is completed and the aggregator can
-        /// stop.
-        ExecutionCompleted {}
     }
 }
 
@@ -170,3 +279,4 @@ impl RunnerEventReporter {
 }
 
 pub type Reporter = RunnerEventReporter;
+pub type TestSpecificReporter = RunnerEventTestSpecificReporter;