Include contract compilation information in report

crates/report/Cargo.toml

@@ -13,6 +13,7 @@ revive-dt-config = { workspace = true }
 revive-dt-format = { workspace = true }
 revive-dt-compiler = { workspace = true }
 
+alloy-primitives = { workspace = true }
 anyhow = { workspace = true }
 paste = { workspace = true }
 indexmap = { workspace = true, features = ["serde"] }
@@ -21,6 +22,7 @@ serde = { workspace = true }
 serde_json = { workspace = true }
 serde_with = { workspace = true }
 tokio = { workspace = true }
+tracing = { workspace = true }
 
 [lints]
 workspace = true

crates/report/src/aggregator.rs

@@ -4,20 +4,24 @@
 use std::{
     collections::{BTreeMap, BTreeSet, HashMap, HashSet},
     fs::OpenOptions,
+    path::PathBuf,
     time::{SystemTime, UNIX_EPOCH},
 };
 
+use alloy_primitives::Address;
 use anyhow::Result;
 use indexmap::IndexMap;
-use revive_dt_compiler::Mode;
+use revive_dt_compiler::{CompilerInput, CompilerOutput, Mode};
 use revive_dt_config::{Arguments, TestingPlatform};
-use revive_dt_format::{case::CaseIdx, corpus::Corpus};
+use revive_dt_format::{case::CaseIdx, corpus::Corpus, metadata::ContractInstance};
+use semver::Version;
 use serde::Serialize;
 use serde_with::{DisplayFromStr, serde_as};
 use tokio::sync::{
     broadcast::{Sender, channel},
     mpsc::{UnboundedReceiver, UnboundedSender, unbounded_channel},
 };
+use tracing::debug;
 
 use crate::*;
 
@@ -54,7 +58,10 @@ impl ReportAggregator {
     }
 
     async fn aggregate(mut self) -> Result<()> {
+        debug!("Starting to aggregate report");
+
         while let Some(event) = self.runner_rx.recv().await {
+            debug!(?event, "Received Event");
             match event {
                 RunnerEvent::SubscribeToEvents(event) => {
                     self.handle_subscribe_to_events_event(*event);
@@ -83,8 +90,24 @@ impl ReportAggregator {
                 RunnerEvent::FollowerNodeAssigned(event) => {
                     self.handle_follower_node_assigned_event(*event);
                 }
+                RunnerEvent::PreLinkContractsCompilationSucceeded(event) => {
+                    self.handle_pre_link_contracts_compilation_succeeded_event(*event)
+                }
+                RunnerEvent::PostLinkContractsCompilationSucceeded(event) => {
+                    self.handle_post_link_contracts_compilation_succeeded_event(*event)
+                }
+                RunnerEvent::PreLinkContractsCompilationFailed(event) => {
+                    self.handle_pre_link_contracts_compilation_failed_event(*event)
+                }
+                RunnerEvent::PostLinkContractsCompilationFailed(event) => {
+                    self.handle_post_link_contracts_compilation_failed_event(*event)
+                }
+                RunnerEvent::LibrariesDeployed(event) => {
+                    self.handle_libraries_deployed_event(*event);
+                }
             }
         }
+        debug!("Report aggregation completed");
 
         let file_name = {
             let current_timestamp = SystemTime::now().duration_since(UNIX_EPOCH)?.as_secs();
@@ -232,6 +255,113 @@ impl ReportAggregator {
             });
     }
 
+    fn handle_pre_link_contracts_compilation_succeeded_event(
+        &mut self,
+        event: PreLinkContractsCompilationSucceededEvent,
+    ) {
+        let include_input = self.report.config.report_include_compiler_input;
+        let include_output = self.report.config.report_include_compiler_output;
+
+        let execution_information = self.execution_information(&event.execution_specifier);
+
+        let compiler_input = if include_input {
+            event.compiler_input
+        } else {
+            None
+        };
+        let compiler_output = if include_output {
+            Some(event.compiler_output)
+        } else {
+            None
+        };
+
+        execution_information.pre_link_compilation_status = Some(CompilationStatus::Success {
+            is_cached: event.is_cached,
+            compiler_version: event.compiler_version,
+            compiler_path: event.compiler_path,
+            compiler_input,
+            compiler_output,
+        });
+    }
+
+    fn handle_post_link_contracts_compilation_succeeded_event(
+        &mut self,
+        event: PostLinkContractsCompilationSucceededEvent,
+    ) {
+        let include_input = self.report.config.report_include_compiler_input;
+        let include_output = self.report.config.report_include_compiler_output;
+
+        let execution_information = self.execution_information(&event.execution_specifier);
+
+        let compiler_input = if include_input {
+            event.compiler_input
+        } else {
+            None
+        };
+        let compiler_output = if include_output {
+            Some(event.compiler_output)
+        } else {
+            None
+        };
+
+        execution_information.post_link_compilation_status = Some(CompilationStatus::Success {
+            is_cached: event.is_cached,
+            compiler_version: event.compiler_version,
+            compiler_path: event.compiler_path,
+            compiler_input,
+            compiler_output,
+        });
+    }
+
+    fn handle_pre_link_contracts_compilation_failed_event(
+        &mut self,
+        event: PreLinkContractsCompilationFailedEvent,
+    ) {
+        let include_input = self.report.config.report_include_compiler_input;
+
+        let execution_information = self.execution_information(&event.execution_specifier);
+
+        let compiler_input = if include_input {
+            event.compiler_input
+        } else {
+            None
+        };
+
+        execution_information.pre_link_compilation_status = Some(CompilationStatus::Failure {
+            reason: event.reason,
+            compiler_version: event.compiler_version,
+            compiler_path: event.compiler_path,
+            compiler_input,
+        });
+    }
+
+    fn handle_post_link_contracts_compilation_failed_event(
+        &mut self,
+        event: PostLinkContractsCompilationFailedEvent,
+    ) {
+        let include_input = self.report.config.report_include_compiler_input;
+
+        let execution_information = self.execution_information(&event.execution_specifier);
+
+        let compiler_input = if include_input {
+            event.compiler_input
+        } else {
+            None
+        };
+
+        execution_information.post_link_compilation_status = Some(CompilationStatus::Failure {
+            reason: event.reason,
+            compiler_version: event.compiler_version,
+            compiler_path: event.compiler_path,
+            compiler_input,
+        });
+    }
+
+    fn handle_libraries_deployed_event(&mut self, event: LibrariesDeployedEvent) {
+        self.execution_information(&event.execution_specifier)
+            .deployed_libraries = Some(event.libraries);
+    }
+
     fn test_case_report(&mut self, specifier: &TestSpecifier) -> &mut TestCaseReport {
         self.report
             .test_case_information
@@ -242,6 +372,21 @@ impl ReportAggregator {
             .entry(specifier.case_idx)
             .or_default()
     }
+
+    fn execution_information(
+        &mut self,
+        specifier: &ExecutionSpecifier,
+    ) -> &mut ExecutionInformation {
+        let test_case_report = self.test_case_report(&specifier.test_specifier);
+        match specifier.node_designation {
+            NodeDesignation::Leader => test_case_report
+                .leader_execution_information
+                .get_or_insert_default(),
+            NodeDesignation::Follower => test_case_report
+                .follower_execution_information
+                .get_or_insert_default(),
+        }
+    }
 }
 
 #[serde_as]
@@ -287,6 +432,12 @@ pub struct TestCaseReport {
     /// Information related to the follower node assigned to this test case.
     #[serde(skip_serializing_if = "Option::is_none")]
     pub follower_node: Option<TestCaseNodeInformation>,
+    /// Information related to the execution on the leader.
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub leader_execution_information: Option<ExecutionInformation>,
+    /// Information related to the execution on the follower.
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub follower_execution_information: Option<ExecutionInformation>,
 }
 
 /// Information related to the status of the test. Could be that the test succeeded, failed, or that
@@ -324,3 +475,57 @@ pub struct TestCaseNodeInformation {
     /// The connection string of the node.
     pub connection_string: String,
 }
+
+/// Execution information tied to the leader or the follower.
+#[derive(Clone, Debug, Default, Serialize)]
+pub struct ExecutionInformation {
+    /// Information on the pre-link compiled contracts.
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub pre_link_compilation_status: Option<CompilationStatus>,
+    /// Information on the post-link compiled contracts.
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub post_link_compilation_status: Option<CompilationStatus>,
+    /// Information on the deployed libraries.
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub deployed_libraries: Option<BTreeMap<ContractInstance, Address>>,
+}
+
+/// Information related to compilation
+#[derive(Clone, Debug, Serialize)]
+#[serde(tag = "status")]
+pub enum CompilationStatus {
+    /// The compilation was successful.
+    Success {
+        /// A flag with information on whether the compilation artifacts were cached or not.
+        is_cached: bool,
+        /// The version of the compiler used to compile the contracts.
+        compiler_version: Version,
+        /// The path of the compiler used to compile the contracts.
+        compiler_path: PathBuf,
+        /// The input provided to the compiler to compile the contracts. This is only included if
+        /// the appropriate flag is set in the CLI configuration and if the contracts were not
+        /// cached and the compiler was invoked.
+        #[serde(skip_serializing_if = "Option::is_none")]
+        compiler_input: Option<CompilerInput>,
+        /// The output of the compiler. This is only included if the appropriate flag is set in the
+        /// CLI configurations.
+        #[serde(skip_serializing_if = "Option::is_none")]
+        compiler_output: Option<CompilerOutput>,
+    },
+    /// The compilation failed.
+    Failure {
+        /// The failure reason.
+        reason: String,
+        /// The version of the compiler used to compile the contracts.
+        #[serde(skip_serializing_if = "Option::is_none")]
+        compiler_version: Option<Version>,
+        /// The path of the compiler used to compile the contracts.
+        #[serde(skip_serializing_if = "Option::is_none")]
+        compiler_path: Option<PathBuf>,
+        /// The input provided to the compiler to compile the contracts. This is only included if
+        /// the appropriate flag is set in the CLI configuration and if the contracts were not
+        /// cached and the compiler was invoked.
+        #[serde(skip_serializing_if = "Option::is_none")]
+        compiler_input: Option<CompilerInput>,
+    },
+}

crates/report/src/common.rs

@@ -1,12 +1,11 @@
 //! Common types and functions used throughout the crate.
 
-use std::path::PathBuf;
+use std::{path::PathBuf, sync::Arc};
 
 use revive_dt_common::define_wrapper_type;
 use revive_dt_compiler::Mode;
 use revive_dt_format::case::CaseIdx;
 use serde::{Deserialize, Serialize};
-use serde_with::{DisplayFromStr, serde_as};
 
 define_wrapper_type!(
     #[derive(Clone, Debug, PartialEq, Eq, PartialOrd, Ord, Hash, Serialize, Deserialize)]
@@ -15,11 +14,24 @@ define_wrapper_type!(
 );
 
 /// An absolute specifier for a test.
-#[serde_as]
-#[derive(Clone, Debug, PartialEq, Eq, Hash, Serialize)]
+#[derive(Clone, Debug, PartialEq, Eq, Hash)]
 pub struct TestSpecifier {
-    #[serde_as(as = "DisplayFromStr")]
     pub solc_mode: Mode,
     pub metadata_file_path: PathBuf,
     pub case_idx: CaseIdx,
 }
+
+/// An absolute path for a test that also includes information about the node that it's assigned to
+/// and whether it's the leader or follower.
+#[derive(Clone, Debug, PartialEq, Eq, Hash)]
+pub struct ExecutionSpecifier {
+    pub test_specifier: Arc<TestSpecifier>,
+    pub node_id: usize,
+    pub node_designation: NodeDesignation,
+}
+
+#[derive(Clone, Debug, PartialEq, Eq, PartialOrd, Ord, Hash)]
+pub enum NodeDesignation {
+    Leader,
+    Follower,
+}

crates/report/src/runner_event.rs

@@ -1,17 +1,20 @@
 //! The types associated with the events sent by the runner to the reporter.
 #![allow(dead_code)]
 
-use std::sync::Arc;
+use std::{collections::BTreeMap, path::PathBuf, sync::Arc};
 
+use alloy_primitives::Address;
 use indexmap::IndexMap;
+use revive_dt_compiler::{CompilerInput, CompilerOutput};
 use revive_dt_config::TestingPlatform;
-use revive_dt_format::corpus::Corpus;
 use revive_dt_format::metadata::Metadata;
+use revive_dt_format::{corpus::Corpus, metadata::ContractInstance};
+use semver::Version;
 use tokio::sync::{broadcast, oneshot};
 
-use crate::{ReporterEvent, TestSpecifier, common::MetadataFilePath};
+use crate::{ExecutionSpecifier, ReporterEvent, TestSpecifier, common::MetadataFilePath};
 
-macro_rules! __report_gen__emit_test_specific {
+macro_rules! __report_gen_emit_test_specific {
     (
         $ident:ident,
         $variant_ident:ident,
@@ -36,21 +39,21 @@ macro_rules! __report_gen__emit_test_specific {
     };
 }
 
-macro_rules! __report_gen__emit_test_specific_by_parse {
+macro_rules! __report_gen_emit_test_specific_by_parse {
     (
         $ident:ident,
         $variant_ident:ident,
         $skip_field:ident;
         $( $bname:ident : $bty:ty, )* ; $( $aname:ident : $aty:ty, )*
     ) => {
-        __report_gen__emit_test_specific!(
+        __report_gen_emit_test_specific!(
             $ident, $variant_ident, $skip_field;
             $( $bname : $bty, )* ; $( $aname : $aty, )*
         );
     };
 }
 
-macro_rules! __report_gen__scan_before {
+macro_rules! __report_gen_scan_before {
     (
         $ident:ident, $variant_ident:ident;
         $( $before:ident : $bty:ty, )*
@@ -59,7 +62,7 @@ macro_rules! __report_gen__scan_before {
         $( $after:ident : $aty:ty, )*
         ;
     ) => {
-        __report_gen__emit_test_specific_by_parse!(
+        __report_gen_emit_test_specific_by_parse!(
             $ident, $variant_ident, test_specifier;
             $( $before : $bty, )* ; $( $after : $aty, )*
         );
@@ -71,7 +74,7 @@ macro_rules! __report_gen__scan_before {
         $name:ident : $ty:ty, $( $after:ident : $aty:ty, )*
         ;
     ) => {
-        __report_gen__scan_before!(
+        __report_gen_scan_before!(
             $ident, $variant_ident;
             $( $before : $bty, )* $name : $ty,
             ;
@@ -97,7 +100,7 @@ macro_rules! __report_gen_for_variant {
         $variant_ident:ident;
         $( $field_ident:ident : $field_ty:ty ),+ $(,)?
     ) => {
-        __report_gen__scan_before!(
+        __report_gen_scan_before!(
             $ident, $variant_ident;
             ;
             $( $field_ident : $field_ty, )*
@@ -106,11 +109,99 @@ macro_rules! __report_gen_for_variant {
     };
 }
 
-macro_rules! keep_if_doc {
-    (#[doc = $doc:expr]) => {
-        #[doc = $doc]
+macro_rules! __report_gen_emit_execution_specific {
+    (
+        $ident:ident,
+        $variant_ident:ident,
+        $skip_field:ident;
+        $( $bname:ident : $bty:ty, )*
+        ;
+        $( $aname:ident : $aty:ty, )*
+    ) => {
+        paste::paste! {
+            pub fn [< report_ $variant_ident:snake _event >](
+                &self
+                $(, $bname: impl Into<$bty> )*
+                $(, $aname: impl Into<$aty> )*
+            ) -> anyhow::Result<()> {
+                self.report([< $variant_ident Event >] {
+                    $skip_field: self.execution_specifier.clone()
+                    $(, $bname: $bname.into() )*
+                    $(, $aname: $aname.into() )*
+                })
+            }
+        }
+    };
+}
+
+macro_rules! __report_gen_emit_execution_specific_by_parse {
+    (
+        $ident:ident,
+        $variant_ident:ident,
+        $skip_field:ident;
+        $( $bname:ident : $bty:ty, )* ; $( $aname:ident : $aty:ty, )*
+    ) => {
+        __report_gen_emit_execution_specific!(
+            $ident, $variant_ident, $skip_field;
+            $( $bname : $bty, )* ; $( $aname : $aty, )*
+        );
+    };
+}
+
+macro_rules! __report_gen_scan_before_exec {
+    (
+        $ident:ident, $variant_ident:ident;
+        $( $before:ident : $bty:ty, )*
+        ;
+        execution_specifier : $skip_ty:ty,
+        $( $after:ident : $aty:ty, )*
+        ;
+    ) => {
+        __report_gen_emit_execution_specific_by_parse!(
+            $ident, $variant_ident, execution_specifier;
+            $( $before : $bty, )* ; $( $after : $aty, )*
+        );
+    };
+    (
+        $ident:ident, $variant_ident:ident;
+        $( $before:ident : $bty:ty, )*
+        ;
+        $name:ident : $ty:ty, $( $after:ident : $aty:ty, )*
+        ;
+    ) => {
+        __report_gen_scan_before_exec!(
+            $ident, $variant_ident;
+            $( $before : $bty, )* $name : $ty,
+            ;
+            $( $after : $aty, )*
+            ;
+        );
+    };
+    (
+        $ident:ident, $variant_ident:ident;
+        $( $before:ident : $bty:ty, )*
+        ;
+        ;
+    ) => {};
+}
+
+macro_rules! __report_gen_for_variant_exec {
+    (
+        $ident:ident,
+        $variant_ident:ident;
+    ) => {};
+    (
+        $ident:ident,
+        $variant_ident:ident;
+        $( $field_ident:ident : $field_ty:ty ),+ $(,)?
+    ) => {
+        __report_gen_scan_before_exec!(
+            $ident, $variant_ident;
+            ;
+            $( $field_ident : $field_ty, )*
+            ;
+        );
     };
-    ( $($_:tt)* ) => {};
 }
 
 /// Defines the runner-event which is sent from the test runners to the report aggregator.
@@ -149,6 +240,7 @@ macro_rules! define_event {
     ) => {
         paste::paste! {
             $(#[$enum_meta])*
+            #[derive(Debug)]
             $vis enum $ident {
                 $(
                     $(#[$variant_meta])*
@@ -157,6 +249,7 @@ macro_rules! define_event {
             }
 
             $(
+                #[derive(Debug)]
                 $(#[$variant_meta])*
                 $vis struct [<$variant_ident Event>] {
                     $(
@@ -204,7 +297,6 @@ macro_rules! define_event {
                 }
 
                 $(
-                    keep_if_doc!($(#[$variant_meta])*);
                     pub fn [< report_ $variant_ident:snake _event >](&self, $($field_ident: impl Into<$field_ty>),*) -> anyhow::Result<()> {
                         self.report([< $variant_ident Event >] {
                             $($field_ident: $field_ident.into()),*
@@ -221,6 +313,21 @@ macro_rules! define_event {
             }
 
             impl [< $ident TestSpecificReporter >] {
+                pub fn execution_specific_reporter(
+                    &self,
+                    node_id: impl Into<usize>,
+                    node_designation: impl Into<$crate::common::NodeDesignation>
+                ) -> [< $ident ExecutionSpecificReporter >] {
+                    [< $ident ExecutionSpecificReporter >] {
+                        reporter: self.reporter.clone(),
+                        execution_specifier: Arc::new($crate::common::ExecutionSpecifier {
+                            test_specifier: self.test_specifier.clone(),
+                            node_id: node_id.into(),
+                            node_designation: node_designation.into(),
+                        })
+                    }
+                }
+
                 fn report(&self, event: impl Into<$ident>) -> anyhow::Result<()> {
                     self.reporter.report(event)
                 }
@@ -229,6 +336,23 @@ macro_rules! define_event {
                     __report_gen_for_variant! { $ident, $variant_ident; $( $field_ident : $field_ty ),* }
                 )*
             }
+
+            /// A reporter that's tied to a specific execution of the test case such as execution on
+            /// a specific node like the leader or follower.
+            #[derive(Clone, Debug)]
+            pub struct [< $ident ExecutionSpecificReporter >] {
+                $vis reporter: [< $ident Reporter >],
+                $vis execution_specifier: std::sync::Arc<$crate::common::ExecutionSpecifier>,
+            }
+
+            impl [< $ident ExecutionSpecificReporter >] {
+                fn report(&self, event: impl Into<$ident>) -> anyhow::Result<()> {
+                    self.reporter.report(event)
+                }
+                $(
+                    __report_gen_for_variant_exec! { $ident, $variant_ident; $( $field_ident : $field_ty ),* }
+                )*
+            }
         }
     };
 }
@@ -303,7 +427,79 @@ define_event! {
             platform: TestingPlatform,
             /// The connection string of the node.
             connection_string: String,
-        }
+        },
+        /// An event emitted by the runners when the compilation of the contracts has succeeded
+        /// on the pre-link contracts.
+        PreLinkContractsCompilationSucceeded {
+            /// A specifier for the execution that's taking place.
+            execution_specifier: Arc<ExecutionSpecifier>,
+            /// The version of the compiler used to compile the contracts.
+            compiler_version: Version,
+            /// The path of the compiler used to compile the contracts.
+            compiler_path: PathBuf,
+            /// A flag of whether the contract bytecode and ABI were cached or if they were compiled
+            /// anew.
+            is_cached: bool,
+            /// The input provided to the compiler - this is optional and not provided if the
+            /// contracts were obtained from the cache.
+            compiler_input: Option<CompilerInput>,
+            /// The output of the compiler.
+            compiler_output: CompilerOutput
+        },
+        /// An event emitted by the runners when the compilation of the contracts has succeeded
+        /// on the post-link contracts.
+        PostLinkContractsCompilationSucceeded {
+            /// A specifier for the execution that's taking place.
+            execution_specifier: Arc<ExecutionSpecifier>,
+            /// The version of the compiler used to compile the contracts.
+            compiler_version: Version,
+            /// The path of the compiler used to compile the contracts.
+            compiler_path: PathBuf,
+            /// A flag of whether the contract bytecode and ABI were cached or if they were compiled
+            /// anew.
+            is_cached: bool,
+            /// The input provided to the compiler - this is optional and not provided if the
+            /// contracts were obtained from the cache.
+            compiler_input: Option<CompilerInput>,
+            /// The output of the compiler.
+            compiler_output: CompilerOutput
+        },
+        /// An event emitted by the runners when the compilation of the pre-link contract has
+        /// failed.
+        PreLinkContractsCompilationFailed {
+            /// A specifier for the execution that's taking place.
+            execution_specifier: Arc<ExecutionSpecifier>,
+            /// The version of the compiler used to compile the contracts.
+            compiler_version: Option<Version>,
+            /// The path of the compiler used to compile the contracts.
+            compiler_path: Option<PathBuf>,
+            /// The input provided to the compiler - this is optional and not provided if the
+            /// contracts were obtained from the cache.
+            compiler_input: Option<CompilerInput>,
+            /// The failure reason.
+            reason: String,
+        },
+        /// An event emitted by the runners when the compilation of the post-link contract has
+        /// failed.
+        PostLinkContractsCompilationFailed {
+            /// A specifier for the execution that's taking place.
+            execution_specifier: Arc<ExecutionSpecifier>,
+            /// The version of the compiler used to compile the contracts.
+            compiler_version: Option<Version>,
+            /// The path of the compiler used to compile the contracts.
+            compiler_path: Option<PathBuf>,
+            /// The input provided to the compiler - this is optional and not provided if the
+            /// contracts were obtained from the cache.
+            compiler_input: Option<CompilerInput>,
+            /// The failure reason.
+            reason: String,
+        },
+        LibrariesDeployed {
+            /// A specifier for the execution that's taking place.
+            execution_specifier: Arc<ExecutionSpecifier>,
+            /// The addresses of the libraries that were deployed.
+            libraries: BTreeMap<ContractInstance, Address>
+        },
     }
 }
 
@@ -318,3 +514,4 @@ impl RunnerEventReporter {
 
 pub type Reporter = RunnerEventReporter;
 pub type TestSpecificReporter = RunnerEventTestSpecificReporter;
+pub type ExecutionSpecificReporter = RunnerEventExecutionSpecificReporter;