Make report format more benchmark friendly

crates/core/src/differential_benchmarks/entry_point.rs

@@ -151,7 +151,9 @@ pub async fn handle_differential_benchmarks(
                     .subscribe_to_full_blocks_information()
                     .await
                     .context("Failed to subscribe to full blocks information from the node")?,
-                reporter.clone(),
+                test_definition
+                    .reporter
+                    .execution_specific_reporter(0usize, platform_identifier),
             );
             let driver = Driver::new(
                 platform_information,

crates/core/src/differential_benchmarks/watcher.rs

@@ -6,7 +6,7 @@ use futures::{Stream, StreamExt};
 use revive_dt_common::types::PlatformIdentifier;
 use revive_dt_format::steps::StepPath;
 use revive_dt_node_interaction::MinedBlockInformation;
-use revive_dt_report::Reporter;
+use revive_dt_report::ExecutionSpecificReporter;
 use tokio::sync::{
     RwLock,
     mpsc::{UnboundedReceiver, UnboundedSender, unbounded_channel},
@@ -29,14 +29,14 @@ pub struct Watcher {
     blocks_stream: Pin<Box<dyn Stream<Item = MinedBlockInformation>>>,
 
     /// The reporter used to send events to the report aggregator.
-    reporter: Reporter,
+    reporter: ExecutionSpecificReporter,
 }
 
 impl Watcher {
     pub fn new(
         platform_identifier: PlatformIdentifier,
         blocks_stream: Pin<Box<dyn Stream<Item = MinedBlockInformation>>>,
-        reporter: Reporter,
+        reporter: ExecutionSpecificReporter,
     ) -> (Self, UnboundedSender<WatcherEvent>) {
         let (tx, rx) = unbounded_channel::<WatcherEvent>();
         (

crates/core/src/helpers/test.rs

@@ -310,10 +310,10 @@ impl<'a> TestDefinition<'a> {
         };
 
         let test_case_status = report
-            .test_case_information
+            .execution_information
             .get(&(self.metadata_file_path.to_path_buf().into()))
-            .and_then(|obj| obj.get(&self.mode))
             .and_then(|obj| obj.get(&self.case_idx))
+            .and_then(|obj| obj.get(&self.mode))
             .and_then(|obj| obj.status.as_ref());
 
         match test_case_status {

crates/report/src/aggregator.rs

@@ -234,17 +234,17 @@ impl ReportAggregator {
 
         let case_status = self
             .report
-            .test_case_information
+            .execution_information
             .entry(specifier.metadata_file_path.clone().into())
             .or_default()
-            .entry(specifier.solc_mode.clone())
-            .or_default()
             .iter()
-            .map(|(case_idx, case_report)| {
+            .flat_map(|(case_idx, mode_to_execution_map)| {
-                (
+                let case_status = mode_to_execution_map
-                    *case_idx,
+                    .get(&specifier.solc_mode)?
-                    case_report.status.clone().expect("Can't be uninitialized"),
+                    .status
-                )
+                    .clone()
+                    .expect("Can't be uninitialized");
+                Some((*case_idx, case_status))
             })
             .collect::<BTreeMap<_, _>>();
         let event = ReporterEvent::MetadataFileSolcModeCombinationExecutionCompleted {
@@ -392,13 +392,13 @@ impl ReportAggregator {
 
     fn test_case_report(&mut self, specifier: &TestSpecifier) -> &mut TestCaseReport {
         self.report
-            .test_case_information
+            .execution_information
             .entry(specifier.metadata_file_path.clone().into())
             .or_default()
-            .entry(specifier.solc_mode.clone())
-            .or_default()
             .entry(specifier.case_idx)
             .or_default()
+            .entry(specifier.solc_mode.clone())
+            .or_default()
     }
 
     fn execution_information(
@@ -424,19 +424,22 @@ pub struct Report {
     pub corpora: Vec<ParsedTestSpecifier>,
     /// The list of metadata files that were found by the tool.
     pub metadata_files: BTreeSet<MetadataFilePath>,
+    /// Metrics from the execution.
+    pub metrics: Option<Metrics>,
     /// Information relating to each test case.
-    #[serde_as(as = "BTreeMap<_, HashMap<DisplayFromStr, BTreeMap<DisplayFromStr, _>>>")]
+    #[serde_as(as = "BTreeMap<_, BTreeMap<DisplayFromStr, HashMap<DisplayFromStr, _>>>")]
-    pub test_case_information:
+    pub execution_information:
-        BTreeMap<MetadataFilePath, HashMap<Mode, BTreeMap<CaseIdx, TestCaseReport>>>,
+        BTreeMap<MetadataFilePath, BTreeMap<CaseIdx, HashMap<Mode, TestCaseReport>>>,
 }
 
 impl Report {
     pub fn new(context: Context) -> Self {
         Self {
             context,
+            metrics: Default::default(),
             corpora: Default::default(),
             metadata_files: Default::default(),
-            test_case_information: Default::default(),
+            execution_information: Default::default(),
         }
     }
 }
@@ -545,3 +548,28 @@ pub enum CompilationStatus {
         compiler_input: Option<CompilerInput>,
     },
 }
+
+/// The metrics we collect for our benchmarks.
+#[derive(Clone, Debug, Serialize, Deserialize)]
+pub struct Metrics {
+    pub transaction_per_second: Metric<u64>,
+    pub gas_per_second: Metric<u64>,
+    pub gas_consumption: Metric<u64>,
+    /* Block Fullness */
+    pub gas_block_fullness: Metric<u64>,
+    pub ref_time_block_fullness: Option<Metric<u64>>,
+    pub proof_size_block_fullness: Option<Metric<u64>>,
+}
+
+/// The data that we store for a given metric (e.g., TPS).
+#[derive(Clone, Debug, Serialize, Deserialize)]
+pub struct Metric<T> {
+    pub minimum: Option<PlatformKeyedInformation<T>>,
+    pub maximum: Option<PlatformKeyedInformation<T>>,
+    pub mean: Option<PlatformKeyedInformation<T>>,
+    pub median: Option<PlatformKeyedInformation<T>>,
+    pub sum: Option<PlatformKeyedInformation<T>>,
+}
+
+/// Information keyed by the platform identifier.
+pub type PlatformKeyedInformation<T> = BTreeMap<PlatformIdentifier, T>;