diff --git a/crates/core/src/differential_benchmarks/entry_point.rs b/crates/core/src/differential_benchmarks/entry_point.rs index 2e5fc64..69cc7f3 100644 --- a/crates/core/src/differential_benchmarks/entry_point.rs +++ b/crates/core/src/differential_benchmarks/entry_point.rs @@ -151,7 +151,9 @@ pub async fn handle_differential_benchmarks( .subscribe_to_full_blocks_information() .await .context("Failed to subscribe to full blocks information from the node")?, - reporter.clone(), + test_definition + .reporter + .execution_specific_reporter(0usize, platform_identifier), ); let driver = Driver::new( platform_information, diff --git a/crates/core/src/differential_benchmarks/watcher.rs b/crates/core/src/differential_benchmarks/watcher.rs index c3c94f3..44762ad 100644 --- a/crates/core/src/differential_benchmarks/watcher.rs +++ b/crates/core/src/differential_benchmarks/watcher.rs @@ -6,7 +6,7 @@ use futures::{Stream, StreamExt}; use revive_dt_common::types::PlatformIdentifier; use revive_dt_format::steps::StepPath; use revive_dt_node_interaction::MinedBlockInformation; -use revive_dt_report::Reporter; +use revive_dt_report::ExecutionSpecificReporter; use tokio::sync::{ RwLock, mpsc::{UnboundedReceiver, UnboundedSender, unbounded_channel}, @@ -29,14 +29,14 @@ pub struct Watcher { blocks_stream: Pin>>, /// The reporter used to send events to the report aggregator. - reporter: Reporter, + reporter: ExecutionSpecificReporter, } impl Watcher { pub fn new( platform_identifier: PlatformIdentifier, blocks_stream: Pin>>, - reporter: Reporter, + reporter: ExecutionSpecificReporter, ) -> (Self, UnboundedSender) { let (tx, rx) = unbounded_channel::(); ( diff --git a/crates/core/src/helpers/test.rs b/crates/core/src/helpers/test.rs index 6571c6b..6c39f74 100644 --- a/crates/core/src/helpers/test.rs +++ b/crates/core/src/helpers/test.rs @@ -310,10 +310,10 @@ impl<'a> TestDefinition<'a> { }; let test_case_status = report - .test_case_information + .execution_information .get(&(self.metadata_file_path.to_path_buf().into())) - .and_then(|obj| obj.get(&self.mode)) .and_then(|obj| obj.get(&self.case_idx)) + .and_then(|obj| obj.get(&self.mode)) .and_then(|obj| obj.status.as_ref()); match test_case_status { diff --git a/crates/report/src/aggregator.rs b/crates/report/src/aggregator.rs index 69d9743..2c0af55 100644 --- a/crates/report/src/aggregator.rs +++ b/crates/report/src/aggregator.rs @@ -234,17 +234,17 @@ impl ReportAggregator { let case_status = self .report - .test_case_information + .execution_information .entry(specifier.metadata_file_path.clone().into()) .or_default() - .entry(specifier.solc_mode.clone()) - .or_default() .iter() - .map(|(case_idx, case_report)| { - ( - *case_idx, - case_report.status.clone().expect("Can't be uninitialized"), - ) + .flat_map(|(case_idx, mode_to_execution_map)| { + let case_status = mode_to_execution_map + .get(&specifier.solc_mode)? + .status + .clone() + .expect("Can't be uninitialized"); + Some((*case_idx, case_status)) }) .collect::>(); let event = ReporterEvent::MetadataFileSolcModeCombinationExecutionCompleted { @@ -392,13 +392,13 @@ impl ReportAggregator { fn test_case_report(&mut self, specifier: &TestSpecifier) -> &mut TestCaseReport { self.report - .test_case_information + .execution_information .entry(specifier.metadata_file_path.clone().into()) .or_default() - .entry(specifier.solc_mode.clone()) - .or_default() .entry(specifier.case_idx) .or_default() + .entry(specifier.solc_mode.clone()) + .or_default() } fn execution_information( @@ -424,19 +424,22 @@ pub struct Report { pub corpora: Vec, /// The list of metadata files that were found by the tool. pub metadata_files: BTreeSet, + /// Metrics from the execution. + pub metrics: Option, /// Information relating to each test case. - #[serde_as(as = "BTreeMap<_, HashMap>>")] - pub test_case_information: - BTreeMap>>, + #[serde_as(as = "BTreeMap<_, BTreeMap>>")] + pub execution_information: + BTreeMap>>, } impl Report { pub fn new(context: Context) -> Self { Self { context, + metrics: Default::default(), corpora: Default::default(), metadata_files: Default::default(), - test_case_information: Default::default(), + execution_information: Default::default(), } } } @@ -545,3 +548,28 @@ pub enum CompilationStatus { compiler_input: Option, }, } + +/// The metrics we collect for our benchmarks. +#[derive(Clone, Debug, Serialize, Deserialize)] +pub struct Metrics { + pub transaction_per_second: Metric, + pub gas_per_second: Metric, + pub gas_consumption: Metric, + /* Block Fullness */ + pub gas_block_fullness: Metric, + pub ref_time_block_fullness: Option>, + pub proof_size_block_fullness: Option>, +} + +/// The data that we store for a given metric (e.g., TPS). +#[derive(Clone, Debug, Serialize, Deserialize)] +pub struct Metric { + pub minimum: Option>, + pub maximum: Option>, + pub mean: Option>, + pub median: Option>, + pub sum: Option>, +} + +/// Information keyed by the platform identifier. +pub type PlatformKeyedInformation = BTreeMap;