pezkuwichain/revive-differential-tests
1
0
Fork 0
mirror of https://github.com/pezkuwichain/revive-differential-tests.git synced 2026-06-14 04:01:07 +00:00
Code Issues Packages Projects Releases Wiki Activity

Wire up reporting to benchmarks (#195)

Browse Source 
* Modify the structure of the `MinedBlockInformation`

* Report the step path to the watcher

* Make report format more benchmark friendly

* make report more benchmarks friendly

* Add more models to the report

* Remove corpus from the report

* Add step information to the benchmark report

* Include the contract information in the report

* Add the block information to the report

* compute metrics in each report

* Cleanup watcher from temp code
This commit is contained in:
Omar
2025-10-24 05:15:29 +03:00
committed by GitHub
parent f1a911545e
commit b71445b632
16 changed files with 670 additions and 196 deletions
Download Patch File Download Diff File Expand all files Collapse all files
crates/report/src/aggregator.rs
+445 -35
View File
@@ -4,17 +4,19 @@
use std::{
collections::{BTreeMap, BTreeSet, HashMap, HashSet},
fs::OpenOptions,
ops::{Add, Div},
path::PathBuf,
time::{SystemTime, UNIX_EPOCH},
};
use alloy::primitives::Address;
use alloy::primitives::{Address, BlockNumber, BlockTimestamp, TxHash};
use anyhow::{Context as _, Result};
use indexmap::IndexMap;
use revive_dt_common::types::{ParsedTestSpecifier, PlatformIdentifier};
use itertools::Itertools;
use revive_dt_common::types::PlatformIdentifier;
use revive_dt_compiler::{CompilerInput, CompilerOutput, Mode};
use revive_dt_config::Context;
use revive_dt_format::{case::CaseIdx, metadata::ContractInstance};
use revive_dt_format::{case::CaseIdx, metadata::ContractInstance, steps::StepPath};
use semver::Version;
use serde::{Deserialize, Serialize};
use serde_with::{DisplayFromStr, serde_as};
@@ -67,9 +69,6 @@ impl ReportAggregator {
RunnerEvent::SubscribeToEvents(event) => {
self.handle_subscribe_to_events_event(*event);
}
RunnerEvent::CorpusDiscovery(event) => {
self.handle_corpus_file_discovered_event(*event)
}
RunnerEvent::MetadataFileDiscovery(event) => {
self.handle_metadata_file_discovery_event(*event);
}
@@ -106,12 +105,20 @@ impl ReportAggregator {
RunnerEvent::ContractDeployed(event) => {
self.handle_contract_deployed_event(*event);
}
RunnerEvent::Completion(event) => {
self.handle_completion(*event);
RunnerEvent::Completion(_) => {
break;
}
/* Benchmarks Events */
RunnerEvent::StepTransactionInformation(event) => {
self.handle_step_transaction_information(*event)
}
RunnerEvent::ContractInformation(event) => {
self.handle_contract_information(*event);
}
RunnerEvent::BlockMined(event) => self.handle_block_mined(*event),
}
}
self.handle_completion(CompletionEvent {});
debug!("Report aggregation completed");
let file_name = {
@@ -152,10 +159,6 @@ impl ReportAggregator {
let _ = event.tx.send(self.listener_tx.subscribe());
}
fn handle_corpus_file_discovered_event(&mut self, event: CorpusDiscoveryEvent) {
self.report.corpora.extend(event.test_specifiers);
}
fn handle_metadata_file_discovery_event(&mut self, event: MetadataFileDiscoveryEvent) {
self.report.metadata_files.insert(event.path.clone());
}
@@ -234,17 +237,19 @@ impl ReportAggregator {
let case_status = self
.report
.test_case_information
.execution_information
.entry(specifier.metadata_file_path.clone().into())
.or_default()
.entry(specifier.solc_mode.clone())
.or_default()
.case_reports
.iter()
.map(|(case_idx, case_report)| {
(
*case_idx,
case_report.status.clone().expect("Can't be uninitialized"),
)
.flat_map(|(case_idx, mode_to_execution_map)| {
let case_status = mode_to_execution_map
.mode_execution_reports
.get(&specifier.solc_mode)?
.status
.clone()
.expect("Can't be uninitialized");
Some((*case_idx, case_status))
})
.collect::<BTreeMap<_, _>>();
let event = ReporterEvent::MetadataFileSolcModeCombinationExecutionCompleted {
@@ -383,22 +388,159 @@ impl ReportAggregator {
self.execution_information(&event.execution_specifier)
.deployed_contracts
.get_or_insert_default()
.insert(event.contract_instance, event.address);
.insert(event.contract_instance.clone(), event.address);
self.test_case_report(&event.execution_specifier.test_specifier)
.contract_addresses
.entry(event.contract_instance)
.or_default()
.entry(event.execution_specifier.platform_identifier)
.or_default()
.push(event.address);
}
fn handle_completion(&mut self, _: CompletionEvent) {
self.runner_rx.close();
self.handle_metrics_computation();
}
fn test_case_report(&mut self, specifier: &TestSpecifier) -> &mut TestCaseReport {
fn handle_metrics_computation(&mut self) {
for report in self.report.execution_information.values_mut() {
for report in report.case_reports.values_mut() {
for report in report.mode_execution_reports.values_mut() {
for (platform_identifier, block_information) in
report.mined_block_information.iter_mut()
{
block_information.sort_by(|a, b| {
a.ethereum_block_information
.block_timestamp
.cmp(&b.ethereum_block_information.block_timestamp)
});
// Computing the TPS.
let tps = block_information
.iter()
.tuple_windows::<(_, _)>()
.map(|(block1, block2)| {
block2.ethereum_block_information.transaction_hashes.len() as u64
/ (block2.ethereum_block_information.block_timestamp
- block1.ethereum_block_information.block_timestamp)
})
.collect::<Vec<_>>();
report
.metrics
.get_or_insert_default()
.transaction_per_second
.with_list(*platform_identifier, tps);
// Computing the GPS.
let gps = block_information
.iter()
.tuple_windows::<(_, _)>()
.map(|(block1, block2)| {
block2.ethereum_block_information.mined_gas as u64
/ (block2.ethereum_block_information.block_timestamp
- block1.ethereum_block_information.block_timestamp)
})
.collect::<Vec<_>>();
report
.metrics
.get_or_insert_default()
.gas_per_second
.with_list(*platform_identifier, gps);
// Computing the gas block fullness
let gas_block_fullness = block_information
.iter()
.map(|block| block.gas_block_fullness_percentage())
.map(|v| v as u64)
.collect::<Vec<_>>();
report
.metrics
.get_or_insert_default()
.gas_block_fullness
.with_list(*platform_identifier, gas_block_fullness);
// Computing the ref-time block fullness
let reftime_block_fullness = block_information
.iter()
.filter_map(|block| block.ref_time_block_fullness_percentage())
.map(|v| v as u64)
.collect::<Vec<_>>();
dbg!(&reftime_block_fullness);
if !reftime_block_fullness.is_empty() {
report
.metrics
.get_or_insert_default()
.ref_time_block_fullness
.get_or_insert_default()
.with_list(*platform_identifier, reftime_block_fullness);
}
// Computing the proof size block fullness
let proof_size_block_fullness = block_information
.iter()
.filter_map(|block| block.proof_size_block_fullness_percentage())
.map(|v| v as u64)
.collect::<Vec<_>>();
dbg!(&proof_size_block_fullness);
if !proof_size_block_fullness.is_empty() {
report
.metrics
.get_or_insert_default()
.proof_size_block_fullness
.get_or_insert_default()
.with_list(*platform_identifier, proof_size_block_fullness);
}
}
}
}
}
}
fn handle_step_transaction_information(&mut self, event: StepTransactionInformationEvent) {
self.test_case_report(&event.execution_specifier.test_specifier)
.steps
.entry(event.step_path)
.or_default()
.transactions
.entry(event.execution_specifier.platform_identifier)
.or_default()
.push(event.transaction_information);
}
fn handle_contract_information(&mut self, event: ContractInformationEvent) {
self.test_case_report(&event.execution_specifier.test_specifier)
.compiled_contracts
.entry(event.source_code_path)
.or_default()
.entry(event.contract_name)
.or_default()
.contract_size
.insert(
event.execution_specifier.platform_identifier,
event.contract_size,
);
}
fn handle_block_mined(&mut self, event: BlockMinedEvent) {
self.test_case_report(&event.execution_specifier.test_specifier)
.mined_block_information
.entry(event.execution_specifier.platform_identifier)
.or_default()
.push(event.mined_block_information);
}
fn test_case_report(&mut self, specifier: &TestSpecifier) -> &mut ExecutionReport {
self.report
.test_case_information
.execution_information
.entry(specifier.metadata_file_path.clone().into())
.or_default()
.entry(specifier.solc_mode.clone())
.or_default()
.case_reports
.entry(specifier.case_idx)
.or_default()
.mode_execution_reports
.entry(specifier.solc_mode.clone())
.or_default()
}
fn execution_information(
@@ -419,35 +561,69 @@ impl ReportAggregator {
pub struct Report {
/// The context that the tool was started up with.
pub context: Context,
/// The list of corpus files that the tool found.
#[serde_as(as = "Vec<DisplayFromStr>")]
pub corpora: Vec<ParsedTestSpecifier>,
/// The list of metadata files that were found by the tool.
pub metadata_files: BTreeSet<MetadataFilePath>,
/// Metrics from the execution.
#[serde(skip_serializing_if = "Option::is_none")]
pub metrics: Option<Metrics>,
/// Information relating to each test case.
#[serde_as(as = "BTreeMap<_, HashMap<DisplayFromStr, BTreeMap<DisplayFromStr, _>>>")]
pub test_case_information:
BTreeMap<MetadataFilePath, HashMap<Mode, BTreeMap<CaseIdx, TestCaseReport>>>,
pub execution_information: BTreeMap<MetadataFilePath, MetadataFileReport>,
}
impl Report {
pub fn new(context: Context) -> Self {
Self {
context,
corpora: Default::default(),
metrics: Default::default(),
metadata_files: Default::default(),
test_case_information: Default::default(),
execution_information: Default::default(),
}
}
}
#[derive(Clone, Debug, Serialize, Deserialize, Default)]
pub struct TestCaseReport {
pub struct MetadataFileReport {
/// Metrics from the execution.
#[serde(skip_serializing_if = "Option::is_none")]
pub metrics: Option<Metrics>,
/// The report of each case keyed by the case idx.
pub case_reports: BTreeMap<CaseIdx, CaseReport>,
}
#[serde_as]
#[derive(Clone, Debug, Serialize, Deserialize, Default)]
pub struct CaseReport {
/// Metrics from the execution.
#[serde(skip_serializing_if = "Option::is_none")]
pub metrics: Option<Metrics>,
/// The [`ExecutionReport`] for each one of the [`Mode`]s.
#[serde_as(as = "HashMap<DisplayFromStr, _>")]
pub mode_execution_reports: HashMap<Mode, ExecutionReport>,
}
#[derive(Clone, Debug, Serialize, Deserialize, Default)]
pub struct ExecutionReport {
/// Information on the status of the test case and whether it succeeded, failed, or was ignored.
#[serde(skip_serializing_if = "Option::is_none")]
pub status: Option<TestCaseStatus>,
/// Metrics from the execution.
#[serde(skip_serializing_if = "Option::is_none")]
pub metrics: Option<Metrics>,
/// Information related to the execution on one of the platforms.
pub platform_execution: BTreeMap<PlatformIdentifier, Option<ExecutionInformation>>,
#[serde(skip_serializing_if = "BTreeMap::is_empty")]
pub platform_execution: PlatformKeyedInformation<Option<ExecutionInformation>>,
/// Information on the compiled contracts.
#[serde(skip_serializing_if = "BTreeMap::is_empty")]
pub compiled_contracts: BTreeMap<PathBuf, BTreeMap<String, ContractInformation>>,
/// The addresses of the deployed contracts
#[serde(skip_serializing_if = "BTreeMap::is_empty")]
pub contract_addresses: BTreeMap<ContractInstance, PlatformKeyedInformation<Vec<Address>>>,
/// Information on the mined blocks as part of this execution.
#[serde(skip_serializing_if = "BTreeMap::is_empty")]
pub mined_block_information: PlatformKeyedInformation<Vec<MinedBlockInformation>>,
/// Information tracked for each step that was executed.
#[serde(skip_serializing_if = "BTreeMap::is_empty")]
pub steps: BTreeMap<StepPath, StepReport>,
}
/// Information related to the status of the test. Could be that the test succeeded, failed, or that
@@ -545,3 +721,237 @@ pub enum CompilationStatus {
compiler_input: Option<CompilerInput>,
},
}
/// Information on each step in the execution.
#[derive(Clone, Debug, Default, Serialize, Deserialize)]
pub struct StepReport {
/// Information on the transactions submitted as part of this step.
transactions: PlatformKeyedInformation<Vec<TransactionInformation>>,
}
#[derive(Clone, Debug, Serialize, Deserialize)]
pub struct TransactionInformation {
/// The hash of the transaction
pub transaction_hash: TxHash,
pub submission_timestamp: u64,
pub block_timestamp: u64,
pub block_number: BlockNumber,
}
/// The metrics we collect for our benchmarks.
#[derive(Clone, Debug, Default, Serialize, Deserialize)]
pub struct Metrics {
pub transaction_per_second: Metric<u64>,
pub gas_per_second: Metric<u64>,
/* Block Fullness */
pub gas_block_fullness: Metric<u64>,
#[serde(skip_serializing_if = "Option::is_none")]
pub ref_time_block_fullness: Option<Metric<u64>>,
#[serde(skip_serializing_if = "Option::is_none")]
pub proof_size_block_fullness: Option<Metric<u64>>,
}
/// The data that we store for a given metric (e.g., TPS).
#[derive(Clone, Debug, Default, Serialize, Deserialize)]
pub struct Metric<T> {
#[serde(skip_serializing_if = "Option::is_none")]
pub minimum: Option<PlatformKeyedInformation<T>>,
#[serde(skip_serializing_if = "Option::is_none")]
pub maximum: Option<PlatformKeyedInformation<T>>,
#[serde(skip_serializing_if = "Option::is_none")]
pub mean: Option<PlatformKeyedInformation<T>>,
#[serde(skip_serializing_if = "Option::is_none")]
pub median: Option<PlatformKeyedInformation<T>>,
#[serde(skip_serializing_if = "Option::is_none")]
pub raw: Option<PlatformKeyedInformation<Vec<T>>>,
}
impl<T> Metric<T>
where
T: Default
+ Copy
+ Ord
+ PartialOrd
+ Add<Output = T>
+ Div<Output = T>
+ TryFrom<usize, Error: std::fmt::Debug>,
{
pub fn new() -> Self {
Default::default()
}
pub fn platform_identifiers(&self) -> BTreeSet<PlatformIdentifier> {
self.minimum
.as_ref()
.map(|m| m.keys())
.into_iter()
.flatten()
.chain(
self.maximum
.as_ref()
.map(|m| m.keys())
.into_iter()
.flatten(),
)
.chain(self.mean.as_ref().map(|m| m.keys()).into_iter().flatten())
.chain(self.median.as_ref().map(|m| m.keys()).into_iter().flatten())
.chain(self.raw.as_ref().map(|m| m.keys()).into_iter().flatten())
.copied()
.collect()
}
pub fn with_list(
&mut self,
platform_identifier: PlatformIdentifier,
mut list: Vec<T>,
) -> &mut Self {
list.sort();
let Some(min) = list.first().copied() else {
return self;
};
let Some(max) = list.last().copied() else {
return self;
};
let sum = list.iter().fold(T::default(), |acc, num| acc + *num);
let mean = sum / TryInto::<T>::try_into(list.len()).unwrap();
let median = match list.len().is_multiple_of(2) {
true => {
let idx = list.len() / 2;
let val1 = *list.get(idx - 1).unwrap();
let val2 = *list.get(idx).unwrap();
(val1 + val2) / TryInto::<T>::try_into(2usize).unwrap()
}
false => {
let idx = list.len() / 2;
*list.get(idx).unwrap()
}
};
self.minimum
.get_or_insert_default()
.insert(platform_identifier, min);
self.maximum
.get_or_insert_default()
.insert(platform_identifier, max);
self.mean
.get_or_insert_default()
.insert(platform_identifier, mean);
self.median
.get_or_insert_default()
.insert(platform_identifier, median);
self.raw
.get_or_insert_default()
.insert(platform_identifier, list);
self
}
pub fn combine(&self, other: &Self) -> Self {
let mut platform_identifiers = self.platform_identifiers();
platform_identifiers.extend(other.platform_identifiers());
let mut this = Self::new();
for platform_identifier in platform_identifiers {
let mut l1 = self
.raw
.as_ref()
.and_then(|m| m.get(&platform_identifier))
.cloned()
.unwrap_or_default();
let l2 = other
.raw
.as_ref()
.and_then(|m| m.get(&platform_identifier))
.cloned()
.unwrap_or_default();
l1.extend(l2);
this.with_list(platform_identifier, l1);
}
this
}
}
#[derive(Clone, Debug, Serialize, Deserialize, Default)]
pub struct ContractInformation {
/// The size of the contract on the various platforms.
pub contract_size: PlatformKeyedInformation<usize>,
}
#[derive(Clone, Debug, PartialEq, Eq, PartialOrd, Ord, Hash, Serialize, Deserialize)]
pub struct MinedBlockInformation {
pub ethereum_block_information: EthereumMinedBlockInformation,
pub substrate_block_information: Option<SubstrateMinedBlockInformation>,
}
impl MinedBlockInformation {
pub fn gas_block_fullness_percentage(&self) -> u8 {
self.ethereum_block_information
.gas_block_fullness_percentage()
}
pub fn ref_time_block_fullness_percentage(&self) -> Option<u8> {
self.substrate_block_information
.as_ref()
.map(|block| block.ref_time_block_fullness_percentage())
}
pub fn proof_size_block_fullness_percentage(&self) -> Option<u8> {
self.substrate_block_information
.as_ref()
.map(|block| block.proof_size_block_fullness_percentage())
}
}
#[derive(Clone, Debug, PartialEq, Eq, PartialOrd, Ord, Hash, Serialize, Deserialize)]
pub struct EthereumMinedBlockInformation {
/// The block number.
pub block_number: BlockNumber,
/// The block timestamp.
pub block_timestamp: BlockTimestamp,
/// The amount of gas mined in the block.
pub mined_gas: u128,
/// The gas limit of the block.
pub block_gas_limit: u128,
/// The hashes of the transactions that were mined as part of the block.
pub transaction_hashes: Vec<TxHash>,
}
impl EthereumMinedBlockInformation {
pub fn gas_block_fullness_percentage(&self) -> u8 {
(self.mined_gas * 100 / self.block_gas_limit) as u8
}
}
#[derive(Clone, Copy, Debug, PartialEq, Eq, PartialOrd, Ord, Hash, Serialize, Deserialize)]
pub struct SubstrateMinedBlockInformation {
/// The ref time for substrate based chains.
pub ref_time: u128,
/// The max ref time for substrate based chains.
pub max_ref_time: u64,
/// The proof size for substrate based chains.
pub proof_size: u128,
/// The max proof size for substrate based chains.
pub max_proof_size: u64,
}
impl SubstrateMinedBlockInformation {
pub fn ref_time_block_fullness_percentage(&self) -> u8 {
(self.ref_time * 100 / self.max_ref_time as u128) as u8
}
pub fn proof_size_block_fullness_percentage(&self) -> u8 {
(self.proof_size * 100 / self.max_proof_size as u128) as u8
}
}
/// Information keyed by the platform identifier.
pub type PlatformKeyedInformation<T> = BTreeMap<PlatformIdentifier, T>;
crates/report/src/runner_event.rs
+32 -7
View File
@@ -6,14 +6,16 @@ use std::{collections::BTreeMap, path::PathBuf, sync::Arc};
use alloy::primitives::Address;
use anyhow::Context as _;
use indexmap::IndexMap;
use revive_dt_common::types::ParsedTestSpecifier;
use revive_dt_common::types::PlatformIdentifier;
use revive_dt_compiler::{CompilerInput, CompilerOutput};
use revive_dt_format::metadata::ContractInstance;
use revive_dt_format::metadata::Metadata;
use revive_dt_format::steps::StepPath;
use semver::Version;
use tokio::sync::{broadcast, oneshot};
use crate::MinedBlockInformation;
use crate::TransactionInformation;
use crate::{ExecutionSpecifier, ReporterEvent, TestSpecifier, common::MetadataFilePath};
macro_rules! __report_gen_emit_test_specific {
@@ -481,11 +483,6 @@ define_event! {
/// The channel that the aggregator is to send the receive side of the channel on.
tx: oneshot::Sender<broadcast::Receiver<ReporterEvent>>
},
/// An event emitted by runners when they've discovered a corpus file.
CorpusDiscovery {
/// The contents of the corpus file.
test_specifiers: Vec<ParsedTestSpecifier>
},
/// An event emitted by runners when they've discovered a metadata file.
MetadataFileDiscovery {
/// The path of the metadata file discovered.
@@ -615,7 +612,35 @@ define_event! {
address: Address
},
/// Reports the completion of the run.
Completion {}
Completion {},
/* Benchmarks Events */
/// An event emitted with information on a transaction that was submitted for a certain step
/// of the execution.
StepTransactionInformation {
/// A specifier for the execution that's taking place.
execution_specifier: Arc<ExecutionSpecifier>,
/// The path of the step that this transaction belongs to.
step_path: StepPath,
/// Information about the transaction
transaction_information: TransactionInformation
},
ContractInformation {
/// A specifier for the execution that's taking place.
execution_specifier: Arc<ExecutionSpecifier>,
/// The path of the solidity source code that contains the contract.
source_code_path: PathBuf,
/// The name of the contract
contract_name: String,
/// The size of the contract
contract_size: usize
},
BlockMined {
/// A specifier for the execution that's taking place.
execution_specifier: Arc<ExecutionSpecifier>,
/// Information on the mined block,
mined_block_information: MinedBlockInformation
}
}
}