From b71445b63269514b23b7b9d355e277f1a759705c Mon Sep 17 00:00:00 2001 From: Omar Date: Fri, 24 Oct 2025 05:15:29 +0300 Subject: [PATCH] Wire up reporting to benchmarks (#195) * Modify the structure of the `MinedBlockInformation` * Report the step path to the watcher * Make report format more benchmark friendly * make report more benchmarks friendly * Add more models to the report * Remove corpus from the report * Add step information to the benchmark report * Include the contract information in the report * Add the block information to the report * compute metrics in each report * Cleanup watcher from temp code --- Cargo.lock | 3 + .../src/differential_benchmarks/driver.rs | 54 +- .../differential_benchmarks/entry_point.rs | 4 +- .../src/differential_benchmarks/watcher.rs | 97 ++-- crates/core/src/helpers/test.rs | 15 +- crates/format/src/steps.rs | 4 +- crates/node-interaction/Cargo.toml | 1 + crates/node-interaction/src/lib.rs | 33 +- crates/node/Cargo.toml | 1 + crates/node/src/node_implementations/geth.rs | 30 +- .../node_implementations/lighthouse_geth.rs | 30 +- .../src/node_implementations/substrate.rs | 37 +- .../src/node_implementations/zombienet.rs | 37 +- crates/report/Cargo.toml | 1 + crates/report/src/aggregator.rs | 480 ++++++++++++++++-- crates/report/src/runner_event.rs | 39 +- 16 files changed, 670 insertions(+), 196 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 3253ee3..6c274ba 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -5692,6 +5692,7 @@ dependencies = [ "revive-dt-config", "revive-dt-format", "revive-dt-node-interaction", + "revive-dt-report", "serde", "serde_json", "serde_with", @@ -5715,6 +5716,7 @@ dependencies = [ "futures", "revive-common", "revive-dt-format", + "revive-dt-report", ] [[package]] @@ -5724,6 +5726,7 @@ dependencies = [ "alloy", "anyhow", "indexmap 2.10.0", + "itertools 0.14.0", "paste", "revive-dt-common", "revive-dt-compiler", diff --git a/crates/core/src/differential_benchmarks/driver.rs b/crates/core/src/differential_benchmarks/driver.rs index eb21fcc..1d13b69 100644 --- a/crates/core/src/differential_benchmarks/driver.rs +++ b/crates/core/src/differential_benchmarks/driver.rs @@ -181,7 +181,7 @@ where code, ); let receipt = self - .execute_transaction(tx) + .execute_transaction(tx, None) .and_then(|(_, receipt_fut)| receipt_fut) .await .inspect_err(|err| { @@ -218,6 +218,22 @@ where .inspect_err(|err| error!(?err, "Post-linking compilation failed")) .context("Failed to compile the post-link contracts")?; + for (contract_path, contract_name_to_info_mapping) in compiler_output.contracts.iter() { + for (contract_name, (contract_bytecode, _)) in contract_name_to_info_mapping.iter() { + let contract_bytecode = hex::decode(contract_bytecode) + .expect("Impossible for us to get an undecodable bytecode after linking"); + + self.platform_information + .reporter + .report_contract_information_event( + contract_path.to_path_buf(), + contract_name.clone(), + contract_bytecode.len(), + ) + .expect("Should not fail"); + } + } + self.execution_state = ExecutionState::new( compiler_output.contracts, deployed_libraries.unwrap_or_default(), @@ -279,15 +295,15 @@ where #[instrument(level = "info", skip_all, fields(driver_id = self.driver_id))] pub async fn execute_function_call( &mut self, - _: &StepPath, + step_path: &StepPath, step: &FunctionCallStep, ) -> Result { let deployment_receipts = self - .handle_function_call_contract_deployment(step) + .handle_function_call_contract_deployment(step_path, step) .await .context("Failed to deploy contracts for the function call step")?; let transaction_hash = self - .handle_function_call_execution(step, deployment_receipts) + .handle_function_call_execution(step_path, step, deployment_receipts) .await .context("Failed to handle the function call execution")?; self.handle_function_call_variable_assignment(step, transaction_hash) @@ -298,6 +314,7 @@ where async fn handle_function_call_contract_deployment( &mut self, + step_path: &StepPath, step: &FunctionCallStep, ) -> Result> { let mut instances_we_must_deploy = IndexMap::::new(); @@ -329,7 +346,13 @@ where .await? }; if let (_, _, Some(receipt)) = self - .get_or_deploy_contract_instance(&instance, caller, calldata, value) + .get_or_deploy_contract_instance( + &instance, + caller, + calldata, + value, + Some(step_path), + ) .await .context("Failed to get or deploy contract instance during input execution")? { @@ -342,6 +365,7 @@ where async fn handle_function_call_execution( &mut self, + step_path: &StepPath, step: &FunctionCallStep, mut deployment_receipts: HashMap, ) -> Result { @@ -356,7 +380,7 @@ where let tx = step .as_transaction(self.resolver.as_ref(), self.default_resolution_context()) .await?; - Ok(self.execute_transaction(tx).await?.0) + Ok(self.execute_transaction(tx, Some(step_path)).await?.0) } } } @@ -520,6 +544,7 @@ where deployer: Address, calldata: Option<&Calldata>, value: Option, + step_path: Option<&StepPath>, ) -> Result<(Address, JsonAbi, Option)> { if let Some((_, address, abi)) = self .execution_state @@ -535,7 +560,7 @@ where } else { info!("Contract instance requires deployment."); let (address, abi, receipt) = self - .deploy_contract(contract_instance, deployer, calldata, value) + .deploy_contract(contract_instance, deployer, calldata, value, step_path) .await .context("Failed to deploy contract")?; info!( @@ -562,6 +587,7 @@ where deployer: Address, calldata: Option<&Calldata>, value: Option, + step_path: Option<&StepPath>, ) -> Result<(Address, JsonAbi, TransactionReceipt)> { let Some(ContractPathAndIdent { contract_source_path, @@ -621,7 +647,7 @@ where }; let receipt = match self - .execute_transaction(tx) + .execute_transaction(tx, step_path) .and_then(|(_, receipt_fut)| receipt_fut) .await { @@ -671,6 +697,7 @@ where async fn execute_transaction( &self, transaction: TransactionRequest, + step_path: Option<&StepPath>, ) -> anyhow::Result<(TxHash, impl Future>)> { let node = self.platform_information.node; let transaction_hash = node @@ -680,9 +707,14 @@ where Span::current().record("transaction_hash", display(transaction_hash)); info!("Submitted transaction"); - self.watcher_tx - .send(WatcherEvent::SubmittedTransaction { transaction_hash }) - .context("Failed to send the transaction hash to the watcher")?; + if let Some(step_path) = step_path { + self.watcher_tx + .send(WatcherEvent::SubmittedTransaction { + transaction_hash, + step_path: step_path.clone(), + }) + .context("Failed to send the transaction hash to the watcher")?; + }; Ok((transaction_hash, async move { info!("Starting to poll for transaction receipt"); diff --git a/crates/core/src/differential_benchmarks/entry_point.rs b/crates/core/src/differential_benchmarks/entry_point.rs index 16c5c21..17b897b 100644 --- a/crates/core/src/differential_benchmarks/entry_point.rs +++ b/crates/core/src/differential_benchmarks/entry_point.rs @@ -145,12 +145,14 @@ pub async fn handle_differential_benchmarks( context.wallet_configuration.highest_private_key_exclusive(), ))); let (watcher, watcher_tx) = Watcher::new( - platform_identifier, platform_information .node .subscribe_to_full_blocks_information() .await .context("Failed to subscribe to full blocks information from the node")?, + test_definition + .reporter + .execution_specific_reporter(0usize, platform_identifier), ); let driver = Driver::new( platform_information, diff --git a/crates/core/src/differential_benchmarks/watcher.rs b/crates/core/src/differential_benchmarks/watcher.rs index 473f134..41a8815 100644 --- a/crates/core/src/differential_benchmarks/watcher.rs +++ b/crates/core/src/differential_benchmarks/watcher.rs @@ -1,10 +1,15 @@ -use std::{collections::HashSet, pin::Pin, sync::Arc}; +use std::{ + collections::HashMap, + pin::Pin, + sync::Arc, + time::{SystemTime, UNIX_EPOCH}, +}; use alloy::primitives::{BlockNumber, TxHash}; use anyhow::Result; use futures::{Stream, StreamExt}; -use revive_dt_common::types::PlatformIdentifier; -use revive_dt_node_interaction::MinedBlockInformation; +use revive_dt_format::steps::StepPath; +use revive_dt_report::{ExecutionSpecificReporter, MinedBlockInformation, TransactionInformation}; use tokio::sync::{ RwLock, mpsc::{UnboundedReceiver, UnboundedSender, unbounded_channel}, @@ -15,9 +20,6 @@ use tracing::{info, instrument}; /// and MUST NOT be re-used between workloads since it holds important internal state for a given /// workload and is not designed for reuse. pub struct Watcher { - /// The identifier of the platform that this watcher is for. - platform_identifier: PlatformIdentifier, - /// The receive side of the channel that all of the drivers and various other parts of the code /// send events to the watcher on. rx: UnboundedReceiver, @@ -25,19 +27,22 @@ pub struct Watcher { /// This is a stream of the blocks that were mined by the node. This is for a single platform /// and a single node from that platform. blocks_stream: Pin>>, + + /// The reporter used to send events to the report aggregator. + reporter: ExecutionSpecificReporter, } impl Watcher { pub fn new( - platform_identifier: PlatformIdentifier, blocks_stream: Pin>>, + reporter: ExecutionSpecificReporter, ) -> (Self, UnboundedSender) { let (tx, rx) = unbounded_channel::(); ( Self { - platform_identifier, rx, blocks_stream, + reporter, }, tx, ) @@ -61,7 +66,8 @@ impl Watcher { // This is the set of the transaction hashes that the watcher should be looking for and // watch for them in the blocks. The watcher will keep watching for blocks until it sees // that all of the transactions that it was watching for has been seen in the mined blocks. - let watch_for_transaction_hashes = Arc::new(RwLock::new(HashSet::::new())); + let watch_for_transaction_hashes = + Arc::new(RwLock::new(HashMap::::new())); // A boolean that keeps track of whether all of the transactions were submitted or if more // txs are expected to come through the receive side of the channel. We do not want to rely @@ -81,11 +87,14 @@ impl Watcher { // contain nested repetitions and therefore there's no use in doing any // action if the repetitions are nested. WatcherEvent::RepetitionStartEvent { .. } => {} - WatcherEvent::SubmittedTransaction { transaction_hash } => { + WatcherEvent::SubmittedTransaction { + transaction_hash, + step_path, + } => { watch_for_transaction_hashes .write() .await - .insert(transaction_hash); + .insert(transaction_hash, (step_path, SystemTime::now())); } WatcherEvent::AllTransactionsSubmitted => { *all_transactions_submitted.write().await = true; @@ -97,25 +106,21 @@ impl Watcher { } } }; + let reporter = self.reporter.clone(); let block_information_watching_task = { let watch_for_transaction_hashes = watch_for_transaction_hashes.clone(); let all_transactions_submitted = all_transactions_submitted.clone(); let mut blocks_information_stream = self.blocks_stream; async move { - let mut mined_blocks_information = Vec::new(); - - // region:TEMPORARY - eprintln!("Watcher information for {}", self.platform_identifier); - eprintln!( - "block_number,block_timestamp,mined_gas,block_gas_limit,tx_count,ref_time,max_ref_time,proof_size,max_proof_size" - ); - // endregion:TEMPORARY while let Some(block) = blocks_information_stream.next().await { // If the block number is equal to or less than the last block before the // repetition then we ignore it and continue on to the next block. - if block.block_number <= ignore_block_before { + if block.ethereum_block_information.block_number <= ignore_block_before { continue; } + reporter + .report_block_mined_event(block.clone()) + .expect("Can't fail"); if *all_transactions_submitted.read().await && watch_for_transaction_hashes.read().await.is_empty() @@ -124,8 +129,8 @@ impl Watcher { } info!( - block_number = block.block_number, - block_tx_count = block.transaction_hashes.len(), + block_number = block.ethereum_block_information.block_number, + block_tx_count = block.ethereum_block_information.transaction_hashes.len(), remaining_transactions = watch_for_transaction_hashes.read().await.len(), "Observed a block" ); @@ -134,33 +139,31 @@ impl Watcher { // are currently watching for. let mut watch_for_transaction_hashes = watch_for_transaction_hashes.write().await; - for tx_hash in block.transaction_hashes.iter() { - watch_for_transaction_hashes.remove(tx_hash); + for tx_hash in block.ethereum_block_information.transaction_hashes.iter() { + let Some((step_path, submission_time)) = + watch_for_transaction_hashes.remove(tx_hash) + else { + continue; + }; + let transaction_information = TransactionInformation { + transaction_hash: *tx_hash, + submission_timestamp: submission_time + .duration_since(UNIX_EPOCH) + .expect("Can't fail") + .as_secs() as _, + block_timestamp: block.ethereum_block_information.block_timestamp, + block_number: block.ethereum_block_information.block_number, + }; + reporter + .report_step_transaction_information_event( + step_path, + transaction_information, + ) + .expect("Can't fail") } - - // region:TEMPORARY - // TODO: The following core is TEMPORARY and will be removed once we have proper - // reporting in place and then it can be removed. This serves as as way of doing - // some very simple reporting for the time being. - eprintln!( - "\"{}\",\"{}\",\"{}\",\"{}\",\"{}\",\"{}\",\"{}\",\"{}\",\"{}\"", - block.block_number, - block.block_timestamp, - block.mined_gas, - block.block_gas_limit, - block.transaction_hashes.len(), - block.ref_time, - block.max_ref_time, - block.proof_size, - block.max_proof_size, - ); - // endregion:TEMPORARY - - mined_blocks_information.push(block); } info!("Watcher's Block Watching Task Finished"); - mined_blocks_information } }; @@ -172,7 +175,7 @@ impl Watcher { } } -#[derive(Clone, Copy, Debug, PartialEq, Eq, PartialOrd, Ord, Hash)] +#[derive(Clone, Debug, PartialEq, Eq, PartialOrd, Ord, Hash)] pub enum WatcherEvent { /// Informs the watcher that it should begin watching for the blocks mined by the platforms. /// Before the watcher receives this event it will not be watching for the mined blocks. The @@ -192,6 +195,8 @@ pub enum WatcherEvent { SubmittedTransaction { /// The hash of the submitted transaction. transaction_hash: TxHash, + /// The step path of the step that the transaction belongs to. + step_path: StepPath, }, /// Informs the watcher that all of the transactions of this benchmark have been submitted and diff --git a/crates/core/src/helpers/test.rs b/crates/core/src/helpers/test.rs index 6571c6b..261f29f 100644 --- a/crates/core/src/helpers/test.rs +++ b/crates/core/src/helpers/test.rs @@ -32,9 +32,18 @@ pub async fn create_test_definitions_stream<'a>( only_execute_failed_tests: Option<&Report>, reporter: Reporter, ) -> impl Stream> { + let cloned_reporter = reporter.clone(); stream::iter( corpus .cases_iterator() + .inspect(move |(metadata_file, ..)| { + cloned_reporter + .report_metadata_file_discovery_event( + metadata_file.metadata_file_path.clone(), + metadata_file.content.clone(), + ) + .unwrap(); + }) .map(move |(metadata_file, case_idx, case, mode)| { let reporter = reporter.clone(); @@ -310,10 +319,10 @@ impl<'a> TestDefinition<'a> { }; let test_case_status = report - .test_case_information + .execution_information .get(&(self.metadata_file_path.to_path_buf().into())) - .and_then(|obj| obj.get(&self.mode)) - .and_then(|obj| obj.get(&self.case_idx)) + .and_then(|obj| obj.case_reports.get(&self.case_idx)) + .and_then(|obj| obj.mode_execution_reports.get(&self.mode)) .and_then(|obj| obj.status.as_ref()); match test_case_status { diff --git a/crates/format/src/steps.rs b/crates/format/src/steps.rs index 19ab310..27f12ca 100644 --- a/crates/format/src/steps.rs +++ b/crates/format/src/steps.rs @@ -45,12 +45,12 @@ pub enum Step { } define_wrapper_type!( - #[derive(Clone, Copy, Debug, PartialEq, Eq, Hash, Serialize, Deserialize)] + #[derive(Clone, Copy, Debug, PartialEq, Eq, PartialOrd, Ord, Hash, Serialize, Deserialize)] pub struct StepIdx(usize) impl Display, FromStr; ); define_wrapper_type!( - #[derive(Clone, Debug, PartialEq, Eq, Hash, Serialize, Deserialize)] + #[derive(Clone, Debug, PartialEq, Eq, PartialOrd, Ord, Hash, Serialize, Deserialize)] #[serde(try_from = "String", into = "String")] pub struct StepPath(Vec); ); diff --git a/crates/node-interaction/Cargo.toml b/crates/node-interaction/Cargo.toml index f9c9261..3886177 100644 --- a/crates/node-interaction/Cargo.toml +++ b/crates/node-interaction/Cargo.toml @@ -12,6 +12,7 @@ rust-version.workspace = true revive-common = { workspace = true } revive-dt-format = { workspace = true } +revive-dt-report = { workspace = true } alloy = { workspace = true } anyhow = { workspace = true } diff --git a/crates/node-interaction/src/lib.rs b/crates/node-interaction/src/lib.rs index 54796a6..7e05c78 100644 --- a/crates/node-interaction/src/lib.rs +++ b/crates/node-interaction/src/lib.rs @@ -4,7 +4,7 @@ use std::pin::Pin; use std::sync::Arc; use alloy::network::Ethereum; -use alloy::primitives::{Address, BlockNumber, BlockTimestamp, StorageKey, TxHash, U256}; +use alloy::primitives::{Address, StorageKey, TxHash, U256}; use alloy::providers::DynProvider; use alloy::rpc::types::trace::geth::{DiffMode, GethDebugTracingOptions, GethTrace}; use alloy::rpc::types::{EIP1186AccountProofResponse, TransactionReceipt, TransactionRequest}; @@ -13,6 +13,7 @@ use anyhow::Result; use futures::Stream; use revive_common::EVMVersion; use revive_dt_format::traits::ResolverApi; +use revive_dt_report::MinedBlockInformation; /// An interface for all interactions with Ethereum compatible nodes. #[allow(clippy::type_complexity)] @@ -80,33 +81,3 @@ pub trait EthereumNode { fn provider(&self) -> Pin>> + '_>>; } - -#[derive(Clone, Debug, PartialEq, Eq, PartialOrd, Ord, Hash)] -pub struct MinedBlockInformation { - /// The block number. - pub block_number: BlockNumber, - - /// The block timestamp. - pub block_timestamp: BlockTimestamp, - - /// The amount of gas mined in the block. - pub mined_gas: u128, - - /// The gas limit of the block. - pub block_gas_limit: u128, - - /// The hashes of the transactions that were mined as part of the block. - pub transaction_hashes: Vec, - - /// The ref time for substrate based chains. - pub ref_time: u128, - - /// The max ref time for substrate based chains. - pub max_ref_time: u64, - - /// The proof size for substrate based chains. - pub proof_size: u128, - - /// The max proof size for substrate based chains. - pub max_proof_size: u64, -} diff --git a/crates/node/Cargo.toml b/crates/node/Cargo.toml index cffca52..739ce8b 100644 --- a/crates/node/Cargo.toml +++ b/crates/node/Cargo.toml @@ -21,6 +21,7 @@ revive-dt-common = { workspace = true } revive-dt-config = { workspace = true } revive-dt-format = { workspace = true } revive-dt-node-interaction = { workspace = true } +revive-dt-report = { workspace = true } serde = { workspace = true } serde_json = { workspace = true } diff --git a/crates/node/src/node_implementations/geth.rs b/crates/node/src/node_implementations/geth.rs index 9e2a5c0..04bb697 100644 --- a/crates/node/src/node_implementations/geth.rs +++ b/crates/node/src/node_implementations/geth.rs @@ -43,7 +43,8 @@ use revive_dt_common::{ }; use revive_dt_config::*; use revive_dt_format::traits::ResolverApi; -use revive_dt_node_interaction::{EthereumNode, MinedBlockInformation}; +use revive_dt_node_interaction::EthereumNode; +use revive_dt_report::{EthereumMinedBlockInformation, MinedBlockInformation}; use crate::{ Node, @@ -526,20 +527,19 @@ impl EthereumNode for GethNode { let mined_block_information_stream = block_stream.filter_map(|block| async { let block = block.ok()?; Some(MinedBlockInformation { - block_number: block.number(), - block_timestamp: block.header.timestamp, - mined_gas: block.header.gas_used as _, - block_gas_limit: block.header.gas_limit as _, - transaction_hashes: block - .transactions - .into_hashes() - .as_hashes() - .expect("Must be hashes") - .to_vec(), - ref_time: 0, - max_ref_time: 0, - proof_size: 0, - max_proof_size: 0, + ethereum_block_information: EthereumMinedBlockInformation { + block_number: block.number(), + block_timestamp: block.header.timestamp, + mined_gas: block.header.gas_used as _, + block_gas_limit: block.header.gas_limit as _, + transaction_hashes: block + .transactions + .into_hashes() + .as_hashes() + .expect("Must be hashes") + .to_vec(), + }, + substrate_block_information: None, }) }); diff --git a/crates/node/src/node_implementations/lighthouse_geth.rs b/crates/node/src/node_implementations/lighthouse_geth.rs index bd8ceab..4021727 100644 --- a/crates/node/src/node_implementations/lighthouse_geth.rs +++ b/crates/node/src/node_implementations/lighthouse_geth.rs @@ -56,7 +56,8 @@ use revive_dt_common::{ }; use revive_dt_config::*; use revive_dt_format::traits::ResolverApi; -use revive_dt_node_interaction::{EthereumNode, MinedBlockInformation}; +use revive_dt_node_interaction::EthereumNode; +use revive_dt_report::{EthereumMinedBlockInformation, MinedBlockInformation}; use crate::{ Node, @@ -757,20 +758,19 @@ impl EthereumNode for LighthouseGethNode { let mined_block_information_stream = block_stream.filter_map(|block| async { let block = block.ok()?; Some(MinedBlockInformation { - block_number: block.number(), - block_timestamp: block.header.timestamp, - mined_gas: block.header.gas_used as _, - block_gas_limit: block.header.gas_limit as _, - transaction_hashes: block - .transactions - .into_hashes() - .as_hashes() - .expect("Must be hashes") - .to_vec(), - ref_time: 0, - max_ref_time: 0, - proof_size: 0, - max_proof_size: 0, + ethereum_block_information: EthereumMinedBlockInformation { + block_number: block.number(), + block_timestamp: block.header.timestamp, + mined_gas: block.header.gas_used as _, + block_gas_limit: block.header.gas_limit as _, + transaction_hashes: block + .transactions + .into_hashes() + .as_hashes() + .expect("Must be hashes") + .to_vec(), + }, + substrate_block_information: None, }) }); diff --git a/crates/node/src/node_implementations/substrate.rs b/crates/node/src/node_implementations/substrate.rs index c2b3d62..faa551f 100644 --- a/crates/node/src/node_implementations/substrate.rs +++ b/crates/node/src/node_implementations/substrate.rs @@ -37,7 +37,10 @@ use sp_core::crypto::Ss58Codec; use sp_runtime::AccountId32; use revive_dt_config::*; -use revive_dt_node_interaction::{EthereumNode, MinedBlockInformation}; +use revive_dt_node_interaction::EthereumNode; +use revive_dt_report::{ + EthereumMinedBlockInformation, MinedBlockInformation, SubstrateMinedBlockInformation, +}; use subxt::{OnlineClient, SubstrateConfig}; use tokio::sync::OnceCell; use tracing::{instrument, trace}; @@ -557,20 +560,24 @@ impl EthereumNode for SubstrateNode { let max_proof_size = limits.max_block.proof_size; Some(MinedBlockInformation { - block_number: substrate_block.number() as _, - block_timestamp: revive_block.header.timestamp, - mined_gas: revive_block.header.gas_used as _, - block_gas_limit: revive_block.header.gas_limit as _, - transaction_hashes: revive_block - .transactions - .into_hashes() - .as_hashes() - .expect("Must be hashes") - .to_vec(), - ref_time: block_ref_time, - max_ref_time, - proof_size: block_proof_size, - max_proof_size, + ethereum_block_information: EthereumMinedBlockInformation { + block_number: revive_block.number(), + block_timestamp: revive_block.header.timestamp, + mined_gas: revive_block.header.gas_used as _, + block_gas_limit: revive_block.header.gas_limit as _, + transaction_hashes: revive_block + .transactions + .into_hashes() + .as_hashes() + .expect("Must be hashes") + .to_vec(), + }, + substrate_block_information: Some(SubstrateMinedBlockInformation { + ref_time: block_ref_time, + max_ref_time, + proof_size: block_proof_size, + max_proof_size, + }), }) } }); diff --git a/crates/node/src/node_implementations/zombienet.rs b/crates/node/src/node_implementations/zombienet.rs index 0803e63..5ed3624 100644 --- a/crates/node/src/node_implementations/zombienet.rs +++ b/crates/node/src/node_implementations/zombienet.rs @@ -60,7 +60,10 @@ use revive_common::EVMVersion; use revive_dt_common::fs::clear_directory; use revive_dt_config::*; use revive_dt_format::traits::ResolverApi; -use revive_dt_node_interaction::{EthereumNode, MinedBlockInformation}; +use revive_dt_node_interaction::*; +use revive_dt_report::{ + EthereumMinedBlockInformation, MinedBlockInformation, SubstrateMinedBlockInformation, +}; use serde_json::json; use sp_core::crypto::Ss58Codec; use sp_runtime::AccountId32; @@ -578,20 +581,24 @@ impl EthereumNode for ZombienetNode { let max_proof_size = limits.max_block.proof_size; Some(MinedBlockInformation { - block_number: substrate_block.number() as _, - block_timestamp: revive_block.header.timestamp, - mined_gas: revive_block.header.gas_used as _, - block_gas_limit: revive_block.header.gas_limit as _, - transaction_hashes: revive_block - .transactions - .into_hashes() - .as_hashes() - .expect("Must be hashes") - .to_vec(), - ref_time: block_ref_time, - max_ref_time, - proof_size: block_proof_size, - max_proof_size, + ethereum_block_information: EthereumMinedBlockInformation { + block_number: revive_block.number(), + block_timestamp: revive_block.header.timestamp, + mined_gas: revive_block.header.gas_used as _, + block_gas_limit: revive_block.header.gas_limit as _, + transaction_hashes: revive_block + .transactions + .into_hashes() + .as_hashes() + .expect("Must be hashes") + .to_vec(), + }, + substrate_block_information: Some(SubstrateMinedBlockInformation { + ref_time: block_ref_time, + max_ref_time, + proof_size: block_proof_size, + max_proof_size, + }), }) } }); diff --git a/crates/report/Cargo.toml b/crates/report/Cargo.toml index 9ba6e41..89890e7 100644 --- a/crates/report/Cargo.toml +++ b/crates/report/Cargo.toml @@ -17,6 +17,7 @@ alloy = { workspace = true } anyhow = { workspace = true } paste = { workspace = true } indexmap = { workspace = true, features = ["serde"] } +itertools = { workspace = true } semver = { workspace = true } serde = { workspace = true } serde_json = { workspace = true } diff --git a/crates/report/src/aggregator.rs b/crates/report/src/aggregator.rs index 69d9743..87cc379 100644 --- a/crates/report/src/aggregator.rs +++ b/crates/report/src/aggregator.rs @@ -4,17 +4,19 @@ use std::{ collections::{BTreeMap, BTreeSet, HashMap, HashSet}, fs::OpenOptions, + ops::{Add, Div}, path::PathBuf, time::{SystemTime, UNIX_EPOCH}, }; -use alloy::primitives::Address; +use alloy::primitives::{Address, BlockNumber, BlockTimestamp, TxHash}; use anyhow::{Context as _, Result}; use indexmap::IndexMap; -use revive_dt_common::types::{ParsedTestSpecifier, PlatformIdentifier}; +use itertools::Itertools; +use revive_dt_common::types::PlatformIdentifier; use revive_dt_compiler::{CompilerInput, CompilerOutput, Mode}; use revive_dt_config::Context; -use revive_dt_format::{case::CaseIdx, metadata::ContractInstance}; +use revive_dt_format::{case::CaseIdx, metadata::ContractInstance, steps::StepPath}; use semver::Version; use serde::{Deserialize, Serialize}; use serde_with::{DisplayFromStr, serde_as}; @@ -67,9 +69,6 @@ impl ReportAggregator { RunnerEvent::SubscribeToEvents(event) => { self.handle_subscribe_to_events_event(*event); } - RunnerEvent::CorpusDiscovery(event) => { - self.handle_corpus_file_discovered_event(*event) - } RunnerEvent::MetadataFileDiscovery(event) => { self.handle_metadata_file_discovery_event(*event); } @@ -106,12 +105,20 @@ impl ReportAggregator { RunnerEvent::ContractDeployed(event) => { self.handle_contract_deployed_event(*event); } - RunnerEvent::Completion(event) => { - self.handle_completion(*event); + RunnerEvent::Completion(_) => { break; } + /* Benchmarks Events */ + RunnerEvent::StepTransactionInformation(event) => { + self.handle_step_transaction_information(*event) + } + RunnerEvent::ContractInformation(event) => { + self.handle_contract_information(*event); + } + RunnerEvent::BlockMined(event) => self.handle_block_mined(*event), } } + self.handle_completion(CompletionEvent {}); debug!("Report aggregation completed"); let file_name = { @@ -152,10 +159,6 @@ impl ReportAggregator { let _ = event.tx.send(self.listener_tx.subscribe()); } - fn handle_corpus_file_discovered_event(&mut self, event: CorpusDiscoveryEvent) { - self.report.corpora.extend(event.test_specifiers); - } - fn handle_metadata_file_discovery_event(&mut self, event: MetadataFileDiscoveryEvent) { self.report.metadata_files.insert(event.path.clone()); } @@ -234,17 +237,19 @@ impl ReportAggregator { let case_status = self .report - .test_case_information + .execution_information .entry(specifier.metadata_file_path.clone().into()) .or_default() - .entry(specifier.solc_mode.clone()) - .or_default() + .case_reports .iter() - .map(|(case_idx, case_report)| { - ( - *case_idx, - case_report.status.clone().expect("Can't be uninitialized"), - ) + .flat_map(|(case_idx, mode_to_execution_map)| { + let case_status = mode_to_execution_map + .mode_execution_reports + .get(&specifier.solc_mode)? + .status + .clone() + .expect("Can't be uninitialized"); + Some((*case_idx, case_status)) }) .collect::>(); let event = ReporterEvent::MetadataFileSolcModeCombinationExecutionCompleted { @@ -383,22 +388,159 @@ impl ReportAggregator { self.execution_information(&event.execution_specifier) .deployed_contracts .get_or_insert_default() - .insert(event.contract_instance, event.address); + .insert(event.contract_instance.clone(), event.address); + self.test_case_report(&event.execution_specifier.test_specifier) + .contract_addresses + .entry(event.contract_instance) + .or_default() + .entry(event.execution_specifier.platform_identifier) + .or_default() + .push(event.address); } fn handle_completion(&mut self, _: CompletionEvent) { self.runner_rx.close(); + self.handle_metrics_computation(); } - fn test_case_report(&mut self, specifier: &TestSpecifier) -> &mut TestCaseReport { + fn handle_metrics_computation(&mut self) { + for report in self.report.execution_information.values_mut() { + for report in report.case_reports.values_mut() { + for report in report.mode_execution_reports.values_mut() { + for (platform_identifier, block_information) in + report.mined_block_information.iter_mut() + { + block_information.sort_by(|a, b| { + a.ethereum_block_information + .block_timestamp + .cmp(&b.ethereum_block_information.block_timestamp) + }); + + // Computing the TPS. + let tps = block_information + .iter() + .tuple_windows::<(_, _)>() + .map(|(block1, block2)| { + block2.ethereum_block_information.transaction_hashes.len() as u64 + / (block2.ethereum_block_information.block_timestamp + - block1.ethereum_block_information.block_timestamp) + }) + .collect::>(); + report + .metrics + .get_or_insert_default() + .transaction_per_second + .with_list(*platform_identifier, tps); + + // Computing the GPS. + let gps = block_information + .iter() + .tuple_windows::<(_, _)>() + .map(|(block1, block2)| { + block2.ethereum_block_information.mined_gas as u64 + / (block2.ethereum_block_information.block_timestamp + - block1.ethereum_block_information.block_timestamp) + }) + .collect::>(); + report + .metrics + .get_or_insert_default() + .gas_per_second + .with_list(*platform_identifier, gps); + + // Computing the gas block fullness + let gas_block_fullness = block_information + .iter() + .map(|block| block.gas_block_fullness_percentage()) + .map(|v| v as u64) + .collect::>(); + report + .metrics + .get_or_insert_default() + .gas_block_fullness + .with_list(*platform_identifier, gas_block_fullness); + + // Computing the ref-time block fullness + let reftime_block_fullness = block_information + .iter() + .filter_map(|block| block.ref_time_block_fullness_percentage()) + .map(|v| v as u64) + .collect::>(); + dbg!(&reftime_block_fullness); + if !reftime_block_fullness.is_empty() { + report + .metrics + .get_or_insert_default() + .ref_time_block_fullness + .get_or_insert_default() + .with_list(*platform_identifier, reftime_block_fullness); + } + + // Computing the proof size block fullness + let proof_size_block_fullness = block_information + .iter() + .filter_map(|block| block.proof_size_block_fullness_percentage()) + .map(|v| v as u64) + .collect::>(); + dbg!(&proof_size_block_fullness); + if !proof_size_block_fullness.is_empty() { + report + .metrics + .get_or_insert_default() + .proof_size_block_fullness + .get_or_insert_default() + .with_list(*platform_identifier, proof_size_block_fullness); + } + } + } + } + } + } + + fn handle_step_transaction_information(&mut self, event: StepTransactionInformationEvent) { + self.test_case_report(&event.execution_specifier.test_specifier) + .steps + .entry(event.step_path) + .or_default() + .transactions + .entry(event.execution_specifier.platform_identifier) + .or_default() + .push(event.transaction_information); + } + + fn handle_contract_information(&mut self, event: ContractInformationEvent) { + self.test_case_report(&event.execution_specifier.test_specifier) + .compiled_contracts + .entry(event.source_code_path) + .or_default() + .entry(event.contract_name) + .or_default() + .contract_size + .insert( + event.execution_specifier.platform_identifier, + event.contract_size, + ); + } + + fn handle_block_mined(&mut self, event: BlockMinedEvent) { + self.test_case_report(&event.execution_specifier.test_specifier) + .mined_block_information + .entry(event.execution_specifier.platform_identifier) + .or_default() + .push(event.mined_block_information); + } + + fn test_case_report(&mut self, specifier: &TestSpecifier) -> &mut ExecutionReport { self.report - .test_case_information + .execution_information .entry(specifier.metadata_file_path.clone().into()) .or_default() - .entry(specifier.solc_mode.clone()) - .or_default() + .case_reports .entry(specifier.case_idx) .or_default() + .mode_execution_reports + .entry(specifier.solc_mode.clone()) + .or_default() } fn execution_information( @@ -419,35 +561,69 @@ impl ReportAggregator { pub struct Report { /// The context that the tool was started up with. pub context: Context, - /// The list of corpus files that the tool found. - #[serde_as(as = "Vec")] - pub corpora: Vec, /// The list of metadata files that were found by the tool. pub metadata_files: BTreeSet, + /// Metrics from the execution. + #[serde(skip_serializing_if = "Option::is_none")] + pub metrics: Option, /// Information relating to each test case. - #[serde_as(as = "BTreeMap<_, HashMap>>")] - pub test_case_information: - BTreeMap>>, + pub execution_information: BTreeMap, } impl Report { pub fn new(context: Context) -> Self { Self { context, - corpora: Default::default(), + metrics: Default::default(), metadata_files: Default::default(), - test_case_information: Default::default(), + execution_information: Default::default(), } } } #[derive(Clone, Debug, Serialize, Deserialize, Default)] -pub struct TestCaseReport { +pub struct MetadataFileReport { + /// Metrics from the execution. + #[serde(skip_serializing_if = "Option::is_none")] + pub metrics: Option, + /// The report of each case keyed by the case idx. + pub case_reports: BTreeMap, +} + +#[serde_as] +#[derive(Clone, Debug, Serialize, Deserialize, Default)] +pub struct CaseReport { + /// Metrics from the execution. + #[serde(skip_serializing_if = "Option::is_none")] + pub metrics: Option, + /// The [`ExecutionReport`] for each one of the [`Mode`]s. + #[serde_as(as = "HashMap")] + pub mode_execution_reports: HashMap, +} + +#[derive(Clone, Debug, Serialize, Deserialize, Default)] +pub struct ExecutionReport { /// Information on the status of the test case and whether it succeeded, failed, or was ignored. #[serde(skip_serializing_if = "Option::is_none")] pub status: Option, + /// Metrics from the execution. + #[serde(skip_serializing_if = "Option::is_none")] + pub metrics: Option, /// Information related to the execution on one of the platforms. - pub platform_execution: BTreeMap>, + #[serde(skip_serializing_if = "BTreeMap::is_empty")] + pub platform_execution: PlatformKeyedInformation>, + /// Information on the compiled contracts. + #[serde(skip_serializing_if = "BTreeMap::is_empty")] + pub compiled_contracts: BTreeMap>, + /// The addresses of the deployed contracts + #[serde(skip_serializing_if = "BTreeMap::is_empty")] + pub contract_addresses: BTreeMap>>, + /// Information on the mined blocks as part of this execution. + #[serde(skip_serializing_if = "BTreeMap::is_empty")] + pub mined_block_information: PlatformKeyedInformation>, + /// Information tracked for each step that was executed. + #[serde(skip_serializing_if = "BTreeMap::is_empty")] + pub steps: BTreeMap, } /// Information related to the status of the test. Could be that the test succeeded, failed, or that @@ -545,3 +721,237 @@ pub enum CompilationStatus { compiler_input: Option, }, } + +/// Information on each step in the execution. +#[derive(Clone, Debug, Default, Serialize, Deserialize)] +pub struct StepReport { + /// Information on the transactions submitted as part of this step. + transactions: PlatformKeyedInformation>, +} + +#[derive(Clone, Debug, Serialize, Deserialize)] +pub struct TransactionInformation { + /// The hash of the transaction + pub transaction_hash: TxHash, + pub submission_timestamp: u64, + pub block_timestamp: u64, + pub block_number: BlockNumber, +} + +/// The metrics we collect for our benchmarks. +#[derive(Clone, Debug, Default, Serialize, Deserialize)] +pub struct Metrics { + pub transaction_per_second: Metric, + pub gas_per_second: Metric, + /* Block Fullness */ + pub gas_block_fullness: Metric, + #[serde(skip_serializing_if = "Option::is_none")] + pub ref_time_block_fullness: Option>, + #[serde(skip_serializing_if = "Option::is_none")] + pub proof_size_block_fullness: Option>, +} + +/// The data that we store for a given metric (e.g., TPS). +#[derive(Clone, Debug, Default, Serialize, Deserialize)] +pub struct Metric { + #[serde(skip_serializing_if = "Option::is_none")] + pub minimum: Option>, + #[serde(skip_serializing_if = "Option::is_none")] + pub maximum: Option>, + #[serde(skip_serializing_if = "Option::is_none")] + pub mean: Option>, + #[serde(skip_serializing_if = "Option::is_none")] + pub median: Option>, + #[serde(skip_serializing_if = "Option::is_none")] + pub raw: Option>>, +} + +impl Metric +where + T: Default + + Copy + + Ord + + PartialOrd + + Add + + Div + + TryFrom, +{ + pub fn new() -> Self { + Default::default() + } + + pub fn platform_identifiers(&self) -> BTreeSet { + self.minimum + .as_ref() + .map(|m| m.keys()) + .into_iter() + .flatten() + .chain( + self.maximum + .as_ref() + .map(|m| m.keys()) + .into_iter() + .flatten(), + ) + .chain(self.mean.as_ref().map(|m| m.keys()).into_iter().flatten()) + .chain(self.median.as_ref().map(|m| m.keys()).into_iter().flatten()) + .chain(self.raw.as_ref().map(|m| m.keys()).into_iter().flatten()) + .copied() + .collect() + } + + pub fn with_list( + &mut self, + platform_identifier: PlatformIdentifier, + mut list: Vec, + ) -> &mut Self { + list.sort(); + let Some(min) = list.first().copied() else { + return self; + }; + let Some(max) = list.last().copied() else { + return self; + }; + let sum = list.iter().fold(T::default(), |acc, num| acc + *num); + let mean = sum / TryInto::::try_into(list.len()).unwrap(); + + let median = match list.len().is_multiple_of(2) { + true => { + let idx = list.len() / 2; + let val1 = *list.get(idx - 1).unwrap(); + let val2 = *list.get(idx).unwrap(); + (val1 + val2) / TryInto::::try_into(2usize).unwrap() + } + false => { + let idx = list.len() / 2; + *list.get(idx).unwrap() + } + }; + + self.minimum + .get_or_insert_default() + .insert(platform_identifier, min); + self.maximum + .get_or_insert_default() + .insert(platform_identifier, max); + self.mean + .get_or_insert_default() + .insert(platform_identifier, mean); + self.median + .get_or_insert_default() + .insert(platform_identifier, median); + self.raw + .get_or_insert_default() + .insert(platform_identifier, list); + + self + } + + pub fn combine(&self, other: &Self) -> Self { + let mut platform_identifiers = self.platform_identifiers(); + platform_identifiers.extend(other.platform_identifiers()); + + let mut this = Self::new(); + for platform_identifier in platform_identifiers { + let mut l1 = self + .raw + .as_ref() + .and_then(|m| m.get(&platform_identifier)) + .cloned() + .unwrap_or_default(); + let l2 = other + .raw + .as_ref() + .and_then(|m| m.get(&platform_identifier)) + .cloned() + .unwrap_or_default(); + l1.extend(l2); + this.with_list(platform_identifier, l1); + } + + this + } +} + +#[derive(Clone, Debug, Serialize, Deserialize, Default)] +pub struct ContractInformation { + /// The size of the contract on the various platforms. + pub contract_size: PlatformKeyedInformation, +} + +#[derive(Clone, Debug, PartialEq, Eq, PartialOrd, Ord, Hash, Serialize, Deserialize)] +pub struct MinedBlockInformation { + pub ethereum_block_information: EthereumMinedBlockInformation, + pub substrate_block_information: Option, +} + +impl MinedBlockInformation { + pub fn gas_block_fullness_percentage(&self) -> u8 { + self.ethereum_block_information + .gas_block_fullness_percentage() + } + + pub fn ref_time_block_fullness_percentage(&self) -> Option { + self.substrate_block_information + .as_ref() + .map(|block| block.ref_time_block_fullness_percentage()) + } + + pub fn proof_size_block_fullness_percentage(&self) -> Option { + self.substrate_block_information + .as_ref() + .map(|block| block.proof_size_block_fullness_percentage()) + } +} + +#[derive(Clone, Debug, PartialEq, Eq, PartialOrd, Ord, Hash, Serialize, Deserialize)] +pub struct EthereumMinedBlockInformation { + /// The block number. + pub block_number: BlockNumber, + + /// The block timestamp. + pub block_timestamp: BlockTimestamp, + + /// The amount of gas mined in the block. + pub mined_gas: u128, + + /// The gas limit of the block. + pub block_gas_limit: u128, + + /// The hashes of the transactions that were mined as part of the block. + pub transaction_hashes: Vec, +} + +impl EthereumMinedBlockInformation { + pub fn gas_block_fullness_percentage(&self) -> u8 { + (self.mined_gas * 100 / self.block_gas_limit) as u8 + } +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq, PartialOrd, Ord, Hash, Serialize, Deserialize)] +pub struct SubstrateMinedBlockInformation { + /// The ref time for substrate based chains. + pub ref_time: u128, + + /// The max ref time for substrate based chains. + pub max_ref_time: u64, + + /// The proof size for substrate based chains. + pub proof_size: u128, + + /// The max proof size for substrate based chains. + pub max_proof_size: u64, +} + +impl SubstrateMinedBlockInformation { + pub fn ref_time_block_fullness_percentage(&self) -> u8 { + (self.ref_time * 100 / self.max_ref_time as u128) as u8 + } + + pub fn proof_size_block_fullness_percentage(&self) -> u8 { + (self.proof_size * 100 / self.max_proof_size as u128) as u8 + } +} + +/// Information keyed by the platform identifier. +pub type PlatformKeyedInformation = BTreeMap; diff --git a/crates/report/src/runner_event.rs b/crates/report/src/runner_event.rs index 9f3d62b..0a142b9 100644 --- a/crates/report/src/runner_event.rs +++ b/crates/report/src/runner_event.rs @@ -6,14 +6,16 @@ use std::{collections::BTreeMap, path::PathBuf, sync::Arc}; use alloy::primitives::Address; use anyhow::Context as _; use indexmap::IndexMap; -use revive_dt_common::types::ParsedTestSpecifier; use revive_dt_common::types::PlatformIdentifier; use revive_dt_compiler::{CompilerInput, CompilerOutput}; use revive_dt_format::metadata::ContractInstance; use revive_dt_format::metadata::Metadata; +use revive_dt_format::steps::StepPath; use semver::Version; use tokio::sync::{broadcast, oneshot}; +use crate::MinedBlockInformation; +use crate::TransactionInformation; use crate::{ExecutionSpecifier, ReporterEvent, TestSpecifier, common::MetadataFilePath}; macro_rules! __report_gen_emit_test_specific { @@ -481,11 +483,6 @@ define_event! { /// The channel that the aggregator is to send the receive side of the channel on. tx: oneshot::Sender> }, - /// An event emitted by runners when they've discovered a corpus file. - CorpusDiscovery { - /// The contents of the corpus file. - test_specifiers: Vec - }, /// An event emitted by runners when they've discovered a metadata file. MetadataFileDiscovery { /// The path of the metadata file discovered. @@ -615,7 +612,35 @@ define_event! { address: Address }, /// Reports the completion of the run. - Completion {} + Completion {}, + + /* Benchmarks Events */ + /// An event emitted with information on a transaction that was submitted for a certain step + /// of the execution. + StepTransactionInformation { + /// A specifier for the execution that's taking place. + execution_specifier: Arc, + /// The path of the step that this transaction belongs to. + step_path: StepPath, + /// Information about the transaction + transaction_information: TransactionInformation + }, + ContractInformation { + /// A specifier for the execution that's taking place. + execution_specifier: Arc, + /// The path of the solidity source code that contains the contract. + source_code_path: PathBuf, + /// The name of the contract + contract_name: String, + /// The size of the contract + contract_size: usize + }, + BlockMined { + /// A specifier for the execution that's taking place. + execution_specifier: Arc, + /// Information on the mined block, + mined_block_information: MinedBlockInformation + } } }