Compare commits

...

12 Commits

Author SHA1 Message Date
Omar Abdulla aa8b057cb3 Cleanup watcher from temp code 2025-10-22 17:03:13 +03:00
Omar Abdulla d6b927ce49 compute metrics in each report 2025-10-22 17:00:15 +03:00
Omar Abdulla 03b09d49ec Add the block information to the report 2025-10-22 15:36:42 +03:00
Omar Abdulla 3c10f5d2f3 Include the contract information in the report 2025-10-22 15:12:46 +03:00
Omar Abdulla a5853f86b7 Add step information to the benchmark report 2025-10-22 14:48:16 +03:00
Omar Abdulla d4019f39a4 Remove corpus from the report 2025-10-22 13:44:31 +03:00
Omar Abdulla b2c9e77187 Merge branch 'main' into feature/benchmarking-infra-2 2025-10-22 13:37:08 +03:00
Omar Abdulla ecf86f873f Add more models to the report 2025-10-20 12:25:30 +03:00
Omar Abdulla 18a27233cd make report more benchmarks friendly 2025-10-20 10:31:49 +03:00
Omar Abdulla ce38a300f4 Make report format more benchmark friendly 2025-10-20 10:19:00 +03:00
Omar Abdulla 325cb3f6e9 Report the step path to the watcher 2025-10-20 09:38:51 +03:00
Omar Abdulla 3d5feea4a4 Modify the structure of the MinedBlockInformation 2025-10-20 09:24:33 +03:00
16 changed files with 670 additions and 196 deletions
Generated
+3
View File
@@ -5692,6 +5692,7 @@ dependencies = [
"revive-dt-config",
"revive-dt-format",
"revive-dt-node-interaction",
"revive-dt-report",
"serde",
"serde_json",
"serde_with",
@@ -5715,6 +5716,7 @@ dependencies = [
"futures",
"revive-common",
"revive-dt-format",
"revive-dt-report",
]
[[package]]
@@ -5724,6 +5726,7 @@ dependencies = [
"alloy",
"anyhow",
"indexmap 2.10.0",
"itertools 0.14.0",
"paste",
"revive-dt-common",
"revive-dt-compiler",
@@ -181,7 +181,7 @@ where
code,
);
let receipt = self
.execute_transaction(tx)
.execute_transaction(tx, None)
.and_then(|(_, receipt_fut)| receipt_fut)
.await
.inspect_err(|err| {
@@ -218,6 +218,22 @@ where
.inspect_err(|err| error!(?err, "Post-linking compilation failed"))
.context("Failed to compile the post-link contracts")?;
for (contract_path, contract_name_to_info_mapping) in compiler_output.contracts.iter() {
for (contract_name, (contract_bytecode, _)) in contract_name_to_info_mapping.iter() {
let contract_bytecode = hex::decode(contract_bytecode)
.expect("Impossible for us to get an undecodable bytecode after linking");
self.platform_information
.reporter
.report_contract_information_event(
contract_path.to_path_buf(),
contract_name.clone(),
contract_bytecode.len(),
)
.expect("Should not fail");
}
}
self.execution_state = ExecutionState::new(
compiler_output.contracts,
deployed_libraries.unwrap_or_default(),
@@ -279,15 +295,15 @@ where
#[instrument(level = "info", skip_all, fields(driver_id = self.driver_id))]
pub async fn execute_function_call(
&mut self,
_: &StepPath,
step_path: &StepPath,
step: &FunctionCallStep,
) -> Result<usize> {
let deployment_receipts = self
.handle_function_call_contract_deployment(step)
.handle_function_call_contract_deployment(step_path, step)
.await
.context("Failed to deploy contracts for the function call step")?;
let transaction_hash = self
.handle_function_call_execution(step, deployment_receipts)
.handle_function_call_execution(step_path, step, deployment_receipts)
.await
.context("Failed to handle the function call execution")?;
self.handle_function_call_variable_assignment(step, transaction_hash)
@@ -298,6 +314,7 @@ where
async fn handle_function_call_contract_deployment(
&mut self,
step_path: &StepPath,
step: &FunctionCallStep,
) -> Result<HashMap<ContractInstance, TransactionReceipt>> {
let mut instances_we_must_deploy = IndexMap::<ContractInstance, bool>::new();
@@ -329,7 +346,13 @@ where
.await?
};
if let (_, _, Some(receipt)) = self
.get_or_deploy_contract_instance(&instance, caller, calldata, value)
.get_or_deploy_contract_instance(
&instance,
caller,
calldata,
value,
Some(step_path),
)
.await
.context("Failed to get or deploy contract instance during input execution")?
{
@@ -342,6 +365,7 @@ where
async fn handle_function_call_execution(
&mut self,
step_path: &StepPath,
step: &FunctionCallStep,
mut deployment_receipts: HashMap<ContractInstance, TransactionReceipt>,
) -> Result<TxHash> {
@@ -356,7 +380,7 @@ where
let tx = step
.as_transaction(self.resolver.as_ref(), self.default_resolution_context())
.await?;
Ok(self.execute_transaction(tx).await?.0)
Ok(self.execute_transaction(tx, Some(step_path)).await?.0)
}
}
}
@@ -520,6 +544,7 @@ where
deployer: Address,
calldata: Option<&Calldata>,
value: Option<EtherValue>,
step_path: Option<&StepPath>,
) -> Result<(Address, JsonAbi, Option<TransactionReceipt>)> {
if let Some((_, address, abi)) = self
.execution_state
@@ -535,7 +560,7 @@ where
} else {
info!("Contract instance requires deployment.");
let (address, abi, receipt) = self
.deploy_contract(contract_instance, deployer, calldata, value)
.deploy_contract(contract_instance, deployer, calldata, value, step_path)
.await
.context("Failed to deploy contract")?;
info!(
@@ -562,6 +587,7 @@ where
deployer: Address,
calldata: Option<&Calldata>,
value: Option<EtherValue>,
step_path: Option<&StepPath>,
) -> Result<(Address, JsonAbi, TransactionReceipt)> {
let Some(ContractPathAndIdent {
contract_source_path,
@@ -621,7 +647,7 @@ where
};
let receipt = match self
.execute_transaction(tx)
.execute_transaction(tx, step_path)
.and_then(|(_, receipt_fut)| receipt_fut)
.await
{
@@ -671,6 +697,7 @@ where
async fn execute_transaction(
&self,
transaction: TransactionRequest,
step_path: Option<&StepPath>,
) -> anyhow::Result<(TxHash, impl Future<Output = Result<TransactionReceipt>>)> {
let node = self.platform_information.node;
let transaction_hash = node
@@ -680,9 +707,14 @@ where
Span::current().record("transaction_hash", display(transaction_hash));
info!("Submitted transaction");
self.watcher_tx
.send(WatcherEvent::SubmittedTransaction { transaction_hash })
.context("Failed to send the transaction hash to the watcher")?;
if let Some(step_path) = step_path {
self.watcher_tx
.send(WatcherEvent::SubmittedTransaction {
transaction_hash,
step_path: step_path.clone(),
})
.context("Failed to send the transaction hash to the watcher")?;
};
Ok((transaction_hash, async move {
info!("Starting to poll for transaction receipt");
@@ -145,12 +145,14 @@ pub async fn handle_differential_benchmarks(
context.wallet_configuration.highest_private_key_exclusive(),
)));
let (watcher, watcher_tx) = Watcher::new(
platform_identifier,
platform_information
.node
.subscribe_to_full_blocks_information()
.await
.context("Failed to subscribe to full blocks information from the node")?,
test_definition
.reporter
.execution_specific_reporter(0usize, platform_identifier),
);
let driver = Driver::new(
platform_information,
@@ -1,10 +1,15 @@
use std::{collections::HashSet, pin::Pin, sync::Arc};
use std::{
collections::HashMap,
pin::Pin,
sync::Arc,
time::{SystemTime, UNIX_EPOCH},
};
use alloy::primitives::{BlockNumber, TxHash};
use anyhow::Result;
use futures::{Stream, StreamExt};
use revive_dt_common::types::PlatformIdentifier;
use revive_dt_node_interaction::MinedBlockInformation;
use revive_dt_format::steps::StepPath;
use revive_dt_report::{ExecutionSpecificReporter, MinedBlockInformation, TransactionInformation};
use tokio::sync::{
RwLock,
mpsc::{UnboundedReceiver, UnboundedSender, unbounded_channel},
@@ -15,9 +20,6 @@ use tracing::{info, instrument};
/// and MUST NOT be re-used between workloads since it holds important internal state for a given
/// workload and is not designed for reuse.
pub struct Watcher {
/// The identifier of the platform that this watcher is for.
platform_identifier: PlatformIdentifier,
/// The receive side of the channel that all of the drivers and various other parts of the code
/// send events to the watcher on.
rx: UnboundedReceiver<WatcherEvent>,
@@ -25,19 +27,22 @@ pub struct Watcher {
/// This is a stream of the blocks that were mined by the node. This is for a single platform
/// and a single node from that platform.
blocks_stream: Pin<Box<dyn Stream<Item = MinedBlockInformation>>>,
/// The reporter used to send events to the report aggregator.
reporter: ExecutionSpecificReporter,
}
impl Watcher {
pub fn new(
platform_identifier: PlatformIdentifier,
blocks_stream: Pin<Box<dyn Stream<Item = MinedBlockInformation>>>,
reporter: ExecutionSpecificReporter,
) -> (Self, UnboundedSender<WatcherEvent>) {
let (tx, rx) = unbounded_channel::<WatcherEvent>();
(
Self {
platform_identifier,
rx,
blocks_stream,
reporter,
},
tx,
)
@@ -61,7 +66,8 @@ impl Watcher {
// This is the set of the transaction hashes that the watcher should be looking for and
// watch for them in the blocks. The watcher will keep watching for blocks until it sees
// that all of the transactions that it was watching for has been seen in the mined blocks.
let watch_for_transaction_hashes = Arc::new(RwLock::new(HashSet::<TxHash>::new()));
let watch_for_transaction_hashes =
Arc::new(RwLock::new(HashMap::<TxHash, (StepPath, SystemTime)>::new()));
// A boolean that keeps track of whether all of the transactions were submitted or if more
// txs are expected to come through the receive side of the channel. We do not want to rely
@@ -81,11 +87,14 @@ impl Watcher {
// contain nested repetitions and therefore there's no use in doing any
// action if the repetitions are nested.
WatcherEvent::RepetitionStartEvent { .. } => {}
WatcherEvent::SubmittedTransaction { transaction_hash } => {
WatcherEvent::SubmittedTransaction {
transaction_hash,
step_path,
} => {
watch_for_transaction_hashes
.write()
.await
.insert(transaction_hash);
.insert(transaction_hash, (step_path, SystemTime::now()));
}
WatcherEvent::AllTransactionsSubmitted => {
*all_transactions_submitted.write().await = true;
@@ -97,25 +106,21 @@ impl Watcher {
}
}
};
let reporter = self.reporter.clone();
let block_information_watching_task = {
let watch_for_transaction_hashes = watch_for_transaction_hashes.clone();
let all_transactions_submitted = all_transactions_submitted.clone();
let mut blocks_information_stream = self.blocks_stream;
async move {
let mut mined_blocks_information = Vec::new();
// region:TEMPORARY
eprintln!("Watcher information for {}", self.platform_identifier);
eprintln!(
"block_number,block_timestamp,mined_gas,block_gas_limit,tx_count,ref_time,max_ref_time,proof_size,max_proof_size"
);
// endregion:TEMPORARY
while let Some(block) = blocks_information_stream.next().await {
// If the block number is equal to or less than the last block before the
// repetition then we ignore it and continue on to the next block.
if block.block_number <= ignore_block_before {
if block.ethereum_block_information.block_number <= ignore_block_before {
continue;
}
reporter
.report_block_mined_event(block.clone())
.expect("Can't fail");
if *all_transactions_submitted.read().await
&& watch_for_transaction_hashes.read().await.is_empty()
@@ -124,8 +129,8 @@ impl Watcher {
}
info!(
block_number = block.block_number,
block_tx_count = block.transaction_hashes.len(),
block_number = block.ethereum_block_information.block_number,
block_tx_count = block.ethereum_block_information.transaction_hashes.len(),
remaining_transactions = watch_for_transaction_hashes.read().await.len(),
"Observed a block"
);
@@ -134,33 +139,31 @@ impl Watcher {
// are currently watching for.
let mut watch_for_transaction_hashes =
watch_for_transaction_hashes.write().await;
for tx_hash in block.transaction_hashes.iter() {
watch_for_transaction_hashes.remove(tx_hash);
for tx_hash in block.ethereum_block_information.transaction_hashes.iter() {
let Some((step_path, submission_time)) =
watch_for_transaction_hashes.remove(tx_hash)
else {
continue;
};
let transaction_information = TransactionInformation {
transaction_hash: *tx_hash,
submission_timestamp: submission_time
.duration_since(UNIX_EPOCH)
.expect("Can't fail")
.as_secs() as _,
block_timestamp: block.ethereum_block_information.block_timestamp,
block_number: block.ethereum_block_information.block_number,
};
reporter
.report_step_transaction_information_event(
step_path,
transaction_information,
)
.expect("Can't fail")
}
// region:TEMPORARY
// TODO: The following core is TEMPORARY and will be removed once we have proper
// reporting in place and then it can be removed. This serves as as way of doing
// some very simple reporting for the time being.
eprintln!(
"\"{}\",\"{}\",\"{}\",\"{}\",\"{}\",\"{}\",\"{}\",\"{}\",\"{}\"",
block.block_number,
block.block_timestamp,
block.mined_gas,
block.block_gas_limit,
block.transaction_hashes.len(),
block.ref_time,
block.max_ref_time,
block.proof_size,
block.max_proof_size,
);
// endregion:TEMPORARY
mined_blocks_information.push(block);
}
info!("Watcher's Block Watching Task Finished");
mined_blocks_information
}
};
@@ -172,7 +175,7 @@ impl Watcher {
}
}
#[derive(Clone, Copy, Debug, PartialEq, Eq, PartialOrd, Ord, Hash)]
#[derive(Clone, Debug, PartialEq, Eq, PartialOrd, Ord, Hash)]
pub enum WatcherEvent {
/// Informs the watcher that it should begin watching for the blocks mined by the platforms.
/// Before the watcher receives this event it will not be watching for the mined blocks. The
@@ -192,6 +195,8 @@ pub enum WatcherEvent {
SubmittedTransaction {
/// The hash of the submitted transaction.
transaction_hash: TxHash,
/// The step path of the step that the transaction belongs to.
step_path: StepPath,
},
/// Informs the watcher that all of the transactions of this benchmark have been submitted and
+12 -3
View File
@@ -32,9 +32,18 @@ pub async fn create_test_definitions_stream<'a>(
only_execute_failed_tests: Option<&Report>,
reporter: Reporter,
) -> impl Stream<Item = TestDefinition<'a>> {
let cloned_reporter = reporter.clone();
stream::iter(
corpus
.cases_iterator()
.inspect(move |(metadata_file, ..)| {
cloned_reporter
.report_metadata_file_discovery_event(
metadata_file.metadata_file_path.clone(),
metadata_file.content.clone(),
)
.unwrap();
})
.map(move |(metadata_file, case_idx, case, mode)| {
let reporter = reporter.clone();
@@ -310,10 +319,10 @@ impl<'a> TestDefinition<'a> {
};
let test_case_status = report
.test_case_information
.execution_information
.get(&(self.metadata_file_path.to_path_buf().into()))
.and_then(|obj| obj.get(&self.mode))
.and_then(|obj| obj.get(&self.case_idx))
.and_then(|obj| obj.case_reports.get(&self.case_idx))
.and_then(|obj| obj.mode_execution_reports.get(&self.mode))
.and_then(|obj| obj.status.as_ref());
match test_case_status {
+2 -2
View File
@@ -45,12 +45,12 @@ pub enum Step {
}
define_wrapper_type!(
#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash, Serialize, Deserialize)]
#[derive(Clone, Copy, Debug, PartialEq, Eq, PartialOrd, Ord, Hash, Serialize, Deserialize)]
pub struct StepIdx(usize) impl Display, FromStr;
);
define_wrapper_type!(
#[derive(Clone, Debug, PartialEq, Eq, Hash, Serialize, Deserialize)]
#[derive(Clone, Debug, PartialEq, Eq, PartialOrd, Ord, Hash, Serialize, Deserialize)]
#[serde(try_from = "String", into = "String")]
pub struct StepPath(Vec<StepIdx>);
);
+1
View File
@@ -12,6 +12,7 @@ rust-version.workspace = true
revive-common = { workspace = true }
revive-dt-format = { workspace = true }
revive-dt-report = { workspace = true }
alloy = { workspace = true }
anyhow = { workspace = true }
+2 -31
View File
@@ -4,7 +4,7 @@ use std::pin::Pin;
use std::sync::Arc;
use alloy::network::Ethereum;
use alloy::primitives::{Address, BlockNumber, BlockTimestamp, StorageKey, TxHash, U256};
use alloy::primitives::{Address, StorageKey, TxHash, U256};
use alloy::providers::DynProvider;
use alloy::rpc::types::trace::geth::{DiffMode, GethDebugTracingOptions, GethTrace};
use alloy::rpc::types::{EIP1186AccountProofResponse, TransactionReceipt, TransactionRequest};
@@ -13,6 +13,7 @@ use anyhow::Result;
use futures::Stream;
use revive_common::EVMVersion;
use revive_dt_format::traits::ResolverApi;
use revive_dt_report::MinedBlockInformation;
/// An interface for all interactions with Ethereum compatible nodes.
#[allow(clippy::type_complexity)]
@@ -80,33 +81,3 @@ pub trait EthereumNode {
fn provider(&self)
-> Pin<Box<dyn Future<Output = anyhow::Result<DynProvider<Ethereum>>> + '_>>;
}
#[derive(Clone, Debug, PartialEq, Eq, PartialOrd, Ord, Hash)]
pub struct MinedBlockInformation {
/// The block number.
pub block_number: BlockNumber,
/// The block timestamp.
pub block_timestamp: BlockTimestamp,
/// The amount of gas mined in the block.
pub mined_gas: u128,
/// The gas limit of the block.
pub block_gas_limit: u128,
/// The hashes of the transactions that were mined as part of the block.
pub transaction_hashes: Vec<TxHash>,
/// The ref time for substrate based chains.
pub ref_time: u128,
/// The max ref time for substrate based chains.
pub max_ref_time: u64,
/// The proof size for substrate based chains.
pub proof_size: u128,
/// The max proof size for substrate based chains.
pub max_proof_size: u64,
}
+1
View File
@@ -21,6 +21,7 @@ revive-dt-common = { workspace = true }
revive-dt-config = { workspace = true }
revive-dt-format = { workspace = true }
revive-dt-node-interaction = { workspace = true }
revive-dt-report = { workspace = true }
serde = { workspace = true }
serde_json = { workspace = true }
+15 -15
View File
@@ -43,7 +43,8 @@ use revive_dt_common::{
};
use revive_dt_config::*;
use revive_dt_format::traits::ResolverApi;
use revive_dt_node_interaction::{EthereumNode, MinedBlockInformation};
use revive_dt_node_interaction::EthereumNode;
use revive_dt_report::{EthereumMinedBlockInformation, MinedBlockInformation};
use crate::{
Node,
@@ -526,20 +527,19 @@ impl EthereumNode for GethNode {
let mined_block_information_stream = block_stream.filter_map(|block| async {
let block = block.ok()?;
Some(MinedBlockInformation {
block_number: block.number(),
block_timestamp: block.header.timestamp,
mined_gas: block.header.gas_used as _,
block_gas_limit: block.header.gas_limit as _,
transaction_hashes: block
.transactions
.into_hashes()
.as_hashes()
.expect("Must be hashes")
.to_vec(),
ref_time: 0,
max_ref_time: 0,
proof_size: 0,
max_proof_size: 0,
ethereum_block_information: EthereumMinedBlockInformation {
block_number: block.number(),
block_timestamp: block.header.timestamp,
mined_gas: block.header.gas_used as _,
block_gas_limit: block.header.gas_limit as _,
transaction_hashes: block
.transactions
.into_hashes()
.as_hashes()
.expect("Must be hashes")
.to_vec(),
},
substrate_block_information: None,
})
});
@@ -56,7 +56,8 @@ use revive_dt_common::{
};
use revive_dt_config::*;
use revive_dt_format::traits::ResolverApi;
use revive_dt_node_interaction::{EthereumNode, MinedBlockInformation};
use revive_dt_node_interaction::EthereumNode;
use revive_dt_report::{EthereumMinedBlockInformation, MinedBlockInformation};
use crate::{
Node,
@@ -757,20 +758,19 @@ impl EthereumNode for LighthouseGethNode {
let mined_block_information_stream = block_stream.filter_map(|block| async {
let block = block.ok()?;
Some(MinedBlockInformation {
block_number: block.number(),
block_timestamp: block.header.timestamp,
mined_gas: block.header.gas_used as _,
block_gas_limit: block.header.gas_limit as _,
transaction_hashes: block
.transactions
.into_hashes()
.as_hashes()
.expect("Must be hashes")
.to_vec(),
ref_time: 0,
max_ref_time: 0,
proof_size: 0,
max_proof_size: 0,
ethereum_block_information: EthereumMinedBlockInformation {
block_number: block.number(),
block_timestamp: block.header.timestamp,
mined_gas: block.header.gas_used as _,
block_gas_limit: block.header.gas_limit as _,
transaction_hashes: block
.transactions
.into_hashes()
.as_hashes()
.expect("Must be hashes")
.to_vec(),
},
substrate_block_information: None,
})
});
@@ -37,7 +37,10 @@ use sp_core::crypto::Ss58Codec;
use sp_runtime::AccountId32;
use revive_dt_config::*;
use revive_dt_node_interaction::{EthereumNode, MinedBlockInformation};
use revive_dt_node_interaction::EthereumNode;
use revive_dt_report::{
EthereumMinedBlockInformation, MinedBlockInformation, SubstrateMinedBlockInformation,
};
use subxt::{OnlineClient, SubstrateConfig};
use tokio::sync::OnceCell;
use tracing::{instrument, trace};
@@ -557,20 +560,24 @@ impl EthereumNode for SubstrateNode {
let max_proof_size = limits.max_block.proof_size;
Some(MinedBlockInformation {
block_number: substrate_block.number() as _,
block_timestamp: revive_block.header.timestamp,
mined_gas: revive_block.header.gas_used as _,
block_gas_limit: revive_block.header.gas_limit as _,
transaction_hashes: revive_block
.transactions
.into_hashes()
.as_hashes()
.expect("Must be hashes")
.to_vec(),
ref_time: block_ref_time,
max_ref_time,
proof_size: block_proof_size,
max_proof_size,
ethereum_block_information: EthereumMinedBlockInformation {
block_number: revive_block.number(),
block_timestamp: revive_block.header.timestamp,
mined_gas: revive_block.header.gas_used as _,
block_gas_limit: revive_block.header.gas_limit as _,
transaction_hashes: revive_block
.transactions
.into_hashes()
.as_hashes()
.expect("Must be hashes")
.to_vec(),
},
substrate_block_information: Some(SubstrateMinedBlockInformation {
ref_time: block_ref_time,
max_ref_time,
proof_size: block_proof_size,
max_proof_size,
}),
})
}
});
@@ -60,7 +60,10 @@ use revive_common::EVMVersion;
use revive_dt_common::fs::clear_directory;
use revive_dt_config::*;
use revive_dt_format::traits::ResolverApi;
use revive_dt_node_interaction::{EthereumNode, MinedBlockInformation};
use revive_dt_node_interaction::*;
use revive_dt_report::{
EthereumMinedBlockInformation, MinedBlockInformation, SubstrateMinedBlockInformation,
};
use serde_json::json;
use sp_core::crypto::Ss58Codec;
use sp_runtime::AccountId32;
@@ -578,20 +581,24 @@ impl EthereumNode for ZombienetNode {
let max_proof_size = limits.max_block.proof_size;
Some(MinedBlockInformation {
block_number: substrate_block.number() as _,
block_timestamp: revive_block.header.timestamp,
mined_gas: revive_block.header.gas_used as _,
block_gas_limit: revive_block.header.gas_limit as _,
transaction_hashes: revive_block
.transactions
.into_hashes()
.as_hashes()
.expect("Must be hashes")
.to_vec(),
ref_time: block_ref_time,
max_ref_time,
proof_size: block_proof_size,
max_proof_size,
ethereum_block_information: EthereumMinedBlockInformation {
block_number: revive_block.number(),
block_timestamp: revive_block.header.timestamp,
mined_gas: revive_block.header.gas_used as _,
block_gas_limit: revive_block.header.gas_limit as _,
transaction_hashes: revive_block
.transactions
.into_hashes()
.as_hashes()
.expect("Must be hashes")
.to_vec(),
},
substrate_block_information: Some(SubstrateMinedBlockInformation {
ref_time: block_ref_time,
max_ref_time,
proof_size: block_proof_size,
max_proof_size,
}),
})
}
});
+1
View File
@@ -17,6 +17,7 @@ alloy = { workspace = true }
anyhow = { workspace = true }
paste = { workspace = true }
indexmap = { workspace = true, features = ["serde"] }
itertools = { workspace = true }
semver = { workspace = true }
serde = { workspace = true }
serde_json = { workspace = true }
+445 -35
View File
@@ -4,17 +4,19 @@
use std::{
collections::{BTreeMap, BTreeSet, HashMap, HashSet},
fs::OpenOptions,
ops::{Add, Div},
path::PathBuf,
time::{SystemTime, UNIX_EPOCH},
};
use alloy::primitives::Address;
use alloy::primitives::{Address, BlockNumber, BlockTimestamp, TxHash};
use anyhow::{Context as _, Result};
use indexmap::IndexMap;
use revive_dt_common::types::{ParsedTestSpecifier, PlatformIdentifier};
use itertools::Itertools;
use revive_dt_common::types::PlatformIdentifier;
use revive_dt_compiler::{CompilerInput, CompilerOutput, Mode};
use revive_dt_config::Context;
use revive_dt_format::{case::CaseIdx, metadata::ContractInstance};
use revive_dt_format::{case::CaseIdx, metadata::ContractInstance, steps::StepPath};
use semver::Version;
use serde::{Deserialize, Serialize};
use serde_with::{DisplayFromStr, serde_as};
@@ -67,9 +69,6 @@ impl ReportAggregator {
RunnerEvent::SubscribeToEvents(event) => {
self.handle_subscribe_to_events_event(*event);
}
RunnerEvent::CorpusDiscovery(event) => {
self.handle_corpus_file_discovered_event(*event)
}
RunnerEvent::MetadataFileDiscovery(event) => {
self.handle_metadata_file_discovery_event(*event);
}
@@ -106,12 +105,20 @@ impl ReportAggregator {
RunnerEvent::ContractDeployed(event) => {
self.handle_contract_deployed_event(*event);
}
RunnerEvent::Completion(event) => {
self.handle_completion(*event);
RunnerEvent::Completion(_) => {
break;
}
/* Benchmarks Events */
RunnerEvent::StepTransactionInformation(event) => {
self.handle_step_transaction_information(*event)
}
RunnerEvent::ContractInformation(event) => {
self.handle_contract_information(*event);
}
RunnerEvent::BlockMined(event) => self.handle_block_mined(*event),
}
}
self.handle_completion(CompletionEvent {});
debug!("Report aggregation completed");
let file_name = {
@@ -152,10 +159,6 @@ impl ReportAggregator {
let _ = event.tx.send(self.listener_tx.subscribe());
}
fn handle_corpus_file_discovered_event(&mut self, event: CorpusDiscoveryEvent) {
self.report.corpora.extend(event.test_specifiers);
}
fn handle_metadata_file_discovery_event(&mut self, event: MetadataFileDiscoveryEvent) {
self.report.metadata_files.insert(event.path.clone());
}
@@ -234,17 +237,19 @@ impl ReportAggregator {
let case_status = self
.report
.test_case_information
.execution_information
.entry(specifier.metadata_file_path.clone().into())
.or_default()
.entry(specifier.solc_mode.clone())
.or_default()
.case_reports
.iter()
.map(|(case_idx, case_report)| {
(
*case_idx,
case_report.status.clone().expect("Can't be uninitialized"),
)
.flat_map(|(case_idx, mode_to_execution_map)| {
let case_status = mode_to_execution_map
.mode_execution_reports
.get(&specifier.solc_mode)?
.status
.clone()
.expect("Can't be uninitialized");
Some((*case_idx, case_status))
})
.collect::<BTreeMap<_, _>>();
let event = ReporterEvent::MetadataFileSolcModeCombinationExecutionCompleted {
@@ -383,22 +388,159 @@ impl ReportAggregator {
self.execution_information(&event.execution_specifier)
.deployed_contracts
.get_or_insert_default()
.insert(event.contract_instance, event.address);
.insert(event.contract_instance.clone(), event.address);
self.test_case_report(&event.execution_specifier.test_specifier)
.contract_addresses
.entry(event.contract_instance)
.or_default()
.entry(event.execution_specifier.platform_identifier)
.or_default()
.push(event.address);
}
fn handle_completion(&mut self, _: CompletionEvent) {
self.runner_rx.close();
self.handle_metrics_computation();
}
fn test_case_report(&mut self, specifier: &TestSpecifier) -> &mut TestCaseReport {
fn handle_metrics_computation(&mut self) {
for report in self.report.execution_information.values_mut() {
for report in report.case_reports.values_mut() {
for report in report.mode_execution_reports.values_mut() {
for (platform_identifier, block_information) in
report.mined_block_information.iter_mut()
{
block_information.sort_by(|a, b| {
a.ethereum_block_information
.block_timestamp
.cmp(&b.ethereum_block_information.block_timestamp)
});
// Computing the TPS.
let tps = block_information
.iter()
.tuple_windows::<(_, _)>()
.map(|(block1, block2)| {
block2.ethereum_block_information.transaction_hashes.len() as u64
/ (block2.ethereum_block_information.block_timestamp
- block1.ethereum_block_information.block_timestamp)
})
.collect::<Vec<_>>();
report
.metrics
.get_or_insert_default()
.transaction_per_second
.with_list(*platform_identifier, tps);
// Computing the GPS.
let gps = block_information
.iter()
.tuple_windows::<(_, _)>()
.map(|(block1, block2)| {
block2.ethereum_block_information.mined_gas as u64
/ (block2.ethereum_block_information.block_timestamp
- block1.ethereum_block_information.block_timestamp)
})
.collect::<Vec<_>>();
report
.metrics
.get_or_insert_default()
.gas_per_second
.with_list(*platform_identifier, gps);
// Computing the gas block fullness
let gas_block_fullness = block_information
.iter()
.map(|block| block.gas_block_fullness_percentage())
.map(|v| v as u64)
.collect::<Vec<_>>();
report
.metrics
.get_or_insert_default()
.gas_block_fullness
.with_list(*platform_identifier, gas_block_fullness);
// Computing the ref-time block fullness
let reftime_block_fullness = block_information
.iter()
.filter_map(|block| block.ref_time_block_fullness_percentage())
.map(|v| v as u64)
.collect::<Vec<_>>();
dbg!(&reftime_block_fullness);
if !reftime_block_fullness.is_empty() {
report
.metrics
.get_or_insert_default()
.ref_time_block_fullness
.get_or_insert_default()
.with_list(*platform_identifier, reftime_block_fullness);
}
// Computing the proof size block fullness
let proof_size_block_fullness = block_information
.iter()
.filter_map(|block| block.proof_size_block_fullness_percentage())
.map(|v| v as u64)
.collect::<Vec<_>>();
dbg!(&proof_size_block_fullness);
if !proof_size_block_fullness.is_empty() {
report
.metrics
.get_or_insert_default()
.proof_size_block_fullness
.get_or_insert_default()
.with_list(*platform_identifier, proof_size_block_fullness);
}
}
}
}
}
}
fn handle_step_transaction_information(&mut self, event: StepTransactionInformationEvent) {
self.test_case_report(&event.execution_specifier.test_specifier)
.steps
.entry(event.step_path)
.or_default()
.transactions
.entry(event.execution_specifier.platform_identifier)
.or_default()
.push(event.transaction_information);
}
fn handle_contract_information(&mut self, event: ContractInformationEvent) {
self.test_case_report(&event.execution_specifier.test_specifier)
.compiled_contracts
.entry(event.source_code_path)
.or_default()
.entry(event.contract_name)
.or_default()
.contract_size
.insert(
event.execution_specifier.platform_identifier,
event.contract_size,
);
}
fn handle_block_mined(&mut self, event: BlockMinedEvent) {
self.test_case_report(&event.execution_specifier.test_specifier)
.mined_block_information
.entry(event.execution_specifier.platform_identifier)
.or_default()
.push(event.mined_block_information);
}
fn test_case_report(&mut self, specifier: &TestSpecifier) -> &mut ExecutionReport {
self.report
.test_case_information
.execution_information
.entry(specifier.metadata_file_path.clone().into())
.or_default()
.entry(specifier.solc_mode.clone())
.or_default()
.case_reports
.entry(specifier.case_idx)
.or_default()
.mode_execution_reports
.entry(specifier.solc_mode.clone())
.or_default()
}
fn execution_information(
@@ -419,35 +561,69 @@ impl ReportAggregator {
pub struct Report {
/// The context that the tool was started up with.
pub context: Context,
/// The list of corpus files that the tool found.
#[serde_as(as = "Vec<DisplayFromStr>")]
pub corpora: Vec<ParsedTestSpecifier>,
/// The list of metadata files that were found by the tool.
pub metadata_files: BTreeSet<MetadataFilePath>,
/// Metrics from the execution.
#[serde(skip_serializing_if = "Option::is_none")]
pub metrics: Option<Metrics>,
/// Information relating to each test case.
#[serde_as(as = "BTreeMap<_, HashMap<DisplayFromStr, BTreeMap<DisplayFromStr, _>>>")]
pub test_case_information:
BTreeMap<MetadataFilePath, HashMap<Mode, BTreeMap<CaseIdx, TestCaseReport>>>,
pub execution_information: BTreeMap<MetadataFilePath, MetadataFileReport>,
}
impl Report {
pub fn new(context: Context) -> Self {
Self {
context,
corpora: Default::default(),
metrics: Default::default(),
metadata_files: Default::default(),
test_case_information: Default::default(),
execution_information: Default::default(),
}
}
}
#[derive(Clone, Debug, Serialize, Deserialize, Default)]
pub struct TestCaseReport {
pub struct MetadataFileReport {
/// Metrics from the execution.
#[serde(skip_serializing_if = "Option::is_none")]
pub metrics: Option<Metrics>,
/// The report of each case keyed by the case idx.
pub case_reports: BTreeMap<CaseIdx, CaseReport>,
}
#[serde_as]
#[derive(Clone, Debug, Serialize, Deserialize, Default)]
pub struct CaseReport {
/// Metrics from the execution.
#[serde(skip_serializing_if = "Option::is_none")]
pub metrics: Option<Metrics>,
/// The [`ExecutionReport`] for each one of the [`Mode`]s.
#[serde_as(as = "HashMap<DisplayFromStr, _>")]
pub mode_execution_reports: HashMap<Mode, ExecutionReport>,
}
#[derive(Clone, Debug, Serialize, Deserialize, Default)]
pub struct ExecutionReport {
/// Information on the status of the test case and whether it succeeded, failed, or was ignored.
#[serde(skip_serializing_if = "Option::is_none")]
pub status: Option<TestCaseStatus>,
/// Metrics from the execution.
#[serde(skip_serializing_if = "Option::is_none")]
pub metrics: Option<Metrics>,
/// Information related to the execution on one of the platforms.
pub platform_execution: BTreeMap<PlatformIdentifier, Option<ExecutionInformation>>,
#[serde(skip_serializing_if = "BTreeMap::is_empty")]
pub platform_execution: PlatformKeyedInformation<Option<ExecutionInformation>>,
/// Information on the compiled contracts.
#[serde(skip_serializing_if = "BTreeMap::is_empty")]
pub compiled_contracts: BTreeMap<PathBuf, BTreeMap<String, ContractInformation>>,
/// The addresses of the deployed contracts
#[serde(skip_serializing_if = "BTreeMap::is_empty")]
pub contract_addresses: BTreeMap<ContractInstance, PlatformKeyedInformation<Vec<Address>>>,
/// Information on the mined blocks as part of this execution.
#[serde(skip_serializing_if = "BTreeMap::is_empty")]
pub mined_block_information: PlatformKeyedInformation<Vec<MinedBlockInformation>>,
/// Information tracked for each step that was executed.
#[serde(skip_serializing_if = "BTreeMap::is_empty")]
pub steps: BTreeMap<StepPath, StepReport>,
}
/// Information related to the status of the test. Could be that the test succeeded, failed, or that
@@ -545,3 +721,237 @@ pub enum CompilationStatus {
compiler_input: Option<CompilerInput>,
},
}
/// Information on each step in the execution.
#[derive(Clone, Debug, Default, Serialize, Deserialize)]
pub struct StepReport {
/// Information on the transactions submitted as part of this step.
transactions: PlatformKeyedInformation<Vec<TransactionInformation>>,
}
#[derive(Clone, Debug, Serialize, Deserialize)]
pub struct TransactionInformation {
/// The hash of the transaction
pub transaction_hash: TxHash,
pub submission_timestamp: u64,
pub block_timestamp: u64,
pub block_number: BlockNumber,
}
/// The metrics we collect for our benchmarks.
#[derive(Clone, Debug, Default, Serialize, Deserialize)]
pub struct Metrics {
pub transaction_per_second: Metric<u64>,
pub gas_per_second: Metric<u64>,
/* Block Fullness */
pub gas_block_fullness: Metric<u64>,
#[serde(skip_serializing_if = "Option::is_none")]
pub ref_time_block_fullness: Option<Metric<u64>>,
#[serde(skip_serializing_if = "Option::is_none")]
pub proof_size_block_fullness: Option<Metric<u64>>,
}
/// The data that we store for a given metric (e.g., TPS).
#[derive(Clone, Debug, Default, Serialize, Deserialize)]
pub struct Metric<T> {
#[serde(skip_serializing_if = "Option::is_none")]
pub minimum: Option<PlatformKeyedInformation<T>>,
#[serde(skip_serializing_if = "Option::is_none")]
pub maximum: Option<PlatformKeyedInformation<T>>,
#[serde(skip_serializing_if = "Option::is_none")]
pub mean: Option<PlatformKeyedInformation<T>>,
#[serde(skip_serializing_if = "Option::is_none")]
pub median: Option<PlatformKeyedInformation<T>>,
#[serde(skip_serializing_if = "Option::is_none")]
pub raw: Option<PlatformKeyedInformation<Vec<T>>>,
}
impl<T> Metric<T>
where
T: Default
+ Copy
+ Ord
+ PartialOrd
+ Add<Output = T>
+ Div<Output = T>
+ TryFrom<usize, Error: std::fmt::Debug>,
{
pub fn new() -> Self {
Default::default()
}
pub fn platform_identifiers(&self) -> BTreeSet<PlatformIdentifier> {
self.minimum
.as_ref()
.map(|m| m.keys())
.into_iter()
.flatten()
.chain(
self.maximum
.as_ref()
.map(|m| m.keys())
.into_iter()
.flatten(),
)
.chain(self.mean.as_ref().map(|m| m.keys()).into_iter().flatten())
.chain(self.median.as_ref().map(|m| m.keys()).into_iter().flatten())
.chain(self.raw.as_ref().map(|m| m.keys()).into_iter().flatten())
.copied()
.collect()
}
pub fn with_list(
&mut self,
platform_identifier: PlatformIdentifier,
mut list: Vec<T>,
) -> &mut Self {
list.sort();
let Some(min) = list.first().copied() else {
return self;
};
let Some(max) = list.last().copied() else {
return self;
};
let sum = list.iter().fold(T::default(), |acc, num| acc + *num);
let mean = sum / TryInto::<T>::try_into(list.len()).unwrap();
let median = match list.len().is_multiple_of(2) {
true => {
let idx = list.len() / 2;
let val1 = *list.get(idx - 1).unwrap();
let val2 = *list.get(idx).unwrap();
(val1 + val2) / TryInto::<T>::try_into(2usize).unwrap()
}
false => {
let idx = list.len() / 2;
*list.get(idx).unwrap()
}
};
self.minimum
.get_or_insert_default()
.insert(platform_identifier, min);
self.maximum
.get_or_insert_default()
.insert(platform_identifier, max);
self.mean
.get_or_insert_default()
.insert(platform_identifier, mean);
self.median
.get_or_insert_default()
.insert(platform_identifier, median);
self.raw
.get_or_insert_default()
.insert(platform_identifier, list);
self
}
pub fn combine(&self, other: &Self) -> Self {
let mut platform_identifiers = self.platform_identifiers();
platform_identifiers.extend(other.platform_identifiers());
let mut this = Self::new();
for platform_identifier in platform_identifiers {
let mut l1 = self
.raw
.as_ref()
.and_then(|m| m.get(&platform_identifier))
.cloned()
.unwrap_or_default();
let l2 = other
.raw
.as_ref()
.and_then(|m| m.get(&platform_identifier))
.cloned()
.unwrap_or_default();
l1.extend(l2);
this.with_list(platform_identifier, l1);
}
this
}
}
#[derive(Clone, Debug, Serialize, Deserialize, Default)]
pub struct ContractInformation {
/// The size of the contract on the various platforms.
pub contract_size: PlatformKeyedInformation<usize>,
}
#[derive(Clone, Debug, PartialEq, Eq, PartialOrd, Ord, Hash, Serialize, Deserialize)]
pub struct MinedBlockInformation {
pub ethereum_block_information: EthereumMinedBlockInformation,
pub substrate_block_information: Option<SubstrateMinedBlockInformation>,
}
impl MinedBlockInformation {
pub fn gas_block_fullness_percentage(&self) -> u8 {
self.ethereum_block_information
.gas_block_fullness_percentage()
}
pub fn ref_time_block_fullness_percentage(&self) -> Option<u8> {
self.substrate_block_information
.as_ref()
.map(|block| block.ref_time_block_fullness_percentage())
}
pub fn proof_size_block_fullness_percentage(&self) -> Option<u8> {
self.substrate_block_information
.as_ref()
.map(|block| block.proof_size_block_fullness_percentage())
}
}
#[derive(Clone, Debug, PartialEq, Eq, PartialOrd, Ord, Hash, Serialize, Deserialize)]
pub struct EthereumMinedBlockInformation {
/// The block number.
pub block_number: BlockNumber,
/// The block timestamp.
pub block_timestamp: BlockTimestamp,
/// The amount of gas mined in the block.
pub mined_gas: u128,
/// The gas limit of the block.
pub block_gas_limit: u128,
/// The hashes of the transactions that were mined as part of the block.
pub transaction_hashes: Vec<TxHash>,
}
impl EthereumMinedBlockInformation {
pub fn gas_block_fullness_percentage(&self) -> u8 {
(self.mined_gas * 100 / self.block_gas_limit) as u8
}
}
#[derive(Clone, Copy, Debug, PartialEq, Eq, PartialOrd, Ord, Hash, Serialize, Deserialize)]
pub struct SubstrateMinedBlockInformation {
/// The ref time for substrate based chains.
pub ref_time: u128,
/// The max ref time for substrate based chains.
pub max_ref_time: u64,
/// The proof size for substrate based chains.
pub proof_size: u128,
/// The max proof size for substrate based chains.
pub max_proof_size: u64,
}
impl SubstrateMinedBlockInformation {
pub fn ref_time_block_fullness_percentage(&self) -> u8 {
(self.ref_time * 100 / self.max_ref_time as u128) as u8
}
pub fn proof_size_block_fullness_percentage(&self) -> u8 {
(self.proof_size * 100 / self.max_proof_size as u128) as u8
}
}
/// Information keyed by the platform identifier.
pub type PlatformKeyedInformation<T> = BTreeMap<PlatformIdentifier, T>;
+32 -7
View File
@@ -6,14 +6,16 @@ use std::{collections::BTreeMap, path::PathBuf, sync::Arc};
use alloy::primitives::Address;
use anyhow::Context as _;
use indexmap::IndexMap;
use revive_dt_common::types::ParsedTestSpecifier;
use revive_dt_common::types::PlatformIdentifier;
use revive_dt_compiler::{CompilerInput, CompilerOutput};
use revive_dt_format::metadata::ContractInstance;
use revive_dt_format::metadata::Metadata;
use revive_dt_format::steps::StepPath;
use semver::Version;
use tokio::sync::{broadcast, oneshot};
use crate::MinedBlockInformation;
use crate::TransactionInformation;
use crate::{ExecutionSpecifier, ReporterEvent, TestSpecifier, common::MetadataFilePath};
macro_rules! __report_gen_emit_test_specific {
@@ -481,11 +483,6 @@ define_event! {
/// The channel that the aggregator is to send the receive side of the channel on.
tx: oneshot::Sender<broadcast::Receiver<ReporterEvent>>
},
/// An event emitted by runners when they've discovered a corpus file.
CorpusDiscovery {
/// The contents of the corpus file.
test_specifiers: Vec<ParsedTestSpecifier>
},
/// An event emitted by runners when they've discovered a metadata file.
MetadataFileDiscovery {
/// The path of the metadata file discovered.
@@ -615,7 +612,35 @@ define_event! {
address: Address
},
/// Reports the completion of the run.
Completion {}
Completion {},
/* Benchmarks Events */
/// An event emitted with information on a transaction that was submitted for a certain step
/// of the execution.
StepTransactionInformation {
/// A specifier for the execution that's taking place.
execution_specifier: Arc<ExecutionSpecifier>,
/// The path of the step that this transaction belongs to.
step_path: StepPath,
/// Information about the transaction
transaction_information: TransactionInformation
},
ContractInformation {
/// A specifier for the execution that's taking place.
execution_specifier: Arc<ExecutionSpecifier>,
/// The path of the solidity source code that contains the contract.
source_code_path: PathBuf,
/// The name of the contract
contract_name: String,
/// The size of the contract
contract_size: usize
},
BlockMined {
/// A specifier for the execution that's taking place.
execution_specifier: Arc<ExecutionSpecifier>,
/// Information on the mined block,
mined_block_information: MinedBlockInformation
}
}
}