mirror of
https://github.com/pezkuwichain/revive-differential-tests.git
synced 2026-06-14 06:21:09 +00:00
Updated Reporting Infrastructure (#151)
* Remove the old reporting infra * Use the Test struct more in the code * Implement the initial set of reporter events * Add more runner events to the reporter and refine the structure * Add reporting infra for reporting ignored tests * Update report to use better map data structures * Add case status information to the report * Integrate the reporting infrastructure with the CLI reporter used by the program. * Include contract compilation information in report * Cleanup report model * Add information on the deployed contracts
This commit is contained in:
Omar
@@ -0,0 +1,550 @@
|
||||
//! Implementation of the report aggregator task which consumes the events sent by the various
|
||||
//! reporters and combines them into a single unified report.
|
||||
|
||||
use std::{
|
||||
collections::{BTreeMap, BTreeSet, HashMap, HashSet},
|
||||
fs::OpenOptions,
|
||||
path::PathBuf,
|
||||
time::{SystemTime, UNIX_EPOCH},
|
||||
};
|
||||
|
||||
use alloy_primitives::Address;
|
||||
use anyhow::Result;
|
||||
use indexmap::IndexMap;
|
||||
use revive_dt_compiler::{CompilerInput, CompilerOutput, Mode};
|
||||
use revive_dt_config::{Arguments, TestingPlatform};
|
||||
use revive_dt_format::{case::CaseIdx, corpus::Corpus, metadata::ContractInstance};
|
||||
use semver::Version;
|
||||
use serde::Serialize;
|
||||
use serde_with::{DisplayFromStr, serde_as};
|
||||
use tokio::sync::{
|
||||
broadcast::{Sender, channel},
|
||||
mpsc::{UnboundedReceiver, UnboundedSender, unbounded_channel},
|
||||
};
|
||||
use tracing::debug;
|
||||
|
||||
use crate::*;
|
||||
|
||||
pub struct ReportAggregator {
|
||||
/* Internal Report State */
|
||||
report: Report,
|
||||
remaining_cases: HashMap<MetadataFilePath, HashMap<Mode, HashSet<CaseIdx>>>,
|
||||
/* Channels */
|
||||
runner_tx: Option<UnboundedSender<RunnerEvent>>,
|
||||
runner_rx: UnboundedReceiver<RunnerEvent>,
|
||||
listener_tx: Sender<ReporterEvent>,
|
||||
}
|
||||
|
||||
impl ReportAggregator {
|
||||
pub fn new(config: Arguments) -> Self {
|
||||
let (runner_tx, runner_rx) = unbounded_channel::<RunnerEvent>();
|
||||
let (listener_tx, _) = channel::<ReporterEvent>(1024);
|
||||
Self {
|
||||
report: Report::new(config),
|
||||
remaining_cases: Default::default(),
|
||||
runner_tx: Some(runner_tx),
|
||||
runner_rx,
|
||||
listener_tx,
|
||||
}
|
||||
}
|
||||
|
||||
pub fn into_task(mut self) -> (Reporter, impl Future<Output = Result<()>>) {
|
||||
let reporter = self
|
||||
.runner_tx
|
||||
.take()
|
||||
.map(Into::into)
|
||||
.expect("Can't fail since this can only be called once");
|
||||
(reporter, async move { self.aggregate().await })
|
||||
}
|
||||
|
||||
async fn aggregate(mut self) -> Result<()> {
|
||||
debug!("Starting to aggregate report");
|
||||
|
||||
while let Some(event) = self.runner_rx.recv().await {
|
||||
debug!(?event, "Received Event");
|
||||
match event {
|
||||
RunnerEvent::SubscribeToEvents(event) => {
|
||||
self.handle_subscribe_to_events_event(*event);
|
||||
}
|
||||
RunnerEvent::CorpusFileDiscovery(event) => {
|
||||
self.handle_corpus_file_discovered_event(*event)
|
||||
}
|
||||
RunnerEvent::MetadataFileDiscovery(event) => {
|
||||
self.handle_metadata_file_discovery_event(*event);
|
||||
}
|
||||
RunnerEvent::TestCaseDiscovery(event) => {
|
||||
self.handle_test_case_discovery(*event);
|
||||
}
|
||||
RunnerEvent::TestSucceeded(event) => {
|
||||
self.handle_test_succeeded_event(*event);
|
||||
}
|
||||
RunnerEvent::TestFailed(event) => {
|
||||
self.handle_test_failed_event(*event);
|
||||
}
|
||||
RunnerEvent::TestIgnored(event) => {
|
||||
self.handle_test_ignored_event(*event);
|
||||
}
|
||||
RunnerEvent::LeaderNodeAssigned(event) => {
|
||||
self.handle_leader_node_assigned_event(*event);
|
||||
}
|
||||
RunnerEvent::FollowerNodeAssigned(event) => {
|
||||
self.handle_follower_node_assigned_event(*event);
|
||||
}
|
||||
RunnerEvent::PreLinkContractsCompilationSucceeded(event) => {
|
||||
self.handle_pre_link_contracts_compilation_succeeded_event(*event)
|
||||
}
|
||||
RunnerEvent::PostLinkContractsCompilationSucceeded(event) => {
|
||||
self.handle_post_link_contracts_compilation_succeeded_event(*event)
|
||||
}
|
||||
RunnerEvent::PreLinkContractsCompilationFailed(event) => {
|
||||
self.handle_pre_link_contracts_compilation_failed_event(*event)
|
||||
}
|
||||
RunnerEvent::PostLinkContractsCompilationFailed(event) => {
|
||||
self.handle_post_link_contracts_compilation_failed_event(*event)
|
||||
}
|
||||
RunnerEvent::LibrariesDeployed(event) => {
|
||||
self.handle_libraries_deployed_event(*event);
|
||||
}
|
||||
RunnerEvent::ContractDeployed(event) => {
|
||||
self.handle_contract_deployed_event(*event);
|
||||
}
|
||||
}
|
||||
}
|
||||
debug!("Report aggregation completed");
|
||||
|
||||
let file_name = {
|
||||
let current_timestamp = SystemTime::now().duration_since(UNIX_EPOCH)?.as_secs();
|
||||
let mut file_name = current_timestamp.to_string();
|
||||
file_name.push_str(".json");
|
||||
file_name
|
||||
};
|
||||
let file_path = self.report.config.directory().join(file_name);
|
||||
let file = OpenOptions::new()
|
||||
.create(true)
|
||||
.write(true)
|
||||
.truncate(true)
|
||||
.read(false)
|
||||
.open(file_path)?;
|
||||
serde_json::to_writer_pretty(file, &self.report)?;
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn handle_subscribe_to_events_event(&self, event: SubscribeToEventsEvent) {
|
||||
let _ = event.tx.send(self.listener_tx.subscribe());
|
||||
}
|
||||
|
||||
fn handle_corpus_file_discovered_event(&mut self, event: CorpusFileDiscoveryEvent) {
|
||||
self.report.corpora.push(event.corpus);
|
||||
}
|
||||
|
||||
fn handle_metadata_file_discovery_event(&mut self, event: MetadataFileDiscoveryEvent) {
|
||||
self.report.metadata_files.insert(event.path.clone());
|
||||
}
|
||||
|
||||
fn handle_test_case_discovery(&mut self, event: TestCaseDiscoveryEvent) {
|
||||
self.remaining_cases
|
||||
.entry(event.test_specifier.metadata_file_path.clone().into())
|
||||
.or_default()
|
||||
.entry(event.test_specifier.solc_mode.clone())
|
||||
.or_default()
|
||||
.insert(event.test_specifier.case_idx);
|
||||
}
|
||||
|
||||
fn handle_test_succeeded_event(&mut self, event: TestSucceededEvent) {
|
||||
// Remove this from the set of cases we're tracking since it has completed.
|
||||
self.remaining_cases
|
||||
.entry(event.test_specifier.metadata_file_path.clone().into())
|
||||
.or_default()
|
||||
.entry(event.test_specifier.solc_mode.clone())
|
||||
.or_default()
|
||||
.remove(&event.test_specifier.case_idx);
|
||||
|
||||
// Add information on the fact that the case was ignored to the report.
|
||||
let test_case_report = self.test_case_report(&event.test_specifier);
|
||||
test_case_report.status = Some(TestCaseStatus::Succeeded {
|
||||
steps_executed: event.steps_executed,
|
||||
});
|
||||
self.handle_post_test_case_status_update(&event.test_specifier);
|
||||
}
|
||||
|
||||
fn handle_test_failed_event(&mut self, event: TestFailedEvent) {
|
||||
// Remove this from the set of cases we're tracking since it has completed.
|
||||
self.remaining_cases
|
||||
.entry(event.test_specifier.metadata_file_path.clone().into())
|
||||
.or_default()
|
||||
.entry(event.test_specifier.solc_mode.clone())
|
||||
.or_default()
|
||||
.remove(&event.test_specifier.case_idx);
|
||||
|
||||
// Add information on the fact that the case was ignored to the report.
|
||||
let test_case_report = self.test_case_report(&event.test_specifier);
|
||||
test_case_report.status = Some(TestCaseStatus::Failed {
|
||||
reason: event.reason,
|
||||
});
|
||||
self.handle_post_test_case_status_update(&event.test_specifier);
|
||||
}
|
||||
|
||||
fn handle_test_ignored_event(&mut self, event: TestIgnoredEvent) {
|
||||
// Remove this from the set of cases we're tracking since it has completed.
|
||||
self.remaining_cases
|
||||
.entry(event.test_specifier.metadata_file_path.clone().into())
|
||||
.or_default()
|
||||
.entry(event.test_specifier.solc_mode.clone())
|
||||
.or_default()
|
||||
.remove(&event.test_specifier.case_idx);
|
||||
|
||||
// Add information on the fact that the case was ignored to the report.
|
||||
let test_case_report = self.test_case_report(&event.test_specifier);
|
||||
test_case_report.status = Some(TestCaseStatus::Ignored {
|
||||
reason: event.reason,
|
||||
additional_fields: event.additional_fields,
|
||||
});
|
||||
self.handle_post_test_case_status_update(&event.test_specifier);
|
||||
}
|
||||
|
||||
fn handle_post_test_case_status_update(&mut self, specifier: &TestSpecifier) {
|
||||
let remaining_cases = self
|
||||
.remaining_cases
|
||||
.entry(specifier.metadata_file_path.clone().into())
|
||||
.or_default()
|
||||
.entry(specifier.solc_mode.clone())
|
||||
.or_default();
|
||||
if !remaining_cases.is_empty() {
|
||||
return;
|
||||
}
|
||||
|
||||
let case_status = self
|
||||
.report
|
||||
.test_case_information
|
||||
.entry(specifier.metadata_file_path.clone().into())
|
||||
.or_default()
|
||||
.entry(specifier.solc_mode.clone())
|
||||
.or_default()
|
||||
.iter()
|
||||
.map(|(case_idx, case_report)| {
|
||||
(
|
||||
*case_idx,
|
||||
case_report.status.clone().expect("Can't be uninitialized"),
|
||||
)
|
||||
})
|
||||
.collect::<BTreeMap<_, _>>();
|
||||
let event = ReporterEvent::MetadataFileSolcModeCombinationExecutionCompleted {
|
||||
metadata_file_path: specifier.metadata_file_path.clone().into(),
|
||||
mode: specifier.solc_mode.clone(),
|
||||
case_status,
|
||||
};
|
||||
|
||||
// According to the documentation on send, the sending fails if there are no more receiver
|
||||
// handles. Therefore, this isn't an error that we want to bubble up or anything. If we fail
|
||||
// to send then we ignore the error.
|
||||
let _ = self.listener_tx.send(event);
|
||||
}
|
||||
|
||||
fn handle_leader_node_assigned_event(&mut self, event: LeaderNodeAssignedEvent) {
|
||||
let execution_information = self.execution_information(&ExecutionSpecifier {
|
||||
test_specifier: event.test_specifier,
|
||||
node_id: event.id,
|
||||
node_designation: NodeDesignation::Leader,
|
||||
});
|
||||
execution_information.node = Some(TestCaseNodeInformation {
|
||||
id: event.id,
|
||||
platform: event.platform,
|
||||
connection_string: event.connection_string,
|
||||
});
|
||||
}
|
||||
|
||||
fn handle_follower_node_assigned_event(&mut self, event: FollowerNodeAssignedEvent) {
|
||||
let execution_information = self.execution_information(&ExecutionSpecifier {
|
||||
test_specifier: event.test_specifier,
|
||||
node_id: event.id,
|
||||
node_designation: NodeDesignation::Follower,
|
||||
});
|
||||
execution_information.node = Some(TestCaseNodeInformation {
|
||||
id: event.id,
|
||||
platform: event.platform,
|
||||
connection_string: event.connection_string,
|
||||
});
|
||||
}
|
||||
|
||||
fn handle_pre_link_contracts_compilation_succeeded_event(
|
||||
&mut self,
|
||||
event: PreLinkContractsCompilationSucceededEvent,
|
||||
) {
|
||||
let include_input = self.report.config.report_include_compiler_input;
|
||||
let include_output = self.report.config.report_include_compiler_output;
|
||||
|
||||
let execution_information = self.execution_information(&event.execution_specifier);
|
||||
|
||||
let compiler_input = if include_input {
|
||||
event.compiler_input
|
||||
} else {
|
||||
None
|
||||
};
|
||||
let compiler_output = if include_output {
|
||||
Some(event.compiler_output)
|
||||
} else {
|
||||
None
|
||||
};
|
||||
|
||||
execution_information.pre_link_compilation_status = Some(CompilationStatus::Success {
|
||||
is_cached: event.is_cached,
|
||||
compiler_version: event.compiler_version,
|
||||
compiler_path: event.compiler_path,
|
||||
compiler_input,
|
||||
compiler_output,
|
||||
});
|
||||
}
|
||||
|
||||
fn handle_post_link_contracts_compilation_succeeded_event(
|
||||
&mut self,
|
||||
event: PostLinkContractsCompilationSucceededEvent,
|
||||
) {
|
||||
let include_input = self.report.config.report_include_compiler_input;
|
||||
let include_output = self.report.config.report_include_compiler_output;
|
||||
|
||||
let execution_information = self.execution_information(&event.execution_specifier);
|
||||
|
||||
let compiler_input = if include_input {
|
||||
event.compiler_input
|
||||
} else {
|
||||
None
|
||||
};
|
||||
let compiler_output = if include_output {
|
||||
Some(event.compiler_output)
|
||||
} else {
|
||||
None
|
||||
};
|
||||
|
||||
execution_information.post_link_compilation_status = Some(CompilationStatus::Success {
|
||||
is_cached: event.is_cached,
|
||||
compiler_version: event.compiler_version,
|
||||
compiler_path: event.compiler_path,
|
||||
compiler_input,
|
||||
compiler_output,
|
||||
});
|
||||
}
|
||||
|
||||
fn handle_pre_link_contracts_compilation_failed_event(
|
||||
&mut self,
|
||||
event: PreLinkContractsCompilationFailedEvent,
|
||||
) {
|
||||
let include_input = self.report.config.report_include_compiler_input;
|
||||
|
||||
let execution_information = self.execution_information(&event.execution_specifier);
|
||||
|
||||
let compiler_input = if include_input {
|
||||
event.compiler_input
|
||||
} else {
|
||||
None
|
||||
};
|
||||
|
||||
execution_information.pre_link_compilation_status = Some(CompilationStatus::Failure {
|
||||
reason: event.reason,
|
||||
compiler_version: event.compiler_version,
|
||||
compiler_path: event.compiler_path,
|
||||
compiler_input,
|
||||
});
|
||||
}
|
||||
|
||||
fn handle_post_link_contracts_compilation_failed_event(
|
||||
&mut self,
|
||||
event: PostLinkContractsCompilationFailedEvent,
|
||||
) {
|
||||
let include_input = self.report.config.report_include_compiler_input;
|
||||
|
||||
let execution_information = self.execution_information(&event.execution_specifier);
|
||||
|
||||
let compiler_input = if include_input {
|
||||
event.compiler_input
|
||||
} else {
|
||||
None
|
||||
};
|
||||
|
||||
execution_information.post_link_compilation_status = Some(CompilationStatus::Failure {
|
||||
reason: event.reason,
|
||||
compiler_version: event.compiler_version,
|
||||
compiler_path: event.compiler_path,
|
||||
compiler_input,
|
||||
});
|
||||
}
|
||||
|
||||
fn handle_libraries_deployed_event(&mut self, event: LibrariesDeployedEvent) {
|
||||
self.execution_information(&event.execution_specifier)
|
||||
.deployed_libraries = Some(event.libraries);
|
||||
}
|
||||
|
||||
fn handle_contract_deployed_event(&mut self, event: ContractDeployedEvent) {
|
||||
self.execution_information(&event.execution_specifier)
|
||||
.deployed_contracts
|
||||
.get_or_insert_default()
|
||||
.insert(event.contract_instance, event.address);
|
||||
}
|
||||
|
||||
fn test_case_report(&mut self, specifier: &TestSpecifier) -> &mut TestCaseReport {
|
||||
self.report
|
||||
.test_case_information
|
||||
.entry(specifier.metadata_file_path.clone().into())
|
||||
.or_default()
|
||||
.entry(specifier.solc_mode.clone())
|
||||
.or_default()
|
||||
.entry(specifier.case_idx)
|
||||
.or_default()
|
||||
}
|
||||
|
||||
fn execution_information(
|
||||
&mut self,
|
||||
specifier: &ExecutionSpecifier,
|
||||
) -> &mut ExecutionInformation {
|
||||
let test_case_report = self.test_case_report(&specifier.test_specifier);
|
||||
match specifier.node_designation {
|
||||
NodeDesignation::Leader => test_case_report
|
||||
.leader_execution_information
|
||||
.get_or_insert_default(),
|
||||
NodeDesignation::Follower => test_case_report
|
||||
.follower_execution_information
|
||||
.get_or_insert_default(),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[serde_as]
|
||||
#[derive(Clone, Debug, Serialize)]
|
||||
pub struct Report {
|
||||
/// The configuration that the tool was started up with.
|
||||
pub config: Arguments,
|
||||
/// The platform of the leader chain.
|
||||
pub leader_platform: TestingPlatform,
|
||||
/// The platform of the follower chain.
|
||||
pub follower_platform: TestingPlatform,
|
||||
/// The list of corpus files that the tool found.
|
||||
pub corpora: Vec<Corpus>,
|
||||
/// The list of metadata files that were found by the tool.
|
||||
pub metadata_files: BTreeSet<MetadataFilePath>,
|
||||
/// Information relating to each test case.
|
||||
#[serde_as(as = "BTreeMap<_, HashMap<DisplayFromStr, BTreeMap<DisplayFromStr, _>>>")]
|
||||
pub test_case_information:
|
||||
BTreeMap<MetadataFilePath, HashMap<Mode, BTreeMap<CaseIdx, TestCaseReport>>>,
|
||||
}
|
||||
|
||||
impl Report {
|
||||
pub fn new(config: Arguments) -> Self {
|
||||
Self {
|
||||
leader_platform: config.leader,
|
||||
follower_platform: config.follower,
|
||||
config,
|
||||
corpora: Default::default(),
|
||||
metadata_files: Default::default(),
|
||||
test_case_information: Default::default(),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, Serialize, Default)]
|
||||
pub struct TestCaseReport {
|
||||
/// Information on the status of the test case and whether it succeeded, failed, or was ignored.
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub status: Option<TestCaseStatus>,
|
||||
/// Information related to the execution on the leader.
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub leader_execution_information: Option<ExecutionInformation>,
|
||||
/// Information related to the execution on the follower.
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub follower_execution_information: Option<ExecutionInformation>,
|
||||
}
|
||||
|
||||
/// Information related to the status of the test. Could be that the test succeeded, failed, or that
|
||||
/// it was ignored.
|
||||
#[derive(Clone, Debug, Serialize)]
|
||||
#[serde(tag = "status")]
|
||||
pub enum TestCaseStatus {
|
||||
/// The test case succeeded.
|
||||
Succeeded {
|
||||
/// The number of steps of the case that were executed.
|
||||
steps_executed: usize,
|
||||
},
|
||||
/// The test case failed.
|
||||
Failed {
|
||||
/// The reason for the failure of the test case.
|
||||
reason: String,
|
||||
},
|
||||
/// The test case was ignored. This variant carries information related to why it was ignored.
|
||||
Ignored {
|
||||
/// The reason behind the test case being ignored.
|
||||
reason: String,
|
||||
/// Additional fields that describe more information on why the test case is ignored.
|
||||
#[serde(flatten)]
|
||||
additional_fields: IndexMap<String, serde_json::Value>,
|
||||
},
|
||||
}
|
||||
|
||||
/// Information related to the leader or follower node that's being used to execute the step.
|
||||
#[derive(Clone, Debug, Serialize)]
|
||||
pub struct TestCaseNodeInformation {
|
||||
/// The ID of the node that this case is being executed on.
|
||||
pub id: usize,
|
||||
/// The platform of the node.
|
||||
pub platform: TestingPlatform,
|
||||
/// The connection string of the node.
|
||||
pub connection_string: String,
|
||||
}
|
||||
|
||||
/// Execution information tied to the leader or the follower.
|
||||
#[derive(Clone, Debug, Default, Serialize)]
|
||||
pub struct ExecutionInformation {
|
||||
/// Information related to the node assigned to this test case.
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub node: Option<TestCaseNodeInformation>,
|
||||
/// Information on the pre-link compiled contracts.
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub pre_link_compilation_status: Option<CompilationStatus>,
|
||||
/// Information on the post-link compiled contracts.
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub post_link_compilation_status: Option<CompilationStatus>,
|
||||
/// Information on the deployed libraries.
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub deployed_libraries: Option<BTreeMap<ContractInstance, Address>>,
|
||||
/// Information on the deployed contracts.
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub deployed_contracts: Option<BTreeMap<ContractInstance, Address>>,
|
||||
}
|
||||
|
||||
/// Information related to compilation
|
||||
#[derive(Clone, Debug, Serialize)]
|
||||
#[serde(tag = "status")]
|
||||
pub enum CompilationStatus {
|
||||
/// The compilation was successful.
|
||||
Success {
|
||||
/// A flag with information on whether the compilation artifacts were cached or not.
|
||||
is_cached: bool,
|
||||
/// The version of the compiler used to compile the contracts.
|
||||
compiler_version: Version,
|
||||
/// The path of the compiler used to compile the contracts.
|
||||
compiler_path: PathBuf,
|
||||
/// The input provided to the compiler to compile the contracts. This is only included if
|
||||
/// the appropriate flag is set in the CLI configuration and if the contracts were not
|
||||
/// cached and the compiler was invoked.
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
compiler_input: Option<CompilerInput>,
|
||||
/// The output of the compiler. This is only included if the appropriate flag is set in the
|
||||
/// CLI configurations.
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
compiler_output: Option<CompilerOutput>,
|
||||
},
|
||||
/// The compilation failed.
|
||||
Failure {
|
||||
/// The failure reason.
|
||||
reason: String,
|
||||
/// The version of the compiler used to compile the contracts.
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
compiler_version: Option<Version>,
|
||||
/// The path of the compiler used to compile the contracts.
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
compiler_path: Option<PathBuf>,
|
||||
/// The input provided to the compiler to compile the contracts. This is only included if
|
||||
/// the appropriate flag is set in the CLI configuration and if the contracts were not
|
||||
/// cached and the compiler was invoked.
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
compiler_input: Option<CompilerInput>,
|
||||
},
|
||||
}
|
||||
@@ -1,81 +0,0 @@
|
||||
//! The report analyzer enriches the raw report data.
|
||||
|
||||
use revive_dt_compiler::CompilerOutput;
|
||||
use serde::{Deserialize, Serialize};
|
||||
|
||||
use crate::reporter::CompilationTask;
|
||||
|
||||
/// Provides insights into how well the compilers perform.
|
||||
#[derive(Clone, Default, Debug, Serialize, Deserialize, PartialEq, PartialOrd)]
|
||||
pub struct CompilerStatistics {
|
||||
/// The sum of contracts observed.
|
||||
pub n_contracts: usize,
|
||||
/// The mean size of compiled contracts.
|
||||
pub mean_code_size: usize,
|
||||
/// The mean size of the optimized YUL IR.
|
||||
pub mean_yul_size: usize,
|
||||
/// Is a proxy because the YUL also contains a lot of comments.
|
||||
pub yul_to_bytecode_size_ratio: f32,
|
||||
}
|
||||
|
||||
impl CompilerStatistics {
|
||||
/// Cumulatively update the statistics with the next compiler task.
|
||||
pub fn sample(&mut self, compilation_task: &CompilationTask) {
|
||||
let Some(CompilerOutput { contracts }) = &compilation_task.json_output else {
|
||||
return;
|
||||
};
|
||||
|
||||
for (_solidity, contracts) in contracts.iter() {
|
||||
for (_name, (bytecode, _)) in contracts.iter() {
|
||||
// The EVM bytecode can be unlinked and thus is not necessarily a decodable hex
|
||||
// string; for our statistics this is a good enough approximation.
|
||||
let bytecode_size = bytecode.len() / 2;
|
||||
|
||||
// TODO: for the time being we set the yul_size to be zero. We need to change this
|
||||
// when we overhaul the reporting.
|
||||
|
||||
self.update_sizes(bytecode_size, 0);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Updates the size statistics cumulatively.
|
||||
fn update_sizes(&mut self, bytecode_size: usize, yul_size: usize) {
|
||||
let n_previous = self.n_contracts;
|
||||
let n_current = self.n_contracts + 1;
|
||||
|
||||
self.n_contracts = n_current;
|
||||
|
||||
self.mean_code_size = (n_previous * self.mean_code_size + bytecode_size) / n_current;
|
||||
self.mean_yul_size = (n_previous * self.mean_yul_size + yul_size) / n_current;
|
||||
|
||||
if self.mean_code_size > 0 {
|
||||
self.yul_to_bytecode_size_ratio =
|
||||
self.mean_yul_size as f32 / self.mean_code_size as f32;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::CompilerStatistics;
|
||||
|
||||
#[test]
|
||||
fn compiler_statistics() {
|
||||
let mut received = CompilerStatistics::default();
|
||||
received.update_sizes(0, 0);
|
||||
received.update_sizes(3, 37);
|
||||
received.update_sizes(123, 456);
|
||||
|
||||
let mean_code_size = 41; // rounding error from integer truncation
|
||||
let mean_yul_size = 164;
|
||||
let expected = CompilerStatistics {
|
||||
n_contracts: 3,
|
||||
mean_code_size,
|
||||
mean_yul_size,
|
||||
yul_to_bytecode_size_ratio: mean_yul_size as f32 / mean_code_size as f32,
|
||||
};
|
||||
|
||||
assert_eq!(received, expected);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,43 @@
|
||||
//! Common types and functions used throughout the crate.
|
||||
|
||||
use std::{path::PathBuf, sync::Arc};
|
||||
|
||||
use revive_dt_common::define_wrapper_type;
|
||||
use revive_dt_compiler::Mode;
|
||||
use revive_dt_format::{case::CaseIdx, input::StepIdx};
|
||||
use serde::{Deserialize, Serialize};
|
||||
|
||||
define_wrapper_type!(
|
||||
#[derive(Clone, Debug, PartialEq, Eq, PartialOrd, Ord, Hash, Serialize, Deserialize)]
|
||||
#[serde(transparent)]
|
||||
pub struct MetadataFilePath(PathBuf);
|
||||
);
|
||||
|
||||
/// An absolute specifier for a test.
|
||||
#[derive(Clone, Debug, PartialEq, Eq, Hash)]
|
||||
pub struct TestSpecifier {
|
||||
pub solc_mode: Mode,
|
||||
pub metadata_file_path: PathBuf,
|
||||
pub case_idx: CaseIdx,
|
||||
}
|
||||
|
||||
/// An absolute path for a test that also includes information about the node that it's assigned to
|
||||
/// and whether it's the leader or follower.
|
||||
#[derive(Clone, Debug, PartialEq, Eq, Hash)]
|
||||
pub struct ExecutionSpecifier {
|
||||
pub test_specifier: Arc<TestSpecifier>,
|
||||
pub node_id: usize,
|
||||
pub node_designation: NodeDesignation,
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, PartialEq, Eq, PartialOrd, Ord, Hash)]
|
||||
pub enum NodeDesignation {
|
||||
Leader,
|
||||
Follower,
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, PartialEq, Eq, Hash)]
|
||||
pub struct StepExecutionSpecifier {
|
||||
pub execution_specifier: Arc<ExecutionSpecifier>,
|
||||
pub step_idx: StepIdx,
|
||||
}
|
||||
@@ -1,4 +1,11 @@
|
||||
//! The revive differential tests reporting facility.
|
||||
//! This crate implements the reporting infrastructure for the differential testing tool.
|
||||
|
||||
pub mod analyzer;
|
||||
pub mod reporter;
|
||||
mod aggregator;
|
||||
mod common;
|
||||
mod reporter_event;
|
||||
mod runner_event;
|
||||
|
||||
pub use aggregator::*;
|
||||
pub use common::*;
|
||||
pub use reporter_event::*;
|
||||
pub use runner_event::*;
|
||||
|
||||
@@ -1,234 +0,0 @@
|
||||
//! The reporter is the central place observing test execution by collecting data.
|
||||
//!
|
||||
//! The data collected gives useful insights into the outcome of the test run
|
||||
//! and helps identifying and reproducing failing cases.
|
||||
|
||||
use std::{
|
||||
collections::HashMap,
|
||||
fs::{self, File, create_dir_all},
|
||||
path::PathBuf,
|
||||
sync::{Mutex, OnceLock},
|
||||
time::{SystemTime, UNIX_EPOCH},
|
||||
};
|
||||
|
||||
use anyhow::Context;
|
||||
use serde::Serialize;
|
||||
|
||||
use revive_dt_common::types::Mode;
|
||||
use revive_dt_compiler::{CompilerInput, CompilerOutput};
|
||||
use revive_dt_config::{Arguments, TestingPlatform};
|
||||
use revive_dt_format::corpus::Corpus;
|
||||
|
||||
use crate::analyzer::CompilerStatistics;
|
||||
|
||||
pub(crate) static REPORTER: OnceLock<Mutex<Report>> = OnceLock::new();
|
||||
|
||||
/// The `Report` datastructure stores all relevant inforamtion required for generating reports.
|
||||
#[derive(Clone, Debug, Default, Serialize)]
|
||||
pub struct Report {
|
||||
/// The configuration used during the test.
|
||||
pub config: Arguments,
|
||||
/// The observed test corpora.
|
||||
pub corpora: Vec<Corpus>,
|
||||
/// The observed test definitions.
|
||||
pub metadata_files: Vec<PathBuf>,
|
||||
/// The observed compilation results.
|
||||
pub compiler_results: HashMap<TestingPlatform, Vec<CompilationResult>>,
|
||||
/// The observed compilation statistics.
|
||||
pub compiler_statistics: HashMap<TestingPlatform, CompilerStatistics>,
|
||||
/// The file name this is serialized to.
|
||||
#[serde(skip)]
|
||||
directory: PathBuf,
|
||||
}
|
||||
|
||||
/// Contains a compiled contract.
|
||||
#[derive(Clone, Debug, Serialize)]
|
||||
pub struct CompilationTask {
|
||||
/// The observed compiler input.
|
||||
pub json_input: CompilerInput,
|
||||
/// The observed compiler output.
|
||||
pub json_output: Option<CompilerOutput>,
|
||||
/// The observed compiler mode.
|
||||
pub mode: Mode,
|
||||
/// The observed compiler version.
|
||||
pub compiler_version: String,
|
||||
/// The observed error, if any.
|
||||
pub error: Option<String>,
|
||||
}
|
||||
|
||||
/// Represents a report about a compilation task.
|
||||
#[derive(Clone, Debug, Serialize)]
|
||||
pub struct CompilationResult {
|
||||
/// The observed compilation task.
|
||||
pub compilation_task: CompilationTask,
|
||||
/// The linked span.
|
||||
pub span: Span,
|
||||
}
|
||||
|
||||
/// The [Span] struct indicates the context of what is being reported.
|
||||
#[derive(Clone, Copy, Debug, Serialize)]
|
||||
pub struct Span {
|
||||
/// The corpus index this belongs to.
|
||||
corpus: usize,
|
||||
/// The metadata file this belongs to.
|
||||
metadata_file: usize,
|
||||
/// The index of the case definition this belongs to.
|
||||
case: usize,
|
||||
/// The index of the case input this belongs to.
|
||||
input: usize,
|
||||
}
|
||||
|
||||
impl Report {
|
||||
/// The file name where this report will be written to.
|
||||
pub const FILE_NAME: &str = "report.json";
|
||||
|
||||
/// The [Span] is expected to initialize the reporter by providing the config.
|
||||
const INITIALIZED_VIA_SPAN: &str = "requires a Span which initializes the reporter";
|
||||
|
||||
/// Create a new [Report].
|
||||
fn new(config: Arguments) -> anyhow::Result<Self> {
|
||||
let now = SystemTime::now()
|
||||
.duration_since(UNIX_EPOCH)
|
||||
.unwrap()
|
||||
.as_millis();
|
||||
|
||||
let directory = config.directory().join("report").join(format!("{now}"));
|
||||
if !directory.exists() {
|
||||
create_dir_all(&directory)?;
|
||||
}
|
||||
|
||||
Ok(Self {
|
||||
config,
|
||||
directory,
|
||||
..Default::default()
|
||||
})
|
||||
}
|
||||
|
||||
/// Add a compilation task to the report.
|
||||
pub fn compilation(span: Span, platform: TestingPlatform, compilation_task: CompilationTask) {
|
||||
let mut report = REPORTER
|
||||
.get()
|
||||
.expect(Report::INITIALIZED_VIA_SPAN)
|
||||
.lock()
|
||||
.unwrap();
|
||||
|
||||
report
|
||||
.compiler_statistics
|
||||
.entry(platform)
|
||||
.or_default()
|
||||
.sample(&compilation_task);
|
||||
|
||||
report
|
||||
.compiler_results
|
||||
.entry(platform)
|
||||
.or_default()
|
||||
.push(CompilationResult {
|
||||
compilation_task,
|
||||
span,
|
||||
});
|
||||
}
|
||||
|
||||
/// Write the report to disk.
|
||||
pub fn save() -> anyhow::Result<()> {
|
||||
let Some(reporter) = REPORTER.get() else {
|
||||
return Ok(());
|
||||
};
|
||||
let report = reporter.lock().unwrap();
|
||||
|
||||
if let Err(error) = report.write_to_file() {
|
||||
anyhow::bail!("can not write report: {error}");
|
||||
}
|
||||
|
||||
if report.config.extract_problems {
|
||||
if let Err(error) = report.save_compiler_problems() {
|
||||
anyhow::bail!("can not write compiler problems: {error}");
|
||||
}
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Write compiler problems to disk for later debugging.
|
||||
pub fn save_compiler_problems(&self) -> anyhow::Result<()> {
|
||||
for (platform, results) in self.compiler_results.iter() {
|
||||
for result in results {
|
||||
// ignore if there were no errors
|
||||
if result.compilation_task.error.is_none() {
|
||||
continue;
|
||||
}
|
||||
|
||||
let path = &self.metadata_files[result.span.metadata_file]
|
||||
.parent()
|
||||
.unwrap()
|
||||
.join(format!("{platform}_errors"));
|
||||
if !path.exists() {
|
||||
create_dir_all(path)?;
|
||||
}
|
||||
|
||||
if let Some(error) = result.compilation_task.error.as_ref() {
|
||||
fs::write(path.join("compiler_error.txt"), error)?;
|
||||
}
|
||||
|
||||
if let Some(errors) = result.compilation_task.json_output.as_ref() {
|
||||
let file = File::create(path.join("compiler_output.txt"))?;
|
||||
serde_json::to_writer_pretty(file, &errors)?;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn write_to_file(&self) -> anyhow::Result<()> {
|
||||
let path = self.directory.join(Self::FILE_NAME);
|
||||
|
||||
let file = File::create(&path).context(path.display().to_string())?;
|
||||
serde_json::to_writer_pretty(file, &self)?;
|
||||
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
impl Span {
|
||||
/// Create a new [Span] with case and input index at 0.
|
||||
///
|
||||
/// Initializes the reporting facility on the first call.
|
||||
pub fn new(corpus: Corpus, config: Arguments) -> anyhow::Result<Self> {
|
||||
let report = Mutex::new(Report::new(config)?);
|
||||
let mut reporter = REPORTER.get_or_init(|| report).lock().unwrap();
|
||||
reporter.corpora.push(corpus);
|
||||
|
||||
Ok(Self {
|
||||
corpus: reporter.corpora.len() - 1,
|
||||
metadata_file: 0,
|
||||
case: 0,
|
||||
input: 0,
|
||||
})
|
||||
}
|
||||
|
||||
/// Advance to the next metadata file: Resets the case input index to 0.
|
||||
pub fn next_metadata(&mut self, metadata_file: PathBuf) {
|
||||
let mut reporter = REPORTER
|
||||
.get()
|
||||
.expect(Report::INITIALIZED_VIA_SPAN)
|
||||
.lock()
|
||||
.unwrap();
|
||||
|
||||
reporter.metadata_files.push(metadata_file);
|
||||
|
||||
self.metadata_file = reporter.metadata_files.len() - 1;
|
||||
self.case = 0;
|
||||
self.input = 0;
|
||||
}
|
||||
|
||||
/// Advance to the next case: Increas the case index by one and resets the input index to 0.
|
||||
pub fn next_case(&mut self) {
|
||||
self.case += 1;
|
||||
self.input = 0;
|
||||
}
|
||||
|
||||
/// Advance to the next input.
|
||||
pub fn next_input(&mut self) {
|
||||
self.input += 1;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,22 @@
|
||||
//! A reporter event sent by the report aggregator to the various listeners.
|
||||
|
||||
use std::collections::BTreeMap;
|
||||
|
||||
use revive_dt_compiler::Mode;
|
||||
use revive_dt_format::case::CaseIdx;
|
||||
|
||||
use crate::{MetadataFilePath, TestCaseStatus};
|
||||
|
||||
#[derive(Clone, Debug)]
|
||||
pub enum ReporterEvent {
|
||||
/// An event sent by the reporter once an entire metadata file and solc mode combination has
|
||||
/// finished execution.
|
||||
MetadataFileSolcModeCombinationExecutionCompleted {
|
||||
/// The path of the metadata file.
|
||||
metadata_file_path: MetadataFilePath,
|
||||
/// The Solc mode that this metadata file was executed in.
|
||||
mode: Mode,
|
||||
/// The status of each one of the cases.
|
||||
case_status: BTreeMap<CaseIdx, TestCaseStatus>,
|
||||
},
|
||||
}
|
||||
@@ -0,0 +1,640 @@
|
||||
//! The types associated with the events sent by the runner to the reporter.
|
||||
#![allow(dead_code)]
|
||||
|
||||
use std::{collections::BTreeMap, path::PathBuf, sync::Arc};
|
||||
|
||||
use alloy_primitives::Address;
|
||||
use indexmap::IndexMap;
|
||||
use revive_dt_compiler::{CompilerInput, CompilerOutput};
|
||||
use revive_dt_config::TestingPlatform;
|
||||
use revive_dt_format::metadata::Metadata;
|
||||
use revive_dt_format::{corpus::Corpus, metadata::ContractInstance};
|
||||
use semver::Version;
|
||||
use tokio::sync::{broadcast, oneshot};
|
||||
|
||||
use crate::{ExecutionSpecifier, ReporterEvent, TestSpecifier, common::MetadataFilePath};
|
||||
|
||||
macro_rules! __report_gen_emit_test_specific {
|
||||
(
|
||||
$ident:ident,
|
||||
$variant_ident:ident,
|
||||
$skip_field:ident;
|
||||
$( $bname:ident : $bty:ty, )*
|
||||
;
|
||||
$( $aname:ident : $aty:ty, )*
|
||||
) => {
|
||||
paste::paste! {
|
||||
pub fn [< report_ $variant_ident:snake _event >](
|
||||
&self
|
||||
$(, $bname: impl Into<$bty> )*
|
||||
$(, $aname: impl Into<$aty> )*
|
||||
) -> anyhow::Result<()> {
|
||||
self.report([< $variant_ident Event >] {
|
||||
$skip_field: self.test_specifier.clone()
|
||||
$(, $bname: $bname.into() )*
|
||||
$(, $aname: $aname.into() )*
|
||||
})
|
||||
}
|
||||
}
|
||||
};
|
||||
}
|
||||
|
||||
macro_rules! __report_gen_emit_test_specific_by_parse {
|
||||
(
|
||||
$ident:ident,
|
||||
$variant_ident:ident,
|
||||
$skip_field:ident;
|
||||
$( $bname:ident : $bty:ty, )* ; $( $aname:ident : $aty:ty, )*
|
||||
) => {
|
||||
__report_gen_emit_test_specific!(
|
||||
$ident, $variant_ident, $skip_field;
|
||||
$( $bname : $bty, )* ; $( $aname : $aty, )*
|
||||
);
|
||||
};
|
||||
}
|
||||
|
||||
macro_rules! __report_gen_scan_before {
|
||||
(
|
||||
$ident:ident, $variant_ident:ident;
|
||||
$( $before:ident : $bty:ty, )*
|
||||
;
|
||||
test_specifier : $skip_ty:ty,
|
||||
$( $after:ident : $aty:ty, )*
|
||||
;
|
||||
) => {
|
||||
__report_gen_emit_test_specific_by_parse!(
|
||||
$ident, $variant_ident, test_specifier;
|
||||
$( $before : $bty, )* ; $( $after : $aty, )*
|
||||
);
|
||||
};
|
||||
(
|
||||
$ident:ident, $variant_ident:ident;
|
||||
$( $before:ident : $bty:ty, )*
|
||||
;
|
||||
$name:ident : $ty:ty, $( $after:ident : $aty:ty, )*
|
||||
;
|
||||
) => {
|
||||
__report_gen_scan_before!(
|
||||
$ident, $variant_ident;
|
||||
$( $before : $bty, )* $name : $ty,
|
||||
;
|
||||
$( $after : $aty, )*
|
||||
;
|
||||
);
|
||||
};
|
||||
(
|
||||
$ident:ident, $variant_ident:ident;
|
||||
$( $before:ident : $bty:ty, )*
|
||||
;
|
||||
;
|
||||
) => {};
|
||||
}
|
||||
|
||||
macro_rules! __report_gen_for_variant {
|
||||
(
|
||||
$ident:ident,
|
||||
$variant_ident:ident;
|
||||
) => {};
|
||||
(
|
||||
$ident:ident,
|
||||
$variant_ident:ident;
|
||||
$( $field_ident:ident : $field_ty:ty ),+ $(,)?
|
||||
) => {
|
||||
__report_gen_scan_before!(
|
||||
$ident, $variant_ident;
|
||||
;
|
||||
$( $field_ident : $field_ty, )*
|
||||
;
|
||||
);
|
||||
};
|
||||
}
|
||||
|
||||
macro_rules! __report_gen_emit_execution_specific {
|
||||
(
|
||||
$ident:ident,
|
||||
$variant_ident:ident,
|
||||
$skip_field:ident;
|
||||
$( $bname:ident : $bty:ty, )*
|
||||
;
|
||||
$( $aname:ident : $aty:ty, )*
|
||||
) => {
|
||||
paste::paste! {
|
||||
pub fn [< report_ $variant_ident:snake _event >](
|
||||
&self
|
||||
$(, $bname: impl Into<$bty> )*
|
||||
$(, $aname: impl Into<$aty> )*
|
||||
) -> anyhow::Result<()> {
|
||||
self.report([< $variant_ident Event >] {
|
||||
$skip_field: self.execution_specifier.clone()
|
||||
$(, $bname: $bname.into() )*
|
||||
$(, $aname: $aname.into() )*
|
||||
})
|
||||
}
|
||||
}
|
||||
};
|
||||
}
|
||||
|
||||
macro_rules! __report_gen_emit_execution_specific_by_parse {
|
||||
(
|
||||
$ident:ident,
|
||||
$variant_ident:ident,
|
||||
$skip_field:ident;
|
||||
$( $bname:ident : $bty:ty, )* ; $( $aname:ident : $aty:ty, )*
|
||||
) => {
|
||||
__report_gen_emit_execution_specific!(
|
||||
$ident, $variant_ident, $skip_field;
|
||||
$( $bname : $bty, )* ; $( $aname : $aty, )*
|
||||
);
|
||||
};
|
||||
}
|
||||
|
||||
macro_rules! __report_gen_scan_before_exec {
|
||||
(
|
||||
$ident:ident, $variant_ident:ident;
|
||||
$( $before:ident : $bty:ty, )*
|
||||
;
|
||||
execution_specifier : $skip_ty:ty,
|
||||
$( $after:ident : $aty:ty, )*
|
||||
;
|
||||
) => {
|
||||
__report_gen_emit_execution_specific_by_parse!(
|
||||
$ident, $variant_ident, execution_specifier;
|
||||
$( $before : $bty, )* ; $( $after : $aty, )*
|
||||
);
|
||||
};
|
||||
(
|
||||
$ident:ident, $variant_ident:ident;
|
||||
$( $before:ident : $bty:ty, )*
|
||||
;
|
||||
$name:ident : $ty:ty, $( $after:ident : $aty:ty, )*
|
||||
;
|
||||
) => {
|
||||
__report_gen_scan_before_exec!(
|
||||
$ident, $variant_ident;
|
||||
$( $before : $bty, )* $name : $ty,
|
||||
;
|
||||
$( $after : $aty, )*
|
||||
;
|
||||
);
|
||||
};
|
||||
(
|
||||
$ident:ident, $variant_ident:ident;
|
||||
$( $before:ident : $bty:ty, )*
|
||||
;
|
||||
;
|
||||
) => {};
|
||||
}
|
||||
|
||||
macro_rules! __report_gen_for_variant_exec {
|
||||
(
|
||||
$ident:ident,
|
||||
$variant_ident:ident;
|
||||
) => {};
|
||||
(
|
||||
$ident:ident,
|
||||
$variant_ident:ident;
|
||||
$( $field_ident:ident : $field_ty:ty ),+ $(,)?
|
||||
) => {
|
||||
__report_gen_scan_before_exec!(
|
||||
$ident, $variant_ident;
|
||||
;
|
||||
$( $field_ident : $field_ty, )*
|
||||
;
|
||||
);
|
||||
};
|
||||
}
|
||||
|
||||
macro_rules! __report_gen_emit_step_execution_specific {
|
||||
(
|
||||
$ident:ident,
|
||||
$variant_ident:ident,
|
||||
$skip_field:ident;
|
||||
$( $bname:ident : $bty:ty, )*
|
||||
;
|
||||
$( $aname:ident : $aty:ty, )*
|
||||
) => {
|
||||
paste::paste! {
|
||||
pub fn [< report_ $variant_ident:snake _event >](
|
||||
&self
|
||||
$(, $bname: impl Into<$bty> )*
|
||||
$(, $aname: impl Into<$aty> )*
|
||||
) -> anyhow::Result<()> {
|
||||
self.report([< $variant_ident Event >] {
|
||||
$skip_field: self.step_specifier.clone()
|
||||
$(, $bname: $bname.into() )*
|
||||
$(, $aname: $aname.into() )*
|
||||
})
|
||||
}
|
||||
}
|
||||
};
|
||||
}
|
||||
|
||||
macro_rules! __report_gen_emit_step_execution_specific_by_parse {
|
||||
(
|
||||
$ident:ident,
|
||||
$variant_ident:ident,
|
||||
$skip_field:ident;
|
||||
$( $bname:ident : $bty:ty, )* ; $( $aname:ident : $aty:ty, )*
|
||||
) => {
|
||||
__report_gen_emit_step_execution_specific!(
|
||||
$ident, $variant_ident, $skip_field;
|
||||
$( $bname : $bty, )* ; $( $aname : $aty, )*
|
||||
);
|
||||
};
|
||||
}
|
||||
|
||||
macro_rules! __report_gen_scan_before_step {
|
||||
(
|
||||
$ident:ident, $variant_ident:ident;
|
||||
$( $before:ident : $bty:ty, )*
|
||||
;
|
||||
step_specifier : $skip_ty:ty,
|
||||
$( $after:ident : $aty:ty, )*
|
||||
;
|
||||
) => {
|
||||
__report_gen_emit_step_execution_specific_by_parse!(
|
||||
$ident, $variant_ident, step_specifier;
|
||||
$( $before : $bty, )* ; $( $after : $aty, )*
|
||||
);
|
||||
};
|
||||
(
|
||||
$ident:ident, $variant_ident:ident;
|
||||
$( $before:ident : $bty:ty, )*
|
||||
;
|
||||
$name:ident : $ty:ty, $( $after:ident : $aty:ty, )*
|
||||
;
|
||||
) => {
|
||||
__report_gen_scan_before_step!(
|
||||
$ident, $variant_ident;
|
||||
$( $before : $bty, )* $name : $ty,
|
||||
;
|
||||
$( $after : $aty, )*
|
||||
;
|
||||
);
|
||||
};
|
||||
(
|
||||
$ident:ident, $variant_ident:ident;
|
||||
$( $before:ident : $bty:ty, )*
|
||||
;
|
||||
;
|
||||
) => {};
|
||||
}
|
||||
|
||||
macro_rules! __report_gen_for_variant_step {
|
||||
(
|
||||
$ident:ident,
|
||||
$variant_ident:ident;
|
||||
) => {};
|
||||
(
|
||||
$ident:ident,
|
||||
$variant_ident:ident;
|
||||
$( $field_ident:ident : $field_ty:ty ),+ $(,)?
|
||||
) => {
|
||||
__report_gen_scan_before_step!(
|
||||
$ident, $variant_ident;
|
||||
;
|
||||
$( $field_ident : $field_ty, )*
|
||||
;
|
||||
);
|
||||
};
|
||||
}
|
||||
|
||||
/// Defines the runner-event which is sent from the test runners to the report aggregator.
|
||||
///
|
||||
/// This macro defines a number of things related to the reporting infrastructure and the interface
|
||||
/// used. First of all, it defines the enum of all of the possible events that the runners can send
|
||||
/// to the aggregator. For each one of the variants it defines a separate struct for it to allow the
|
||||
/// variant field in the enum to be put in a [`Box`].
|
||||
///
|
||||
/// In addition to the above, it defines [`From`] implementations for the various event types for
|
||||
/// the [`RunnerEvent`] enum essentially allowing for events such as [`CorpusFileDiscoveryEvent`] to
|
||||
/// be converted into a [`RunnerEvent`].
|
||||
///
|
||||
/// In addition to the above, it also defines the [`RunnerEventReporter`] which is a wrapper around
|
||||
/// an [`UnboundedSender`] allowing for events to be sent to the report aggregator.
|
||||
///
|
||||
/// With the above description, we can see that this macro defines almost all of the interface of
|
||||
/// the reporting infrastructure, from the enum itself, to its associated types, and also to the
|
||||
/// reporter that's used to report events to the aggregator.
|
||||
///
|
||||
/// [`UnboundedSender`]: tokio::sync::mpsc::UnboundedSender
|
||||
macro_rules! define_event {
|
||||
(
|
||||
$(#[$enum_meta: meta])*
|
||||
$vis: vis enum $ident: ident {
|
||||
$(
|
||||
$(#[$variant_meta: meta])*
|
||||
$variant_ident: ident {
|
||||
$(
|
||||
$(#[$field_meta: meta])*
|
||||
$field_ident: ident: $field_ty: ty
|
||||
),* $(,)?
|
||||
}
|
||||
),* $(,)?
|
||||
}
|
||||
) => {
|
||||
paste::paste! {
|
||||
$(#[$enum_meta])*
|
||||
#[derive(Debug)]
|
||||
$vis enum $ident {
|
||||
$(
|
||||
$(#[$variant_meta])*
|
||||
$variant_ident(Box<[<$variant_ident Event>]>)
|
||||
),*
|
||||
}
|
||||
|
||||
$(
|
||||
#[derive(Debug)]
|
||||
$(#[$variant_meta])*
|
||||
$vis struct [<$variant_ident Event>] {
|
||||
$(
|
||||
$(#[$field_meta])*
|
||||
$vis $field_ident: $field_ty
|
||||
),*
|
||||
}
|
||||
)*
|
||||
|
||||
$(
|
||||
impl From<[<$variant_ident Event>]> for $ident {
|
||||
fn from(value: [<$variant_ident Event>]) -> Self {
|
||||
Self::$variant_ident(Box::new(value))
|
||||
}
|
||||
}
|
||||
)*
|
||||
|
||||
/// Provides a way to report events to the aggregator.
|
||||
///
|
||||
/// Under the hood, this is a wrapper around an [`UnboundedSender`] which abstracts away
|
||||
/// the fact that channels are used and that implements high-level methods for reporting
|
||||
/// various events to the aggregator.
|
||||
#[derive(Clone, Debug)]
|
||||
pub struct [< $ident Reporter >]($vis tokio::sync::mpsc::UnboundedSender<$ident>);
|
||||
|
||||
impl From<tokio::sync::mpsc::UnboundedSender<$ident>> for [< $ident Reporter >] {
|
||||
fn from(value: tokio::sync::mpsc::UnboundedSender<$ident>) -> Self {
|
||||
Self(value)
|
||||
}
|
||||
}
|
||||
|
||||
impl [< $ident Reporter >] {
|
||||
pub fn test_specific_reporter(
|
||||
&self,
|
||||
test_specifier: impl Into<std::sync::Arc<crate::common::TestSpecifier>>
|
||||
) -> [< $ident TestSpecificReporter >] {
|
||||
[< $ident TestSpecificReporter >] {
|
||||
reporter: self.clone(),
|
||||
test_specifier: test_specifier.into(),
|
||||
}
|
||||
}
|
||||
|
||||
fn report(&self, event: impl Into<$ident>) -> anyhow::Result<()> {
|
||||
self.0.send(event.into()).map_err(Into::into)
|
||||
}
|
||||
|
||||
$(
|
||||
pub fn [< report_ $variant_ident:snake _event >](&self, $($field_ident: impl Into<$field_ty>),*) -> anyhow::Result<()> {
|
||||
self.report([< $variant_ident Event >] {
|
||||
$($field_ident: $field_ident.into()),*
|
||||
})
|
||||
}
|
||||
)*
|
||||
}
|
||||
|
||||
/// A reporter that's tied to a specific test case.
|
||||
#[derive(Clone, Debug)]
|
||||
pub struct [< $ident TestSpecificReporter >] {
|
||||
$vis reporter: [< $ident Reporter >],
|
||||
$vis test_specifier: std::sync::Arc<crate::common::TestSpecifier>,
|
||||
}
|
||||
|
||||
impl [< $ident TestSpecificReporter >] {
|
||||
pub fn execution_specific_reporter(
|
||||
&self,
|
||||
node_id: impl Into<usize>,
|
||||
node_designation: impl Into<$crate::common::NodeDesignation>
|
||||
) -> [< $ident ExecutionSpecificReporter >] {
|
||||
[< $ident ExecutionSpecificReporter >] {
|
||||
reporter: self.reporter.clone(),
|
||||
execution_specifier: Arc::new($crate::common::ExecutionSpecifier {
|
||||
test_specifier: self.test_specifier.clone(),
|
||||
node_id: node_id.into(),
|
||||
node_designation: node_designation.into(),
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
fn report(&self, event: impl Into<$ident>) -> anyhow::Result<()> {
|
||||
self.reporter.report(event)
|
||||
}
|
||||
|
||||
$(
|
||||
__report_gen_for_variant! { $ident, $variant_ident; $( $field_ident : $field_ty ),* }
|
||||
)*
|
||||
}
|
||||
|
||||
/// A reporter that's tied to a specific execution of the test case such as execution on
|
||||
/// a specific node like the leader or follower.
|
||||
#[derive(Clone, Debug)]
|
||||
pub struct [< $ident ExecutionSpecificReporter >] {
|
||||
$vis reporter: [< $ident Reporter >],
|
||||
$vis execution_specifier: std::sync::Arc<$crate::common::ExecutionSpecifier>,
|
||||
}
|
||||
|
||||
impl [< $ident ExecutionSpecificReporter >] {
|
||||
fn report(&self, event: impl Into<$ident>) -> anyhow::Result<()> {
|
||||
self.reporter.report(event)
|
||||
}
|
||||
|
||||
$(
|
||||
__report_gen_for_variant_exec! { $ident, $variant_ident; $( $field_ident : $field_ty ),* }
|
||||
)*
|
||||
}
|
||||
|
||||
/// A reporter that's tied to a specific step execution
|
||||
#[derive(Clone, Debug)]
|
||||
pub struct [< $ident StepExecutionSpecificReporter >] {
|
||||
$vis reporter: [< $ident Reporter >],
|
||||
$vis step_specifier: std::sync::Arc<$crate::common::StepExecutionSpecifier>,
|
||||
}
|
||||
|
||||
impl [< $ident StepExecutionSpecificReporter >] {
|
||||
fn report(&self, event: impl Into<$ident>) -> anyhow::Result<()> {
|
||||
self.reporter.report(event)
|
||||
}
|
||||
|
||||
$(
|
||||
__report_gen_for_variant_step! { $ident, $variant_ident; $( $field_ident : $field_ty ),* }
|
||||
)*
|
||||
}
|
||||
}
|
||||
};
|
||||
}
|
||||
|
||||
define_event! {
|
||||
/// An event type that's sent by the test runners/drivers to the report aggregator.
|
||||
pub(crate) enum RunnerEvent {
|
||||
/// An event emitted by the reporter when it wishes to listen to events emitted by the
|
||||
/// aggregator.
|
||||
SubscribeToEvents {
|
||||
/// The channel that the aggregator is to send the receive side of the channel on.
|
||||
tx: oneshot::Sender<broadcast::Receiver<ReporterEvent>>
|
||||
},
|
||||
/// An event emitted by runners when they've discovered a corpus file.
|
||||
CorpusFileDiscovery {
|
||||
/// The contents of the corpus file.
|
||||
corpus: Corpus
|
||||
},
|
||||
/// An event emitted by runners when they've discovered a metadata file.
|
||||
MetadataFileDiscovery {
|
||||
/// The path of the metadata file discovered.
|
||||
path: MetadataFilePath,
|
||||
/// The content of the metadata file.
|
||||
metadata: Metadata
|
||||
},
|
||||
/// An event emitted by the runners when they discover a test case.
|
||||
TestCaseDiscovery {
|
||||
/// A specifier for the test that was discovered.
|
||||
test_specifier: Arc<TestSpecifier>,
|
||||
},
|
||||
/// An event emitted by the runners when a test case is ignored.
|
||||
TestIgnored {
|
||||
/// A specifier for the test that's been ignored.
|
||||
test_specifier: Arc<TestSpecifier>,
|
||||
/// A reason for the test to be ignored.
|
||||
reason: String,
|
||||
/// Additional fields that describe more information on why the test was ignored.
|
||||
additional_fields: IndexMap<String, serde_json::Value>
|
||||
},
|
||||
/// An event emitted by the runners when a test case has succeeded.
|
||||
TestSucceeded {
|
||||
/// A specifier for the test that succeeded.
|
||||
test_specifier: Arc<TestSpecifier>,
|
||||
/// The number of steps of the case that were executed by the driver.
|
||||
steps_executed: usize,
|
||||
},
|
||||
/// An event emitted by the runners when a test case has failed.
|
||||
TestFailed {
|
||||
/// A specifier for the test that succeeded.
|
||||
test_specifier: Arc<TestSpecifier>,
|
||||
/// A reason for the failure of the test.
|
||||
reason: String,
|
||||
},
|
||||
/// An event emitted when the test case is assigned a leader node.
|
||||
LeaderNodeAssigned {
|
||||
/// A specifier for the test that the assignment is for.
|
||||
test_specifier: Arc<TestSpecifier>,
|
||||
/// The ID of the node that this case is being executed on.
|
||||
id: usize,
|
||||
/// The platform of the node.
|
||||
platform: TestingPlatform,
|
||||
/// The connection string of the node.
|
||||
connection_string: String,
|
||||
},
|
||||
/// An event emitted when the test case is assigned a follower node.
|
||||
FollowerNodeAssigned {
|
||||
/// A specifier for the test that the assignment is for.
|
||||
test_specifier: Arc<TestSpecifier>,
|
||||
/// The ID of the node that this case is being executed on.
|
||||
id: usize,
|
||||
/// The platform of the node.
|
||||
platform: TestingPlatform,
|
||||
/// The connection string of the node.
|
||||
connection_string: String,
|
||||
},
|
||||
/// An event emitted by the runners when the compilation of the contracts has succeeded
|
||||
/// on the pre-link contracts.
|
||||
PreLinkContractsCompilationSucceeded {
|
||||
/// A specifier for the execution that's taking place.
|
||||
execution_specifier: Arc<ExecutionSpecifier>,
|
||||
/// The version of the compiler used to compile the contracts.
|
||||
compiler_version: Version,
|
||||
/// The path of the compiler used to compile the contracts.
|
||||
compiler_path: PathBuf,
|
||||
/// A flag of whether the contract bytecode and ABI were cached or if they were compiled
|
||||
/// anew.
|
||||
is_cached: bool,
|
||||
/// The input provided to the compiler - this is optional and not provided if the
|
||||
/// contracts were obtained from the cache.
|
||||
compiler_input: Option<CompilerInput>,
|
||||
/// The output of the compiler.
|
||||
compiler_output: CompilerOutput
|
||||
},
|
||||
/// An event emitted by the runners when the compilation of the contracts has succeeded
|
||||
/// on the post-link contracts.
|
||||
PostLinkContractsCompilationSucceeded {
|
||||
/// A specifier for the execution that's taking place.
|
||||
execution_specifier: Arc<ExecutionSpecifier>,
|
||||
/// The version of the compiler used to compile the contracts.
|
||||
compiler_version: Version,
|
||||
/// The path of the compiler used to compile the contracts.
|
||||
compiler_path: PathBuf,
|
||||
/// A flag of whether the contract bytecode and ABI were cached or if they were compiled
|
||||
/// anew.
|
||||
is_cached: bool,
|
||||
/// The input provided to the compiler - this is optional and not provided if the
|
||||
/// contracts were obtained from the cache.
|
||||
compiler_input: Option<CompilerInput>,
|
||||
/// The output of the compiler.
|
||||
compiler_output: CompilerOutput
|
||||
},
|
||||
/// An event emitted by the runners when the compilation of the pre-link contract has
|
||||
/// failed.
|
||||
PreLinkContractsCompilationFailed {
|
||||
/// A specifier for the execution that's taking place.
|
||||
execution_specifier: Arc<ExecutionSpecifier>,
|
||||
/// The version of the compiler used to compile the contracts.
|
||||
compiler_version: Option<Version>,
|
||||
/// The path of the compiler used to compile the contracts.
|
||||
compiler_path: Option<PathBuf>,
|
||||
/// The input provided to the compiler - this is optional and not provided if the
|
||||
/// contracts were obtained from the cache.
|
||||
compiler_input: Option<CompilerInput>,
|
||||
/// The failure reason.
|
||||
reason: String,
|
||||
},
|
||||
/// An event emitted by the runners when the compilation of the post-link contract has
|
||||
/// failed.
|
||||
PostLinkContractsCompilationFailed {
|
||||
/// A specifier for the execution that's taking place.
|
||||
execution_specifier: Arc<ExecutionSpecifier>,
|
||||
/// The version of the compiler used to compile the contracts.
|
||||
compiler_version: Option<Version>,
|
||||
/// The path of the compiler used to compile the contracts.
|
||||
compiler_path: Option<PathBuf>,
|
||||
/// The input provided to the compiler - this is optional and not provided if the
|
||||
/// contracts were obtained from the cache.
|
||||
compiler_input: Option<CompilerInput>,
|
||||
/// The failure reason.
|
||||
reason: String,
|
||||
},
|
||||
/// An event emitted by the runners when a library has been deployed.
|
||||
LibrariesDeployed {
|
||||
/// A specifier for the execution that's taking place.
|
||||
execution_specifier: Arc<ExecutionSpecifier>,
|
||||
/// The addresses of the libraries that were deployed.
|
||||
libraries: BTreeMap<ContractInstance, Address>
|
||||
},
|
||||
/// An event emitted by the runners when they've deployed a new contract.
|
||||
ContractDeployed {
|
||||
/// A specifier for the execution that's taking place.
|
||||
execution_specifier: Arc<ExecutionSpecifier>,
|
||||
/// The instance name of the contract.
|
||||
contract_instance: ContractInstance,
|
||||
/// The address of the contract.
|
||||
address: Address
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
/// An extension to the [`Reporter`] implemented by the macro.
|
||||
impl RunnerEventReporter {
|
||||
pub async fn subscribe(&self) -> anyhow::Result<broadcast::Receiver<ReporterEvent>> {
|
||||
let (tx, rx) = oneshot::channel::<broadcast::Receiver<ReporterEvent>>();
|
||||
self.report_subscribe_to_events_event(tx)?;
|
||||
rx.await.map_err(Into::into)
|
||||
}
|
||||
}
|
||||
|
||||
pub type Reporter = RunnerEventReporter;
|
||||
pub type TestSpecificReporter = RunnerEventTestSpecificReporter;
|
||||
pub type ExecutionSpecificReporter = RunnerEventExecutionSpecificReporter;
|
||||
Reference in New Issue
Block a user