pezkuwichain/revive-differential-tests
1
0
Fork 0
mirror of https://github.com/pezkuwichain/revive-differential-tests.git synced 2026-05-07 23:07:56 +00:00
Code Issues Packages Projects Releases Wiki Activity

Add reporting infra for reporting ignored tests

Browse Source
This commit is contained in:
Omar Abdulla
2025-08-24 22:50:40 +03:00
parent 4d7d7ed843
commit 38e3915169
5 changed files with 280 additions and 55 deletions
Download Patch File Download Diff File Expand all files Collapse all files
crates/common/src/macros/define_wrapper_type.rs
+2
View File
@@ -1,6 +1,7 @@
#[macro_export]
macro_rules! impl_for_wrapper {
(Display, $ident: ident) => {
#[automatically_derived]
impl std::fmt::Display for $ident {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
std::fmt::Display::fmt(&self.0, f)
@@ -8,6 +9,7 @@ macro_rules! impl_for_wrapper {
}
};
(FromStr, $ident: ident) => {
#[automatically_derived]
impl std::str::FromStr for $ident {
type Err = anyhow::Error;
crates/core/src/main.rs
+105 -33
View File
@@ -1,7 +1,7 @@
mod cached_compiler;
use std::{
collections::{BTreeMap, HashMap},
collections::HashMap,
io::{BufWriter, Write, stderr},
path::Path,
sync::{Arc, LazyLock},
@@ -14,11 +14,10 @@ use alloy::{
};
use anyhow::Context;
use clap::Parser;
use futures::{FutureExt, stream};
use futures::stream;
use futures::{Stream, StreamExt};
use indexmap::IndexMap;
use revive_dt_node_interaction::EthereumNode;
use revive_dt_report::{ReportAggregator, Reporter};
use revive_dt_report::{ReportAggregator, Reporter, TestSpecificReporter, TestSpecifier};
use temp_dir::TempDir;
use tokio::{join, sync::mpsc, try_join};
use tracing::{debug, info, info_span, instrument};
@@ -53,6 +52,7 @@ struct Test<'a> {
mode: Mode,
case_idx: CaseIdx,
case: &'a Case,
reporter: TestSpecificReporter,
}
/// This represents the results that we gather from running test cases.
@@ -183,16 +183,11 @@ where
{
let (report_tx, report_rx) = mpsc::unbounded_channel::<(Test<'_>, CaseResult)>();
let tests = prepare_tests::<L, F>(args, metadata_files);
let tests = prepare_tests::<L, F>(args, metadata_files, reporter);
let driver_task = start_driver_task::<L, F>(args, tests, report_tx).await?;
let status_reporter_task = start_reporter_task(report_rx);
drop(reporter);
let (_, _, rtn) = tokio::join!(
status_reporter_task.inspect(|_| info!("Status reporter completed")),
driver_task.inspect(|_| info!("Driver completed")),
report_aggregator_task.inspect(|_| info!("Report aggregator completed"))
);
let (_, _, rtn) = tokio::join!(status_reporter_task, driver_task, report_aggregator_task,);
rtn?;
Ok(())
@@ -201,6 +196,7 @@ where
fn prepare_tests<'a, L, F>(
args: &Arguments,
metadata_files: &'a [MetadataFile],
reporter: Reporter,
) -> impl Stream<Item = Test<'a>>
where
L: Platform,
@@ -227,27 +223,25 @@ where
.into_iter()
.map(move |mode| (metadata_file, case_idx, case, mode))
})
.fold(
IndexMap::<_, BTreeMap<_, Vec<_>>>::new(),
|mut map, (metadata_file, case_idx, case, mode)| {
let test = Test {
metadata: metadata_file,
metadata_file_path: metadata_file.metadata_file_path.as_path(),
mode: mode.clone(),
case_idx: CaseIdx::new(case_idx),
case,
};
map.entry(mode)
.or_default()
.entry(test.case_idx)
.or_default()
.push(test);
map
},
)
.into_values()
.flatten()
.flat_map(|(_, value)| value.into_iter())
.map(move |(metadata_file, case_idx, case, mode)| Test {
metadata: metadata_file,
metadata_file_path: metadata_file.metadata_file_path.as_path(),
mode: mode.clone(),
case_idx: CaseIdx::new(case_idx),
case,
reporter: reporter.test_specific_reporter(Arc::new(TestSpecifier {
solc_mode: mode.clone(),
metadata_file_path: metadata_file.metadata_file_path.clone(),
case_idx: CaseIdx::new(case_idx),
})),
})
.inspect(|test| {
test.reporter
.report_test_case_discovery_event()
.expect("Can't fail")
})
.collect::<Vec<_>>()
.into_iter()
// Filter the test out if the leader and follower do not support the target.
.filter(|test| {
let leader_support =
@@ -262,7 +256,30 @@ where
leader_support,
follower_support,
"Target is not supported, throwing metadata file out"
)
);
test
.reporter
.report_test_ignored_event(
"Either the leader or the follower do not support the target desired by the test",
HashMap::from_iter([
(
"test_desired_targets".to_string(),
serde_json::to_value(test.metadata.targets.as_ref())
.expect("Can't fail")
),
(
"leader_support".to_string(),
serde_json::to_value(leader_support)
.expect("Can't fail")
),
(
"follower_support".to_string(),
serde_json::to_value(follower_support)
.expect("Can't fail")
)
])
)
.expect("Can't fail");
}
is_allowed
@@ -274,6 +291,13 @@ where
file_path = %test.metadata.relative_path().display(),
"Metadata file is ignored, throwing case out"
);
test
.reporter
.report_test_ignored_event(
"Metadata file is ignored, therefore all cases are ignored",
HashMap::new(),
)
.expect("Can't fail");
false
} else {
true
@@ -287,6 +311,13 @@ where
case_idx = %test.case_idx,
"Case is ignored, throwing case out"
);
test
.reporter
.report_test_ignored_event(
"Case is ignored",
HashMap::new(),
)
.expect("Can't fail");
false
} else {
true
@@ -309,6 +340,29 @@ where
follower_compatibility,
"EVM Version is incompatible, throwing case out"
);
test
.reporter
.report_test_ignored_event(
"EVM version is incompatible with either the leader or the follower",
HashMap::from_iter([
(
"test_desired_evm_version".to_string(),
serde_json::to_value(test.metadata.required_evm_version)
.expect("Can't fail")
),
(
"leader_compatibility".to_string(),
serde_json::to_value(leader_compatibility)
.expect("Can't fail")
),
(
"follower_compatibility".to_string(),
serde_json::to_value(follower_compatibility)
.expect("Can't fail")
)
])
)
.expect("Can't fail");
}
is_allowed
@@ -337,6 +391,24 @@ where
follower_support,
"Compilers do not support this, throwing case out"
);
test
.reporter
.report_test_ignored_event(
"Compilers do not support this mode either for the leader or for the follower.",
HashMap::from_iter([
(
"leader_support".to_string(),
serde_json::to_value(leader_support)
.expect("Can't fail")
),
(
"follower_support".to_string(),
serde_json::to_value(follower_support)
.expect("Can't fail")
)
])
)
.expect("Can't fail");
}
is_allowed.then_some(test)
crates/report/src/aggregator.rs
+57 -17
View File
@@ -19,7 +19,7 @@ use tokio::sync::{
};
use crate::{
SubscribeToEventsEvent, TestIgnoredEvent,
SubscribeToEventsEvent, TestCaseDiscoveryEvent, TestIgnoredEvent, TestSpecifier,
common::MetadataFilePath,
reporter_event::ReporterEvent,
runner_event::{CorpusFileDiscoveryEvent, MetadataFileDiscoveryEvent, Reporter, RunnerEvent},
@@ -28,7 +28,7 @@ use crate::{
pub struct ReportAggregator {
/* Internal Report State */
report: Report,
remaining_cases: HashMap<MetadataFilePath, HashSet<CaseIdx>>,
remaining_cases: HashMap<MetadataFilePath, HashMap<Mode, HashSet<CaseIdx>>>,
/* Channels */
runner_tx: Option<UnboundedSender<RunnerEvent>>,
runner_rx: UnboundedReceiver<RunnerEvent>,
@@ -60,7 +60,6 @@ impl ReportAggregator {
async fn aggregate(mut self) -> Result<()> {
while let Some(event) = self.runner_rx.recv().await {
match event {
RunnerEvent::ExecutionCompleted(..) => break,
RunnerEvent::SubscribeToEvents(event) => {
self.handle_subscribe_to_events_event(*event);
}
@@ -70,6 +69,9 @@ impl ReportAggregator {
RunnerEvent::MetadataFileDiscovery(event) => {
self.handle_metadata_file_discovery_event(*event);
}
RunnerEvent::TestCaseDiscovery(event) => {
self.handle_test_case_discovery(*event);
}
RunnerEvent::TestIgnored(event) => {
self.handle_test_ignored_event(*event);
}
@@ -104,21 +106,44 @@ impl ReportAggregator {
fn handle_metadata_file_discovery_event(&mut self, event: MetadataFileDiscoveryEvent) {
self.report.metadata_files.insert(event.path.clone());
self.remaining_cases.insert(
event.path,
event
.metadata
.cases
.iter()
.enumerate()
.map(|(id, _)| id)
.map(CaseIdx::new)
.collect(),
);
}
fn handle_test_ignored_event(&mut self, _: TestIgnoredEvent) {
todo!()
fn handle_test_case_discovery(&mut self, event: TestCaseDiscoveryEvent) {
self.remaining_cases
.entry(event.test_specifier.metadata_file_path.clone().into())
.or_default()
.entry(event.test_specifier.solc_mode.clone())
.or_default()
.insert(event.test_specifier.case_idx);
}
fn handle_test_ignored_event(&mut self, event: TestIgnoredEvent) {
// Remove this from the set of cases we're tracking
self.remaining_cases
.entry(event.test_specifier.metadata_file_path.clone().into())
.or_default()
.entry(event.test_specifier.solc_mode.clone())
.or_default()
.remove(&event.test_specifier.case_idx);
// Add information on the fact that the case was ignored to the report.
let test_case_report = self.test_case_report(&event.test_specifier);
test_case_report.ignore = Some(TestCaseIgnoreInformation {
is_ignored: true,
reason: event.reason,
additional_fields: event.additional_fields,
});
}
fn test_case_report(&mut self, specifier: &TestSpecifier) -> &mut TestCaseReport {
self.report
.test_case_information
.entry(specifier.metadata_file_path.clone().into())
.or_default()
.entry(specifier.solc_mode.clone())
.or_default()
.entry(specifier.case_idx)
.or_default()
}
}
@@ -148,5 +173,20 @@ impl Report {
}
}
#[derive(Clone, Debug, Serialize, Default)]
pub struct TestCaseReport {
/// Information related to the test case being ignored and why it's ignored.
ignore: Option<TestCaseIgnoreInformation>,
}
/// Information related to the test case being ignored and why it's ignored.
#[derive(Clone, Debug, Serialize)]
pub struct TestCaseReport {}
pub struct TestCaseIgnoreInformation {
/// A boolean that defines if the test case is ignored or not.
pub is_ignored: bool,
/// The reason behind the test case being ignored.
pub reason: String,
/// Additional fields that describe more information on why the test case is ignored.
#[serde(flatten)]
pub additional_fields: HashMap<String, serde_json::Value>,
}
crates/report/src/lib.rs
+1
View File
@@ -6,5 +6,6 @@ mod reporter_event;
mod runner_event;
pub use aggregator::*;
pub use common::*;
pub use reporter_event::*;
pub use runner_event::*;
crates/report/src/runner_event.rs
+115 -5
View File
@@ -1,11 +1,108 @@
//! The types associated with the events sent by the runner to the reporter.
#![allow(dead_code)]
use std::{collections::HashMap, sync::Arc};
use revive_dt_format::corpus::Corpus;
use revive_dt_format::metadata::Metadata;
use tokio::sync::{broadcast, oneshot};
use crate::{ReporterEvent, common::MetadataFilePath};
use crate::{ReporterEvent, TestSpecifier, common::MetadataFilePath};
macro_rules! __report_gen__emit_test_specific {
(
$ident:ident,
$variant_ident:ident,
$skip_field:ident;
$( $bname:ident : $bty:ty, )*
;
$( $aname:ident : $aty:ty, )*
) => {
paste::paste! {
pub fn [< report_ $variant_ident:snake _event >](
&self
$(, $bname: impl Into<$bty> )*
$(, $aname: impl Into<$aty> )*
) -> anyhow::Result<()> {
self.report([< $variant_ident Event >] {
$skip_field: self.test_specifier.clone()
$(, $bname: $bname.into() )*
$(, $aname: $aname.into() )*
})
}
}
};
}
macro_rules! __report_gen__emit_test_specific_by_parse {
(
$ident:ident,
$variant_ident:ident,
$skip_field:ident;
$( $bname:ident : $bty:ty, )* ; $( $aname:ident : $aty:ty, )*
) => {
__report_gen__emit_test_specific!(
$ident, $variant_ident, $skip_field;
$( $bname : $bty, )* ; $( $aname : $aty, )*
);
};
}
macro_rules! __report_gen__scan_before {
(
$ident:ident, $variant_ident:ident;
$( $before:ident : $bty:ty, )*
;
test_specifier : $skip_ty:ty,
$( $after:ident : $aty:ty, )*
;
) => {
__report_gen__emit_test_specific_by_parse!(
$ident, $variant_ident, test_specifier;
$( $before : $bty, )* ; $( $after : $aty, )*
);
};
(
$ident:ident, $variant_ident:ident;
$( $before:ident : $bty:ty, )*
;
$name:ident : $ty:ty, $( $after:ident : $aty:ty, )*
;
) => {
__report_gen__scan_before!(
$ident, $variant_ident;
$( $before : $bty, )* $name : $ty,
;
$( $after : $aty, )*
;
);
};
(
$ident:ident, $variant_ident:ident;
$( $before:ident : $bty:ty, )*
;
;
) => {};
}
macro_rules! __report_gen_for_variant {
(
$ident:ident,
$variant_ident:ident;
) => {};
(
$ident:ident,
$variant_ident:ident;
$( $field_ident:ident : $field_ty:ty ),+ $(,)?
) => {
__report_gen__scan_before!(
$ident, $variant_ident;
;
$( $field_ident : $field_ty, )*
;
);
};
}
macro_rules! keep_if_doc {
(#[doc = $doc:expr]) => {
@@ -115,6 +212,7 @@ macro_rules! define_event {
}
/// A reporter that's tied to a specific test case.
#[derive(Clone, Debug)]
pub struct [< $ident TestSpecificReporter >] {
$vis reporter: [< $ident Reporter >],
$vis test_specifier: std::sync::Arc<crate::common::TestSpecifier>,
@@ -124,6 +222,10 @@ macro_rules! define_event {
fn report(&self, event: impl Into<$ident>) -> anyhow::Result<()> {
self.reporter.report(event)
}
$(
__report_gen_for_variant! { $ident, $variant_ident; $( $field_ident : $field_ty ),* }
)*
}
}
};
@@ -150,13 +252,20 @@ define_event! {
/// The content of the metadata file.
metadata: Metadata
},
/// An event emitted by the runners when they discover a test case.
TestCaseDiscovery {
/// A specifier for the test that was discovered.
test_specifier: Arc<TestSpecifier>,
},
/// An event emitted by the runners when a test case is ignored.
TestIgnored {
/// A specifier for the test that's been ignored.
test_specifier: Arc<TestSpecifier>,
/// A reason for the test to be ignored.
reason: String,
/// Additional fields that describe more information on why the test was ignored.
additional_fields: HashMap<String, serde_json::Value>
},
/// An event emitted by the runners when the execution is completed and the aggregator can
/// stop.
ExecutionCompleted {}
}
}
@@ -170,3 +279,4 @@ impl RunnerEventReporter {
}
pub type Reporter = RunnerEventReporter;
pub type TestSpecificReporter = RunnerEventTestSpecificReporter;