mirror of
https://github.com/pezkuwichain/revive-differential-tests.git
synced 2026-06-09 20:21:04 +00:00
Update retester CI to check expectations (#225)
* Add a report processing tool * Add expectations tests to the CI action * Fix an issue with CI * Fix CI * Fix the path of the workdir in CI * Fix CI issue with the paths * Update the format of the expectations file
This commit is contained in:
Omar
@@ -0,0 +1,328 @@
|
||||
use std::{
|
||||
borrow::Cow,
|
||||
collections::{BTreeMap, BTreeSet},
|
||||
fmt::Display,
|
||||
fs::{File, OpenOptions},
|
||||
ops::{Deref, DerefMut},
|
||||
path::{Path, PathBuf},
|
||||
str::FromStr,
|
||||
};
|
||||
|
||||
use anyhow::{Context as _, Error, Result, bail};
|
||||
use clap::Parser;
|
||||
use serde::{Deserialize, Serialize, de::DeserializeOwned};
|
||||
|
||||
use revive_dt_common::types::{Mode, ParsedTestSpecifier};
|
||||
use revive_dt_report::{Report, TestCaseStatus};
|
||||
|
||||
fn main() -> Result<()> {
|
||||
let cli = Cli::try_parse().context("Failed to parse the CLI arguments")?;
|
||||
|
||||
match cli {
|
||||
Cli::GenerateExpectationsFile {
|
||||
report_path,
|
||||
output_path: output_file,
|
||||
remove_prefix,
|
||||
} => {
|
||||
let remove_prefix = remove_prefix
|
||||
.into_iter()
|
||||
.map(|path| path.canonicalize().context("Failed to canonicalize path"))
|
||||
.collect::<Result<Vec<_>>>()?;
|
||||
|
||||
let expectations = report_path
|
||||
.execution_information
|
||||
.iter()
|
||||
.flat_map(|(metadata_file_path, metadata_file_report)| {
|
||||
metadata_file_report
|
||||
.case_reports
|
||||
.iter()
|
||||
.map(move |(case_idx, case_report)| {
|
||||
(metadata_file_path, case_idx, case_report)
|
||||
})
|
||||
})
|
||||
.flat_map(|(metadata_file_path, case_idx, case_report)| {
|
||||
case_report.mode_execution_reports.iter().map(
|
||||
move |(mode, execution_report)| {
|
||||
(
|
||||
metadata_file_path,
|
||||
case_idx,
|
||||
mode,
|
||||
execution_report.status.as_ref(),
|
||||
)
|
||||
},
|
||||
)
|
||||
})
|
||||
.filter_map(|(metadata_file_path, case_idx, mode, status)| {
|
||||
status.map(|status| (metadata_file_path, case_idx, mode, status))
|
||||
})
|
||||
.map(|(metadata_file_path, case_idx, mode, status)| {
|
||||
(
|
||||
TestSpecifier {
|
||||
metadata_file_path: Cow::Borrowed(
|
||||
remove_prefix
|
||||
.iter()
|
||||
.filter_map(|prefix| {
|
||||
metadata_file_path.as_inner().strip_prefix(prefix).ok()
|
||||
})
|
||||
.next()
|
||||
.unwrap_or(metadata_file_path.as_inner()),
|
||||
),
|
||||
case_idx: case_idx.into_inner(),
|
||||
mode: Cow::Borrowed(mode),
|
||||
},
|
||||
Status::from(status),
|
||||
)
|
||||
})
|
||||
.collect::<Expectations>();
|
||||
|
||||
let output_file = OpenOptions::new()
|
||||
.truncate(true)
|
||||
.create(true)
|
||||
.write(true)
|
||||
.open(output_file)
|
||||
.context("Failed to create the output file")?;
|
||||
serde_json::to_writer_pretty(output_file, &expectations)
|
||||
.context("Failed to write the expectations to file")?;
|
||||
}
|
||||
Cli::CompareExpectationFiles {
|
||||
base_expectation_path,
|
||||
other_expectation_path,
|
||||
} => {
|
||||
let keys = base_expectation_path
|
||||
.keys()
|
||||
.chain(other_expectation_path.keys())
|
||||
.collect::<BTreeSet<_>>();
|
||||
|
||||
for key in keys {
|
||||
let base_status = base_expectation_path.get(key).context(format!(
|
||||
"Entry not found in the base expectations: \"{}\"",
|
||||
key
|
||||
))?;
|
||||
let other_status = other_expectation_path.get(key).context(format!(
|
||||
"Entry not found in the other expectations: \"{}\"",
|
||||
key
|
||||
))?;
|
||||
|
||||
if base_status != other_status {
|
||||
bail!(
|
||||
"Expectations for entry \"{}\" have changed. They were {:?} and now they are {:?}",
|
||||
key,
|
||||
base_status,
|
||||
other_status
|
||||
)
|
||||
}
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
type Expectations<'a> = BTreeMap<TestSpecifier<'a>, Status>;
|
||||
|
||||
/// A tool that's used to process the reports generated by the retester binary in various ways.
|
||||
#[derive(Clone, Debug, Parser)]
|
||||
#[command(name = "retester", term_width = 100)]
|
||||
pub enum Cli {
|
||||
/// Generates an expectation file out of a given report.
|
||||
GenerateExpectationsFile {
|
||||
/// The path of the report's JSON file to generate the expectation's file for.
|
||||
#[clap(long)]
|
||||
report_path: JsonFile<Report>,
|
||||
|
||||
/// The path of the output file to generate.
|
||||
///
|
||||
/// Note that we expect that:
|
||||
/// 1. The provided path points to a JSON file.
|
||||
/// 1. The ancestor's of the provided path already exist such that no directory creations
|
||||
/// are required.
|
||||
#[clap(long)]
|
||||
output_path: PathBuf,
|
||||
|
||||
/// Prefix paths to remove from the paths in the final expectations file.
|
||||
#[clap(long)]
|
||||
remove_prefix: Vec<PathBuf>,
|
||||
},
|
||||
|
||||
/// Compares two expectation files to ensure that they match each other.
|
||||
CompareExpectationFiles {
|
||||
/// The path of the base expectation file.
|
||||
#[clap(long)]
|
||||
base_expectation_path: JsonFile<Expectations<'static>>,
|
||||
|
||||
/// The path of the other expectation file.
|
||||
#[clap(long)]
|
||||
other_expectation_path: JsonFile<Expectations<'static>>,
|
||||
},
|
||||
}
|
||||
|
||||
#[derive(Clone, Copy, Debug, PartialEq, Eq, PartialOrd, Ord, Hash, Serialize, Deserialize)]
|
||||
pub enum Status {
|
||||
Succeeded,
|
||||
Failed,
|
||||
Ignored,
|
||||
}
|
||||
|
||||
impl From<TestCaseStatus> for Status {
|
||||
fn from(value: TestCaseStatus) -> Self {
|
||||
match value {
|
||||
TestCaseStatus::Succeeded { .. } => Self::Succeeded,
|
||||
TestCaseStatus::Failed { .. } => Self::Failed,
|
||||
TestCaseStatus::Ignored { .. } => Self::Ignored,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl<'a> From<&'a TestCaseStatus> for Status {
|
||||
fn from(value: &'a TestCaseStatus) -> Self {
|
||||
match value {
|
||||
TestCaseStatus::Succeeded { .. } => Self::Succeeded,
|
||||
TestCaseStatus::Failed { .. } => Self::Failed,
|
||||
TestCaseStatus::Ignored { .. } => Self::Ignored,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, PartialEq, Eq, PartialOrd, Ord, Hash)]
|
||||
pub struct JsonFile<T> {
|
||||
path: PathBuf,
|
||||
content: Box<T>,
|
||||
}
|
||||
|
||||
impl<T> Deref for JsonFile<T> {
|
||||
type Target = T;
|
||||
|
||||
fn deref(&self) -> &Self::Target {
|
||||
&self.content
|
||||
}
|
||||
}
|
||||
|
||||
impl<T> DerefMut for JsonFile<T> {
|
||||
fn deref_mut(&mut self) -> &mut Self::Target {
|
||||
&mut self.content
|
||||
}
|
||||
}
|
||||
|
||||
impl<T> FromStr for JsonFile<T>
|
||||
where
|
||||
T: DeserializeOwned,
|
||||
{
|
||||
type Err = Error;
|
||||
|
||||
fn from_str(s: &str) -> Result<Self, Self::Err> {
|
||||
let path = PathBuf::from(s);
|
||||
let file = File::open(&path).context("Failed to open the file")?;
|
||||
serde_json::from_reader(&file)
|
||||
.map(|content| Self { path, content })
|
||||
.context(format!(
|
||||
"Failed to deserialize file's content as {}",
|
||||
std::any::type_name::<T>()
|
||||
))
|
||||
}
|
||||
}
|
||||
|
||||
impl<T> Display for JsonFile<T> {
|
||||
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
|
||||
Display::fmt(&self.path.display(), f)
|
||||
}
|
||||
}
|
||||
|
||||
impl<T> From<JsonFile<T>> for String {
|
||||
fn from(value: JsonFile<T>) -> Self {
|
||||
value.to_string()
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, PartialEq, Eq, PartialOrd, Ord, Hash)]
|
||||
pub struct TestSpecifier<'a> {
|
||||
pub metadata_file_path: Cow<'a, Path>,
|
||||
pub case_idx: usize,
|
||||
pub mode: Cow<'a, Mode>,
|
||||
}
|
||||
|
||||
impl<'a> Display for TestSpecifier<'a> {
|
||||
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
|
||||
write!(
|
||||
f,
|
||||
"{}::{}::{}",
|
||||
self.metadata_file_path.display(),
|
||||
self.case_idx,
|
||||
self.mode
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
impl<'a> From<TestSpecifier<'a>> for ParsedTestSpecifier {
|
||||
fn from(
|
||||
TestSpecifier {
|
||||
metadata_file_path,
|
||||
case_idx,
|
||||
mode,
|
||||
}: TestSpecifier,
|
||||
) -> Self {
|
||||
Self::CaseWithMode {
|
||||
metadata_file_path: metadata_file_path.to_path_buf(),
|
||||
case_idx,
|
||||
mode: mode.into_owned(),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl TryFrom<ParsedTestSpecifier> for TestSpecifier<'static> {
|
||||
type Error = Error;
|
||||
|
||||
fn try_from(value: ParsedTestSpecifier) -> Result<Self> {
|
||||
let ParsedTestSpecifier::CaseWithMode {
|
||||
metadata_file_path,
|
||||
case_idx,
|
||||
mode,
|
||||
} = value
|
||||
else {
|
||||
bail!("Expected a full test case specifier")
|
||||
};
|
||||
Ok(Self {
|
||||
metadata_file_path: Cow::Owned(metadata_file_path),
|
||||
case_idx,
|
||||
mode: Cow::Owned(mode),
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
impl<'a> Serialize for TestSpecifier<'a> {
|
||||
fn serialize<S>(&self, serializer: S) -> std::result::Result<S::Ok, S::Error>
|
||||
where
|
||||
S: serde::Serializer,
|
||||
{
|
||||
self.to_string().serialize(serializer)
|
||||
}
|
||||
}
|
||||
|
||||
impl<'d, 'a> Deserialize<'d> for TestSpecifier<'a> {
|
||||
fn deserialize<D>(deserializer: D) -> std::result::Result<Self, D::Error>
|
||||
where
|
||||
D: serde::Deserializer<'d>,
|
||||
{
|
||||
let string = String::deserialize(deserializer)?;
|
||||
let mut splitted = string.split("::");
|
||||
let (Some(metadata_file_path), Some(case_idx), Some(mode), None) = (
|
||||
splitted.next(),
|
||||
splitted.next(),
|
||||
splitted.next(),
|
||||
splitted.next(),
|
||||
) else {
|
||||
return Err(serde::de::Error::custom(
|
||||
"Test specifier doesn't contain the components required",
|
||||
));
|
||||
};
|
||||
let metadata_file_path = PathBuf::from(metadata_file_path);
|
||||
let case_idx = usize::from_str(case_idx)
|
||||
.map_err(|_| serde::de::Error::custom("Case idx is not a usize"))?;
|
||||
let mode = Mode::from_str(mode).map_err(|_| serde::de::Error::custom("Invalid mode"))?;
|
||||
|
||||
Ok(Self {
|
||||
metadata_file_path: Cow::Owned(metadata_file_path),
|
||||
case_idx,
|
||||
mode: Cow::Owned(mode),
|
||||
})
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user