diff --git a/crates/core/src/main.rs b/crates/core/src/main.rs index f74101d..507a9eb 100644 --- a/crates/core/src/main.rs +++ b/crates/core/src/main.rs @@ -39,7 +39,6 @@ use revive_dt_format::{ mode::ParsedMode, }; use revive_dt_node::{Node, pool::NodePool}; -use revive_dt_report::reporter::{Report, Span}; use crate::cached_compiler::CachedCompiler; @@ -70,13 +69,11 @@ fn main() -> anyhow::Result<()> { ); let body = async { - for (corpus, tests) in collect_corpora(&args)? { - let span = Span::new(corpus, args.clone())?; + for (_, tests) in collect_corpora(&args)? { match &args.compile_only { - Some(platform) => compile_corpus(&args, &tests, platform, span).await, - None => execute_corpus(&args, &tests, span).await?, + Some(platform) => compile_corpus(&args, &tests, platform).await, + None => execute_corpus(&args, &tests).await?, } - Report::save()?; } Ok(()) }; @@ -150,11 +147,7 @@ fn collect_corpora(args: &Arguments) -> anyhow::Result( - args: &Arguments, - metadata_files: &[MetadataFile], - span: Span, -) -> anyhow::Result<()> +async fn run_driver(args: &Arguments, metadata_files: &[MetadataFile]) -> anyhow::Result<()> where L: Platform, F: Platform, @@ -164,7 +157,7 @@ where let (report_tx, report_rx) = mpsc::unbounded_channel::<(Test<'_>, CaseResult)>(); let tests = prepare_tests::(args, metadata_files); - let driver_task = start_driver_task::(args, tests, span, report_tx).await?; + let driver_task = start_driver_task::(args, tests, report_tx).await?; let status_reporter_task = start_reporter_task(report_rx); tokio::join!(status_reporter_task, driver_task); @@ -336,7 +329,6 @@ async fn does_compiler_support_mode( async fn start_driver_task<'a, L, F>( args: &Arguments, tests: impl Stream>, - span: Span, report_tx: mpsc::UnboundedSender<(Test<'a>, CaseResult)>, ) -> anyhow::Result> where @@ -385,7 +377,6 @@ where cached_compiler, leader_node, follower_node, - span, ) .await; @@ -492,7 +483,6 @@ async fn handle_case_driver( cached_compiler: Arc, leader_node: &L::Blockchain, follower_node: &F::Blockchain, - _: Span, ) -> anyhow::Result where L: Platform, @@ -676,17 +666,13 @@ where .inspect(|steps_executed| info!(steps_executed, "Case succeeded")) } -async fn execute_corpus( - args: &Arguments, - tests: &[MetadataFile], - span: Span, -) -> anyhow::Result<()> { +async fn execute_corpus(args: &Arguments, tests: &[MetadataFile]) -> anyhow::Result<()> { match (&args.leader, &args.follower) { (TestingPlatform::Geth, TestingPlatform::Kitchensink) => { - run_driver::(args, tests, span).await? + run_driver::(args, tests).await? } (TestingPlatform::Geth, TestingPlatform::Geth) => { - run_driver::(args, tests, span).await? + run_driver::(args, tests).await? } _ => unimplemented!(), } @@ -694,12 +680,7 @@ async fn execute_corpus( Ok(()) } -async fn compile_corpus( - config: &Arguments, - tests: &[MetadataFile], - platform: &TestingPlatform, - _: Span, -) { +async fn compile_corpus(config: &Arguments, tests: &[MetadataFile], platform: &TestingPlatform) { let tests = tests.iter().flat_map(|metadata| { metadata .solc_modes() diff --git a/crates/report/src/analyzer.rs b/crates/report/src/analyzer.rs deleted file mode 100644 index 52fd360..0000000 --- a/crates/report/src/analyzer.rs +++ /dev/null @@ -1,81 +0,0 @@ -//! The report analyzer enriches the raw report data. - -use revive_dt_compiler::CompilerOutput; -use serde::{Deserialize, Serialize}; - -use crate::reporter::CompilationTask; - -/// Provides insights into how well the compilers perform. -#[derive(Clone, Default, Debug, Serialize, Deserialize, PartialEq, PartialOrd)] -pub struct CompilerStatistics { - /// The sum of contracts observed. - pub n_contracts: usize, - /// The mean size of compiled contracts. - pub mean_code_size: usize, - /// The mean size of the optimized YUL IR. - pub mean_yul_size: usize, - /// Is a proxy because the YUL also contains a lot of comments. - pub yul_to_bytecode_size_ratio: f32, -} - -impl CompilerStatistics { - /// Cumulatively update the statistics with the next compiler task. - pub fn sample(&mut self, compilation_task: &CompilationTask) { - let Some(CompilerOutput { contracts }) = &compilation_task.json_output else { - return; - }; - - for (_solidity, contracts) in contracts.iter() { - for (_name, (bytecode, _)) in contracts.iter() { - // The EVM bytecode can be unlinked and thus is not necessarily a decodable hex - // string; for our statistics this is a good enough approximation. - let bytecode_size = bytecode.len() / 2; - - // TODO: for the time being we set the yul_size to be zero. We need to change this - // when we overhaul the reporting. - - self.update_sizes(bytecode_size, 0); - } - } - } - - /// Updates the size statistics cumulatively. - fn update_sizes(&mut self, bytecode_size: usize, yul_size: usize) { - let n_previous = self.n_contracts; - let n_current = self.n_contracts + 1; - - self.n_contracts = n_current; - - self.mean_code_size = (n_previous * self.mean_code_size + bytecode_size) / n_current; - self.mean_yul_size = (n_previous * self.mean_yul_size + yul_size) / n_current; - - if self.mean_code_size > 0 { - self.yul_to_bytecode_size_ratio = - self.mean_yul_size as f32 / self.mean_code_size as f32; - } - } -} - -#[cfg(test)] -mod tests { - use super::CompilerStatistics; - - #[test] - fn compiler_statistics() { - let mut received = CompilerStatistics::default(); - received.update_sizes(0, 0); - received.update_sizes(3, 37); - received.update_sizes(123, 456); - - let mean_code_size = 41; // rounding error from integer truncation - let mean_yul_size = 164; - let expected = CompilerStatistics { - n_contracts: 3, - mean_code_size, - mean_yul_size, - yul_to_bytecode_size_ratio: mean_yul_size as f32 / mean_code_size as f32, - }; - - assert_eq!(received, expected); - } -} diff --git a/crates/report/src/lib.rs b/crates/report/src/lib.rs index 04ceeed..d104cba 100644 --- a/crates/report/src/lib.rs +++ b/crates/report/src/lib.rs @@ -1,4 +1 @@ -//! The revive differential tests reporting facility. - -pub mod analyzer; -pub mod reporter; +//! This crate implements the reporting infrastructure for the differential testing tool. diff --git a/crates/report/src/reporter.rs b/crates/report/src/reporter.rs deleted file mode 100644 index 5313ac7..0000000 --- a/crates/report/src/reporter.rs +++ /dev/null @@ -1,234 +0,0 @@ -//! The reporter is the central place observing test execution by collecting data. -//! -//! The data collected gives useful insights into the outcome of the test run -//! and helps identifying and reproducing failing cases. - -use std::{ - collections::HashMap, - fs::{self, File, create_dir_all}, - path::PathBuf, - sync::{Mutex, OnceLock}, - time::{SystemTime, UNIX_EPOCH}, -}; - -use anyhow::Context; -use serde::Serialize; - -use revive_dt_common::types::Mode; -use revive_dt_compiler::{CompilerInput, CompilerOutput}; -use revive_dt_config::{Arguments, TestingPlatform}; -use revive_dt_format::corpus::Corpus; - -use crate::analyzer::CompilerStatistics; - -pub(crate) static REPORTER: OnceLock> = OnceLock::new(); - -/// The `Report` datastructure stores all relevant inforamtion required for generating reports. -#[derive(Clone, Debug, Default, Serialize)] -pub struct Report { - /// The configuration used during the test. - pub config: Arguments, - /// The observed test corpora. - pub corpora: Vec, - /// The observed test definitions. - pub metadata_files: Vec, - /// The observed compilation results. - pub compiler_results: HashMap>, - /// The observed compilation statistics. - pub compiler_statistics: HashMap, - /// The file name this is serialized to. - #[serde(skip)] - directory: PathBuf, -} - -/// Contains a compiled contract. -#[derive(Clone, Debug, Serialize)] -pub struct CompilationTask { - /// The observed compiler input. - pub json_input: CompilerInput, - /// The observed compiler output. - pub json_output: Option, - /// The observed compiler mode. - pub mode: Mode, - /// The observed compiler version. - pub compiler_version: String, - /// The observed error, if any. - pub error: Option, -} - -/// Represents a report about a compilation task. -#[derive(Clone, Debug, Serialize)] -pub struct CompilationResult { - /// The observed compilation task. - pub compilation_task: CompilationTask, - /// The linked span. - pub span: Span, -} - -/// The [Span] struct indicates the context of what is being reported. -#[derive(Clone, Copy, Debug, Serialize)] -pub struct Span { - /// The corpus index this belongs to. - corpus: usize, - /// The metadata file this belongs to. - metadata_file: usize, - /// The index of the case definition this belongs to. - case: usize, - /// The index of the case input this belongs to. - input: usize, -} - -impl Report { - /// The file name where this report will be written to. - pub const FILE_NAME: &str = "report.json"; - - /// The [Span] is expected to initialize the reporter by providing the config. - const INITIALIZED_VIA_SPAN: &str = "requires a Span which initializes the reporter"; - - /// Create a new [Report]. - fn new(config: Arguments) -> anyhow::Result { - let now = SystemTime::now() - .duration_since(UNIX_EPOCH) - .unwrap() - .as_millis(); - - let directory = config.directory().join("report").join(format!("{now}")); - if !directory.exists() { - create_dir_all(&directory)?; - } - - Ok(Self { - config, - directory, - ..Default::default() - }) - } - - /// Add a compilation task to the report. - pub fn compilation(span: Span, platform: TestingPlatform, compilation_task: CompilationTask) { - let mut report = REPORTER - .get() - .expect(Report::INITIALIZED_VIA_SPAN) - .lock() - .unwrap(); - - report - .compiler_statistics - .entry(platform) - .or_default() - .sample(&compilation_task); - - report - .compiler_results - .entry(platform) - .or_default() - .push(CompilationResult { - compilation_task, - span, - }); - } - - /// Write the report to disk. - pub fn save() -> anyhow::Result<()> { - let Some(reporter) = REPORTER.get() else { - return Ok(()); - }; - let report = reporter.lock().unwrap(); - - if let Err(error) = report.write_to_file() { - anyhow::bail!("can not write report: {error}"); - } - - if report.config.extract_problems { - if let Err(error) = report.save_compiler_problems() { - anyhow::bail!("can not write compiler problems: {error}"); - } - } - - Ok(()) - } - - /// Write compiler problems to disk for later debugging. - pub fn save_compiler_problems(&self) -> anyhow::Result<()> { - for (platform, results) in self.compiler_results.iter() { - for result in results { - // ignore if there were no errors - if result.compilation_task.error.is_none() { - continue; - } - - let path = &self.metadata_files[result.span.metadata_file] - .parent() - .unwrap() - .join(format!("{platform}_errors")); - if !path.exists() { - create_dir_all(path)?; - } - - if let Some(error) = result.compilation_task.error.as_ref() { - fs::write(path.join("compiler_error.txt"), error)?; - } - - if let Some(errors) = result.compilation_task.json_output.as_ref() { - let file = File::create(path.join("compiler_output.txt"))?; - serde_json::to_writer_pretty(file, &errors)?; - } - } - } - - Ok(()) - } - - fn write_to_file(&self) -> anyhow::Result<()> { - let path = self.directory.join(Self::FILE_NAME); - - let file = File::create(&path).context(path.display().to_string())?; - serde_json::to_writer_pretty(file, &self)?; - - Ok(()) - } -} - -impl Span { - /// Create a new [Span] with case and input index at 0. - /// - /// Initializes the reporting facility on the first call. - pub fn new(corpus: Corpus, config: Arguments) -> anyhow::Result { - let report = Mutex::new(Report::new(config)?); - let mut reporter = REPORTER.get_or_init(|| report).lock().unwrap(); - reporter.corpora.push(corpus); - - Ok(Self { - corpus: reporter.corpora.len() - 1, - metadata_file: 0, - case: 0, - input: 0, - }) - } - - /// Advance to the next metadata file: Resets the case input index to 0. - pub fn next_metadata(&mut self, metadata_file: PathBuf) { - let mut reporter = REPORTER - .get() - .expect(Report::INITIALIZED_VIA_SPAN) - .lock() - .unwrap(); - - reporter.metadata_files.push(metadata_file); - - self.metadata_file = reporter.metadata_files.len() - 1; - self.case = 0; - self.input = 0; - } - - /// Advance to the next case: Increas the case index by one and resets the input index to 0. - pub fn next_case(&mut self) { - self.case += 1; - self.input = 0; - } - - /// Advance to the next input. - pub fn next_input(&mut self) { - self.input += 1; - } -}