diff --git a/crates/common/src/types/mod.rs b/crates/common/src/types/mod.rs index 2811086..8019524 100644 --- a/crates/common/src/types/mod.rs +++ b/crates/common/src/types/mod.rs @@ -1,9 +1,11 @@ mod identifiers; mod mode; mod private_key_allocator; +mod round_robin_pool; mod version_or_requirement; pub use identifiers::*; pub use mode::*; pub use private_key_allocator::*; +pub use round_robin_pool::*; pub use version_or_requirement::*; diff --git a/crates/common/src/types/round_robin_pool.rs b/crates/common/src/types/round_robin_pool.rs new file mode 100644 index 0000000..81882c1 --- /dev/null +++ b/crates/common/src/types/round_robin_pool.rs @@ -0,0 +1,24 @@ +use std::sync::atomic::{AtomicUsize, Ordering}; + +pub struct RoundRobinPool { + next_index: AtomicUsize, + items: Vec, +} + +impl RoundRobinPool { + pub fn new(items: Vec) -> Self { + Self { + next_index: Default::default(), + items, + } + } + + pub fn round_robin(&self) -> &T { + let current = self.next_index.fetch_add(1, Ordering::SeqCst) % self.items.len(); + self.items.get(current).unwrap() + } + + pub fn iter(&self) -> impl Iterator { + self.items.iter() + } +} diff --git a/crates/compiler/src/revive_resolc.rs b/crates/compiler/src/revive_resolc.rs index 79a32fb..f0025ea 100644 --- a/crates/compiler/src/revive_resolc.rs +++ b/crates/compiler/src/revive_resolc.rs @@ -16,6 +16,7 @@ use revive_solc_json_interface::{ SolcStandardJsonInputSettingsOptimizer, SolcStandardJsonInputSettingsSelection, SolcStandardJsonOutput, }; +use tracing::{Span, field::display}; use crate::{ CompilerInput, CompilerOutput, ModeOptimizerSetting, ModePipeline, SolidityCompiler, solc::Solc, @@ -80,6 +81,16 @@ impl SolidityCompiler for Resolc { } #[tracing::instrument(level = "debug", ret)] + #[tracing::instrument( + level = "error", + skip_all, + fields( + resolc_version = %self.version(), + solc_version = %self.0.solc.version(), + json_in = tracing::field::Empty + ), + err(Debug) + )] fn build( &self, CompilerInput { @@ -141,6 +152,7 @@ impl SolidityCompiler for Resolc { polkavm: None, }, }; + Span::current().record("json_in", display(serde_json::to_string(&input).unwrap())); let path = &self.0.resolc_path; let mut command = AsyncCommand::new(path); @@ -148,6 +160,8 @@ impl SolidityCompiler for Resolc { .stdin(Stdio::piped()) .stdout(Stdio::piped()) .stderr(Stdio::piped()) + .arg("--solc") + .arg(self.0.solc.path()) .arg("--standard-json"); if let Some(ref base_path) = base_path { diff --git a/crates/compiler/src/solc.rs b/crates/compiler/src/solc.rs index defdb19..9a825ad 100644 --- a/crates/compiler/src/solc.rs +++ b/crates/compiler/src/solc.rs @@ -10,8 +10,9 @@ use std::{ use dashmap::DashMap; use revive_dt_common::types::VersionOrRequirement; -use revive_dt_config::{ResolcConfiguration, SolcConfiguration, WorkingDirectoryConfiguration}; +use revive_dt_config::{SolcConfiguration, WorkingDirectoryConfiguration}; use revive_dt_solc_binaries::download_solc; +use tracing::{Span, field::display, info}; use crate::{CompilerInput, CompilerOutput, ModeOptimizerSetting, ModePipeline, SolidityCompiler}; @@ -39,9 +40,7 @@ struct SolcInner { impl Solc { pub async fn new( - context: impl AsRef - + AsRef - + AsRef, + context: impl AsRef + AsRef, version: impl Into>, ) -> Result { // This is a cache for the compiler objects so that whenever the same compiler version is @@ -69,6 +68,11 @@ impl Solc { Ok(COMPILERS_CACHE .entry((path.clone(), version.clone())) .or_insert_with(|| { + info!( + solc_path = %path.display(), + solc_version = %version, + "Created a new solc compiler object" + ); Self(Arc::new(SolcInner { solc_path: path, solc_version: version, @@ -88,6 +92,12 @@ impl SolidityCompiler for Solc { } #[tracing::instrument(level = "debug", ret)] + #[tracing::instrument( + level = "error", + skip_all, + fields(json_in = tracing::field::Empty), + err(Debug) + )] fn build( &self, CompilerInput { @@ -166,12 +176,14 @@ impl SolidityCompiler for Solc { }, }; + Span::current().record("json_in", display(serde_json::to_string(&input).unwrap())); + let path = &self.0.solc_path; let mut command = AsyncCommand::new(path); command .stdin(Stdio::piped()) .stdout(Stdio::piped()) - .stderr(Stdio::piped()) + .stderr(Stdio::null()) .arg("--standard-json"); if let Some(ref base_path) = base_path { @@ -205,20 +217,18 @@ impl SolidityCompiler for Solc { if !output.status.success() { let json_in = serde_json::to_string_pretty(&input) .context("Failed to pretty-print Standard JSON input for logging")?; - let message = String::from_utf8_lossy(&output.stderr); tracing::error!( status = %output.status, - message = %message, json_input = json_in, "Compilation using solc failed" ); - anyhow::bail!("Compilation failed with an error: {message}"); + anyhow::bail!("Compilation failed"); } let parsed = serde_json::from_slice::(&output.stdout) .map_err(|e| { anyhow::anyhow!( - "failed to parse resolc JSON output: {e}\nstderr: {}", + "failed to parse resolc JSON output: {e}\nstdout: {}", String::from_utf8_lossy(&output.stdout) ) }) diff --git a/crates/core/src/differential_tests/driver.rs b/crates/core/src/differential_tests/driver.rs index a10ae10..b02087a 100644 --- a/crates/core/src/differential_tests/driver.rs +++ b/crates/core/src/differential_tests/driver.rs @@ -1,4 +1,7 @@ -use std::{collections::HashMap, sync::Arc}; +use std::{ + collections::{BTreeMap, HashMap}, + sync::Arc, +}; use alloy::{ consensus::EMPTY_ROOT_HASH, @@ -17,7 +20,7 @@ use alloy::{ use anyhow::{Context as _, Result, bail}; use futures::TryStreamExt; use indexmap::IndexMap; -use revive_dt_common::types::PrivateKeyAllocator; +use revive_dt_common::types::{PlatformIdentifier, PrivateKeyAllocator}; use revive_dt_format::{ metadata::{ContractInstance, ContractPathAndIdent}, steps::{ @@ -35,6 +38,92 @@ use crate::{ helpers::{CachedCompiler, TestDefinition, TestPlatformInformation}, }; +type StepsIterator = std::vec::IntoIter<(StepPath, Step)>; + +pub struct DifferentialTestsDriver<'a, I> { + /// The drivers for the various platforms that we're executing the tests on. + platform_drivers: BTreeMap>, +} + +impl<'a, I> DifferentialTestsDriver<'a, I> where I: Iterator {} + +impl<'a> DifferentialTestsDriver<'a, StepsIterator> { + // region:Constructors + pub async fn new_root( + test_definition: &'a TestDefinition<'a>, + private_key_allocator: Arc>, + cached_compiler: &CachedCompiler<'a>, + ) -> Result { + let platform_drivers = futures::future::try_join_all(test_definition.platforms.iter().map( + |(identifier, information)| { + let identifier = *identifier; + let private_key_allocator = private_key_allocator.clone(); + async move { + Self::create_platform_driver( + identifier, + information, + test_definition, + private_key_allocator, + cached_compiler, + ) + .await + .map(|driver| (identifier, driver)) + } + }, + )) + .await + .context("Failed to create the drivers for the various platforms")? + .into_iter() + .collect::>(); + + Ok(Self { platform_drivers }) + } + + async fn create_platform_driver( + identifier: PlatformIdentifier, + information: &'a TestPlatformInformation<'a>, + test_definition: &'a TestDefinition<'a>, + private_key_allocator: Arc>, + cached_compiler: &CachedCompiler<'a>, + ) -> Result> { + let steps: Vec<(StepPath, Step)> = test_definition + .case + .steps_iterator() + .enumerate() + .map(|(step_idx, step)| -> (StepPath, Step) { + (StepPath::new(vec![StepIdx::new(step_idx)]), step) + }) + .collect(); + let steps_iterator: StepsIterator = steps.into_iter(); + + DifferentialTestsPlatformDriver::new( + information, + test_definition, + private_key_allocator, + cached_compiler, + steps_iterator, + ) + .await + .context(format!("Failed to create driver for {identifier}")) + } + // endregion:Constructors + + // region:Execution + #[instrument(level = "info", skip_all)] + pub async fn execute_all(mut self) -> Result { + let platform_drivers = std::mem::take(&mut self.platform_drivers); + let results = futures::future::try_join_all( + platform_drivers + .into_values() + .map(|driver| driver.execute_all()), + ) + .await + .context("Failed to execute all of the steps on the driver")?; + Ok(results.first().copied().unwrap_or_default()) + } + // endregion:Execution +} + /// The differential tests driver for a single platform. pub struct DifferentialTestsPlatformDriver<'a, I> { /// The information of the platform that this driver is for. @@ -50,6 +139,9 @@ pub struct DifferentialTestsPlatformDriver<'a, I> { /// The execution state associated with the platform. execution_state: ExecutionState, + /// The number of steps that were executed on the driver. + steps_executed: usize, + /// This is the queue of steps that are to be executed by the driver for this test case. Each /// time `execute_step` is called one of the steps is executed. steps_iterator: I, @@ -60,28 +152,6 @@ where I: Iterator, { // region:Constructors & Initialization - pub async fn new_root( - platform_information: &'a TestPlatformInformation<'a>, - test_definition: &'a TestDefinition<'a>, - private_key_allocator: Arc>, - cached_compiler: &CachedCompiler<'a>, - ) -> Result>> { - let steps_iterator = test_definition - .case - .steps - .clone() - .into_iter() - .enumerate() - .map(|(step_idx, step)| (StepPath::new(vec![StepIdx::new(step_idx)]), step)); - DifferentialTestsPlatformDriver::new( - platform_information, - test_definition, - private_key_allocator, - cached_compiler, - steps_iterator, - ) - .await - } pub async fn new( platform_information: &'a TestPlatformInformation<'a>, @@ -99,6 +169,7 @@ where test_definition, private_key_allocator, execution_state, + steps_executed: 0, steps_iterator: steps, }) } @@ -242,11 +313,11 @@ where // region:Step Handling #[instrument(level = "info", skip_all)] - pub async fn execute_all(mut self) -> Result<()> { + pub async fn execute_all(mut self) -> Result { while let Some(result) = self.execute_next_step().await { result? } - Ok(()) + Ok(self.steps_executed) } #[instrument( @@ -272,7 +343,7 @@ where err(Debug), )] async fn execute_step(&mut self, step_path: &StepPath, step: &Step) -> Result<()> { - match step { + let steps_executed = match step { Step::FunctionCall(step) => self .execute_function_call(step_path, step.as_ref()) .await @@ -293,7 +364,9 @@ where .execute_account_allocation(step_path, step.as_ref()) .await .context("Account Allocation Step Failed"), - } + }?; + self.steps_executed += steps_executed; + Ok(()) } #[instrument(level = "info", skip_all)] @@ -301,7 +374,7 @@ where &mut self, _: &StepPath, step: &FunctionCallStep, - ) -> Result<()> { + ) -> Result { let deployment_receipts = self .handle_function_call_contract_deployment(step) .await @@ -317,7 +390,10 @@ where self.handle_function_call_variable_assignment(step, &tracing_result) .await .context("Failed to handle function call variable assignment")?; - todo!() + self.handle_function_call_assertions(step, &execution_receipt, &tracing_result) + .await + .context("Failed to handle function call assertions")?; + Ok(1) } async fn handle_function_call_contract_deployment( @@ -651,7 +727,7 @@ where &mut self, _: &StepPath, step: &BalanceAssertionStep, - ) -> anyhow::Result<()> { + ) -> anyhow::Result { self.step_address_auto_deployment(&step.address) .await .context("Failed to perform auto-deployment for the step address")?; @@ -677,7 +753,7 @@ where ) } - Ok(()) + Ok(1) } #[instrument(level = "info", skip_all, err(Debug))] @@ -685,7 +761,7 @@ where &mut self, _: &StepPath, step: &StorageEmptyAssertionStep, - ) -> Result<()> { + ) -> Result { self.step_address_auto_deployment(&step.address) .await .context("Failed to perform auto-deployment for the step address")?; @@ -717,32 +793,43 @@ where ) }; - Ok(()) + Ok(1) } #[instrument(level = "info", skip_all, err(Debug))] - async fn execute_repeat_step(&mut self, step_path: &StepPath, step: &RepeatStep) -> Result<()> { - let tasks = - (0..step.repeat) - .map(|_| DifferentialTestsPlatformDriver { - platform_information: self.platform_information, - test_definition: self.test_definition, - private_key_allocator: self.private_key_allocator.clone(), - execution_state: self.execution_state.clone(), - steps_iterator: step.steps.iter().cloned().enumerate().map( - |(step_idx, step)| { + async fn execute_repeat_step( + &mut self, + step_path: &StepPath, + step: &RepeatStep, + ) -> Result { + let tasks = (0..step.repeat) + .map(|_| DifferentialTestsPlatformDriver { + platform_information: self.platform_information, + test_definition: self.test_definition, + private_key_allocator: self.private_key_allocator.clone(), + execution_state: self.execution_state.clone(), + steps_executed: 0, + steps_iterator: { + let steps: Vec<(StepPath, Step)> = step + .steps + .iter() + .cloned() + .enumerate() + .map(|(step_idx, step)| { let step_idx = StepIdx::new(step_idx); let step_path = step_path.append(step_idx); (step_path, step) - }, - ), - }) - .map(|driver| driver.execute_all()) - .collect::>(); - futures::future::try_join_all(tasks) + }) + .collect(); + steps.into_iter() + }, + }) + .map(|driver| driver.execute_all()) + .collect::>(); + let res = futures::future::try_join_all(tasks) .await .context("Repetition execution failed")?; - Ok(()) + Ok(res.first().copied().unwrap_or_default()) } #[instrument(level = "info", skip_all, err(Debug))] @@ -750,7 +837,7 @@ where &mut self, _: &StepPath, step: &AllocateAccountStep, - ) -> Result<()> { + ) -> Result { let Some(variable_name) = step.variable_name.strip_prefix("$VARIABLE:") else { bail!("Account allocation must start with $VARIABLE:"); }; @@ -763,7 +850,7 @@ where .variables .insert(variable_name.to_string(), variable); - Ok(()) + Ok(1) } // endregion:Step Handling diff --git a/crates/core/src/differential_tests/entry_point.rs b/crates/core/src/differential_tests/entry_point.rs index 5f8f6dc..2d4a99c 100644 --- a/crates/core/src/differential_tests/entry_point.rs +++ b/crates/core/src/differential_tests/entry_point.rs @@ -1,15 +1,26 @@ //! The main entry point into differential testing. -use std::collections::BTreeMap; +use std::{ + collections::BTreeMap, + io::{BufWriter, Write, stderr}, + sync::Arc, + time::Instant, +}; use anyhow::Context as _; +use futures::{FutureExt, StreamExt}; +use revive_dt_common::types::PrivateKeyAllocator; use revive_dt_core::Platform; -use tracing::{error, instrument}; +use tokio::sync::Mutex; +use tracing::{Instrument, error, info, info_span, instrument}; use revive_dt_config::{Context, TestExecutionContext}; -use revive_dt_report::Reporter; +use revive_dt_report::{Reporter, ReporterEvent, TestCaseStatus}; -use crate::helpers::{NodePool, collect_metadata_files}; +use crate::{ + differential_tests::DifferentialTestsDriver, + helpers::{CachedCompiler, NodePool, collect_metadata_files, create_test_definitions_stream}, +}; /// Handles the differential testing executing it according to the information defined in the /// context @@ -18,9 +29,12 @@ pub async fn handle_differential_tests( context: TestExecutionContext, reporter: Reporter, ) -> anyhow::Result<()> { + let reporter_clone = reporter.clone(); + // Discover all of the metadata files that are defined in the context. let metadata_files = collect_metadata_files(&context) .context("Failed to collect metadata files for differential testing")?; + info!(len = metadata_files.len(), "Discovered metadata files"); // Discover the list of platforms that the tests should run on based on the context. let platforms = context @@ -54,6 +68,175 @@ pub async fn handle_differential_tests( map }; + info!("Spawned the platform nodes"); - todo!() + // Preparing test definitions. + let full_context = Context::ExecuteTests(Box::new(context.clone())); + let test_definitions = create_test_definitions_stream( + &full_context, + metadata_files.iter(), + &platforms_and_nodes, + reporter.clone(), + ) + .await + .collect::>() + .await; + info!(len = test_definitions.len(), "Created test definitions"); + + // Creating everything else required for the driver to run. + let cached_compiler = CachedCompiler::new( + context + .working_directory + .as_path() + .join("compilation_cache"), + context + .compilation_configuration + .invalidate_compilation_cache, + ) + .await + .map(Arc::new) + .context("Failed to initialize cached compiler")?; + let private_key_allocator = Arc::new(Mutex::new(PrivateKeyAllocator::new( + context.wallet_configuration.highest_private_key_exclusive(), + ))); + + // Creating the driver and executing all of the steps. + let driver_task = futures::future::join_all(test_definitions.iter().map(|test_definition| { + let private_key_allocator = private_key_allocator.clone(); + let cached_compiler = cached_compiler.clone(); + let mode = test_definition.mode.clone(); + let span = info_span!( + "Executing Test Case", + metadata_file_path = %test_definition.metadata_file_path.display(), + case_idx = %test_definition.case_idx, + mode = %mode + ); + async move { + let driver = match DifferentialTestsDriver::new_root( + test_definition, + private_key_allocator, + &cached_compiler, + ) + .await + { + Ok(driver) => driver, + Err(error) => { + test_definition + .reporter + .report_test_failed_event(format!("{error:#}")) + .expect("Can't fail"); + error!("Test Case Failed"); + return; + } + }; + info!("Created the driver for the test case"); + + match driver.execute_all().await { + Ok(steps_executed) => test_definition + .reporter + .report_test_succeeded_event(steps_executed) + .expect("Can't fail"), + Err(error) => { + test_definition + .reporter + .report_test_failed_event(format!("{error:#}")) + .expect("Can't fail"); + error!("Test Case Failed"); + } + }; + info!("Finished the execution of the test case") + } + .instrument(span) + })) + .inspect(|_| { + reporter_clone + .report_completion_event() + .expect("Can't fail") + }); + let cli_reporting_task = start_cli_reporting_task(reporter); + + futures::future::join(driver_task, cli_reporting_task).await; + + Ok(()) +} + +#[allow(irrefutable_let_patterns, clippy::uninlined_format_args)] +async fn start_cli_reporting_task(reporter: Reporter) { + let mut aggregator_events_rx = reporter.subscribe().await.expect("Can't fail"); + drop(reporter); + + let start = Instant::now(); + + const GREEN: &str = "\x1B[32m"; + const RED: &str = "\x1B[31m"; + const GREY: &str = "\x1B[90m"; + const COLOR_RESET: &str = "\x1B[0m"; + const BOLD: &str = "\x1B[1m"; + const BOLD_RESET: &str = "\x1B[22m"; + + let mut number_of_successes = 0; + let mut number_of_failures = 0; + + let mut buf = BufWriter::new(stderr()); + while let Ok(event) = aggregator_events_rx.recv().await { + let ReporterEvent::MetadataFileSolcModeCombinationExecutionCompleted { + metadata_file_path, + mode, + case_status, + } = event + else { + continue; + }; + + let _ = writeln!(buf, "{} - {}", mode, metadata_file_path.display()); + for (case_idx, case_status) in case_status.into_iter() { + let _ = write!(buf, "\tCase Index {case_idx:>3}: "); + let _ = match case_status { + TestCaseStatus::Succeeded { steps_executed } => { + number_of_successes += 1; + writeln!( + buf, + "{}{}Case Succeeded{} - Steps Executed: {}{}", + GREEN, BOLD, BOLD_RESET, steps_executed, COLOR_RESET + ) + } + TestCaseStatus::Failed { reason } => { + number_of_failures += 1; + writeln!( + buf, + "{}{}Case Failed{} - Reason: {}{}", + RED, + BOLD, + BOLD_RESET, + reason.trim(), + COLOR_RESET, + ) + } + TestCaseStatus::Ignored { reason, .. } => writeln!( + buf, + "{}{}Case Ignored{} - Reason: {}{}", + GREY, + BOLD, + BOLD_RESET, + reason.trim(), + COLOR_RESET, + ), + }; + } + let _ = writeln!(buf); + } + + // Summary at the end. + let _ = writeln!( + buf, + "{} cases: {}{}{} cases succeeded, {}{}{} cases failed in {} seconds", + number_of_successes + number_of_failures, + GREEN, + number_of_successes, + COLOR_RESET, + RED, + number_of_failures, + COLOR_RESET, + start.elapsed().as_secs() + ); } diff --git a/crates/core/src/helpers/cached_compiler.rs b/crates/core/src/helpers/cached_compiler.rs index c10f7e1..93017d8 100644 --- a/crates/core/src/helpers/cached_compiler.rs +++ b/crates/core/src/helpers/cached_compiler.rs @@ -5,7 +5,7 @@ use std::{ borrow::Cow, collections::HashMap, path::{Path, PathBuf}, - sync::Arc, + sync::{Arc, LazyLock}, }; use futures::FutureExt; @@ -19,7 +19,7 @@ use anyhow::{Context as _, Error, Result}; use revive_dt_report::ExecutionSpecificReporter; use semver::Version; use serde::{Deserialize, Serialize}; -use tokio::sync::{Mutex, RwLock}; +use tokio::sync::{Mutex, RwLock, Semaphore}; use tracing::{Instrument, debug, debug_span, instrument}; pub struct CachedCompiler<'a> { @@ -165,10 +165,22 @@ impl<'a> CachedCompiler<'a> { cache_value.compiler_output } None => { - compilation_callback() + let compiler_output = compilation_callback() .await .context("Compilation callback failed (cache miss path)")? - .compiler_output + .compiler_output; + self.artifacts_cache + .insert( + &cache_key, + &CacheValue { + compiler_output: compiler_output.clone(), + }, + ) + .await + .context( + "Failed to write the cached value of the compilation artifacts", + )?; + compiler_output } } } @@ -186,6 +198,12 @@ async fn compile_contracts( compiler: &dyn SolidityCompiler, reporter: &ExecutionSpecificReporter, ) -> Result { + // Puts a limit on how many compilations we can perform at any given instance which helps us + // with some of the errors we've been seeing with high concurrency on MacOS (we have not tried + // it on Linux so we don't know if these issues also persist there or not.) + static SPAWN_GATE: LazyLock = LazyLock::new(|| Semaphore::new(100)); + let _permit = SPAWN_GATE.acquire().await?; + let all_sources_in_dir = FilesWithExtensionIterator::new(metadata_directory.as_ref()) .with_allowed_extension("sol") .with_use_cached_fs(true) diff --git a/crates/core/src/helpers/test.rs b/crates/core/src/helpers/test.rs index 71b5e08..ec16b91 100644 --- a/crates/core/src/helpers/test.rs +++ b/crates/core/src/helpers/test.rs @@ -19,12 +19,12 @@ use revive_dt_format::{ use revive_dt_node_interaction::EthereumNode; use revive_dt_report::{ExecutionSpecificReporter, Reporter}; use revive_dt_report::{TestSpecificReporter, TestSpecifier}; -use tracing::{debug, error}; +use tracing::{debug, error, info}; use crate::Platform; use crate::helpers::NodePool; -async fn create_test_definitions_stream<'a>( +pub async fn create_test_definitions_stream<'a>( // This is only required for creating the compiler objects and is not used anywhere else in the // function. context: &Context, @@ -165,6 +165,14 @@ async fn create_test_definitions_stream<'a>( } } }) + .inspect(|test| { + info!( + metadata_file_path = %test.metadata_file_path.display(), + case_idx = %test.case_idx, + mode = %test.mode, + "Created a test case definition" + ); + }) } /// This is a full description of a differential test to run alongside the full metadata file, the @@ -228,15 +236,8 @@ impl<'a> TestDefinition<'a> { for (_, platform_information) in self.platforms.iter() { let is_allowed_for_platform = match self.metadata.targets.as_ref() { None => true, - Some(targets) => { - let mut target_matches = false; - for target in targets.iter() { - if &platform_information.platform.vm_identifier() == target { - target_matches = true; - break; - } - } - target_matches + Some(required_vm_identifiers) => { + required_vm_identifiers.contains(&platform_information.platform.vm_identifier()) } }; is_allowed &= is_allowed_for_platform; diff --git a/crates/core/src/main.rs b/crates/core/src/main.rs index f179e06..8f350bf 100644 --- a/crates/core/src/main.rs +++ b/crates/core/src/main.rs @@ -1,54 +1,17 @@ mod differential_tests; mod helpers; -use std::{ - borrow::Cow, - collections::{BTreeSet, HashMap}, - io::{BufWriter, Write, stderr}, - path::Path, - sync::Arc, - time::Instant, -}; - -use alloy::{ - network::{Ethereum, TransactionBuilder}, - rpc::types::TransactionRequest, -}; -use anyhow::Context as _; use clap::Parser; -use futures::stream; -use futures::{Stream, StreamExt}; -use indexmap::{IndexMap, indexmap}; -use revive_dt_node_interaction::EthereumNode; -use revive_dt_report::{ - ExecutionSpecificReporter, ReportAggregator, Reporter, ReporterEvent, TestCaseStatus, - TestSpecificReporter, TestSpecifier, -}; +use revive_dt_report::ReportAggregator; use schemars::schema_for; -use serde_json::{Value, json}; -use tokio::sync::Mutex; -use tracing::{debug, error, info, info_span, instrument}; +use tracing::info; use tracing_subscriber::{EnvFilter, FmtSubscriber}; -use revive_dt_common::{ - iterators::EitherIter, - types::{Mode, PrivateKeyAllocator}, -}; -use revive_dt_compiler::SolidityCompiler; -use revive_dt_config::{Context, *}; -use revive_dt_core::{ - Platform, - driver::{CaseDriver, CaseState}, -}; -use revive_dt_format::{ - case::{Case, CaseIdx}, - corpus::Corpus, - metadata::{ContractPathAndIdent, Metadata, MetadataFile}, - mode::ParsedMode, - steps::{FunctionCallStep, Step}, -}; +use revive_dt_config::Context; +use revive_dt_core::Platform; +use revive_dt_format::metadata::Metadata; -use crate::helpers::*; +use crate::differential_tests::handle_differential_tests; fn main() -> anyhow::Result<()> { let (writer, _guard) = tracing_appender::non_blocking::NonBlockingBuilder::default() @@ -74,37 +37,20 @@ fn main() -> anyhow::Result<()> { let (reporter, report_aggregator_task) = ReportAggregator::new(context.clone()).into_task(); match context { - Context::ExecuteTests(context) => { - let tests = collect_corpora(&context) - .context("Failed to collect corpus files from provided arguments")? - .into_iter() - .inspect(|(corpus, _)| { - reporter - .report_corpus_file_discovery_event(corpus.clone()) - .expect("Can't fail") - }) - .flat_map(|(_, files)| files.into_iter()) - .inspect(|metadata_file| { - reporter - .report_metadata_file_discovery_event( - metadata_file.metadata_file_path.clone(), - metadata_file.content.clone(), - ) - .expect("Can't fail") - }) - .collect::>(); + Context::ExecuteTests(context) => tokio::runtime::Builder::new_multi_thread() + .worker_threads(context.concurrency_configuration.number_of_threads) + .enable_all() + .build() + .expect("Failed building the Runtime") + .block_on(async move { + let differential_tests_handling_task = + handle_differential_tests(*context, reporter); - tokio::runtime::Builder::new_multi_thread() - .worker_threads(context.concurrency_configuration.number_of_threads) - .enable_all() - .build() - .expect("Failed building the Runtime") - .block_on(async move { - execute_corpus(*context, &tests, reporter, report_aggregator_task) - .await - .context("Failed to execute corpus") - }) - } + futures::future::try_join(differential_tests_handling_task, report_aggregator_task) + .await?; + + Ok(()) + }), Context::ExportJsonSchema => { let schema = schema_for!(Metadata); println!("{}", serde_json::to_string_pretty(&schema).unwrap()); @@ -112,672 +58,3 @@ fn main() -> anyhow::Result<()> { } } } - -#[instrument(level = "debug", name = "Collecting Corpora", skip_all)] -fn collect_corpora( - context: &TestExecutionContext, -) -> anyhow::Result>> { - let mut corpora = HashMap::new(); - - for path in &context.corpus_configuration.paths { - let span = info_span!("Processing corpus file", path = %path.display()); - let _guard = span.enter(); - - let corpus = Corpus::try_from_path(path)?; - info!( - name = corpus.name(), - number_of_contained_paths = corpus.path_count(), - "Deserialized corpus file" - ); - let tests = corpus.enumerate_tests(); - corpora.insert(corpus, tests); - } - - Ok(corpora) -} - -async fn run_driver( - context: TestExecutionContext, - metadata_files: &[MetadataFile], - reporter: Reporter, - report_aggregator_task: impl Future>, - platforms: Vec<&dyn Platform>, -) -> anyhow::Result<()> { - let mut nodes = Vec::<(&dyn Platform, NodePool)>::new(); - for platform in platforms.into_iter() { - let pool = NodePool::new(Context::ExecuteTests(Box::new(context.clone())), platform) - .await - .inspect_err(|err| { - error!( - ?err, - platform_identifier = %platform.platform_identifier(), - "Failed to initialize the node pool for the platform." - ) - }) - .context("Failed to initialize the node pool")?; - nodes.push((platform, pool)); - } - - let tests_stream = tests_stream( - &context, - metadata_files.iter(), - nodes.as_slice(), - reporter.clone(), - ) - .await; - let driver_task = start_driver_task(&context, tests_stream) - .await - .context("Failed to start driver task")?; - let cli_reporting_task = start_cli_reporting_task(reporter); - - let (_, _, rtn) = tokio::join!(cli_reporting_task, driver_task, report_aggregator_task); - rtn?; - - Ok(()) -} - -async fn tests_stream<'a>( - args: &TestExecutionContext, - metadata_files: impl IntoIterator + Clone, - nodes: &'a [(&dyn Platform, NodePool)], - reporter: Reporter, -) -> impl Stream> { - let tests = metadata_files - .into_iter() - .flat_map(|metadata_file| { - metadata_file - .cases - .iter() - .enumerate() - .map(move |(case_idx, case)| (metadata_file, case_idx, case)) - }) - // Flatten over the modes, prefer the case modes over the metadata file modes. - .flat_map(|(metadata_file, case_idx, case)| { - let reporter = reporter.clone(); - - let modes = case.modes.as_ref().or(metadata_file.modes.as_ref()); - let modes = match modes { - Some(modes) => EitherIter::A( - ParsedMode::many_to_modes(modes.iter()).map(Cow::<'static, _>::Owned), - ), - None => EitherIter::B(Mode::all().map(Cow::<'static, _>::Borrowed)), - }; - - modes.into_iter().map(move |mode| { - ( - metadata_file, - case_idx, - case, - mode.clone(), - reporter.test_specific_reporter(Arc::new(TestSpecifier { - solc_mode: mode.as_ref().clone(), - metadata_file_path: metadata_file.metadata_file_path.clone(), - case_idx: CaseIdx::new(case_idx), - })), - ) - }) - }) - .collect::>(); - - // Note: before we do any kind of filtering or process the iterator in any way, we need to - // inform the report aggregator of all of the cases that were found as it keeps a state of the - // test cases for its internal use. - for (_, _, _, _, reporter) in tests.iter() { - reporter - .report_test_case_discovery_event() - .expect("Can't fail") - } - - stream::iter(tests.into_iter()) - .filter_map( - move |(metadata_file, case_idx, case, mode, reporter)| async move { - let mut platforms = Vec::new(); - for (platform, node_pool) in nodes.iter() { - let node = node_pool.round_robbin(); - let compiler = platform - .new_compiler( - Context::ExecuteTests(Box::new(args.clone())), - mode.version.clone().map(Into::into), - ) - .await - .inspect_err(|err| { - error!( - ?err, - platform_identifier = %platform.platform_identifier(), - "Failed to instantiate the compiler" - ) - }) - .ok()?; - - let reporter = reporter - .execution_specific_reporter(node.id(), platform.platform_identifier()); - platforms.push((*platform, node, compiler, reporter)); - } - - Some(Test { - metadata: metadata_file, - metadata_file_path: metadata_file.metadata_file_path.as_path(), - mode: mode.clone(), - case_idx: CaseIdx::new(case_idx), - case, - platforms, - reporter, - }) - }, - ) - .filter_map(move |test| async move { - match test.check_compatibility() { - Ok(()) => Some(test), - Err((reason, additional_information)) => { - debug!( - metadata_file_path = %test.metadata.metadata_file_path.display(), - case_idx = %test.case_idx, - mode = %test.mode, - reason, - additional_information = - serde_json::to_string(&additional_information).unwrap(), - - "Ignoring Test Case" - ); - test.reporter - .report_test_ignored_event( - reason.to_string(), - additional_information - .into_iter() - .map(|(k, v)| (k.into(), v)) - .collect::>(), - ) - .expect("Can't fail"); - None - } - } - }) -} - -async fn start_driver_task<'a>( - context: &TestExecutionContext, - tests: impl Stream>, -) -> anyhow::Result> { - info!("Starting driver task"); - - let cached_compiler = Arc::new( - CachedCompiler::new( - context - .working_directory - .as_path() - .join("compilation_cache"), - context - .compilation_configuration - .invalidate_compilation_cache, - ) - .await - .context("Failed to initialize cached compiler")?, - ); - - Ok(tests.for_each_concurrent( - context.concurrency_configuration.concurrency_limit(), - move |test| { - let cached_compiler = cached_compiler.clone(); - - async move { - for (platform, node, _, _) in test.platforms.iter() { - test.reporter - .report_node_assigned_event( - node.id(), - platform.platform_identifier(), - node.connection_string(), - ) - .expect("Can't fail"); - } - - let private_key_allocator = Arc::new(Mutex::new(PrivateKeyAllocator::new( - context.wallet_configuration.highest_private_key_exclusive(), - ))); - - let reporter = test.reporter.clone(); - let result = - handle_case_driver(&test, cached_compiler, private_key_allocator).await; - - match result { - Ok(steps_executed) => reporter - .report_test_succeeded_event(steps_executed) - .expect("Can't fail"), - Err(error) => reporter - .report_test_failed_event(format!("{error:#}")) - .expect("Can't fail"), - } - } - }, - )) -} - -#[allow(irrefutable_let_patterns, clippy::uninlined_format_args)] -async fn start_cli_reporting_task(reporter: Reporter) { - let mut aggregator_events_rx = reporter.subscribe().await.expect("Can't fail"); - drop(reporter); - - let start = Instant::now(); - - const GREEN: &str = "\x1B[32m"; - const RED: &str = "\x1B[31m"; - const GREY: &str = "\x1B[90m"; - const COLOR_RESET: &str = "\x1B[0m"; - const BOLD: &str = "\x1B[1m"; - const BOLD_RESET: &str = "\x1B[22m"; - - let mut number_of_successes = 0; - let mut number_of_failures = 0; - - let mut buf = BufWriter::new(stderr()); - while let Ok(event) = aggregator_events_rx.recv().await { - let ReporterEvent::MetadataFileSolcModeCombinationExecutionCompleted { - metadata_file_path, - mode, - case_status, - } = event - else { - continue; - }; - - let _ = writeln!(buf, "{} - {}", mode, metadata_file_path.display()); - for (case_idx, case_status) in case_status.into_iter() { - let _ = write!(buf, "\tCase Index {case_idx:>3}: "); - let _ = match case_status { - TestCaseStatus::Succeeded { steps_executed } => { - number_of_successes += 1; - writeln!( - buf, - "{}{}Case Succeeded{} - Steps Executed: {}{}", - GREEN, BOLD, BOLD_RESET, steps_executed, COLOR_RESET - ) - } - TestCaseStatus::Failed { reason } => { - number_of_failures += 1; - writeln!( - buf, - "{}{}Case Failed{} - Reason: {}{}", - RED, - BOLD, - BOLD_RESET, - reason.trim(), - COLOR_RESET, - ) - } - TestCaseStatus::Ignored { reason, .. } => writeln!( - buf, - "{}{}Case Ignored{} - Reason: {}{}", - GREY, - BOLD, - BOLD_RESET, - reason.trim(), - COLOR_RESET, - ), - }; - } - let _ = writeln!(buf); - } - - // Summary at the end. - let _ = writeln!( - buf, - "{} cases: {}{}{} cases succeeded, {}{}{} cases failed in {} seconds", - number_of_successes + number_of_failures, - GREEN, - number_of_successes, - COLOR_RESET, - RED, - number_of_failures, - COLOR_RESET, - start.elapsed().as_secs() - ); -} - -#[allow(clippy::too_many_arguments)] -#[instrument( - level = "info", - name = "Handling Case" - skip_all, - fields( - metadata_file_path = %test.metadata.relative_path().display(), - mode = %test.mode, - case_idx = %test.case_idx, - case_name = test.case.name.as_deref().unwrap_or("Unnamed Case"), - ) -)] -async fn handle_case_driver<'a>( - test: &Test<'a>, - cached_compiler: Arc>, - private_key_allocator: Arc>, -) -> anyhow::Result { - let platform_state = stream::iter(test.platforms.iter()) - // Compiling the pre-link contracts. - .filter_map(|(platform, node, compiler, reporter)| { - let cached_compiler = cached_compiler.clone(); - - async move { - let compiler_output = cached_compiler - .compile_contracts( - test.metadata, - test.metadata_file_path, - test.mode.clone(), - None, - compiler.as_ref(), - *platform, - reporter, - ) - .await - .inspect_err(|err| { - error!( - ?err, - platform_identifier = %platform.platform_identifier(), - "Pre-linking compilation failed" - ) - }) - .ok()?; - Some((test, platform, node, compiler, reporter, compiler_output)) - } - }) - // Deploying the libraries for the platform. - .filter_map( - |(test, platform, node, compiler, reporter, compiler_output)| async move { - let mut deployed_libraries = None::>; - let mut contract_sources = test - .metadata - .contract_sources() - .inspect_err(|err| { - error!( - ?err, - platform_identifier = %platform.platform_identifier(), - "Failed to retrieve contract sources from metadata" - ) - }) - .ok()?; - for library_instance in test - .metadata - .libraries - .iter() - .flatten() - .flat_map(|(_, map)| map.values()) - { - debug!(%library_instance, "Deploying Library Instance"); - - let ContractPathAndIdent { - contract_source_path: library_source_path, - contract_ident: library_ident, - } = contract_sources.remove(library_instance)?; - - let (code, abi) = compiler_output - .contracts - .get(&library_source_path) - .and_then(|contracts| contracts.get(library_ident.as_str()))?; - - let code = alloy::hex::decode(code).ok()?; - - // Getting the deployer address from the cases themselves. This is to ensure - // that we're doing the deployments from different accounts and therefore we're - // not slowed down by the nonce. - let deployer_address = test - .case - .steps - .iter() - .filter_map(|step| match step { - Step::FunctionCall(input) => input.caller.as_address().copied(), - Step::BalanceAssertion(..) => None, - Step::StorageEmptyAssertion(..) => None, - Step::Repeat(..) => None, - Step::AllocateAccount(..) => None, - }) - .next() - .unwrap_or(FunctionCallStep::default_caller_address()); - let tx = TransactionBuilder::::with_deploy_code( - TransactionRequest::default().from(deployer_address), - code, - ); - let receipt = node - .execute_transaction(tx) - .await - .inspect_err(|err| { - error!( - ?err, - %library_instance, - platform_identifier = %platform.platform_identifier(), - "Failed to deploy the library" - ) - }) - .ok()?; - - debug!( - ?library_instance, - platform_identifier = %platform.platform_identifier(), - "Deployed library" - ); - - let library_address = receipt.contract_address?; - - deployed_libraries.get_or_insert_default().insert( - library_instance.clone(), - (library_ident.clone(), library_address, abi.clone()), - ); - } - - Some(( - test, - platform, - node, - compiler, - reporter, - compiler_output, - deployed_libraries, - )) - }, - ) - // Compiling the post-link contracts. - .filter_map( - |(test, platform, node, compiler, reporter, _, deployed_libraries)| { - let cached_compiler = cached_compiler.clone(); - let private_key_allocator = private_key_allocator.clone(); - - async move { - let compiler_output = cached_compiler - .compile_contracts( - test.metadata, - test.metadata_file_path, - test.mode.clone(), - deployed_libraries.as_ref(), - compiler.as_ref(), - *platform, - reporter, - ) - .await - .inspect_err(|err| { - error!( - ?err, - platform_identifier = %platform.platform_identifier(), - "Pre-linking compilation failed" - ) - }) - .ok()?; - - let case_state = CaseState::new( - compiler.version().clone(), - compiler_output.contracts, - deployed_libraries.unwrap_or_default(), - reporter.clone(), - private_key_allocator, - ); - - Some((*node, platform.platform_identifier(), case_state)) - } - }, - ) - // Collect - .collect::>() - .await; - - let mut driver = CaseDriver::new(test.metadata, test.case, platform_state); - driver - .execute() - .await - .inspect(|steps_executed| info!(steps_executed, "Case succeeded")) -} - -async fn execute_corpus( - context: TestExecutionContext, - tests: &[MetadataFile], - reporter: Reporter, - report_aggregator_task: impl Future>, -) -> anyhow::Result<()> { - let platforms = context - .platforms - .iter() - .copied() - .collect::>() - .into_iter() - .map(Into::<&dyn Platform>::into) - .collect::>(); - - run_driver(context, tests, reporter, report_aggregator_task, platforms).await?; - - Ok(()) -} - -/// this represents a single "test"; a mode, path and collection of cases. -#[allow(clippy::type_complexity)] -struct Test<'a> { - metadata: &'a MetadataFile, - metadata_file_path: &'a Path, - mode: Cow<'a, Mode>, - case_idx: CaseIdx, - case: &'a Case, - platforms: Vec<( - &'a dyn Platform, - &'a dyn EthereumNode, - Box, - ExecutionSpecificReporter, - )>, - reporter: TestSpecificReporter, -} - -impl<'a> Test<'a> { - /// Checks if this test can be ran with the current configuration. - pub fn check_compatibility(&self) -> TestCheckFunctionResult { - self.check_metadata_file_ignored()?; - self.check_case_file_ignored()?; - self.check_target_compatibility()?; - self.check_evm_version_compatibility()?; - self.check_compiler_compatibility()?; - Ok(()) - } - - /// Checks if the metadata file is ignored or not. - fn check_metadata_file_ignored(&self) -> TestCheckFunctionResult { - if self.metadata.ignore.is_some_and(|ignore| ignore) { - Err(("Metadata file is ignored.", indexmap! {})) - } else { - Ok(()) - } - } - - /// Checks if the case file is ignored or not. - fn check_case_file_ignored(&self) -> TestCheckFunctionResult { - if self.case.ignore.is_some_and(|ignore| ignore) { - Err(("Case is ignored.", indexmap! {})) - } else { - Ok(()) - } - } - - /// Checks if the platforms all support the desired targets in the metadata file. - fn check_target_compatibility(&self) -> TestCheckFunctionResult { - let mut error_map = indexmap! { - "test_desired_targets" => json!(self.metadata.targets.as_ref()), - }; - let mut is_allowed = true; - for (platform, ..) in self.platforms.iter() { - let is_allowed_for_platform = match self.metadata.targets.as_ref() { - None => true, - Some(targets) => { - let mut target_matches = false; - for target in targets.iter() { - if &platform.vm_identifier() == target { - target_matches = true; - break; - } - } - target_matches - } - }; - is_allowed &= is_allowed_for_platform; - error_map.insert( - platform.platform_identifier().into(), - json!(is_allowed_for_platform), - ); - } - - if is_allowed { - Ok(()) - } else { - Err(( - "One of the platforms do do not support the targets allowed by the test.", - error_map, - )) - } - } - - // Checks for the compatibility of the EVM version with the platforms specified. - fn check_evm_version_compatibility(&self) -> TestCheckFunctionResult { - let Some(evm_version_requirement) = self.metadata.required_evm_version else { - return Ok(()); - }; - - let mut error_map = indexmap! { - "test_desired_evm_version" => json!(self.metadata.required_evm_version), - }; - let mut is_allowed = true; - for (platform, node, ..) in self.platforms.iter() { - let is_allowed_for_platform = evm_version_requirement.matches(&node.evm_version()); - is_allowed &= is_allowed_for_platform; - error_map.insert( - platform.platform_identifier().into(), - json!(is_allowed_for_platform), - ); - } - - if is_allowed { - Ok(()) - } else { - Err(( - "EVM version is incompatible for the platforms specified", - error_map, - )) - } - } - - /// Checks if the platforms compilers support the mode that the test is for. - fn check_compiler_compatibility(&self) -> TestCheckFunctionResult { - let mut error_map = indexmap! { - "test_desired_evm_version" => json!(self.metadata.required_evm_version), - }; - let mut is_allowed = true; - for (platform, _, compiler, ..) in self.platforms.iter() { - let is_allowed_for_platform = - compiler.supports_mode(self.mode.optimize_setting, self.mode.pipeline); - is_allowed &= is_allowed_for_platform; - error_map.insert( - platform.platform_identifier().into(), - json!(is_allowed_for_platform), - ); - } - - if is_allowed { - Ok(()) - } else { - Err(( - "Compilers do not support this mode either for the provided platforms.", - error_map, - )) - } - } -} - -type TestCheckFunctionResult = Result<(), (&'static str, IndexMap<&'static str, Value>)>; diff --git a/crates/report/src/aggregator.rs b/crates/report/src/aggregator.rs index b3a693d..642237d 100644 --- a/crates/report/src/aggregator.rs +++ b/crates/report/src/aggregator.rs @@ -106,6 +106,7 @@ impl ReportAggregator { RunnerEvent::ContractDeployed(event) => { self.handle_contract_deployed_event(*event); } + RunnerEvent::Completion(event) => self.handle_completion(*event), } } debug!("Report aggregation completed"); @@ -382,6 +383,10 @@ impl ReportAggregator { .insert(event.contract_instance, event.address); } + fn handle_completion(&mut self, _: CompletionEvent) { + self.runner_rx.close(); + } + fn test_case_report(&mut self, specifier: &TestSpecifier) -> &mut TestCaseReport { self.report .test_case_information diff --git a/crates/report/src/runner_event.rs b/crates/report/src/runner_event.rs index 361555c..184dcdd 100644 --- a/crates/report/src/runner_event.rs +++ b/crates/report/src/runner_event.rs @@ -613,6 +613,8 @@ define_event! { /// The address of the contract. address: Address }, + /// Reports the completion of the run. + Completion {} } } diff --git a/run_tests.sh b/run_tests.sh index 5e33e87..daf7511 100755 --- a/run_tests.sh +++ b/run_tests.sh @@ -91,11 +91,11 @@ echo "" # Run the tool RUST_LOG="info,alloy_pubsub::service=error" cargo run --release -- execute-tests \ --platform geth-evm-solc \ - --platform lighthouse-geth-evm-solc \ + --platform revive-dev-node-polkavm-resolc \ --corpus "$CORPUS_FILE" \ --working-directory "$WORKDIR" \ - --concurrency.number-of-nodes 3 \ - --wallet.additional-keys 10000 \ + --concurrency.number-of-nodes 5 \ + --wallet.additional-keys 100000 \ --kitchensink.path "$SUBSTRATE_NODE_BIN" \ --revive-dev-node.path "$REVIVE_DEV_NODE_BIN" \ --eth-rpc.path "$ETH_RPC_BIN" \