//! The test driver handles the compilation and execution of the test cases. use alloy::json_abi::JsonAbi; use alloy::network::{Ethereum, TransactionBuilder}; use alloy::rpc::types::TransactionReceipt; use alloy::rpc::types::trace::geth::GethTrace; use alloy::{ primitives::{Address, map::HashMap}, rpc::types::{ TransactionRequest, trace::geth::{AccountState, DiffMode}, }, }; use revive_dt_compiler::{Compiler, CompilerInput, SolidityCompiler}; use revive_dt_config::Arguments; use revive_dt_format::{input::Input, metadata::Metadata, mode::SolcMode}; use revive_dt_node_interaction::EthereumNode; use revive_dt_report::reporter::{CompilationTask, Report, Span}; use revive_solc_json_interface::SolcStandardJsonOutput; use serde_json::Value; use std::collections::HashMap as StdHashMap; use std::fmt::Debug; use crate::Platform; type Contracts = HashMap< CompilerInput<<::Compiler as SolidityCompiler>::Options>, SolcStandardJsonOutput, >; pub struct State<'a, T: Platform> { config: &'a Arguments, span: Span, contracts: Contracts, deployed_contracts: StdHashMap, deployed_abis: StdHashMap, } impl<'a, T> State<'a, T> where T: Platform, { pub fn new(config: &'a Arguments, span: Span) -> Self { Self { config, span, contracts: Default::default(), deployed_contracts: Default::default(), deployed_abis: Default::default(), } } /// Returns a copy of the current span. fn span(&self) -> Span { self.span } pub fn build_contracts(&mut self, mode: &SolcMode, metadata: &Metadata) -> anyhow::Result<()> { let mut span = self.span(); span.next_metadata( metadata .file_path .as_ref() .expect("metadata should have been read from a file") .clone(), ); let Some(version) = mode.last_patch_version(&self.config.solc) else { anyhow::bail!("unsupported solc version: {:?}", &mode.solc_version); }; let compiler = Compiler::::new() .allow_path(metadata.directory()?) .solc_optimizer(mode.solc_optimize()); let compiler = FilesWithExtensionIterator::new(metadata.directory()?) .with_allowed_extension("sol") .try_fold(compiler, |compiler, path| compiler.with_source(&path))?; let mut task = CompilationTask { json_input: compiler.input(), json_output: None, mode: mode.clone(), compiler_version: format!("{}", &version), error: None, }; let compiler_path = T::Compiler::get_compiler_executable(self.config, version)?; match compiler.try_build(compiler_path) { Ok(output) => { task.json_output = Some(output.output.clone()); task.error = output.error; self.contracts.insert(output.input, output.output); if let Some(last_output) = self.contracts.values().last() { if let Some(contracts) = &last_output.contracts { for (file, contracts_map) in contracts { for contract_name in contracts_map.keys() { tracing::debug!( "Compiled contract: {contract_name} from file: {file}" ); } } } else { tracing::warn!("Compiled contracts field is None"); } } Report::compilation(span, T::config_id(), task); Ok(()) } Err(error) => { tracing::error!("Failed to compile contract: {:?}", error.to_string()); task.error = Some(error.to_string()); Err(error) } } } pub fn execute_input( &mut self, input: &Input, node: &T::Blockchain, ) -> anyhow::Result<(TransactionReceipt, GethTrace, DiffMode)> { tracing::trace!("Calling execute_input for input: {input:?}"); let nonce = node.fetch_add_nonce(input.caller)?; tracing::debug!( "Nonce calculated on the execute contract, calculated nonce {}, for contract {}, having address {} on node: {}", &nonce, &input.instance, &input.caller, std::any::type_name::() ); let tx = match input.legacy_transaction( nonce, &self.deployed_contracts, &self.deployed_abis, node, ) { Ok(tx) => { tracing::debug!("Legacy transaction data: {tx:#?}"); tx } Err(err) => { tracing::error!("Failed to construct legacy transaction: {err:?}"); return Err(err); } }; tracing::trace!("Executing transaction for input: {input:?}"); let receipt = match node.execute_transaction(tx) { Ok(receipt) => receipt, Err(err) => { tracing::error!( "Failed to execute transaction when executing the contract: {}, {:?}", &input.instance, err ); return Err(err); } }; tracing::trace!( "Transaction receipt for executed contract: {} - {:?}", &input.instance, receipt, ); let trace = node.trace_transaction(receipt.clone())?; tracing::trace!( "Trace result for contract: {} - {:?}", &input.instance, trace ); let diff = node.state_diff(receipt.clone())?; Ok((receipt, trace, diff)) } pub fn deploy_contracts(&mut self, input: &Input, node: &T::Blockchain) -> anyhow::Result<()> { let tracing_span = tracing::debug_span!( "Deploying contracts", ?input, node = std::any::type_name::() ); let _guard = tracing_span.enter(); tracing::debug!(number_of_contracts_to_deploy = self.contracts.len()); for output in self.contracts.values() { let Some(contract_map) = &output.contracts else { tracing::debug!( "No contracts in output — skipping deployment for this input {}", &input.instance ); continue; }; for contracts in contract_map.values() { for (contract_name, contract) in contracts { let tracing_span = tracing::info_span!("Deploying contract", contract_name); let _guard = tracing_span.enter(); tracing::debug!( "Contract name is: {:?} and the input name is: {:?}", &contract_name, &input.instance ); let bytecode = contract .evm .as_ref() .and_then(|evm| evm.bytecode.as_ref()) .map(|b| b.object.clone()); let Some(code) = bytecode else { tracing::error!("no bytecode for contract {contract_name}"); continue; }; let nonce = match node.fetch_add_nonce(input.caller) { Ok(nonce) => nonce, Err(error) => { tracing::error!( caller = ?input.caller, ?error, "Failed to get the nonce for the caller" ); return Err(error); } }; tracing::debug!( "Calculated nonce {}, for contract {}, having address {} on node: {}", &nonce, &input.instance, &input.caller, std::any::type_name::() ); // We are using alloy for building and submitting the transactions and it will // automatically fill in all of the missing fields from the provider that we // are using. let code = match alloy::hex::decode(&code) { Ok(code) => code, Err(error) => { tracing::error!( code, ?error, "Failed to hex-decode the code of the contract. (This could possibly mean that it contains '_' and therefore it requires linking to be performed)" ); return Err(error.into()); } }; let tx = { let tx = TransactionRequest::default() .nonce(nonce) .from(input.caller); TransactionBuilder::::with_deploy_code(tx, code) }; let receipt = match node.execute_transaction(tx) { Ok(receipt) => receipt, Err(err) => { tracing::error!( "Failed to execute transaction when deploying the contract on node : {:?}, {:?}, {:?}", std::any::type_name::(), &contract_name, err ); return Err(err); } }; tracing::debug!( "Deployment tx sent for {} with nonce {} → tx hash: {:?}, on node: {:?}", contract_name, nonce, receipt.transaction_hash, std::any::type_name::(), ); tracing::trace!( "Deployed transaction receipt for contract: {} - {:?}, on node: {:?}", &contract_name, receipt, std::any::type_name::(), ); let Some(address) = receipt.contract_address else { tracing::error!( "contract {contract_name} deployment did not return an address" ); continue; }; self.deployed_contracts .insert(contract_name.clone(), address); tracing::trace!( "deployed contract `{}` at {:?}, on node {:?}", contract_name, address, std::any::type_name::() ); let Some(Value::String(metadata)) = &contract.metadata else { tracing::error!(?contract, "Contract does not have a metadata field"); anyhow::bail!("Contract does not have a metadata field: {contract:?}"); }; // Deserialize the solc metadata into a JSON object so we can get the ABI of the // contracts. If we fail to perform the deserialization then we return an error // as there's no other way to handle this. let Ok(metadata) = serde_json::from_str::(metadata) else { tracing::error!(%metadata, "Failed to parse solc metadata into a structured value"); anyhow::bail!( "Failed to parse solc metadata into a structured value {metadata}" ); }; // Accessing the ABI on the solc metadata and erroring if the accessing failed let Some(abi) = metadata.get("output").and_then(|value| value.get("abi")) else { tracing::error!(%metadata, "Failed to access the .output.abi field of the solc metadata"); anyhow::bail!( "Failed to access the .output.abi field of the solc metadata {metadata}" ); }; // Deserialize the ABI object that we got from the unstructured JSON into a // structured ABI object and error out if we fail. let Ok(abi) = serde_json::from_value::(abi.clone()) else { tracing::error!(%metadata, "Failed to deserialize ABI into a structured format"); anyhow::bail!( "Failed to deserialize ABI into a structured format {metadata}" ); }; self.deployed_abis.insert(contract_name.clone(), abi); } } } tracing::debug!("Available contracts: {:?}", self.deployed_contracts.keys()); Ok(()) } } pub struct Driver<'a, Leader: Platform, Follower: Platform> { metadata: &'a Metadata, config: &'a Arguments, leader_node: &'a Leader::Blockchain, follower_node: &'a Follower::Blockchain, } impl<'a, L, F> Driver<'a, L, F> where L: Platform, F: Platform, { pub fn new( metadata: &'a Metadata, config: &'a Arguments, leader_node: &'a L::Blockchain, follower_node: &'a F::Blockchain, ) -> Driver<'a, L, F> { Self { metadata, config, leader_node, follower_node, } } pub fn trace_diff_mode(label: &str, diff: &DiffMode) { tracing::trace!("{label} - PRE STATE:"); for (addr, state) in &diff.pre { Self::trace_account_state(" [pre]", addr, state); } tracing::trace!("{label} - POST STATE:"); for (addr, state) in &diff.post { Self::trace_account_state(" [post]", addr, state); } } fn trace_account_state(prefix: &str, addr: &Address, state: &AccountState) { tracing::trace!("{prefix} 0x{addr:x}"); if let Some(balance) = &state.balance { tracing::trace!("{prefix} balance: {balance}"); } if let Some(nonce) = &state.nonce { tracing::trace!("{prefix} nonce: {nonce}"); } if let Some(code) = &state.code { tracing::trace!("{prefix} code: {code}"); } } // A note on this function and the choice of how we handle errors that happen here. This is not // a doc comment since it's a comment for the maintainers of this code and not for the users of // this code. // // This function does a few things: it builds the contracts for the various SOLC modes needed. // It deploys the contracts to the chain, and it executes the various inputs that are specified // for the test cases. // // In most functions in the codebase, it's fine to just say "If we encounter an error just // bubble it up to the caller", but this isn't a good idea to do here and we need an elaborate // way to report errors all while being graceful and continuing execution where we can. For // example, if one of the inputs of one of the cases fail to execute, then we should not just // bubble that error up immediately. Instead, we should note it down and continue to the next // case as the next case might succeed. // // Therefore, this method returns an `ExecutionResult` object, and not just a normal `Result`. // This object is fully typed to contain information about what exactly in the execution was a // success and what failed. // // The above then allows us to have better logging and better information in the caller of this // function as we have a more detailed view of what worked and what didn't. pub fn execute(&mut self, span: Span) -> ExecutionResult { // This is the execution result object that all of the execution information will be // collected into and returned at the end of the execution. let mut execution_result = ExecutionResult::default(); let tracing_span = tracing::info_span!("Handling metadata file"); let _guard = tracing_span.enter(); for mode in self.metadata.solc_modes() { let tracing_span = tracing::info_span!("With solc mode", solc_mode = ?mode); let _guard = tracing_span.enter(); let mut leader_state = State::::new(self.config, span); let mut follower_state = State::::new(self.config, span); // We build the contracts. If building the contracts for the metadata file fails then we // have no other option but to keep note of this error and move on to the next solc mode // and NOT just bail out of the execution as a whole. let build_result = tracing::info_span!("Building contracts").in_scope(|| { match leader_state.build_contracts(&mode, self.metadata) { Ok(_) => { tracing::debug!(target = ?Target::Leader, "Contract building succeeded"); execution_result.add_successful_build(Target::Leader, mode.clone()); }, Err(error) => { tracing::error!(target = ?Target::Leader, ?error, "Contract building failed"); execution_result.add_failed_build(Target::Leader, mode.clone(), error); return Err(()); } } match follower_state.build_contracts(&mode, self.metadata) { Ok(_) => { tracing::debug!(target = ?Target::Follower, "Contract building succeeded"); execution_result.add_successful_build(Target::Follower, mode.clone()); }, Err(error) => { tracing::error!(target = ?Target::Follower, ?error, "Contract building failed"); execution_result.add_failed_build(Target::Follower, mode.clone(), error); return Err(()); } } Ok(()) }); if build_result.is_err() { // Note: We skip to the next solc mode as there's nothing that we can do at this // point, the building has failed. We do NOT bail out of the execution as a whole. continue; } // For cases if one of the inputs fail then we move on to the next case and we do NOT // bail out of the whole thing. 'case_loop: for (case_idx, case) in self.metadata.cases.iter().enumerate() { let tracing_span = tracing::info_span!( "Handling case", case_name = case.name, case_idx = case_idx ); let _guard = tracing_span.enter(); // For inputs if one of the inputs fail we move on to the next case (we do not move // on to the next input as it doesn't make sense. It depends on the previous one). for (input_idx, input) in case.inputs.iter().enumerate() { let tracing_span = tracing::info_span!("Handling input", input_idx); let _guard = tracing_span.enter(); // TODO: verify if this is correct, I doubt that we need to do contract redeploy // for each input. It doesn't quite look to be correct but we need to cross // check with the matterlabs implementation. This matches our implementation but // I have doubts around its correctness. let deployment_result = tracing::info_span!( "Deploying contracts", contract_name = input.instance ) .in_scope(|| { if let Err(error) = leader_state.deploy_contracts(input, self.leader_node) { tracing::error!(target = ?Target::Leader, ?error, "Contract deployment failed"); execution_result.add_failed_case( Target::Leader, mode.clone(), case.name.clone().unwrap_or("no case name".to_owned()), case_idx, input_idx, anyhow::Error::msg( format!("Failed to deploy contracts, {error}") ) ); return Err(error) }; if let Err(error) = follower_state.deploy_contracts(input, self.follower_node) { tracing::error!(target = ?Target::Follower, ?error, "Contract deployment failed"); execution_result.add_failed_case( Target::Follower, mode.clone(), case.name.clone().unwrap_or("no case name".to_owned()), case_idx, input_idx, anyhow::Error::msg( format!("Failed to deploy contracts, {error}") ) ); return Err(error) }; Ok(()) }); if deployment_result.is_err() { // Noting it again here: if something in the input fails we do not move on // to the next input, we move to the next case completely. continue 'case_loop; } let execution_result = tracing::info_span!("Executing input", contract_name = input.instance) .in_scope(|| { let (leader_receipt, _, leader_diff) = match leader_state.execute_input(input, self.leader_node) { Ok(result) => result, Err(error) => { tracing::error!( target = ?Target::Leader, ?error, "Contract execution failed" ); return Err(error); } }; let (follower_receipt, _, follower_diff) = match follower_state.execute_input(input, self.follower_node) { Ok(result) => result, Err(error) => { tracing::error!( target = ?Target::Follower, ?error, "Contract execution failed" ); return Err(error); } }; Ok((leader_receipt, leader_diff, follower_receipt, follower_diff)) }); let Ok((leader_receipt, leader_diff, follower_receipt, follower_diff)) = execution_result else { // Noting it again here: if something in the input fails we do not move on // to the next input, we move to the next case completely. continue 'case_loop; }; if leader_diff == follower_diff { tracing::debug!("State diffs match between leader and follower."); } else { tracing::debug!("State diffs mismatch between leader and follower."); Self::trace_diff_mode("Leader", &leader_diff); Self::trace_diff_mode("Follower", &follower_diff); } if leader_receipt.logs() != follower_receipt.logs() { tracing::debug!("Log/event mismatch between leader and follower."); tracing::trace!("Leader logs: {:?}", leader_receipt.logs()); tracing::trace!("Follower logs: {:?}", follower_receipt.logs()); } if leader_receipt.status() != follower_receipt.status() { tracing::debug!( "Mismatch in status: leader = {}, follower = {}", leader_receipt.status(), follower_receipt.status() ); } } // Note: Only consider the case as having been successful after we have processed // all of the inputs and completed the entire loop over the input. execution_result.add_successful_case( Target::Leader, mode.clone(), case.name.clone().unwrap_or("no case name".to_owned()), case_idx, ); execution_result.add_successful_case( Target::Follower, mode.clone(), case.name.clone().unwrap_or("no case name".to_owned()), case_idx, ); } } execution_result } } #[derive(Debug, Default)] pub struct ExecutionResult { pub results: Vec>, pub successful_cases_count: usize, pub failed_cases_count: usize, } impl ExecutionResult { pub fn new() -> Self { Self { results: Default::default(), successful_cases_count: Default::default(), failed_cases_count: Default::default(), } } pub fn add_successful_build(&mut self, target: Target, solc_mode: SolcMode) { self.results .push(Box::new(BuildResult::Success { target, solc_mode })); } pub fn add_failed_build(&mut self, target: Target, solc_mode: SolcMode, error: anyhow::Error) { self.results.push(Box::new(BuildResult::Failure { target, solc_mode, error, })); } pub fn add_successful_case( &mut self, target: Target, solc_mode: SolcMode, case_name: String, case_idx: usize, ) { self.successful_cases_count += 1; self.results.push(Box::new(CaseResult::Success { target, solc_mode, case_name, case_idx, })); } pub fn add_failed_case( &mut self, target: Target, solc_mode: SolcMode, case_name: String, case_idx: usize, input_idx: usize, error: anyhow::Error, ) { self.failed_cases_count += 1; self.results.push(Box::new(CaseResult::Failure { target, solc_mode, case_name, case_idx, error, input_idx, })); } } pub trait ExecutionResultItem: Debug { /// Converts this result item into an [`anyhow::Result`]. fn as_result(&self) -> Result<(), &anyhow::Error>; /// Provides information on whether the provided result item is of a success or failure. fn is_success(&self) -> bool; /// Provides information of the target that this result is for. fn target(&self) -> &Target; /// Provides information on the [`SolcMode`] mode that we being used for this result item. fn solc_mode(&self) -> &SolcMode; /// Provides information on the case name and number that this result item pertains to. This is /// [`None`] if the error doesn't belong to any case (e.g., if it's a build error outside of any /// of the cases.). fn case_name_and_index(&self) -> Option<(&str, usize)>; /// Provides information on the input number that this result item pertains to. This is [`None`] /// if the error doesn't belong to any input (e.g., if it's a build error outside of any of the /// inputs.). fn input_index(&self) -> Option; } #[derive(Clone, Copy, Debug, PartialEq, Eq, PartialOrd, Ord, Hash)] pub enum Target { Leader, Follower, } #[derive(Debug)] pub enum BuildResult { Success { target: Target, solc_mode: SolcMode, }, Failure { target: Target, solc_mode: SolcMode, error: anyhow::Error, }, } impl ExecutionResultItem for BuildResult { fn as_result(&self) -> Result<(), &anyhow::Error> { match self { Self::Success { .. } => Ok(()), Self::Failure { error, .. } => Err(error)?, } } fn is_success(&self) -> bool { match self { Self::Success { .. } => true, Self::Failure { .. } => false, } } fn target(&self) -> &Target { match self { Self::Success { target, .. } | Self::Failure { target, .. } => target, } } fn solc_mode(&self) -> &SolcMode { match self { Self::Success { solc_mode, .. } | Self::Failure { solc_mode, .. } => solc_mode, } } fn case_name_and_index(&self) -> Option<(&str, usize)> { None } fn input_index(&self) -> Option { None } } #[derive(Debug)] pub enum CaseResult { Success { target: Target, solc_mode: SolcMode, case_name: String, case_idx: usize, }, Failure { target: Target, solc_mode: SolcMode, case_name: String, case_idx: usize, input_idx: usize, error: anyhow::Error, }, } impl ExecutionResultItem for CaseResult { fn as_result(&self) -> Result<(), &anyhow::Error> { match self { Self::Success { .. } => Ok(()), Self::Failure { error, .. } => Err(error)?, } } fn is_success(&self) -> bool { match self { Self::Success { .. } => true, Self::Failure { .. } => false, } } fn target(&self) -> &Target { match self { Self::Success { target, .. } | Self::Failure { target, .. } => target, } } fn solc_mode(&self) -> &SolcMode { match self { Self::Success { solc_mode, .. } | Self::Failure { solc_mode, .. } => solc_mode, } } fn case_name_and_index(&self) -> Option<(&str, usize)> { match self { Self::Success { case_name, case_idx, .. } | Self::Failure { case_name, case_idx, .. } => Some((case_name, *case_idx)), } } fn input_index(&self) -> Option { match self { CaseResult::Success { .. } => None, CaseResult::Failure { input_idx, .. } => Some(*input_idx), } } } /// An iterator that finds files of a certain extension in the provided directory. You can think of /// this a glob pattern similar to: `${path}/**/*.md` struct FilesWithExtensionIterator { /// The set of allowed extensions that that match the requirement and that should be returned /// when found. allowed_extensions: std::collections::HashSet>, /// The set of directories to visit next. This iterator does BFS and so these directories will /// only be visited if we can't find any files in our state. directories_to_search: Vec, /// The set of files matching the allowed extensions that were found. If there are entries in /// this vector then they will be returned when the [`Iterator::next`] method is called. If not /// then we visit one of the next directories to visit. /// /// [`Iterator`]: std::iter::Iterator files_matching_allowed_extensions: Vec, } impl FilesWithExtensionIterator { fn new(root_directory: std::path::PathBuf) -> Self { Self { allowed_extensions: Default::default(), directories_to_search: vec![root_directory], files_matching_allowed_extensions: Default::default(), } } fn with_allowed_extension( mut self, allowed_extension: impl Into>, ) -> Self { self.allowed_extensions.insert(allowed_extension.into()); self } } impl Iterator for FilesWithExtensionIterator { type Item = std::path::PathBuf; fn next(&mut self) -> Option { if let Some(file_path) = self.files_matching_allowed_extensions.pop() { return Some(file_path); }; let directory_to_search = self.directories_to_search.pop()?; // Read all of the entries in the directory. If we failed to read this dir's entires then we // elect to just ignore it and look in the next directory, we do that by calling the next // method again on the iterator, which is an intentional decision that we made here instead // of panicking. let Ok(dir_entries) = std::fs::read_dir(directory_to_search) else { return self.next(); }; for entry in dir_entries.flatten() { let entry_path = entry.path(); if entry_path.is_dir() { self.directories_to_search.push(entry_path) } else if entry_path.is_file() && entry_path.extension().is_some_and(|ext| { self.allowed_extensions .iter() .any(|allowed| ext.eq_ignore_ascii_case(allowed.as_ref())) }) { self.files_matching_allowed_extensions.push(entry_path) } } self.next() } }