Parallelize execution

crates/config/src/lib.rs

@@ -163,7 +163,9 @@ impl Default for Arguments {
 /// The Solidity compatible node implementation.
 ///
 /// This describes the solutions to be tested against on a high level.
-#[derive(Clone, Copy, Debug, Eq, Hash, PartialEq, ValueEnum, Serialize, Deserialize)]
+#[derive(
+    Clone, Copy, Debug, PartialEq, Eq, PartialOrd, Ord, Hash, ValueEnum, Serialize, Deserialize,
+)]
 #[clap(rename_all = "lower")]
 pub enum TestingPlatform {
     /// The go-ethereum reference full node EVM implementation.

crates/core/src/driver/mod.rs

@@ -1,7 +1,6 @@
 //! The test driver handles the compilation and execution of the test cases.
 
 use std::collections::HashMap;
-use std::fmt::Debug;
 use std::marker::PhantomData;
 use std::path::PathBuf;
 
@@ -24,213 +23,63 @@ use anyhow::Context;
 use indexmap::IndexMap;
 use semver::Version;
 
-use revive_dt_common::iterators::FilesWithExtensionIterator;
-use revive_dt_compiler::{Compiler, SolidityCompiler};
-use revive_dt_config::Arguments;
 use revive_dt_format::case::{Case, CaseIdx};
 use revive_dt_format::input::{Calldata, EtherValue, Expected, ExpectedOutput, Method};
 use revive_dt_format::metadata::{ContractInstance, ContractPathAndIdent};
-use revive_dt_format::{input::Input, metadata::Metadata, mode::SolcMode};
+use revive_dt_format::{input::Input, metadata::Metadata};
 use revive_dt_node::Node;
 use revive_dt_node_interaction::EthereumNode;
-use revive_dt_report::reporter::{CompilationTask, Report, Span};
 
 use crate::Platform;
 
-pub struct CaseState<'a, T: Platform> {
+pub struct CaseState<T: Platform> {
-    /// The configuration that the framework was started with.
-    ///
-    /// This is currently used to get certain information from it such as the solc mode and other
-    /// information used at runtime.
-    config: &'a Arguments,
-
-    /// The [`Span`] used in reporting.
-    span: Span,
-
     /// A map of all of the compiled contracts for the given metadata file.
     compiled_contracts: HashMap<PathBuf, HashMap<String, (String, JsonAbi)>>,
 
-    /// This map stores the contracts deployments that have been made for each case within a
+    /// This map stores the contracts deployments for this case.
-    /// metadata file. Note, this means that the state can't be reused between different metadata
+    deployed_contracts: HashMap<ContractInstance, (Address, JsonAbi)>,
-    /// files.
-    deployed_contracts: HashMap<CaseIdx, HashMap<ContractInstance, (Address, JsonAbi)>>,
 
     /// This map stores the variables used for each one of the cases contained in the metadata
     /// file.
-    variables: HashMap<CaseIdx, HashMap<String, U256>>,
+    variables: HashMap<String, U256>,
 
-    /// This is a map of the deployed libraries.
+    /// Stores the version used for the current case.
-    ///
+    compiler_version: Version,
-    /// This map is not per case, but rather, per metadata file. This means that we do not redeploy
-    /// the libraries with each case.
-    deployed_libraries: HashMap<ContractInstance, (Address, JsonAbi)>,
-
-    /// Stores the version of the compiler used for the given Solc mode.
-    compiler_version: HashMap<&'a SolcMode, Version>,
 
     phantom: PhantomData<T>,
 }
 
-impl<'a, T> CaseState<'a, T>
+impl<T> CaseState<T>
 where
     T: Platform,
 {
-    pub fn new(config: &'a Arguments, span: Span) -> Self {
+    pub fn new(
+        compiler_version: Version,
+        compiled_contracts: HashMap<PathBuf, HashMap<String, (String, JsonAbi)>>,
+        deployed_contracts: HashMap<ContractInstance, (Address, JsonAbi)>,
+    ) -> Self {
         Self {
-            config,
+            compiled_contracts,
-            span,
+            deployed_contracts,
-            compiled_contracts: Default::default(),
-            deployed_contracts: Default::default(),
             variables: Default::default(),
-            deployed_libraries: Default::default(),
+            compiler_version,
-            compiler_version: Default::default(),
+            phantom: PhantomData,
-            phantom: Default::default(),
         }
     }
 
-    /// Returns a copy of the current span.
-    fn span(&self) -> Span {
-        self.span
-    }
-
-    pub fn build_contracts(
-        &mut self,
-        mode: &'a SolcMode,
-        metadata: &Metadata,
-    ) -> anyhow::Result<()> {
-        let mut span = self.span();
-        span.next_metadata(
-            metadata
-                .file_path
-                .as_ref()
-                .expect("metadata should have been read from a file")
-                .clone(),
-        );
-
-        let compiler_version_or_requirement =
-            mode.compiler_version_to_use(self.config.solc.clone());
-        let compiler_path =
-            T::Compiler::get_compiler_executable(self.config, compiler_version_or_requirement)?;
-        let compiler_version = T::Compiler::new(compiler_path.clone()).version()?;
-        self.compiler_version.insert(mode, compiler_version.clone());
-
-        tracing::info!(%compiler_version, "Resolved the compiler version to use");
-
-        let compiler = Compiler::<T::Compiler>::new()
-            .with_allow_path(metadata.directory()?)
-            .with_optimization(mode.solc_optimize());
-        let mut compiler = metadata
-            .files_to_compile()?
-            .try_fold(compiler, |compiler, path| compiler.with_source(&path))?;
-        for (library_instance, (library_address, _)) in self.deployed_libraries.iter() {
-            let library_ident = &metadata
-                .contracts
-                .as_ref()
-                .and_then(|contracts| contracts.get(library_instance))
-                .expect("Impossible for library to not be found in contracts")
-                .contract_ident;
-
-            // Note the following: we need to tell solc which files require the libraries to be
-            // linked into them. We do not have access to this information and therefore we choose
-            // an easier, yet more compute intensive route, of telling solc that all of the files
-            // need to link the library and it will only perform the linking for the files that do
-            // actually need the library.
-            compiler = FilesWithExtensionIterator::new(metadata.directory()?)
-                .with_allowed_extension("sol")
-                .fold(compiler, |compiler, path| {
-                    compiler.with_library(&path, library_ident.as_str(), *library_address)
-                });
-        }
-
-        let mut task = CompilationTask {
-            json_input: compiler.input(),
-            json_output: None,
-            mode: mode.clone(),
-            compiler_version: format!("{}", &compiler_version),
-            error: None,
-        };
-
-        match compiler.try_build(compiler_path) {
-            Ok(output) => {
-                task.json_output = Some(output.clone());
-
-                for (contract_path, contracts) in output.contracts.into_iter() {
-                    let map = self
-                        .compiled_contracts
-                        .entry(contract_path.clone())
-                        .or_default();
-                    for (contract_name, contract_info) in contracts.into_iter() {
-                        tracing::debug!(
-                            contract_path = %contract_path.display(),
-                            contract_name = contract_name,
-                            "Compiled contract"
-                        );
-
-                        map.insert(contract_name, contract_info);
-                    }
-                }
-
-                Report::compilation(span, T::config_id(), task);
-                Ok(())
-            }
-            Err(error) => {
-                tracing::error!("Failed to compile contract: {:?}", error.to_string());
-                task.error = Some(error.to_string());
-                Err(error)
-            }
-        }
-    }
-
-    pub fn build_and_publish_libraries(
-        &mut self,
-        metadata: &Metadata,
-        mode: &'a SolcMode,
-        node: &T::Blockchain,
-    ) -> anyhow::Result<()> {
-        self.build_contracts(mode, metadata)?;
-
-        for library_instance in metadata
-            .libraries
-            .iter()
-            .flatten()
-            .flat_map(|(_, map)| map.values())
-        {
-            self.get_or_deploy_contract_instance(
-                library_instance,
-                metadata,
-                None,
-                Input::default_caller(),
-                None,
-                None,
-                node,
-            )?;
-        }
-
-        Ok(())
-    }
-
     pub fn handle_input(
         &mut self,
         metadata: &Metadata,
         case_idx: CaseIdx,
         input: &Input,
         node: &T::Blockchain,
-        mode: &SolcMode,
     ) -> anyhow::Result<(TransactionReceipt, GethTrace, DiffMode)> {
         let deployment_receipts =
             self.handle_contract_deployment(metadata, case_idx, input, node)?;
-        let execution_receipt =
+        let execution_receipt = self.handle_input_execution(input, deployment_receipts, node)?;
-            self.handle_input_execution(case_idx, input, deployment_receipts, node)?;
         let tracing_result = self.handle_input_call_frame_tracing(&execution_receipt, node)?;
-        self.handle_input_variable_assignment(case_idx, input, &tracing_result)?;
+        self.handle_input_variable_assignment(input, &tracing_result)?;
-        self.handle_input_expectations(
+        self.handle_input_expectations(input, &execution_receipt, node, &tracing_result)?;
-            case_idx,
-            input,
-            &execution_receipt,
-            node,
-            mode,
-            &tracing_result,
-        )?;
         self.handle_input_diff(case_idx, execution_receipt, node)
     }
 
@@ -251,12 +100,7 @@ where
 
         let mut instances_we_must_deploy = IndexMap::<ContractInstance, bool>::new();
         for instance in input.find_all_contract_instances().into_iter() {
-            if !self
+            if !self.deployed_contracts.contains_key(&instance) {
-                .deployed_contracts
-                .entry(case_idx)
-                .or_insert_with(|| self.deployed_libraries.clone())
-                .contains_key(&instance)
-            {
                 instances_we_must_deploy.entry(instance).or_insert(false);
             }
         }
@@ -280,7 +124,6 @@ where
             if let (_, _, Some(receipt)) = self.get_or_deploy_contract_instance(
                 &instance,
                 metadata,
-                case_idx,
                 input.caller,
                 calldata,
                 value,
@@ -296,7 +139,6 @@ where
     /// Handles the execution of the input in terms of the calls that need to be made.
     fn handle_input_execution(
         &mut self,
-        case_idx: CaseIdx,
         input: &Input,
         mut deployment_receipts: HashMap<ContractInstance, TransactionReceipt>,
         node: &T::Blockchain,
@@ -308,22 +150,18 @@ where
                 .remove(&input.instance)
                 .context("Failed to find deployment receipt"),
             Method::Fallback | Method::FunctionName(_) => {
-                let tx = match input.legacy_transaction(
+                let tx =
-                    self.deployed_contracts
+                    match input.legacy_transaction(&self.deployed_contracts, &self.variables, node)
-                        .entry(case_idx)
+                    {
-                        .or_insert_with(|| self.deployed_libraries.clone()),
+                        Ok(tx) => {
-                    &*self.variables.entry(case_idx).or_default(),
+                            tracing::debug!("Legacy transaction data: {tx:#?}");
-                    node,
+                            tx
-                ) {
+                        }
-                    Ok(tx) => {
+                        Err(err) => {
-                        tracing::debug!("Legacy transaction data: {tx:#?}");
+                            tracing::error!("Failed to construct legacy transaction: {err:?}");
-                        tx
+                            return Err(err);
-                    }
+                        }
-                    Err(err) => {
+                    };
-                        tracing::error!("Failed to construct legacy transaction: {err:?}");
-                        return Err(err);
-                    }
-                };
 
                 tracing::trace!("Executing transaction for input: {input:?}");
 
@@ -365,7 +203,6 @@ where
 
     fn handle_input_variable_assignment(
         &mut self,
-        case_idx: CaseIdx,
         input: &Input,
         tracing_result: &CallFrame,
     ) -> anyhow::Result<()> {
@@ -383,10 +220,7 @@ where
                 .chunks(32),
         ) {
             let value = U256::from_be_slice(output_word);
-            self.variables
+            self.variables.insert(variable_name.clone(), value);
-                .entry(case_idx)
-                .or_default()
-                .insert(variable_name.clone(), value);
         }
 
         Ok(())
@@ -394,11 +228,9 @@ where
 
     fn handle_input_expectations(
         &mut self,
-        case_idx: CaseIdx,
         input: &Input,
         execution_receipt: &TransactionReceipt,
         node: &T::Blockchain,
-        mode: &SolcMode,
         tracing_result: &CallFrame,
     ) -> anyhow::Result<()> {
         let span = tracing::info_span!("Handling input expectations");
@@ -434,12 +266,10 @@ where
 
         for expectation in expectations.iter() {
             self.handle_input_expectation_item(
-                case_idx,
                 execution_receipt,
                 node,
                 expectation,
                 tracing_result,
-                mode,
             )?;
         }
 
@@ -448,28 +278,19 @@ where
 
     fn handle_input_expectation_item(
         &mut self,
-        case_idx: CaseIdx,
         execution_receipt: &TransactionReceipt,
         node: &T::Blockchain,
         expectation: &ExpectedOutput,
         tracing_result: &CallFrame,
-        mode: &SolcMode,
     ) -> anyhow::Result<()> {
         if let Some(ref version_requirement) = expectation.compiler_version {
-            let compiler_version = self
+            if !version_requirement.matches(&self.compiler_version) {
-                .compiler_version
-                .get(mode)
-                .context("Failed to find the compiler version fo the solc mode")?;
-            if !version_requirement.matches(compiler_version) {
                 return Ok(());
             }
         }
 
-        let deployed_contracts = self
+        let deployed_contracts = &mut self.deployed_contracts;
-            .deployed_contracts
+        let variables = &mut self.variables;
-            .entry(case_idx)
-            .or_insert_with(|| self.deployed_libraries.clone());
-        let variables = self.variables.entry(case_idx).or_default();
         let chain_state_provider = node;
 
         // Handling the receipt state assertion.
@@ -627,23 +448,12 @@ where
         &mut self,
         contract_instance: &ContractInstance,
         metadata: &Metadata,
-        case_idx: impl Into<Option<CaseIdx>>,
         deployer: Address,
         calldata: Option<&Calldata>,
         value: Option<EtherValue>,
         node: &T::Blockchain,
     ) -> anyhow::Result<(Address, JsonAbi, Option<TransactionReceipt>)> {
-        let case_idx = case_idx.into();
+        if let Some((address, abi)) = self.deployed_contracts.get(contract_instance) {
-
-        let deployed_contracts = match case_idx {
-            Some(case_idx) => self
-                .deployed_contracts
-                .entry(case_idx)
-                .or_insert_with(|| self.deployed_libraries.clone()),
-            None => &mut self.deployed_libraries,
-        };
-
-        if let Some((address, abi)) = deployed_contracts.get(contract_instance) {
             return Ok((*address, abi.clone(), None));
         }
 
@@ -690,7 +500,7 @@ where
         };
 
         if let Some(calldata) = calldata {
-            let calldata = calldata.calldata(deployed_contracts, None, node)?;
+            let calldata = calldata.calldata(&self.deployed_contracts, None, node)?;
             code.extend(calldata);
         }
 
@@ -725,7 +535,8 @@ where
             "Deployed contract"
         );
 
-        deployed_contracts.insert(contract_instance.clone(), (address, abi.clone()));
+        self.deployed_contracts
+            .insert(contract_instance.clone(), (address, abi.clone()));
 
         Ok((address, abi, Some(receipt)))
     }
@@ -735,9 +546,10 @@ pub struct CaseDriver<'a, Leader: Platform, Follower: Platform> {
     metadata: &'a Metadata,
     case: &'a Case,
     case_idx: CaseIdx,
-    config: &'a Arguments,
     leader_node: &'a Leader::Blockchain,
     follower_node: &'a Follower::Blockchain,
+    leader_state: CaseState<Leader>,
+    follower_state: CaseState<Follower>,
 }
 
 impl<'a, L, F> CaseDriver<'a, L, F>
@@ -745,21 +557,24 @@ where
     L: Platform,
     F: Platform,
 {
+    #[allow(clippy::too_many_arguments)]
     pub fn new(
         metadata: &'a Metadata,
         case: &'a Case,
         case_idx: impl Into<CaseIdx>,
-        config: &'a Arguments,
         leader_node: &'a L::Blockchain,
         follower_node: &'a F::Blockchain,
+        leader_state: CaseState<L>,
+        follower_state: CaseState<F>,
     ) -> CaseDriver<'a, L, F> {
         Self {
             metadata,
             case,
             case_idx: case_idx.into(),
-            config,
             leader_node,
             follower_node,
+            leader_state,
+            follower_state,
         }
     }
 
@@ -789,37 +604,7 @@ where
         }
     }
 
-    // A note on this function and the choice of how we handle errors that happen here. This is not
+    pub fn execute(&mut self) -> anyhow::Result<usize> {
-    // a doc comment since it's a comment for the maintainers of this code and not for the users of
-    // this code.
-    //
-    // This function does a few things: it builds the contracts for the various SOLC modes needed.
-    // It deploys the contracts to the chain, and it executes the various inputs that are specified
-    // for the test cases.
-    //
-    // In most functions in the codebase, it's fine to just say "If we encounter an error just
-    // bubble it up to the caller", but this isn't a good idea to do here and we need an elaborate
-    // way to report errors all while being graceful and continuing execution where we can. For
-    // example, if one of the inputs of one of the cases fail to execute, then we should not just
-    // bubble that error up immediately. Instead, we should note it down and continue to the next
-    // case as the next case might succeed.
-    //
-    // Therefore, this method returns an `ExecutionResult` object, and not just a normal `Result`.
-    // This object is fully typed to contain information about what exactly in the execution was a
-    // success and what failed.
-    //
-    // The above then allows us to have better logging and better information in the caller of this
-    // function as we have a more detailed view of what worked and what didn't.
-    pub fn execute(&mut self, span: Span) -> ExecutionResult {
-        // This is the execution result object that all of the execution information will be
-        // collected into and returned at the end of the execution.
-        let mut execution_result = ExecutionResult::default();
-
-        let tracing_span = tracing::info_span!("Handling metadata file");
-        let _guard = tracing_span.enter();
-
-        // We only execute this input if it's valid for the leader and the follower. Otherwise, we
-        // skip it with a warning.
         if !self
             .leader_node
             .matches_target(self.metadata.targets.as_deref())
@@ -831,410 +616,44 @@ where
                 targets = ?self.metadata.targets,
                 "Either the leader or follower node do not support the targets of the file"
             );
-            return execution_result;
+            return Ok(0);
         }
 
-        for mode in self.metadata.solc_modes() {
+        let mut inputs_executed = 0;
-            let tracing_span = tracing::info_span!("With solc mode", solc_mode = ?mode);
+        for (input_idx, input) in self.case.inputs_iterator().enumerate() {
+            let tracing_span = tracing::info_span!("Handling input", input_idx);
             let _guard = tracing_span.enter();
 
-            let mut leader_state = CaseState::<L>::new(self.config, span);
+            let (leader_receipt, _, leader_diff) = self.leader_state.handle_input(
-            let mut follower_state = CaseState::<F>::new(self.config, span);
+                self.metadata,
+                self.case_idx,
+                &input,
+                self.leader_node,
+            )?;
+            let (follower_receipt, _, follower_diff) = self.follower_state.handle_input(
+                self.metadata,
+                self.case_idx,
+                &input,
+                self.follower_node,
+            )?;
 
-            // Note: we are currently forced to do two compilation passes due to linking. In the
+            if leader_diff == follower_diff {
-            // first compilation pass we compile the libraries and publish them to the chain. In the
+                tracing::debug!("State diffs match between leader and follower.");
-            // second compilation pass we compile the contracts with the library addresses so that
+            } else {
-            // they're linked at compile-time.
+                tracing::debug!("State diffs mismatch between leader and follower.");
-            let build_result = tracing::info_span!("Building and publishing libraries")
+                Self::trace_diff_mode("Leader", &leader_diff);
-                .in_scope(|| {
+                Self::trace_diff_mode("Follower", &follower_diff);
-                    match leader_state.build_and_publish_libraries(self.metadata, &mode, self.leader_node) {
-                        Ok(_) => {
-                            tracing::debug!(target = ?Target::Leader, "Library building succeeded");
-                            execution_result.add_successful_build(Target::Leader, mode.clone());
-                        },
-                        Err(error) => {
-                            tracing::error!(target = ?Target::Leader, ?error, "Library building failed");
-                            execution_result.add_failed_build(Target::Leader, mode.clone(), error);
-                            return Err(());
-                        }
-                    }
-                    match follower_state.build_and_publish_libraries(self.metadata, &mode, self.follower_node) {
-                        Ok(_) => {
-                            tracing::debug!(target = ?Target::Follower, "Library building succeeded");
-                            execution_result.add_successful_build(Target::Follower, mode.clone());
-                        },
-                        Err(error) => {
-                            tracing::error!(target = ?Target::Follower, ?error, "Library building failed");
-                            execution_result.add_failed_build(Target::Follower, mode.clone(), error);
-                            return Err(());
-                        }
-                    }
-                    Ok(())
-                });
-            if build_result.is_err() {
-                // Note: We skip to the next solc mode as there's nothing that we can do at this
-                // point, the building has failed. We do NOT bail out of the execution as a whole.
-                continue;
             }
 
-            // We build the contracts. If building the contracts for the metadata file fails then we
+            if leader_receipt.logs() != follower_receipt.logs() {
-            // have no other option but to keep note of this error and move on to the next solc mode
+                tracing::debug!("Log/event mismatch between leader and follower.");
-            // and NOT just bail out of the execution as a whole.
+                tracing::trace!("Leader logs: {:?}", leader_receipt.logs());
-            let build_result = tracing::info_span!("Building contracts").in_scope(|| {
+                tracing::trace!("Follower logs: {:?}", follower_receipt.logs());
-                match leader_state.build_contracts(&mode, self.metadata) {
-                    Ok(_) => {
-                        tracing::debug!(target = ?Target::Leader, "Contract building succeeded");
-                        execution_result.add_successful_build(Target::Leader, mode.clone());
-                    },
-                    Err(error) => {
-                        tracing::error!(target = ?Target::Leader, ?error, "Contract building failed");
-                        execution_result.add_failed_build(Target::Leader, mode.clone(), error);
-                        return Err(());
-                    }
-                }
-                match follower_state.build_contracts(&mode, self.metadata) {
-                    Ok(_) => {
-                        tracing::debug!(target = ?Target::Follower, "Contract building succeeded");
-                        execution_result.add_successful_build(Target::Follower, mode.clone());
-                    },
-                    Err(error) => {
-                        tracing::error!(target = ?Target::Follower, ?error, "Contract building failed");
-                        execution_result.add_failed_build(Target::Follower, mode.clone(), error);
-                        return Err(());
-                    }
-                }
-                Ok(())
-            });
-            if build_result.is_err() {
-                // Note: We skip to the next solc mode as there's nothing that we can do at this
-                // point, the building has failed. We do NOT bail out of the execution as a whole.
-                continue;
             }
 
-            // For cases if one of the inputs fail then we move on to the next case and we do NOT
+            inputs_executed += 1;
-            // bail out of the whole thing.
-            let case = self.case;
-            let case_idx = self.case_idx;
-            let tracing_span =
-                tracing::info_span!("Handling case", case_name = case.name, case_idx = *case_idx);
-            let _guard = tracing_span.enter();
-
-            let case_idx = CaseIdx::new(case_idx);
-
-            // For inputs if one of the inputs fail we move on to the next case (we do not move
-            // on to the next input as it doesn't make sense. It depends on the previous one).
-            for (input_idx, input) in case.inputs_iterator().enumerate() {
-                let tracing_span = tracing::info_span!("Handling input", input_idx);
-                let _guard = tracing_span.enter();
-
-                let input_execution_result =
-                    tracing::info_span!("Executing input", contract_name = ?input.instance)
-                        .in_scope(|| {
-                            let (leader_receipt, _, leader_diff) = match leader_state.handle_input(
-                                self.metadata,
-                                case_idx,
-                                &input,
-                                self.leader_node,
-                                &mode,
-                            ) {
-                                Ok(result) => result,
-                                Err(error) => {
-                                    tracing::error!(
-                                        target = ?Target::Leader,
-                                        ?error,
-                                        "Contract execution failed"
-                                    );
-                                    execution_result.add_failed_case(
-                                        Target::Leader,
-                                        mode.clone(),
-                                        case.name.as_deref().unwrap_or("no case name").to_owned(),
-                                        case_idx,
-                                        input_idx,
-                                        anyhow::Error::msg(format!("{error}")),
-                                    );
-                                    return Err(error);
-                                }
-                            };
-
-                            let (follower_receipt, _, follower_diff) = match follower_state
-                                .handle_input(
-                                    self.metadata,
-                                    case_idx,
-                                    &input,
-                                    self.follower_node,
-                                    &mode,
-                                ) {
-                                Ok(result) => result,
-                                Err(error) => {
-                                    tracing::error!(
-                                        target = ?Target::Follower,
-                                        ?error,
-                                        "Contract execution failed"
-                                    );
-                                    execution_result.add_failed_case(
-                                        Target::Follower,
-                                        mode.clone(),
-                                        case.name.as_deref().unwrap_or("no case name").to_owned(),
-                                        case_idx,
-                                        input_idx,
-                                        anyhow::Error::msg(format!("{error}")),
-                                    );
-                                    return Err(error);
-                                }
-                            };
-
-                            Ok((leader_receipt, leader_diff, follower_receipt, follower_diff))
-                        });
-                let Ok((leader_receipt, leader_diff, follower_receipt, follower_diff)) =
-                    input_execution_result
-                else {
-                    return execution_result;
-                };
-
-                if leader_diff == follower_diff {
-                    tracing::debug!("State diffs match between leader and follower.");
-                } else {
-                    tracing::debug!("State diffs mismatch between leader and follower.");
-                    Self::trace_diff_mode("Leader", &leader_diff);
-                    Self::trace_diff_mode("Follower", &follower_diff);
-                }
-
-                if leader_receipt.logs() != follower_receipt.logs() {
-                    tracing::debug!("Log/event mismatch between leader and follower.");
-                    tracing::trace!("Leader logs: {:?}", leader_receipt.logs());
-                    tracing::trace!("Follower logs: {:?}", follower_receipt.logs());
-                }
-            }
-
-            // Note: Only consider the case as having been successful after we have processed
-            // all of the inputs and completed the entire loop over the input.
-            execution_result.add_successful_case(
-                Target::Leader,
-                mode.clone(),
-                case.name.clone().unwrap_or("no case name".to_owned()),
-                case_idx,
-            );
-            execution_result.add_successful_case(
-                Target::Follower,
-                mode.clone(),
-                case.name.clone().unwrap_or("no case name".to_owned()),
-                case_idx,
-            );
         }
 
-        execution_result
+        Ok(inputs_executed)
-    }
-}
-
-#[derive(Debug, Default)]
-pub struct ExecutionResult {
-    pub results: Vec<Box<dyn ExecutionResultItem>>,
-    pub successful_cases_count: usize,
-    pub failed_cases_count: usize,
-}
-
-impl ExecutionResult {
-    pub fn new() -> Self {
-        Self {
-            results: Default::default(),
-            successful_cases_count: Default::default(),
-            failed_cases_count: Default::default(),
-        }
-    }
-
-    pub fn add_successful_build(&mut self, target: Target, solc_mode: SolcMode) {
-        self.results
-            .push(Box::new(BuildResult::Success { target, solc_mode }));
-    }
-
-    pub fn add_failed_build(&mut self, target: Target, solc_mode: SolcMode, error: anyhow::Error) {
-        self.results.push(Box::new(BuildResult::Failure {
-            target,
-            solc_mode,
-            error,
-        }));
-    }
-
-    pub fn add_successful_case(
-        &mut self,
-        target: Target,
-        solc_mode: SolcMode,
-        case_name: String,
-        case_idx: CaseIdx,
-    ) {
-        self.successful_cases_count += 1;
-        self.results.push(Box::new(CaseResult::Success {
-            target,
-            solc_mode,
-            case_name,
-            case_idx,
-        }));
-    }
-
-    pub fn add_failed_case(
-        &mut self,
-        target: Target,
-        solc_mode: SolcMode,
-        case_name: String,
-        case_idx: CaseIdx,
-        input_idx: usize,
-        error: anyhow::Error,
-    ) {
-        self.failed_cases_count += 1;
-        self.results.push(Box::new(CaseResult::Failure {
-            target,
-            solc_mode,
-            case_name,
-            case_idx,
-            error,
-            input_idx,
-        }));
-    }
-}
-
-pub trait ExecutionResultItem: Debug {
-    /// Converts this result item into an [`anyhow::Result`].
-    fn as_result(&self) -> Result<(), &anyhow::Error>;
-
-    /// Provides information on whether the provided result item is of a success or failure.
-    fn is_success(&self) -> bool;
-
-    /// Provides information of the target that this result is for.
-    fn target(&self) -> &Target;
-
-    /// Provides information on the [`SolcMode`] mode that we being used for this result item.
-    fn solc_mode(&self) -> &SolcMode;
-
-    /// Provides information on the case name and number that this result item pertains to. This is
-    /// [`None`] if the error doesn't belong to any case (e.g., if it's a build error outside of any
-    /// of the cases.).
-    fn case_name_and_index(&self) -> Option<(&str, &CaseIdx)>;
-
-    /// Provides information on the input number that this result item pertains to. This is [`None`]
-    /// if the error doesn't belong to any input (e.g., if it's a build error outside of any of the
-    /// inputs.).
-    fn input_index(&self) -> Option<usize>;
-}
-
-#[derive(Clone, Copy, Debug, PartialEq, Eq, PartialOrd, Ord, Hash)]
-pub enum Target {
-    Leader,
-    Follower,
-}
-
-#[derive(Debug)]
-pub enum BuildResult {
-    Success {
-        target: Target,
-        solc_mode: SolcMode,
-    },
-    Failure {
-        target: Target,
-        solc_mode: SolcMode,
-        error: anyhow::Error,
-    },
-}
-
-impl ExecutionResultItem for BuildResult {
-    fn as_result(&self) -> Result<(), &anyhow::Error> {
-        match self {
-            Self::Success { .. } => Ok(()),
-            Self::Failure { error, .. } => Err(error)?,
-        }
-    }
-
-    fn is_success(&self) -> bool {
-        match self {
-            Self::Success { .. } => true,
-            Self::Failure { .. } => false,
-        }
-    }
-
-    fn target(&self) -> &Target {
-        match self {
-            Self::Success { target, .. } | Self::Failure { target, .. } => target,
-        }
-    }
-
-    fn solc_mode(&self) -> &SolcMode {
-        match self {
-            Self::Success { solc_mode, .. } | Self::Failure { solc_mode, .. } => solc_mode,
-        }
-    }
-
-    fn case_name_and_index(&self) -> Option<(&str, &CaseIdx)> {
-        None
-    }
-
-    fn input_index(&self) -> Option<usize> {
-        None
-    }
-}
-
-#[derive(Debug)]
-pub enum CaseResult {
-    Success {
-        target: Target,
-        solc_mode: SolcMode,
-        case_name: String,
-        case_idx: CaseIdx,
-    },
-    Failure {
-        target: Target,
-        solc_mode: SolcMode,
-        case_name: String,
-        case_idx: CaseIdx,
-        input_idx: usize,
-        error: anyhow::Error,
-    },
-}
-
-impl ExecutionResultItem for CaseResult {
-    fn as_result(&self) -> Result<(), &anyhow::Error> {
-        match self {
-            Self::Success { .. } => Ok(()),
-            Self::Failure { error, .. } => Err(error)?,
-        }
-    }
-
-    fn is_success(&self) -> bool {
-        match self {
-            Self::Success { .. } => true,
-            Self::Failure { .. } => false,
-        }
-    }
-
-    fn target(&self) -> &Target {
-        match self {
-            Self::Success { target, .. } | Self::Failure { target, .. } => target,
-        }
-    }
-
-    fn solc_mode(&self) -> &SolcMode {
-        match self {
-            Self::Success { solc_mode, .. } | Self::Failure { solc_mode, .. } => solc_mode,
-        }
-    }
-
-    fn case_name_and_index(&self) -> Option<(&str, &CaseIdx)> {
-        match self {
-            Self::Success {
-                case_name,
-                case_idx,
-                ..
-            }
-            | Self::Failure {
-                case_name,
-                case_idx,
-                ..
-            } => Some((case_name, case_idx)),
-        }
-    }
-
-    fn input_index(&self) -> Option<usize> {
-        match self {
-            CaseResult::Success { .. } => None,
-            CaseResult::Failure { input_idx, .. } => Some(*input_idx),
-        }
     }
 }

crates/core/src/main.rs

@@ -1,22 +1,53 @@
-use std::{collections::HashMap, sync::LazyLock};
+use std::{
+    collections::HashMap,
+    path::Path,
+    sync::{Arc, LazyLock, Mutex, RwLock},
+};
 
+use alloy::{
+    json_abi::JsonAbi,
+    network::{Ethereum, TransactionBuilder},
+    primitives::Address,
+    rpc::types::TransactionRequest,
+};
+use anyhow::Context;
 use clap::Parser;
 use rayon::{ThreadPoolBuilder, prelude::*};
+use revive_dt_common::iterators::FilesWithExtensionIterator;
+use revive_dt_node_interaction::EthereumNode;
+use semver::Version;
+use temp_dir::TempDir;
+use tracing::Level;
+use tracing_subscriber::{EnvFilter, FmtSubscriber};
 
+use revive_dt_compiler::SolidityCompiler;
+use revive_dt_compiler::{Compiler, CompilerOutput};
 use revive_dt_config::*;
 use revive_dt_core::{
     Geth, Kitchensink, Platform,
     driver::{CaseDriver, CaseState},
 };
-use revive_dt_format::{corpus::Corpus, metadata::MetadataFile};
+use revive_dt_format::{
+    case::{Case, CaseIdx},
+    corpus::Corpus,
+    input::Input,
+    metadata::{ContractInstance, ContractPathAndIdent, Metadata, MetadataFile},
+    mode::SolcMode,
+};
 use revive_dt_node::pool::NodePool;
 use revive_dt_report::reporter::{Report, Span};
-use temp_dir::TempDir;
-use tracing::Level;
-use tracing_subscriber::{EnvFilter, FmtSubscriber};
 
 static TEMP_DIR: LazyLock<TempDir> = LazyLock::new(|| TempDir::new().unwrap());
 
+type CompilationCache<'a> = Arc<
+    RwLock<
+        HashMap<
+            (&'a Path, SolcMode, TestingPlatform),
+            Arc<Mutex<Option<Arc<(Version, CompilerOutput)>>>>,
+        >,
+    >,
+>;
+
 fn main() -> anyhow::Result<()> {
     let args = init_cli()?;
 
@@ -104,54 +135,365 @@ where
                     .cases
                     .iter()
                     .enumerate()
-                    .map(move |(case_idx, case)| (path, metadata, case_idx, case))
+                    .flat_map(move |(case_idx, case)| {
+                        metadata
+                            .solc_modes()
+                            .into_iter()
+                            .map(move |solc_mode| (path, metadata, case_idx, case, solc_mode))
+                    })
             },
         )
         .collect::<Vec<_>>();
 
-    test_cases
+    let compilation_cache = Arc::new(RwLock::new(HashMap::new()));
-        .into_par_iter()
+    test_cases.into_par_iter().for_each(
-        .for_each(|(metadata_file_path, metadata, case_idx, case)| {
+        |(metadata_file_path, metadata, case_idx, case, solc_mode)| {
             let tracing_span = tracing::span!(
                 Level::INFO,
                 "Running driver",
-                metadata_file_path = metadata_file_path.display().to_string(),
+                metadata_file_path = %metadata_file_path.display(),
+                case_idx = case_idx,
+                solc_mode = ?solc_mode,
             );
             let _guard = tracing_span.enter();
 
-            let mut driver = CaseDriver::<L, F>::new(
+            let result = handle_case_driver::<L, F>(
+                metadata_file_path.as_path(),
                 metadata,
+                case_idx.into(),
                 case,
-                case_idx,
+                solc_mode,
                 args,
+                compilation_cache.clone(),
                 leader_nodes.round_robbin(),
                 follower_nodes.round_robbin(),
+                span,
             );
-
+            match result {
-            let execution_result = driver.execute(span);
+                Ok(inputs_executed) => tracing::info!(inputs_executed, "Execution succeeded"),
-            tracing::info!(
+                Err(error) => tracing::info!(%error, "Execution failed"),
-                case_success_count = execution_result.successful_cases_count,
-                case_failure_count = execution_result.failed_cases_count,
-                "Execution completed"
-            );
-
-            let mut error_count = 0;
-            for result in execution_result.results.iter() {
-                if !result.is_success() {
-                    tracing::error!(execution_error = ?result, "Encountered an error");
-                    error_count += 1;
-                }
             }
-            if error_count == 0 {
+            tracing::info!("Execution completed");
-                tracing::info!("Execution succeeded");
+        },
-            } else {
+    );
-                tracing::info!("Execution failed");
-            }
-        });
 
     Ok(())
 }
 
+#[allow(clippy::too_many_arguments)]
+fn handle_case_driver<'a, L, F>(
+    metadata_file_path: &'a Path,
+    metadata: &'a Metadata,
+    case_idx: CaseIdx,
+    case: &Case,
+    mode: SolcMode,
+    config: &Arguments,
+    compilation_cache: CompilationCache<'a>,
+    leader_node: &L::Blockchain,
+    follower_node: &F::Blockchain,
+    _: Span,
+) -> anyhow::Result<usize>
+where
+    L: Platform,
+    F: Platform,
+    L::Blockchain: revive_dt_node::Node + Send + Sync + 'static,
+    F::Blockchain: revive_dt_node::Node + Send + Sync + 'static,
+{
+    let leader_pre_link_contracts = get_or_build_contracts::<L>(
+        metadata,
+        metadata_file_path,
+        mode.clone(),
+        config,
+        compilation_cache.clone(),
+        &HashMap::new(),
+    )?;
+    let follower_pre_link_contracts = get_or_build_contracts::<F>(
+        metadata,
+        metadata_file_path,
+        mode.clone(),
+        config,
+        compilation_cache.clone(),
+        &HashMap::new(),
+    )?;
+
+    let mut leader_deployed_libraries = HashMap::new();
+    let mut follower_deployed_libraries = HashMap::new();
+    let mut contract_sources = metadata.contract_sources()?;
+    for library_instance in metadata
+        .libraries
+        .iter()
+        .flatten()
+        .flat_map(|(_, map)| map.values())
+    {
+        let ContractPathAndIdent {
+            contract_source_path: library_source_path,
+            contract_ident: library_ident,
+        } = contract_sources
+            .remove(library_instance)
+            .context("Failed to find the contract source")?;
+
+        let (leader_code, leader_abi) = leader_pre_link_contracts
+            .1
+            .contracts
+            .get(&library_source_path)
+            .and_then(|contracts| contracts.get(library_ident.as_str()))
+            .context("Declared library was not compiled")?;
+        let (follower_code, follower_abi) = follower_pre_link_contracts
+            .1
+            .contracts
+            .get(&library_source_path)
+            .and_then(|contracts| contracts.get(library_ident.as_str()))
+            .context("Declared library was not compiled")?;
+
+        let leader_code = match alloy::hex::decode(leader_code) {
+            Ok(code) => code,
+            Err(error) => {
+                tracing::error!(
+                    ?error,
+                    contract_source_path = library_source_path.display().to_string(),
+                    contract_ident = library_ident.as_ref(),
+                    "Failed to hex-decode byte code - This could possibly mean that the bytecode requires linking"
+                );
+                anyhow::bail!("Failed to hex-decode the byte code {}", error)
+            }
+        };
+        let follower_code = match alloy::hex::decode(follower_code) {
+            Ok(code) => code,
+            Err(error) => {
+                tracing::error!(
+                    ?error,
+                    contract_source_path = library_source_path.display().to_string(),
+                    contract_ident = library_ident.as_ref(),
+                    "Failed to hex-decode byte code - This could possibly mean that the bytecode requires linking"
+                );
+                anyhow::bail!("Failed to hex-decode the byte code {}", error)
+            }
+        };
+
+        // Getting the deployer address from the cases themselves. This is to ensure that we're
+        // doing the deployments from different accounts and therefore we're not slowed down by
+        // the nonce.
+        let deployer_address = case
+            .inputs
+            .iter()
+            .map(|input| input.caller)
+            .next()
+            .unwrap_or(Input::default_caller());
+        let leader_tx = TransactionBuilder::<Ethereum>::with_deploy_code(
+            TransactionRequest::default().from(deployer_address),
+            leader_code,
+        );
+        let follower_tx = TransactionBuilder::<Ethereum>::with_deploy_code(
+            TransactionRequest::default().from(deployer_address),
+            follower_code,
+        );
+
+        let leader_receipt = match leader_node.execute_transaction(leader_tx) {
+            Ok(receipt) => receipt,
+            Err(error) => {
+                tracing::error!(
+                    node = std::any::type_name::<L>(),
+                    ?error,
+                    "Contract deployment transaction failed."
+                );
+                return Err(error);
+            }
+        };
+        let follower_receipt = match follower_node.execute_transaction(follower_tx) {
+            Ok(receipt) => receipt,
+            Err(error) => {
+                tracing::error!(
+                    node = std::any::type_name::<F>(),
+                    ?error,
+                    "Contract deployment transaction failed."
+                );
+                return Err(error);
+            }
+        };
+
+        let Some(leader_library_address) = leader_receipt.contract_address else {
+            tracing::error!("Contract deployment transaction didn't return an address");
+            anyhow::bail!("Contract deployment didn't return an address");
+        };
+        let Some(follower_library_address) = follower_receipt.contract_address else {
+            tracing::error!("Contract deployment transaction didn't return an address");
+            anyhow::bail!("Contract deployment didn't return an address");
+        };
+
+        leader_deployed_libraries.insert(
+            library_instance.clone(),
+            (leader_library_address, leader_abi.clone()),
+        );
+        follower_deployed_libraries.insert(
+            library_instance.clone(),
+            (follower_library_address, follower_abi.clone()),
+        );
+    }
+
+    let metadata_file_contains_libraries = metadata
+        .libraries
+        .iter()
+        .flat_map(|map| map.iter())
+        .flat_map(|(_, value)| value.iter())
+        .next()
+        .is_some();
+    let compiled_contracts_require_linking = leader_pre_link_contracts
+        .1
+        .contracts
+        .values()
+        .chain(follower_pre_link_contracts.1.contracts.values())
+        .flat_map(|value| value.values())
+        .any(|(code, _)| !code.chars().all(|char| char.is_ascii_hexdigit()));
+    let (leader_compiled_contracts, follower_compiled_contracts) =
+        if metadata_file_contains_libraries && compiled_contracts_require_linking {
+            let leader_key = (metadata_file_path, mode.clone(), L::config_id());
+            let follower_key = (metadata_file_path, mode.clone(), L::config_id());
+            {
+                let mut cache = compilation_cache.write().expect("Poisoned");
+                cache.remove(&leader_key);
+                cache.remove(&follower_key);
+            }
+
+            let leader_post_link_contracts = get_or_build_contracts::<L>(
+                metadata,
+                metadata_file_path,
+                mode.clone(),
+                config,
+                compilation_cache.clone(),
+                &leader_deployed_libraries,
+            )?;
+            let follower_post_link_contracts = get_or_build_contracts::<F>(
+                metadata,
+                metadata_file_path,
+                mode.clone(),
+                config,
+                compilation_cache,
+                &follower_deployed_libraries,
+            )?;
+
+            (leader_post_link_contracts, follower_post_link_contracts)
+        } else {
+            (leader_pre_link_contracts, follower_pre_link_contracts)
+        };
+
+    let leader_state = CaseState::<L>::new(
+        leader_compiled_contracts.0.clone(),
+        leader_compiled_contracts.1.contracts.clone(),
+        leader_deployed_libraries,
+    );
+    let follower_state = CaseState::<F>::new(
+        follower_compiled_contracts.0.clone(),
+        follower_compiled_contracts.1.contracts.clone(),
+        follower_deployed_libraries,
+    );
+
+    let mut driver = CaseDriver::<L, F>::new(
+        metadata,
+        case,
+        case_idx,
+        leader_node,
+        follower_node,
+        leader_state,
+        follower_state,
+    );
+    driver.execute()
+}
+
+fn get_or_build_contracts<'a, P: Platform>(
+    metadata: &'a Metadata,
+    metadata_file_path: &'a Path,
+    mode: SolcMode,
+    config: &Arguments,
+    compilation_cache: CompilationCache<'a>,
+    deployed_libraries: &HashMap<ContractInstance, (Address, JsonAbi)>,
+) -> anyhow::Result<Arc<(Version, CompilerOutput)>> {
+    let key = (metadata_file_path, mode.clone(), P::config_id());
+    if let Some(compilation_artifact) = compilation_cache
+        .read()
+        .expect("Poisoned")
+        .get(&key)
+        .cloned()
+    {
+        let mut compilation_artifact = compilation_artifact.lock().expect("Poisoned");
+        match *compilation_artifact {
+            Some(ref compiled_contracts) => {
+                tracing::debug!(?key, "Compiled contracts cache hit");
+                return Ok(compiled_contracts.clone());
+            }
+            None => {
+                tracing::debug!(?key, "Compiled contracts cache miss");
+                let compiled_contracts = Arc::new(compile_contracts::<P>(
+                    metadata,
+                    &mode,
+                    config,
+                    deployed_libraries,
+                )?);
+                *compilation_artifact = Some(compiled_contracts.clone());
+                return Ok(compiled_contracts.clone());
+            }
+        }
+    };
+
+    tracing::debug!(?key, "Compiled contracts cache miss");
+    let mutex = {
+        let mut compilation_cache = compilation_cache.write().expect("Poisoned");
+        let mutex = Arc::new(Mutex::new(None));
+        compilation_cache.insert(key, mutex.clone());
+        mutex
+    };
+    let mut compilation_artifact = mutex.lock().expect("Poisoned");
+    let compiled_contracts = Arc::new(compile_contracts::<P>(
+        metadata,
+        &mode,
+        config,
+        deployed_libraries,
+    )?);
+    *compilation_artifact = Some(compiled_contracts.clone());
+    Ok(compiled_contracts.clone())
+}
+
+fn compile_contracts<P: Platform>(
+    metadata: &Metadata,
+    mode: &SolcMode,
+    config: &Arguments,
+    deployed_libraries: &HashMap<ContractInstance, (Address, JsonAbi)>,
+) -> anyhow::Result<(Version, CompilerOutput)> {
+    let compiler_version_or_requirement = mode.compiler_version_to_use(config.solc.clone());
+    let compiler_path =
+        P::Compiler::get_compiler_executable(config, compiler_version_or_requirement)?;
+    let compiler_version = P::Compiler::new(compiler_path.clone()).version()?;
+
+    let compiler = Compiler::<P::Compiler>::new()
+        .with_allow_path(metadata.directory()?)
+        .with_optimization(mode.solc_optimize());
+    let mut compiler = metadata
+        .files_to_compile()?
+        .try_fold(compiler, |compiler, path| compiler.with_source(&path))?;
+    for (library_instance, (library_address, _)) in deployed_libraries.iter() {
+        let library_ident = &metadata
+            .contracts
+            .as_ref()
+            .and_then(|contracts| contracts.get(library_instance))
+            .expect("Impossible for library to not be found in contracts")
+            .contract_ident;
+
+        // Note the following: we need to tell solc which files require the libraries to be
+        // linked into them. We do not have access to this information and therefore we choose
+        // an easier, yet more compute intensive route, of telling solc that all of the files
+        // need to link the library and it will only perform the linking for the files that do
+        // actually need the library.
+        compiler = FilesWithExtensionIterator::new(metadata.directory()?)
+            .with_allowed_extension("sol")
+            .fold(compiler, |compiler, path| {
+                compiler.with_library(&path, library_ident.as_str(), *library_address)
+            });
+    }
+
+    let compiler_output = compiler.try_build(compiler_path)?;
+
+    Ok((compiler_version, compiler_output))
+}
+
 fn execute_corpus(args: &Arguments, tests: &[MetadataFile], span: Span) -> anyhow::Result<()> {
     match (&args.leader, &args.follower) {
         (TestingPlatform::Geth, TestingPlatform::Kitchensink) => {
@@ -166,22 +508,25 @@ fn execute_corpus(args: &Arguments, tests: &[MetadataFile], span: Span) -> anyho
     Ok(())
 }
 
-fn compile_corpus(
+fn compile_corpus(config: &Arguments, tests: &[MetadataFile], platform: &TestingPlatform, _: Span) {
-    config: &Arguments,
-    tests: &[MetadataFile],
-    platform: &TestingPlatform,
-    span: Span,
-) {
     tests.par_iter().for_each(|metadata| {
         for mode in &metadata.solc_modes() {
             match platform {
                 TestingPlatform::Geth => {
-                    let mut state = CaseState::<Geth>::new(config, span);
+                    let _ = compile_contracts::<Geth>(
-                    let _ = state.build_contracts(mode, metadata);
+                        &metadata.content,
+                        mode,
+                        config,
+                        &Default::default(),
+                    );
                 }
                 TestingPlatform::Kitchensink => {
-                    let mut state = CaseState::<Kitchensink>::new(config, span);
+                    let _ = compile_contracts::<Geth>(
-                    let _ = state.build_contracts(mode, metadata);
+                        &metadata.content,
+                        mode,
+                        config,
+                        &Default::default(),
+                    );
                 }
             };
         }