diff --git a/Cargo.lock b/Cargo.lock index 8b6cabc..a272260 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1452,6 +1452,26 @@ dependencies = [ "parity-scale-codec", ] +[[package]] +name = "bincode" +version = "2.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "36eaf5d7b090263e8150820482d5d93cd964a81e4019913c972f4edcc6edb740" +dependencies = [ + "bincode_derive", + "serde", + "unty", +] + +[[package]] +name = "bincode_derive" +version = "2.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bf95709a440f45e986983918d0e8a1f30a9b1df04918fc828670606804ac3c09" +dependencies = [ + "virtue", +] + [[package]] name = "bit-set" version = "0.8.0" @@ -4485,10 +4505,12 @@ version = "0.1.0" dependencies = [ "alloy", "anyhow", + "bincode", "cacache", "clap", "futures", "indexmap 2.10.0", + "once_cell", "revive-dt-common", "revive-dt-compiler", "revive-dt-config", @@ -4500,6 +4522,7 @@ dependencies = [ "serde", "serde_json", "temp-dir", + "tempfile", "tokio", "tracing", "tracing-subscriber", @@ -6261,6 +6284,12 @@ version = "0.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8ecb6da28b8a351d773b68d5825ac39017e680750f980f3a1a85cd8dd28a47c1" +[[package]] +name = "unty" +version = "0.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6d49784317cd0d1ee7ec5c716dd598ec5b4483ea832a2dced265471cc0f690ae" + [[package]] name = "url" version = "2.5.4" @@ -6319,6 +6348,12 @@ version = "0.9.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a" +[[package]] +name = "virtue" +version = "0.0.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "051eb1abcf10076295e815102942cc58f9d5e3b4560e46e53c21e8ff6f3af7b1" + [[package]] name = "w3f-bls" version = "0.1.9" diff --git a/Cargo.toml b/Cargo.toml index 59cded5..e9464f7 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -25,7 +25,7 @@ revive-dt-solc-binaries = { version = "0.1.0", path = "crates/solc-binaries" } alloy-primitives = "1.2.1" alloy-sol-types = "1.2.1" anyhow = "1.0" -bincode = { version = "2.0.1" } +bincode = { version = "2.0.1", features = ["serde"] } cacache = { version = "13.1.0" } clap = { version = "4", features = ["derive"] } foundry-compilers-artifacts = { version = "0.18.0" } diff --git a/crates/common/src/types/mode.rs b/crates/common/src/types/mode.rs index c380209..535add1 100644 --- a/crates/common/src/types/mode.rs +++ b/crates/common/src/types/mode.rs @@ -10,7 +10,7 @@ use std::str::FromStr; /// in its requirements, and then expanding it out into a list of [`Mode`]s. /// /// Use [`ParsedMode::to_test_modes()`] to do this. -#[derive(Clone, Debug, PartialEq, Eq, Hash, Serialize)] +#[derive(Clone, Debug, PartialEq, Eq, Hash, Serialize, Deserialize)] pub struct Mode { pub pipeline: ModePipeline, pub optimize_setting: ModeOptimizerSetting, diff --git a/crates/compiler/src/lib.rs b/crates/compiler/src/lib.rs index 551d101..799124e 100644 --- a/crates/compiler/src/lib.rs +++ b/crates/compiler/src/lib.rs @@ -169,6 +169,14 @@ where self } + pub fn then(self, callback: impl FnOnce(Self) -> Self) -> Self { + callback(self) + } + + pub fn try_then(self, callback: impl FnOnce(Self) -> Result) -> Result { + callback(self) + } + pub async fn try_build( self, compiler_path: impl AsRef, diff --git a/crates/config/src/lib.rs b/crates/config/src/lib.rs index e4fb409..86ba19e 100644 --- a/crates/config/src/lib.rs +++ b/crates/config/src/lib.rs @@ -120,6 +120,10 @@ pub struct Arguments { /// By default it uses `eth-rpc` binary found in `$PATH`. #[arg(short = 'p', long = "eth_proxy", default_value = "eth-rpc")] pub eth_proxy: PathBuf, + + /// Controls if the compilation cache should be invalidated or not. + #[arg(short, long)] + pub invalidate_compilation_cache: bool, } impl Arguments { diff --git a/crates/core/Cargo.toml b/crates/core/Cargo.toml index cdf2238..c44c153 100644 --- a/crates/core/Cargo.toml +++ b/crates/core/Cargo.toml @@ -23,10 +23,12 @@ revive-dt-report = { workspace = true } alloy = { workspace = true } anyhow = { workspace = true } +bincode = { workspace = true } cacache = { workspace = true } clap = { workspace = true } futures = { workspace = true } indexmap = { workspace = true } +once_cell = { workspace = true } tokio = { workspace = true } tracing = { workspace = true } tracing-subscriber = { workspace = true } @@ -34,3 +36,4 @@ semver = { workspace = true } serde = { workspace = true } serde_json = { workspace = true } temp-dir = { workspace = true } +tempfile = { workspace = true } diff --git a/crates/core/src/cached_compiler.rs b/crates/core/src/cached_compiler.rs index bce091a..b9e1dfb 100644 --- a/crates/core/src/cached_compiler.rs +++ b/crates/core/src/cached_compiler.rs @@ -1,11 +1,175 @@ //! A wrapper around the compiler which allows for caching of compilation artifacts so that they can //! be reused between runs. -use std::path::{Path, PathBuf}; +use std::{ + collections::HashMap, + path::{Path, PathBuf}, + sync::Arc, +}; +use futures::FutureExt; +use revive_dt_common::iterators::FilesWithExtensionIterator; +use revive_dt_compiler::{Compiler, CompilerOutput, Mode, SolidityCompiler}; +use revive_dt_config::Arguments; +use revive_dt_format::metadata::{ContractIdent, ContractInstance, Metadata}; + +use alloy::{hex::ToHexExt, json_abi::JsonAbi, primitives::Address}; use anyhow::{Error, Result}; -use revive_dt_format::mode::SolcMode; +use once_cell::sync::Lazy; +use semver::Version; use serde::{Deserialize, Serialize}; +use tokio::sync::{Mutex, RwLock}; +use tracing::{debug, debug_span, instrument}; + +use crate::Platform; + +pub struct CachedCompiler(ArtifactsCache); + +impl CachedCompiler { + pub async fn new(path: impl AsRef, invalidate_cache: bool) -> Result { + let mut cache = ArtifactsCache::new(path); + if invalidate_cache { + cache = cache.with_invalidated_cache().await?; + } + Ok(Self(cache)) + } + + /// Compiles or gets the compilation artifacts from the cache. + #[instrument( + level = "debug", + skip_all, + fields( + metadata_file_path = %metadata_file_path.as_ref().display(), + %mode + ), + err + )] + pub async fn compile_contracts( + &self, + metadata: &Metadata, + metadata_file_path: impl AsRef, + mode: &Mode, + config: &Arguments, + deployed_libraries: Option<&HashMap>, + ) -> Result<(CompilerOutput, Version)> { + static CACHE_KEY_LOCK: Lazy>>>> = + Lazy::new(Default::default); + + let compiler_version_or_requirement = mode.compiler_version_to_use(config.solc.clone()); + let compiler_path = ::get_compiler_executable( + config, + compiler_version_or_requirement, + ) + .await?; + let compiler_version = + ::new(compiler_path.clone()).version()?; + + let cache_key = CacheKey { + platform_key: P::config_id().to_string(), + compiler_version: compiler_version.clone(), + metadata_file_path: metadata_file_path.as_ref().to_path_buf(), + solc_mode: mode.clone(), + }; + debug_span!( + "Running compilation for the cache key", + cache_key.platform_key = %cache_key.platform_key, + cache_key.compiler_version = %cache_key.compiler_version, + cache_key.metadata_file_path = %cache_key.metadata_file_path.display(), + cache_key.solc_mode = %cache_key.solc_mode, + ); + + let compilation_callback = || async move { + compile_contracts::

( + metadata.directory()?, + compiler_path, + metadata.files_to_compile()?, + mode, + deployed_libraries, + ) + .map(|compilation_result| compilation_result.map(CacheValue::new)) + .await + }; + + let compiled_contracts = match deployed_libraries { + // If deployed libraries have been specified then we will re-compile the contract as it + // means that linking is required in this case. + Some(_) => { + debug!("Deployed libraries defined, recompilation must take place"); + compilation_callback().await?.compiler_output + } + // If no deployed libraries are specified then we can follow the cached flow and attempt + // to lookup the compilation artifacts in the cache. + None => { + debug!("Deployed contract undefined, attempting to make use of cache"); + + // Lock this specific cache key such that we do not get inconsistent state. We want + // that when multiple cases come in asking for the compilation artifacts then they + // don't all trigger a compilation if there's a cache miss. Hence, the lock here. + let read_guard = CACHE_KEY_LOCK.read().await; + let mutex = match read_guard.get(&cache_key).cloned() { + Some(value) => value, + None => { + drop(read_guard); + CACHE_KEY_LOCK + .write() + .await + .entry(cache_key.clone()) + .or_default() + .clone() + } + }; + let _guard = mutex.lock().await; + + self.0 + .get_or_insert_with(&cache_key, compilation_callback) + .await + .map(|value| value.compiler_output)? + } + }; + + Ok((compiled_contracts, compiler_version)) + } +} + +async fn compile_contracts( + metadata_directory: impl AsRef, + compiler_path: impl AsRef, + mut files_to_compile: impl Iterator, + mode: &Mode, + deployed_libraries: Option<&HashMap>, +) -> Result { + let all_sources_in_dir = FilesWithExtensionIterator::new(metadata_directory.as_ref()) + .with_allowed_extension("sol") + .with_use_cached_fs(true) + .collect::>(); + + Compiler::::new() + .with_allow_path(metadata_directory) + // Handling the modes + .with_optimization(mode.optimize_setting) + .with_pipeline(mode.pipeline) + // Adding the contract sources to the compiler. + .try_then(|compiler| { + files_to_compile.try_fold(compiler, |compiler, path| compiler.with_source(path)) + })? + // Adding the deployed libraries to the compiler. + .then(|compiler| { + deployed_libraries + .iter() + .flat_map(|value| value.iter()) + .map(|(instance, (ident, address, abi))| (instance, ident, address, abi)) + .flat_map(|(_, ident, address, _)| { + all_sources_in_dir + .iter() + .map(move |path| (ident, address, path)) + }) + .fold(compiler, |compiler, (ident, address, path)| { + compiler.with_library(path, ident.as_str(), *address) + }) + }) + .try_build(compiler_path) + .await +} struct ArtifactsCache { path: PathBuf, @@ -18,13 +182,75 @@ impl ArtifactsCache { } } - pub fn with_invalidated_cache(self) -> Result { - cacache::clear_sync(self.path.as_path()).map_err(Into::::into)?; + pub async fn with_invalidated_cache(self) -> Result { + cacache::clear(self.path.as_path()) + .await + .map_err(Into::::into)?; Ok(self) } + + pub async fn insert(&self, key: CacheKey, value: CacheValue) -> Result<()> { + let key = bincode::serde::encode_to_vec(key, bincode::config::standard())?; + let value = bincode::serde::encode_to_vec(value, bincode::config::standard())?; + cacache::write(self.path.as_path(), key.encode_hex(), value).await?; + Ok(()) + } + + pub async fn get(&self, key: &CacheKey) -> Option { + let key = bincode::serde::encode_to_vec(key, bincode::config::standard()).ok()?; + let value = cacache::read(self.path.as_path(), key.encode_hex()) + .await + .ok()?; + let (value, _) = + bincode::serde::decode_from_slice::(&value, bincode::config::standard()) + .ok()?; + Some(value) + } + + pub async fn get_or_insert_with( + &self, + key: &CacheKey, + callback: impl AsyncFnOnce() -> Result, + ) -> Result { + match self.get(key).await { + Some(value) => { + debug!("Cache hit"); + Ok(value) + } + None => { + debug!("Cache miss"); + let value = callback().await?; + self.insert(key.clone(), value.clone()).await?; + Ok(value) + } + } + } } +#[derive(Clone, Debug, PartialEq, Eq, Hash, Serialize, Deserialize)] struct CacheKey { + /// The platform name that this artifact was compiled for. For example, this could be EVM or + /// PVM. + platform_key: String, + + /// The version of the compiler that was used to compile the artifacts. + compiler_version: Version, + + /// The path of the metadata file that the compilation artifacts are for. metadata_file_path: PathBuf, - solc_mode: SolcMode, + + /// The mode that the compilation artifacts where compiled with. + solc_mode: Mode, +} + +#[derive(Clone, Debug, Serialize, Deserialize)] +struct CacheValue { + /// The compiler output from the compilation run. + compiler_output: CompilerOutput, +} + +impl CacheValue { + pub fn new(compiler_output: CompilerOutput) -> Self { + Self { compiler_output } + } } diff --git a/crates/core/src/driver/mod.rs b/crates/core/src/driver/mod.rs index a8e908e..7cee6bc 100644 --- a/crates/core/src/driver/mod.rs +++ b/crates/core/src/driver/mod.rs @@ -31,7 +31,7 @@ use revive_dt_format::input::{ BalanceAssertion, Calldata, EtherValue, Expected, ExpectedOutput, Input, Method, StorageEmptyAssertion, }; -use revive_dt_format::metadata::{ContractInstance, ContractPathAndIdent}; +use revive_dt_format::metadata::{ContractIdent, ContractInstance, ContractPathAndIdent}; use revive_dt_format::{input::Step, metadata::Metadata}; use revive_dt_node::Node; use revive_dt_node_interaction::EthereumNode; @@ -44,7 +44,7 @@ pub struct CaseState { compiled_contracts: HashMap>, /// This map stores the contracts deployments for this case. - deployed_contracts: HashMap, + deployed_contracts: HashMap, /// This map stores the variables used for each one of the cases contained in the metadata /// file. @@ -63,7 +63,7 @@ where pub fn new( compiler_version: Version, compiled_contracts: HashMap>, - deployed_contracts: HashMap, + deployed_contracts: HashMap, ) -> Self { Self { compiled_contracts, @@ -662,7 +662,7 @@ where value: Option, node: &T::Blockchain, ) -> anyhow::Result<(Address, JsonAbi, Option)> { - if let Some((address, abi)) = self.deployed_contracts.get(contract_instance) { + if let Some((_, address, abi)) = self.deployed_contracts.get(contract_instance) { return Ok((*address, abi.clone(), None)); } @@ -746,8 +746,10 @@ where "Deployed contract" ); - self.deployed_contracts - .insert(contract_instance.clone(), (address, abi.clone())); + self.deployed_contracts.insert( + contract_instance.clone(), + (contract_ident, address, abi.clone()), + ); Ok((address, abi, Some(receipt))) } diff --git a/crates/core/src/lib.rs b/crates/core/src/lib.rs index 4fac5e6..1e5e5c9 100644 --- a/crates/core/src/lib.rs +++ b/crates/core/src/lib.rs @@ -9,7 +9,6 @@ use revive_dt_format::traits::ResolverApi; use revive_dt_node::{Node, geth, kitchensink::KitchensinkNode}; use revive_dt_node_interaction::EthereumNode; -mod cached_compiler; pub mod driver; /// One platform can be tested differentially against another. diff --git a/crates/core/src/main.rs b/crates/core/src/main.rs index a1a9429..6f4d812 100644 --- a/crates/core/src/main.rs +++ b/crates/core/src/main.rs @@ -1,3 +1,5 @@ +mod cached_compiler; + use std::{ collections::HashMap, path::{Path, PathBuf}, @@ -6,26 +8,21 @@ use std::{ }; use alloy::{ - json_abi::JsonAbi, network::{Ethereum, TransactionBuilder}, - primitives::Address, rpc::types::TransactionRequest, }; use anyhow::Context; use clap::Parser; use futures::stream::futures_unordered::FuturesUnordered; use futures::{Stream, StreamExt}; -use revive_dt_common::iterators::FilesWithExtensionIterator; use revive_dt_node_interaction::EthereumNode; -use semver::Version; use temp_dir::TempDir; -use tokio::sync::{Mutex, RwLock, mpsc}; +use tokio::sync::mpsc; use tracing::{Instrument, Level}; use tracing_subscriber::{EnvFilter, FmtSubscriber}; use revive_dt_common::types::Mode; use revive_dt_compiler::SolidityCompiler; -use revive_dt_compiler::{Compiler, CompilerOutput}; use revive_dt_config::*; use revive_dt_core::{ Geth, Kitchensink, Platform, @@ -35,21 +32,14 @@ use revive_dt_format::{ case::{Case, CaseIdx}, corpus::Corpus, input::{Input, Step}, - metadata::{ContractInstance, ContractPathAndIdent, Metadata, MetadataFile}, + metadata::{ContractPathAndIdent, Metadata, MetadataFile}, }; use revive_dt_node::pool::NodePool; use revive_dt_report::reporter::{Report, Span}; -static TEMP_DIR: LazyLock = LazyLock::new(|| TempDir::new().unwrap()); +use crate::cached_compiler::CachedCompiler; -type CompilationCache = Arc< - RwLock< - HashMap< - (PathBuf, Mode, TestingPlatform), - Arc>>>, - >, - >, ->; +static TEMP_DIR: LazyLock = LazyLock::new(|| TempDir::new().unwrap()); /// this represents a single "test"; a mode, path and collection of cases. #[derive(Clone)] @@ -146,7 +136,7 @@ where let (report_tx, report_rx) = mpsc::unbounded_channel::<(Test, CaseResult)>(); let tests = prepare_tests::(args, metadata_files); - let driver_task = start_driver_task::(args, tests, span, report_tx)?; + let driver_task = start_driver_task::(args, tests, span, report_tx).await?; let status_reporter_task = start_reporter_task(report_rx); tokio::join!(status_reporter_task, driver_task); @@ -279,7 +269,7 @@ async fn does_compiler_support_mode( )) } -fn start_driver_task( +async fn start_driver_task( args: &Arguments, tests: impl Stream, span: Span, @@ -293,8 +283,14 @@ where { let leader_nodes = Arc::new(NodePool::::new(args)?); let follower_nodes = Arc::new(NodePool::::new(args)?); - let compilation_cache = Arc::new(RwLock::new(HashMap::new())); let number_concurrent_tasks = args.number_of_concurrent_tasks(); + let cached_compiler = Arc::new( + CachedCompiler::new( + args.directory().join("compilation_cache"), + args.invalidate_compilation_cache, + ) + .await?, + ); Ok(tests.for_each_concurrent( // We want to limit the concurrent tasks here because: @@ -308,8 +304,8 @@ where move |test| { let leader_nodes = leader_nodes.clone(); let follower_nodes = follower_nodes.clone(); - let compilation_cache = compilation_cache.clone(); let report_tx = report_tx.clone(); + let cached_compiler = cached_compiler.clone(); async move { let leader_node = leader_nodes.round_robbin(); @@ -330,7 +326,7 @@ where &test.case, test.mode.clone(), args, - compilation_cache.clone(), + cached_compiler, leader_node, follower_node, span, @@ -420,7 +416,7 @@ async fn handle_case_driver( case: &Case, mode: Mode, config: &Arguments, - compilation_cache: CompilationCache, + cached_compiler: Arc, leader_node: &L::Blockchain, follower_node: &F::Blockchain, _: Span, @@ -431,27 +427,19 @@ where L::Blockchain: revive_dt_node::Node + Send + Sync + 'static, F::Blockchain: revive_dt_node::Node + Send + Sync + 'static, { - let leader_pre_link_contracts = get_or_build_contracts::( - metadata, - metadata_file_path, - mode.clone(), - config, - compilation_cache.clone(), - &HashMap::new(), - ) - .await?; - let follower_pre_link_contracts = get_or_build_contracts::( - metadata, - metadata_file_path, - mode.clone(), - config, - compilation_cache.clone(), - &HashMap::new(), - ) - .await?; + let leader_pre_link_contracts = cached_compiler + .compile_contracts::(metadata, metadata_file_path, &mode, config, None) + .await? + .0 + .contracts; + let follower_pre_link_contracts = cached_compiler + .compile_contracts::(metadata, metadata_file_path, &mode, config, None) + .await? + .0 + .contracts; - let mut leader_deployed_libraries = HashMap::new(); - let mut follower_deployed_libraries = HashMap::new(); + let mut leader_deployed_libraries = None::>; + let mut follower_deployed_libraries = None::>; let mut contract_sources = metadata.contract_sources()?; for library_instance in metadata .libraries @@ -467,14 +455,10 @@ where .context("Failed to find the contract source")?; let (leader_code, leader_abi) = leader_pre_link_contracts - .1 - .contracts .get(&library_source_path) .and_then(|contracts| contracts.get(library_ident.as_str())) .context("Declared library was not compiled")?; let (follower_code, follower_abi) = follower_pre_link_contracts - .1 - .contracts .get(&library_source_path) .and_then(|contracts| contracts.get(library_ident.as_str())) .context("Declared library was not compiled")?; @@ -567,81 +551,52 @@ where anyhow::bail!("Contract deployment didn't return an address"); }; - leader_deployed_libraries.insert( + leader_deployed_libraries.get_or_insert_default().insert( library_instance.clone(), - (leader_library_address, leader_abi.clone()), + ( + library_ident.clone(), + leader_library_address, + leader_abi.clone(), + ), ); - follower_deployed_libraries.insert( + follower_deployed_libraries.get_or_insert_default().insert( library_instance.clone(), - (follower_library_address, follower_abi.clone()), + ( + library_ident, + follower_library_address, + follower_abi.clone(), + ), ); } - let metadata_file_contains_libraries = metadata - .libraries - .iter() - .flat_map(|map| map.iter()) - .flat_map(|(_, value)| value.iter()) - .next() - .is_some(); - let compiled_contracts_require_linking = leader_pre_link_contracts - .1 - .contracts - .values() - .chain(follower_pre_link_contracts.1.contracts.values()) - .flat_map(|value| value.values()) - .any(|(code, _)| !code.chars().all(|char| char.is_ascii_hexdigit())); - let (leader_compiled_contracts, follower_compiled_contracts) = - if metadata_file_contains_libraries && compiled_contracts_require_linking { - let leader_key = ( - metadata_file_path.to_path_buf(), - mode.clone(), - L::config_id(), - ); - let follower_key = ( - metadata_file_path.to_path_buf(), - mode.clone(), - F::config_id(), - ); - { - let mut cache = compilation_cache.write().await; - cache.remove(&leader_key); - cache.remove(&follower_key); - } - - let leader_post_link_contracts = get_or_build_contracts::( - metadata, - metadata_file_path, - mode.clone(), - config, - compilation_cache.clone(), - &leader_deployed_libraries, - ) - .await?; - let follower_post_link_contracts = get_or_build_contracts::( - metadata, - metadata_file_path, - mode.clone(), - config, - compilation_cache, - &follower_deployed_libraries, - ) - .await?; - - (leader_post_link_contracts, follower_post_link_contracts) - } else { - (leader_pre_link_contracts, follower_pre_link_contracts) - }; + let (leader_post_link_contracts, leader_compiler_version) = cached_compiler + .compile_contracts::( + metadata, + metadata_file_path, + &mode, + config, + leader_deployed_libraries.as_ref(), + ) + .await?; + let (follower_post_link_contracts, follower_compiler_version) = cached_compiler + .compile_contracts::( + metadata, + metadata_file_path, + &mode, + config, + follower_deployed_libraries.as_ref(), + ) + .await?; let leader_state = CaseState::::new( - leader_compiled_contracts.0.clone(), - leader_compiled_contracts.1.contracts.clone(), - leader_deployed_libraries, + leader_compiler_version, + leader_post_link_contracts.contracts, + leader_deployed_libraries.unwrap_or_default(), ); let follower_state = CaseState::::new( - follower_compiled_contracts.0.clone(), - follower_compiled_contracts.1.contracts.clone(), - follower_deployed_libraries, + follower_compiler_version, + follower_post_link_contracts.contracts, + follower_deployed_libraries.unwrap_or_default(), ); let mut driver = CaseDriver::::new( @@ -656,119 +611,6 @@ where driver.execute().await } -async fn get_or_build_contracts( - metadata: &Metadata, - metadata_file_path: &Path, - mode: Mode, - config: &Arguments, - compilation_cache: CompilationCache, - deployed_libraries: &HashMap, -) -> anyhow::Result> { - let key = ( - metadata_file_path.to_path_buf(), - mode.clone(), - P::config_id(), - ); - if let Some(compilation_artifact) = compilation_cache.read().await.get(&key).cloned() { - let mut compilation_artifact = compilation_artifact.lock().await; - match *compilation_artifact { - Some(ref compiled_contracts) => { - tracing::debug!(?key, "Compiled contracts cache hit"); - return Ok(compiled_contracts.clone()); - } - None => { - tracing::debug!(?key, "Compiled contracts cache miss"); - let compiled_contracts = compile_contracts::

( - metadata, - metadata_file_path, - &mode, - config, - deployed_libraries, - ) - .await?; - let compiled_contracts = Arc::new(compiled_contracts); - - *compilation_artifact = Some(compiled_contracts.clone()); - return Ok(compiled_contracts.clone()); - } - } - }; - - tracing::debug!(?key, "Compiled contracts cache miss"); - let mutex = { - let mut compilation_cache = compilation_cache.write().await; - let mutex = Arc::new(Mutex::new(None)); - compilation_cache.insert(key, mutex.clone()); - mutex - }; - let mut compilation_artifact = mutex.lock().await; - - let compiled_contracts = compile_contracts::

( - metadata, - metadata_file_path, - &mode, - config, - deployed_libraries, - ) - .await?; - let compiled_contracts = Arc::new(compiled_contracts); - - *compilation_artifact = Some(compiled_contracts.clone()); - Ok(compiled_contracts.clone()) -} - -async fn compile_contracts( - metadata: &Metadata, - metadata_file_path: &Path, - mode: &Mode, - config: &Arguments, - deployed_libraries: &HashMap, -) -> anyhow::Result<(Version, CompilerOutput)> { - let compiler_version_or_requirement = mode.compiler_version_to_use(config.solc.clone()); - let compiler_path = - P::Compiler::get_compiler_executable(config, compiler_version_or_requirement).await?; - let compiler_version = P::Compiler::new(compiler_path.clone()).version()?; - - tracing::info!( - %compiler_version, - metadata_file_path = %metadata_file_path.display(), - mode = ?mode, - "Compiling contracts" - ); - - let compiler = Compiler::::new() - .with_allow_path(metadata.directory()?) - .with_optimization(mode.optimize_setting) - .with_pipeline(mode.pipeline); - let mut compiler = metadata - .files_to_compile()? - .try_fold(compiler, |compiler, path| compiler.with_source(&path))?; - for (library_instance, (library_address, _)) in deployed_libraries.iter() { - let library_ident = &metadata - .contracts - .as_ref() - .and_then(|contracts| contracts.get(library_instance)) - .expect("Impossible for library to not be found in contracts") - .contract_ident; - - // Note the following: we need to tell solc which files require the libraries to be linked - // into them. We do not have access to this information and therefore we choose an easier, - // yet more compute intensive route, of telling solc that all of the files need to link the - // library and it will only perform the linking for the files that do actually need the - // library. - compiler = FilesWithExtensionIterator::new(metadata.directory()?) - .with_allowed_extension("sol") - .with_use_cached_fs(true) - .fold(compiler, |compiler, path| { - compiler.with_library(&path, library_ident.as_str(), *library_address) - }); - } - - let compiler_output = compiler.try_build(compiler_path).await?; - - Ok((compiler_version, compiler_output)) -} - async fn execute_corpus( args: &Arguments, tests: &[MetadataFile], @@ -800,28 +642,40 @@ async fn compile_corpus( .map(move |solc_mode| (metadata, solc_mode)) }); + let file = tempfile::NamedTempFile::new().expect("Failed to create temp file"); + let cached_compiler = CachedCompiler::new(file.path(), false) + .await + .map(Arc::new) + .expect("Failed to create the cached compiler"); + futures::stream::iter(tests) - .for_each_concurrent(None, |(metadata, mode)| async move { - match platform { - TestingPlatform::Geth => { - let _ = compile_contracts::( - &metadata.content, - &metadata.path, - &mode, - config, - &Default::default(), - ) - .await; - } - TestingPlatform::Kitchensink => { - let _ = compile_contracts::( - &metadata.content, - &metadata.path, - &mode, - config, - &Default::default(), - ) - .await; + .for_each_concurrent(None, |(metadata, mode)| { + let cached_compiler = cached_compiler.clone(); + + async move { + match platform { + TestingPlatform::Geth => { + let _ = cached_compiler + .compile_contracts::( + metadata, + metadata.path.as_path(), + &mode, + config, + None, + ) + .await; + } + TestingPlatform::Kitchensink => { + let _ = cached_compiler + .compile_contracts::( + metadata, + metadata.path.as_path(), + &mode, + config, + None, + ) + .await; + } } } }) diff --git a/crates/format/src/input.rs b/crates/format/src/input.rs index aace626..36899b0 100644 --- a/crates/format/src/input.rs +++ b/crates/format/src/input.rs @@ -739,12 +739,14 @@ impl<'de> Deserialize<'de> for EtherValue { #[cfg(test)] mod tests { - use super::*; use alloy::{eips::BlockNumberOrTag, json_abi::JsonAbi}; use alloy_primitives::{BlockHash, BlockNumber, BlockTimestamp, ChainId, TxHash, address}; use alloy_sol_types::SolValue; use std::collections::HashMap; + use super::*; + use crate::metadata::ContractIdent; + struct MockResolver; impl ResolverApi for MockResolver { @@ -818,11 +820,11 @@ mod tests { let mut contracts = HashMap::new(); contracts.insert( ContractInstance::new("Contract"), - (Address::ZERO, parsed_abi), + (ContractIdent::new("Contract"), Address::ZERO, parsed_abi), ); let resolver = MockResolver; - let context = ResolutionContext::new_from_parts(&contracts, None, None, None); + let context = ResolutionContext::default().with_deployed_contracts(&contracts); let encoded = input.encoded_input(&resolver, context).await.unwrap(); assert!(encoded.0.starts_with(&selector)); @@ -862,11 +864,11 @@ mod tests { let mut contracts = HashMap::new(); contracts.insert( ContractInstance::new("Contract"), - (Address::ZERO, parsed_abi), + (ContractIdent::new("Contract"), Address::ZERO, parsed_abi), ); let resolver = MockResolver; - let context = ResolutionContext::new_from_parts(&contracts, None, None, None); + let context = ResolutionContext::default().with_deployed_contracts(&contracts); let encoded = input.encoded_input(&resolver, context).await.unwrap(); assert!(encoded.0.starts_with(&selector)); @@ -909,11 +911,11 @@ mod tests { let mut contracts = HashMap::new(); contracts.insert( ContractInstance::new("Contract"), - (Address::ZERO, parsed_abi), + (ContractIdent::new("Contract"), Address::ZERO, parsed_abi), ); let resolver = MockResolver; - let context = ResolutionContext::new_from_parts(&contracts, None, None, None); + let context = ResolutionContext::default().with_deployed_contracts(&contracts); let encoded = input.encoded_input(&resolver, context).await.unwrap(); assert!(encoded.0.starts_with(&selector)); @@ -927,10 +929,10 @@ mod tests { async fn resolve_calldata_item( input: &str, - deployed_contracts: &HashMap, + deployed_contracts: &HashMap, resolver: &impl ResolverApi, ) -> anyhow::Result { - let context = ResolutionContext::new_from_parts(deployed_contracts, None, None, None); + let context = ResolutionContext::default().with_deployed_contracts(deployed_contracts); CalldataItem::new(input).resolve(resolver, context).await } diff --git a/crates/format/src/traits.rs b/crates/format/src/traits.rs index 057239c..1ad6cc3 100644 --- a/crates/format/src/traits.rs +++ b/crates/format/src/traits.rs @@ -6,7 +6,7 @@ use alloy::primitives::{Address, BlockHash, BlockNumber, BlockTimestamp, ChainId use alloy_primitives::TxHash; use anyhow::Result; -use crate::metadata::ContractInstance; +use crate::metadata::{ContractIdent, ContractInstance}; /// A trait of the interface are required to implement to be used by the resolution logic that this /// crate implements to go from string calldata and into the bytes calldata. @@ -48,7 +48,7 @@ pub trait ResolverApi { /// Contextual information required by the code that's performing the resolution. pub struct ResolutionContext<'a> { /// When provided the contracts provided here will be used for resolutions. - deployed_contracts: Option<&'a HashMap>, + deployed_contracts: Option<&'a HashMap>, /// When provided the variables in here will be used for performing resolutions. variables: Option<&'a HashMap>, @@ -66,7 +66,9 @@ impl<'a> ResolutionContext<'a> { } pub fn new_from_parts( - deployed_contracts: impl Into>>, + deployed_contracts: impl Into< + Option<&'a HashMap>, + >, variables: impl Into>>, block_number: impl Into>, transaction_hash: impl Into>, @@ -81,7 +83,9 @@ impl<'a> ResolutionContext<'a> { pub fn with_deployed_contracts( mut self, - deployed_contracts: impl Into>>, + deployed_contracts: impl Into< + Option<&'a HashMap>, + >, ) -> Self { self.deployed_contracts = deployed_contracts.into(); self @@ -122,17 +126,20 @@ impl<'a> ResolutionContext<'a> { } } - pub fn deployed_contract(&self, instance: &ContractInstance) -> Option<&(Address, JsonAbi)> { + pub fn deployed_contract( + &self, + instance: &ContractInstance, + ) -> Option<&(ContractIdent, Address, JsonAbi)> { self.deployed_contracts .and_then(|deployed_contracts| deployed_contracts.get(instance)) } pub fn deployed_contract_address(&self, instance: &ContractInstance) -> Option<&Address> { - self.deployed_contract(instance).map(|(a, _)| a) + self.deployed_contract(instance).map(|(_, a, _)| a) } pub fn deployed_contract_abi(&self, instance: &ContractInstance) -> Option<&JsonAbi> { - self.deployed_contract(instance).map(|(_, a)| a) + self.deployed_contract(instance).map(|(_, _, a)| a) } pub fn variable(&self, name: impl AsRef) -> Option<&U256> {