diff --git a/crates/compiler/src/lib.rs b/crates/compiler/src/lib.rs index bb8934c..3651eb3 100644 --- a/crates/compiler/src/lib.rs +++ b/crates/compiler/src/lib.rs @@ -3,7 +3,11 @@ //! - Polkadot revive resolc compiler //! - Polkadot revive Wasm compiler -use std::{fs::read_to_string, hash::Hash, path::Path}; +use std::{ + fs::read_to_string, + hash::Hash, + path::{Path, PathBuf}, +}; use revive_common::EVMVersion; use revive_solc_json_interface::{ @@ -28,7 +32,7 @@ pub trait SolidityCompiler { input: CompilerInput, ) -> anyhow::Result>; - fn new(solc_version: &Version) -> Self; + fn new(solc_executable: PathBuf) -> Self; } /// The generic compilation input configuration. @@ -142,8 +146,8 @@ where self } - pub fn try_build(self, solc_version: &Version) -> anyhow::Result> { - T::new(solc_version).build(CompilerInput { + pub fn try_build(self, solc_path: PathBuf) -> anyhow::Result> { + T::new(solc_path).build(CompilerInput { extra_options: self.extra_options, input: self.input, }) diff --git a/crates/compiler/src/solc.rs b/crates/compiler/src/solc.rs index f715d29..28e578b 100644 --- a/crates/compiler/src/solc.rs +++ b/crates/compiler/src/solc.rs @@ -11,7 +11,7 @@ use semver::Version; use crate::{CompilerInput, CompilerOutput, SolidityCompiler}; pub struct Solc { - binary_path: PathBuf, + solc_path: PathBuf, } impl SolidityCompiler for Solc { @@ -21,7 +21,7 @@ impl SolidityCompiler for Solc { &self, input: CompilerInput, ) -> anyhow::Result> { - let mut child = Command::new(&self.binary_path) + let mut child = Command::new(&self.solc_path) .stdin(Stdio::piped()) .stdout(Stdio::piped()) .stderr(Stdio::piped()) @@ -38,9 +38,7 @@ impl SolidityCompiler for Solc { }) } - fn new(_solc_version: &Version) -> Self { - Self { - binary_path: "solc".into(), - } + fn new(solc_path: PathBuf) -> Self { + Self { solc_path } } } diff --git a/crates/config/Cargo.toml b/crates/config/Cargo.toml index 9251a3f..3ab45c0 100644 --- a/crates/config/Cargo.toml +++ b/crates/config/Cargo.toml @@ -10,3 +10,5 @@ rust-version.workspace = true [dependencies] clap = { workspace = true } +semver = { workspace = true } +temp-dir = { workspace = true } diff --git a/crates/config/src/lib.rs b/crates/config/src/lib.rs index 9c9e65f..a073281 100644 --- a/crates/config/src/lib.rs +++ b/crates/config/src/lib.rs @@ -1,15 +1,30 @@ //! The global configuration used accross all revive differential testing crates. -use std::path::PathBuf; +use std::{ + env, + path::{Path, PathBuf}, +}; -use clap::{Parser, ValueEnum}; +use clap::{Arg, Parser, ValueEnum}; +use semver::Version; +use temp_dir::TempDir; #[derive(Debug, Parser, Clone)] #[command(name = "retester")] pub struct Arguments { + /// The `solc` version to use if the test didn't specify it explicitly. + #[arg(long = "solc", short, default_value = "0.8.29")] + pub solc: Version, + + /// Use the Wasm compiler versions. + #[arg(long = "wasm")] + pub wasm: bool, + /// The path to the `resolc` executable to be tested. /// /// By default it uses the `resolc` binary found in `$PATH`. + /// + /// If `--wasm` is set, this should point to the resolc Wasm ile. #[arg(long = "resolc", short, default_value = "resolc")] pub resolc: PathBuf, @@ -23,6 +38,12 @@ pub struct Arguments { #[arg(long = "workdir", short)] pub working_directory: Option, + /// Add a tempdir manually if `working_directory` was not given. + /// + /// We attach it here because [TempDir] prunes itself on drop. + #[clap(skip)] + pub temp_dir: Option, + /// The path to the `geth` executable. /// /// By default it uses `geth` binary found in `$PATH`. @@ -58,10 +79,24 @@ pub struct Arguments { pub follower: TestingPlatform, /// Only compile against this testing platform (doesn't execute the tests). - #[arg(short, long = "compile-only")] + #[arg(long = "compile-only")] pub compile_only: bool, } +impl Arguments { + pub fn directory(&self) -> &Path { + if let Some(path) = &self.working_directory { + return path.as_path(); + } + + if let Some(temp_dir) = &self.temp_dir { + return temp_dir.path(); + } + + panic!("should have a workdir configured") + } +} + impl Default for Arguments { fn default() -> Self { Arguments::parse_from(["retester"]) diff --git a/crates/core/Cargo.toml b/crates/core/Cargo.toml index f256d20..f5785c7 100644 --- a/crates/core/Cargo.toml +++ b/crates/core/Cargo.toml @@ -16,9 +16,9 @@ path = "src/main.rs" revive-dt-compiler = { workspace = true } revive-dt-config = { workspace = true } revive-dt-format = { workspace = true } -revive-solc-json-interface = { workspace = true } revive-dt-node = { workspace = true } revive-dt-node-interaction = { workspace = true } +revive-dt-solc-binaries = { workspace = true } alloy = { workspace = true } anyhow = { workspace = true } @@ -26,6 +26,7 @@ clap = { workspace = true } log = { workspace = true } env_logger = { workspace = true } rayon = { workspace = true } +revive-solc-json-interface = { workspace = true } semver = { workspace = true } serde = { workspace = true, features = [ "derive" ] } serde_json = { workspace = true } diff --git a/crates/core/src/driver/mod.rs b/crates/core/src/driver/mod.rs index 1a1686b..1756bd6 100644 --- a/crates/core/src/driver/mod.rs +++ b/crates/core/src/driver/mod.rs @@ -4,10 +4,12 @@ use alloy::primitives::{Address, map::HashMap}; use revive_dt_compiler::{Compiler, CompilerInput, SolidityCompiler}; use revive_dt_config::Arguments; use revive_dt_format::{ + case::Case, metadata::Metadata, mode::{Mode, SolcMode}, }; use revive_dt_node::Node; +use revive_dt_solc_binaries::download_solc; use revive_solc_json_interface::SolcStandardJsonOutput; use semver::Version; @@ -18,54 +20,48 @@ type Contracts = HashMap< SolcStandardJsonOutput, >; -pub struct State { +pub struct State<'a, T: Platform> { + config: &'a Arguments, contracts: Contracts, deployed_contracts: HashMap, node: T::Blockchain, } -impl State +impl<'a, T> State<'a, T> where T: Platform, { - fn new(config: &Arguments) -> Self { + fn new(config: &'a Arguments) -> Self { Self { + config, contracts: Default::default(), deployed_contracts: Default::default(), node: ::new(config), } } - pub fn build_contracts(&mut self, metadata: &Metadata) -> anyhow::Result<()> { + pub fn build_contracts(&mut self, mode: &SolcMode, metadata: &Metadata) -> anyhow::Result<()> { let sources = metadata.contract_sources()?; let base_path = metadata.directory()?.display().to_string(); - let modes = metadata - .modes - .to_owned() - .unwrap_or_else(|| vec![Mode::Solidity(Default::default())]); - let mut result = HashMap::new(); - for mode in modes { - let mut compiler = Compiler::::new().base_path(base_path.clone()); - for (file, _contract) in sources.values() { - compiler = compiler.with_source(file)?; - } - - match mode { - Mode::Solidity(SolcMode { - solc_version: _, - solc_optimize, - llvm_optimizer_settings: _, - }) => { - let optimizer = solc_optimize.unwrap_or(true); - let version = Version::new(0, 8, 29); - let output = compiler.solc_optimizer(optimizer).try_build(&version)?; - result.insert(output.input, output.output); - } - Mode::Unknown(mode) => log::debug!("compiler: ignoring unknown mode '{mode}'"), - } + let mut compiler = Compiler::::new().base_path(base_path.clone()); + for (file, _contract) in sources.values() { + compiler = compiler.with_source(file)?; } + let version = Version::new(0, 8, 29); + let solc_path = download_solc(self.config.directory(), version, self.config.wasm)?; + let output = compiler + .solc_optimizer(mode.solc_optimize()) + .try_build(solc_path)?; + self.contracts.insert(output.input, output.output); + + Ok(()) + } + + pub fn execute_case(&mut self, case: &Case) -> anyhow::Result<()> { + for input in &case.inputs {} + Ok(()) } } @@ -73,8 +69,8 @@ where pub struct Driver<'a, Leader: Platform, Follower: Platform> { metadata: &'a Metadata, config: &'a Arguments, - leader: State, - follower: State, + leader: State<'a, Leader>, + follower: State<'a, Follower>, } impl<'a, L, F> Driver<'a, L, F> @@ -92,13 +88,36 @@ where } pub fn execute(&mut self) -> anyhow::Result<()> { - self.leader.build_contracts(self.metadata)?; - self.follower.build_contracts(self.metadata)?; + for mode in self.modes() { + self.leader.build_contracts(&mode, self.metadata)?; + self.follower.build_contracts(&mode, self.metadata)?; - if self.config.compile_only { - return Ok(()); + if self.config.compile_only { + continue; + } + + for case in &self.metadata.cases { + self.leader.execute_case(case)?; + } + + *self = Self::new(self.metadata, self.config); } - todo!() + Ok(()) + } + + fn modes(&self) -> Vec { + self.metadata + .modes() + .iter() + .filter_map(|mode| match mode { + Mode::Solidity(solc_mode) => Some(solc_mode), + Mode::Unknown(mode) => { + log::debug!("compiler: ignoring unknown mode '{mode}'"); + None + } + }) + .cloned() + .collect() } } diff --git a/crates/core/src/main.rs b/crates/core/src/main.rs index 0abf9e6..ed05737 100644 --- a/crates/core/src/main.rs +++ b/crates/core/src/main.rs @@ -15,8 +15,9 @@ fn main() -> anyhow::Result<()> { if args.corpus.is_empty() { anyhow::bail!("no test corpus specified"); } - let temp_dir = TempDir::new()?; - args.working_directory.get_or_insert(temp_dir.path().into()); + if args.working_directory.is_none() { + args.temp_dir = TempDir::new()?.into() + } for path in args.corpus.iter().collect::>() { log::trace!("attempting corpus {path:?}"); @@ -38,7 +39,7 @@ fn main() -> anyhow::Result<()> { Ok(build) => { log::info!( "metadata {} success", - metadata.file_path.as_ref().unwrap().display() + metadata.directory().as_ref().unwrap().display() ); build } diff --git a/crates/format/src/case.rs b/crates/format/src/case.rs index 4c67581..5516406 100644 --- a/crates/format/src/case.rs +++ b/crates/format/src/case.rs @@ -8,4 +8,5 @@ pub struct Case { pub comment: Option, pub modes: Option>, pub inputs: Vec, + pub group: Option, } diff --git a/crates/format/src/input.rs b/crates/format/src/input.rs index c027eb0..63f0abb 100644 --- a/crates/format/src/input.rs +++ b/crates/format/src/input.rs @@ -3,6 +3,27 @@ use semver::VersionReq; use serde::{Deserialize, de::Deserializer}; use serde_json::Value; +/* fn deserialize_calldata<'de, D>(deserializer: D) -> Result, D::Error> +where + D: Deserializer<'de>, +{ + let calldata_strings: Vec = Vec::deserialize(deserializer)?; + let mut result = Vec::with_capacity(calldata_strings.len() * 32); + + for calldata_string in &calldata_strings { + match calldata_string.parse::() { + Ok(parsed) => result.extend_from_slice(&parsed.to_be_bytes::<32>()), + Err(error) => { + return Err(serde::de::Error::custom(format!( + "parsing U256 {calldata_string} error: {error}" + ))); + } + }; + } + + Ok(result) +} */ + #[derive(Clone, Debug, Default, Deserialize, Eq, PartialEq)] pub struct Input { pub instance: Option, diff --git a/crates/format/src/metadata.rs b/crates/format/src/metadata.rs index 99ebeef..f072907 100644 --- a/crates/format/src/metadata.rs +++ b/crates/format/src/metadata.rs @@ -17,11 +17,18 @@ pub struct Metadata { pub contracts: Option>, pub libraries: Option>>, pub ignore: Option, - pub modes: Option>, + modes: Option>, pub file_path: Option, } impl Metadata { + /// Returns the modes of this metadata, inserting a default mode if not present. + pub fn modes(&self) -> Vec { + self.modes + .to_owned() + .unwrap_or_else(|| vec![Mode::Solidity(Default::default())]) + } + /// Returns the base directory of this metadata. pub fn directory(&self) -> anyhow::Result { Ok(self diff --git a/crates/format/src/mode.rs b/crates/format/src/mode.rs index 75649f6..6f58ac0 100644 --- a/crates/format/src/mode.rs +++ b/crates/format/src/mode.rs @@ -12,7 +12,7 @@ pub enum Mode { #[derive(Debug, Default, Clone, Eq, PartialEq)] pub struct SolcMode { pub solc_version: Option, - pub solc_optimize: Option, + solc_optimize: Option, pub llvm_optimizer_settings: Vec, } @@ -52,6 +52,11 @@ impl SolcMode { Some(result) } + + /// Returns whether to enable the solc optimizer. + pub fn solc_optimize(&self) -> bool { + self.solc_optimize.unwrap_or(true) + } } impl<'de> Deserialize<'de> for Mode { diff --git a/crates/node/src/geth.rs b/crates/node/src/geth.rs index 17be4c6..a46a4c8 100644 --- a/crates/node/src/geth.rs +++ b/crates/node/src/geth.rs @@ -1,7 +1,7 @@ //! The go-ethereum node implementation. use std::{ - fs::{File, create_dir, exists, remove_dir_all}, + fs::{File, create_dir_all, remove_dir_all}, io::{BufRead, BufReader, Read, Write}, path::PathBuf, process::{Child, Command, Stdio}, @@ -54,11 +54,7 @@ impl Instance { /// Create the node directory and call `geth init` to configure the genesis. fn init(&mut self, genesis: String) -> anyhow::Result<&mut Self> { - let geth_directory = self.base_directory.parent().expect("the id should be set"); - if !exists(geth_directory)? { - create_dir(geth_directory)?; - } - create_dir(&self.base_directory)?; + create_dir_all(&self.base_directory)?; let genesis_path = self.base_directory.join(Self::GENESIS_JSON_FILE); File::create(&genesis_path)?.write_all(genesis.as_bytes())?; @@ -161,11 +157,7 @@ impl EthereumNode for Instance { impl Node for Instance { fn new(config: &Arguments) -> Self { - let geth_directory = config - .working_directory - .as_ref() - .expect("config should provide working directory") - .join(Self::BASE_DIRECTORY); + let geth_directory = config.directory().join(Self::BASE_DIRECTORY); let id = NODE_COUNT.fetch_add(1, Ordering::SeqCst); let base_directory = geth_directory.join(id.to_string()); diff --git a/crates/solc-binaries/Cargo.toml b/crates/solc-binaries/Cargo.toml index f1a3a8b..13351a3 100644 --- a/crates/solc-binaries/Cargo.toml +++ b/crates/solc-binaries/Cargo.toml @@ -8,15 +8,11 @@ edition.workspace = true repository.workspace = true rust-version.workspace = true -[features] -default = ["download"] -download = ["reqwest"] - [dependencies] anyhow = { workspace = true } hex = { workspace = true } once_cell = { workspace = true } -reqwest = { workspace = true, optional = true } +reqwest = { workspace = true } semver = { workspace = true } serde = { workspace = true } sha2 = { workspace = true } diff --git a/crates/solc-binaries/src/cache.rs b/crates/solc-binaries/src/cache.rs new file mode 100644 index 0000000..eab0154 --- /dev/null +++ b/crates/solc-binaries/src/cache.rs @@ -0,0 +1,67 @@ +//! Helper for caching the solc binaries. + +use std::{ + cell::OnceCell, + collections::HashSet, + fs::{File, create_dir_all}, + io::Write, + path::{Path, PathBuf}, + sync::Mutex, +}; + +use crate::download::GHDownloader; + +pub const SOLC_CACHE_DIRECTORY: &str = "solc"; +pub const SOLC_CACHER: OnceCell> = OnceCell::new(); + +pub fn get_or_download( + working_directory: &Path, + downloader: &GHDownloader, +) -> anyhow::Result { + SOLC_CACHER + .get_or_init(|| { + Mutex::new(SolcCacher::new( + working_directory.join(SOLC_CACHE_DIRECTORY), + )) + }) + .lock() + .unwrap() + .get_or_download(downloader) +} + +pub struct SolcCacher { + cache_directory: PathBuf, + cached_binaries: HashSet, +} + +impl SolcCacher { + fn new(cache_directory: PathBuf) -> Self { + Self { + cache_directory, + cached_binaries: Default::default(), + } + } + + fn get_or_download(&mut self, downloader: &GHDownloader) -> anyhow::Result { + let directory = self.cache_directory.join(downloader.version.to_string()); + let file_path = directory.join(downloader.target); + + if self.cached_binaries.contains(&file_path) { + return Ok(file_path); + } + + if file_path.exists() { + self.cached_binaries.insert(file_path.clone()); + return Ok(file_path); + } + + create_dir_all(directory)?; + + let buf = downloader.download()?; + File::create_new(&file_path) + .expect("should not exist because of above early return") + .write_all(&buf)?; + + Ok(file_path) + } +} diff --git a/crates/solc-binaries/src/download.rs b/crates/solc-binaries/src/download.rs index baa7707..08859ad 100644 --- a/crates/solc-binaries/src/download.rs +++ b/crates/solc-binaries/src/download.rs @@ -35,9 +35,9 @@ impl List { /// Download solc binaries from GitHub releases (IPFS links aren't reliable). pub struct GHDownloader { - version: Version, - target: &'static str, - list: &'static str, + pub version: Version, + pub target: &'static str, + pub list: &'static str, } impl GHDownloader { @@ -48,36 +48,33 @@ impl GHDownloader { pub const WINDOWS_NAME: &str = "solc-windows.exe"; pub const WASM_NAME: &str = "soljson.js"; - pub fn linux(version: Version) -> Self { + fn new(version: Version, target: &'static str, list: &'static str) -> Self { Self { version, - target: Self::LINUX_NAME, - list: List::LINUX_URL, + target, + list, } } + pub fn linux(version: Version) -> Self { + Self::new(version, Self::LINUX_NAME, List::LINUX_URL) + } + pub fn macosx(version: Version) -> Self { - Self { - version, - target: Self::MACOSX_NAME, - list: List::MACOSX_URL, - } + Self::new(version, Self::MACOSX_NAME, List::MACOSX_URL) } pub fn windows(version: Version) -> Self { - Self { - version, - target: Self::WINDOWS_NAME, - list: List::WINDOWS_URL, - } + Self::new(version, Self::WINDOWS_NAME, List::WINDOWS_URL) } pub fn wasm(version: Version) -> Self { - Self { - version, - target: Self::WASM_NAME, - list: List::WASM_URL, - } + Self::new(version, Self::WASM_NAME, List::WASM_URL) + } + + /// Returns the download link. + pub fn url(&self) -> String { + format!("{}/v{}/{}", Self::BASE_URL, &self.version, &self.target) } /// Download the solc binary. @@ -92,8 +89,7 @@ impl GHDownloader { .ok_or_else(|| anyhow::anyhow!("solc v{} not found builds", self.version)) .map(|b| b.sha256.strip_prefix("0x").unwrap_or(&b.sha256).to_string())?; - let url = format!("{}/v{}/{}", Self::BASE_URL, self.version, self.target); - let file = reqwest::blocking::get(&url)?.bytes()?.to_vec(); + let file = reqwest::blocking::get(self.url())?.bytes()?.to_vec(); if hex::encode(Sha256::digest(&file)) != expected_digest { anyhow::bail!("sha256 mismatch for solc version {}", self.version); diff --git a/crates/solc-binaries/src/lib.rs b/crates/solc-binaries/src/lib.rs index a987d02..84ac3b3 100644 --- a/crates/solc-binaries/src/lib.rs +++ b/crates/solc-binaries/src/lib.rs @@ -1,8 +1,39 @@ -//! This crates provides serializable Rust type definitions for the [solc binary lists][0]. -//! The `download` feature enables helpers to download and cache solc binaries. +//! This crates provides serializable Rust type definitions for the [solc binary lists][0] +//! and download helpers. //! //! [0]: https://binaries.soliditylang.org -#[cfg(feature = "download")] +use std::path::{Path, PathBuf}; + +use cache::get_or_download; +use download::GHDownloader; +use semver::Version; + +pub mod cache; pub mod download; pub mod list; + +/// Downloads the solc binary for Wasm is `wasm` is set, otherwise for +/// the target platform. +/// +/// Subsequent calls for the same version will use a cached artifact +/// and not download it again. +pub fn download_solc( + working_directory: &Path, + version: Version, + wasm: bool, +) -> anyhow::Result { + let downloader = if wasm { + GHDownloader::wasm(version) + } else if cfg!(target_os = "linux") { + GHDownloader::linux(version) + } else if cfg!(target_os = "macos") { + GHDownloader::macosx(version) + } else if cfg!(target_os = "windows") { + GHDownloader::windows(version) + } else { + unimplemented!() + }; + + get_or_download(working_directory, &downloader) +}