the solc download per target helper

Signed-off-by: Cyrill Leutwiler <bigcyrill@hotmail.com>

crates/compiler/src/lib.rs

@@ -3,7 +3,11 @@
 //! - Polkadot revive resolc compiler
 //! - Polkadot revive Wasm compiler
 
-use std::{fs::read_to_string, hash::Hash, path::Path};
+use std::{
+    fs::read_to_string,
+    hash::Hash,
+    path::{Path, PathBuf},
+};
 
 use revive_common::EVMVersion;
 use revive_solc_json_interface::{
@@ -28,7 +32,7 @@ pub trait SolidityCompiler {
         input: CompilerInput<Self::Options>,
     ) -> anyhow::Result<CompilerOutput<Self::Options>>;
 
-    fn new(solc_version: &Version) -> Self;
+    fn new(solc_executable: PathBuf) -> Self;
 }
 
 /// The generic compilation input configuration.
@@ -142,8 +146,8 @@ where
         self
     }
 
-    pub fn try_build(self, solc_version: &Version) -> anyhow::Result<CompilerOutput<T::Options>> {
-        T::new(solc_version).build(CompilerInput {
+    pub fn try_build(self, solc_path: PathBuf) -> anyhow::Result<CompilerOutput<T::Options>> {
+        T::new(solc_path).build(CompilerInput {
             extra_options: self.extra_options,
             input: self.input,
         })

crates/compiler/src/solc.rs

@@ -11,7 +11,7 @@ use semver::Version;
 use crate::{CompilerInput, CompilerOutput, SolidityCompiler};
 
 pub struct Solc {
-    binary_path: PathBuf,
+    solc_path: PathBuf,
 }
 
 impl SolidityCompiler for Solc {
@@ -21,7 +21,7 @@ impl SolidityCompiler for Solc {
         &self,
         input: CompilerInput<Self::Options>,
     ) -> anyhow::Result<CompilerOutput<Self::Options>> {
-        let mut child = Command::new(&self.binary_path)
+        let mut child = Command::new(&self.solc_path)
             .stdin(Stdio::piped())
             .stdout(Stdio::piped())
             .stderr(Stdio::piped())
@@ -38,9 +38,7 @@ impl SolidityCompiler for Solc {
         })
     }
 
-    fn new(_solc_version: &Version) -> Self {
-        Self {
-            binary_path: "solc".into(),
-        }
+    fn new(solc_path: PathBuf) -> Self {
+        Self { solc_path }
     }
 }

crates/config/Cargo.toml

@@ -10,3 +10,5 @@ rust-version.workspace = true
 
 [dependencies]
 clap = { workspace = true }
+semver = { workspace = true }
+temp-dir = { workspace = true }

crates/config/src/lib.rs

@@ -1,15 +1,30 @@
 //! The global configuration used accross all revive differential testing crates.
 
-use std::path::PathBuf;
+use std::{
+    env,
+    path::{Path, PathBuf},
+};
 
-use clap::{Parser, ValueEnum};
+use clap::{Arg, Parser, ValueEnum};
+use semver::Version;
+use temp_dir::TempDir;
 
 #[derive(Debug, Parser, Clone)]
 #[command(name = "retester")]
 pub struct Arguments {
+    /// The `solc` version to use if the test didn't specify it explicitly.
+    #[arg(long = "solc", short, default_value = "0.8.29")]
+    pub solc: Version,
+
+    /// Use the Wasm compiler versions.
+    #[arg(long = "wasm")]
+    pub wasm: bool,
+
     /// The path to the `resolc` executable to be tested.
     ///
     /// By default it uses the `resolc` binary found in `$PATH`.
+    ///
+    /// If `--wasm` is set, this should point to the resolc Wasm ile.
     #[arg(long = "resolc", short, default_value = "resolc")]
     pub resolc: PathBuf,
 
@@ -23,6 +38,12 @@ pub struct Arguments {
     #[arg(long = "workdir", short)]
     pub working_directory: Option<PathBuf>,
 
+    /// Add a tempdir manually if `working_directory` was not given.
+    ///
+    /// We attach it here because [TempDir] prunes itself on drop.
+    #[clap(skip)]
+    pub temp_dir: Option<TempDir>,
+
     /// The path to the `geth` executable.
     ///
     /// By default it uses `geth` binary found in `$PATH`.
@@ -58,10 +79,24 @@ pub struct Arguments {
     pub follower: TestingPlatform,
 
     /// Only compile against this testing platform (doesn't execute the tests).
-    #[arg(short, long = "compile-only")]
+    #[arg(long = "compile-only")]
     pub compile_only: bool,
 }
 
+impl Arguments {
+    pub fn directory(&self) -> &Path {
+        if let Some(path) = &self.working_directory {
+            return path.as_path();
+        }
+
+        if let Some(temp_dir) = &self.temp_dir {
+            return temp_dir.path();
+        }
+
+        panic!("should have a workdir configured")
+    }
+}
+
 impl Default for Arguments {
     fn default() -> Self {
         Arguments::parse_from(["retester"])

crates/core/Cargo.toml

@@ -16,9 +16,9 @@ path = "src/main.rs"
 revive-dt-compiler = { workspace = true }
 revive-dt-config = { workspace = true }
 revive-dt-format = { workspace = true }
-revive-solc-json-interface = { workspace = true }
 revive-dt-node = { workspace = true }
 revive-dt-node-interaction = { workspace = true }
+revive-dt-solc-binaries = { workspace = true }
 
 alloy = { workspace = true }
 anyhow = { workspace = true }
@@ -26,6 +26,7 @@ clap = { workspace = true }
 log = { workspace = true }
 env_logger = { workspace = true }
 rayon = { workspace = true }
+revive-solc-json-interface = { workspace = true }
 semver = { workspace = true }
 serde = { workspace = true, features = [ "derive" ] }
 serde_json = { workspace = true }

crates/core/src/driver/mod.rs

@@ -4,10 +4,12 @@ use alloy::primitives::{Address, map::HashMap};
 use revive_dt_compiler::{Compiler, CompilerInput, SolidityCompiler};
 use revive_dt_config::Arguments;
 use revive_dt_format::{
+    case::Case,
     metadata::Metadata,
     mode::{Mode, SolcMode},
 };
 use revive_dt_node::Node;
+use revive_dt_solc_binaries::download_solc;
 use revive_solc_json_interface::SolcStandardJsonOutput;
 use semver::Version;
 
@@ -18,54 +20,48 @@ type Contracts<T> = HashMap<
     SolcStandardJsonOutput,
 >;
 
-pub struct State<T: Platform> {
+pub struct State<'a, T: Platform> {
+    config: &'a Arguments,
     contracts: Contracts<T>,
     deployed_contracts: HashMap<String, Address>,
     node: T::Blockchain,
 }
 
-impl<T> State<T>
+impl<'a, T> State<'a, T>
 where
     T: Platform,
 {
-    fn new(config: &Arguments) -> Self {
+    fn new(config: &'a Arguments) -> Self {
         Self {
+            config,
             contracts: Default::default(),
             deployed_contracts: Default::default(),
             node: <T::Blockchain as Node>::new(config),
         }
     }
 
-    pub fn build_contracts(&mut self, metadata: &Metadata) -> anyhow::Result<()> {
+    pub fn build_contracts(&mut self, mode: &SolcMode, metadata: &Metadata) -> anyhow::Result<()> {
         let sources = metadata.contract_sources()?;
         let base_path = metadata.directory()?.display().to_string();
-        let modes = metadata
-            .modes
-            .to_owned()
-            .unwrap_or_else(|| vec![Mode::Solidity(Default::default())]);
 
-        let mut result = HashMap::new();
-        for mode in modes {
-            let mut compiler = Compiler::<T::Compiler>::new().base_path(base_path.clone());
-            for (file, _contract) in sources.values() {
-                compiler = compiler.with_source(file)?;
-            }
-
-            match mode {
-                Mode::Solidity(SolcMode {
-                    solc_version: _,
-                    solc_optimize,
-                    llvm_optimizer_settings: _,
-                }) => {
-                    let optimizer = solc_optimize.unwrap_or(true);
-                    let version = Version::new(0, 8, 29);
-                    let output = compiler.solc_optimizer(optimizer).try_build(&version)?;
-                    result.insert(output.input, output.output);
-                }
-                Mode::Unknown(mode) => log::debug!("compiler: ignoring unknown mode '{mode}'"),
-            }
+        let mut compiler = Compiler::<T::Compiler>::new().base_path(base_path.clone());
+        for (file, _contract) in sources.values() {
+            compiler = compiler.with_source(file)?;
         }
 
+        let version = Version::new(0, 8, 29);
+        let solc_path = download_solc(self.config.directory(), version, self.config.wasm)?;
+        let output = compiler
+            .solc_optimizer(mode.solc_optimize())
+            .try_build(solc_path)?;
+        self.contracts.insert(output.input, output.output);
+
+        Ok(())
+    }
+
+    pub fn execute_case(&mut self, case: &Case) -> anyhow::Result<()> {
+        for input in &case.inputs {}
+
         Ok(())
     }
 }
@@ -73,8 +69,8 @@ where
 pub struct Driver<'a, Leader: Platform, Follower: Platform> {
     metadata: &'a Metadata,
     config: &'a Arguments,
-    leader: State<Leader>,
-    follower: State<Follower>,
+    leader: State<'a, Leader>,
+    follower: State<'a, Follower>,
 }
 
 impl<'a, L, F> Driver<'a, L, F>
@@ -92,13 +88,36 @@ where
     }
 
     pub fn execute(&mut self) -> anyhow::Result<()> {
-        self.leader.build_contracts(self.metadata)?;
-        self.follower.build_contracts(self.metadata)?;
+        for mode in self.modes() {
+            self.leader.build_contracts(&mode, self.metadata)?;
+            self.follower.build_contracts(&mode, self.metadata)?;
 
-        if self.config.compile_only {
-            return Ok(());
+            if self.config.compile_only {
+                continue;
+            }
+
+            for case in &self.metadata.cases {
+                self.leader.execute_case(case)?;
+            }
+
+            *self = Self::new(self.metadata, self.config);
         }
 
-        todo!()
+        Ok(())
+    }
+
+    fn modes(&self) -> Vec<SolcMode> {
+        self.metadata
+            .modes()
+            .iter()
+            .filter_map(|mode| match mode {
+                Mode::Solidity(solc_mode) => Some(solc_mode),
+                Mode::Unknown(mode) => {
+                    log::debug!("compiler: ignoring unknown mode '{mode}'");
+                    None
+                }
+            })
+            .cloned()
+            .collect()
     }
 }

crates/core/src/main.rs

@@ -15,8 +15,9 @@ fn main() -> anyhow::Result<()> {
     if args.corpus.is_empty() {
         anyhow::bail!("no test corpus specified");
     }
-    let temp_dir = TempDir::new()?;
-    args.working_directory.get_or_insert(temp_dir.path().into());
+    if args.working_directory.is_none() {
+        args.temp_dir = TempDir::new()?.into()
+    }
 
     for path in args.corpus.iter().collect::<BTreeSet<_>>() {
         log::trace!("attempting corpus {path:?}");
@@ -38,7 +39,7 @@ fn main() -> anyhow::Result<()> {
                 Ok(build) => {
                     log::info!(
                         "metadata {} success",
-                        metadata.file_path.as_ref().unwrap().display()
+                        metadata.directory().as_ref().unwrap().display()
                     );
                     build
                 }

crates/format/src/case.rs

@@ -8,4 +8,5 @@ pub struct Case {
     pub comment: Option<String>,
     pub modes: Option<Vec<Mode>>,
     pub inputs: Vec<Input>,
+    pub group: Option<String>,
 }

crates/format/src/input.rs

@@ -3,6 +3,27 @@ use semver::VersionReq;
 use serde::{Deserialize, de::Deserializer};
 use serde_json::Value;
 
+/* fn deserialize_calldata<'de, D>(deserializer: D) -> Result<Vec<u8>, D::Error>
+where
+    D: Deserializer<'de>,
+{
+    let calldata_strings: Vec<String> = Vec::deserialize(deserializer)?;
+    let mut result = Vec::with_capacity(calldata_strings.len() * 32);
+
+    for calldata_string in &calldata_strings {
+        match calldata_string.parse::<U256>() {
+            Ok(parsed) => result.extend_from_slice(&parsed.to_be_bytes::<32>()),
+            Err(error) => {
+                return Err(serde::de::Error::custom(format!(
+                    "parsing U256 {calldata_string} error: {error}"
+                )));
+            }
+        };
+    }
+
+    Ok(result)
+} */
+
 #[derive(Clone, Debug, Default, Deserialize, Eq, PartialEq)]
 pub struct Input {
     pub instance: Option<String>,

crates/format/src/metadata.rs

@@ -17,11 +17,18 @@ pub struct Metadata {
     pub contracts: Option<BTreeMap<String, String>>,
     pub libraries: Option<BTreeMap<String, BTreeMap<String, String>>>,
     pub ignore: Option<bool>,
-    pub modes: Option<Vec<Mode>>,
+    modes: Option<Vec<Mode>>,
     pub file_path: Option<PathBuf>,
 }
 
 impl Metadata {
+    /// Returns the modes of this metadata, inserting a default mode if not present.
+    pub fn modes(&self) -> Vec<Mode> {
+        self.modes
+            .to_owned()
+            .unwrap_or_else(|| vec![Mode::Solidity(Default::default())])
+    }
+
     /// Returns the base directory of this metadata.
     pub fn directory(&self) -> anyhow::Result<PathBuf> {
         Ok(self

crates/format/src/mode.rs

@@ -12,7 +12,7 @@ pub enum Mode {
 #[derive(Debug, Default, Clone, Eq, PartialEq)]
 pub struct SolcMode {
     pub solc_version: Option<semver::VersionReq>,
-    pub solc_optimize: Option<bool>,
+    solc_optimize: Option<bool>,
     pub llvm_optimizer_settings: Vec<String>,
 }
 
@@ -52,6 +52,11 @@ impl SolcMode {
 
         Some(result)
     }
+
+    /// Returns whether to enable the solc optimizer.
+    pub fn solc_optimize(&self) -> bool {
+        self.solc_optimize.unwrap_or(true)
+    }
 }
 
 impl<'de> Deserialize<'de> for Mode {

crates/node/src/geth.rs

@@ -1,7 +1,7 @@
 //! The go-ethereum node implementation.
 
 use std::{
-    fs::{File, create_dir, exists, remove_dir_all},
+    fs::{File, create_dir_all, remove_dir_all},
     io::{BufRead, BufReader, Read, Write},
     path::PathBuf,
     process::{Child, Command, Stdio},
@@ -54,11 +54,7 @@ impl Instance {
 
     /// Create the node directory and call `geth init` to configure the genesis.
     fn init(&mut self, genesis: String) -> anyhow::Result<&mut Self> {
-        let geth_directory = self.base_directory.parent().expect("the id should be set");
-        if !exists(geth_directory)? {
-            create_dir(geth_directory)?;
-        }
-        create_dir(&self.base_directory)?;
+        create_dir_all(&self.base_directory)?;
 
         let genesis_path = self.base_directory.join(Self::GENESIS_JSON_FILE);
         File::create(&genesis_path)?.write_all(genesis.as_bytes())?;
@@ -161,11 +157,7 @@ impl EthereumNode for Instance {
 
 impl Node for Instance {
     fn new(config: &Arguments) -> Self {
-        let geth_directory = config
-            .working_directory
-            .as_ref()
-            .expect("config should provide working directory")
-            .join(Self::BASE_DIRECTORY);
+        let geth_directory = config.directory().join(Self::BASE_DIRECTORY);
         let id = NODE_COUNT.fetch_add(1, Ordering::SeqCst);
         let base_directory = geth_directory.join(id.to_string());
 

crates/solc-binaries/Cargo.toml

@@ -8,15 +8,11 @@ edition.workspace = true
 repository.workspace = true
 rust-version.workspace = true
 
-[features]
-default = ["download"]
-download = ["reqwest"]
-
 [dependencies]
 anyhow = { workspace = true }
 hex = { workspace = true }
 once_cell = { workspace = true }
-reqwest = { workspace = true, optional = true }
+reqwest = { workspace = true }
 semver = { workspace = true }
 serde = { workspace = true }
 sha2 = { workspace = true }

crates/solc-binaries/src/cache.rs

@@ -0,0 +1,67 @@
+//! Helper for caching the solc binaries.
+
+use std::{
+    cell::OnceCell,
+    collections::HashSet,
+    fs::{File, create_dir_all},
+    io::Write,
+    path::{Path, PathBuf},
+    sync::Mutex,
+};
+
+use crate::download::GHDownloader;
+
+pub const SOLC_CACHE_DIRECTORY: &str = "solc";
+pub const SOLC_CACHER: OnceCell<Mutex<SolcCacher>> = OnceCell::new();
+
+pub fn get_or_download(
+    working_directory: &Path,
+    downloader: &GHDownloader,
+) -> anyhow::Result<PathBuf> {
+    SOLC_CACHER
+        .get_or_init(|| {
+            Mutex::new(SolcCacher::new(
+                working_directory.join(SOLC_CACHE_DIRECTORY),
+            ))
+        })
+        .lock()
+        .unwrap()
+        .get_or_download(downloader)
+}
+
+pub struct SolcCacher {
+    cache_directory: PathBuf,
+    cached_binaries: HashSet<PathBuf>,
+}
+
+impl SolcCacher {
+    fn new(cache_directory: PathBuf) -> Self {
+        Self {
+            cache_directory,
+            cached_binaries: Default::default(),
+        }
+    }
+
+    fn get_or_download(&mut self, downloader: &GHDownloader) -> anyhow::Result<PathBuf> {
+        let directory = self.cache_directory.join(downloader.version.to_string());
+        let file_path = directory.join(downloader.target);
+
+        if self.cached_binaries.contains(&file_path) {
+            return Ok(file_path);
+        }
+
+        if file_path.exists() {
+            self.cached_binaries.insert(file_path.clone());
+            return Ok(file_path);
+        }
+
+        create_dir_all(directory)?;
+
+        let buf = downloader.download()?;
+        File::create_new(&file_path)
+            .expect("should not exist because of above early return")
+            .write_all(&buf)?;
+
+        Ok(file_path)
+    }
+}

crates/solc-binaries/src/download.rs

@@ -35,9 +35,9 @@ impl List {
 
 /// Download solc binaries from GitHub releases (IPFS links aren't reliable).
 pub struct GHDownloader {
-    version: Version,
-    target: &'static str,
-    list: &'static str,
+    pub version: Version,
+    pub target: &'static str,
+    pub list: &'static str,
 }
 
 impl GHDownloader {
@@ -48,36 +48,33 @@ impl GHDownloader {
     pub const WINDOWS_NAME: &str = "solc-windows.exe";
     pub const WASM_NAME: &str = "soljson.js";
 
-    pub fn linux(version: Version) -> Self {
+    fn new(version: Version, target: &'static str, list: &'static str) -> Self {
         Self {
             version,
-            target: Self::LINUX_NAME,
-            list: List::LINUX_URL,
+            target,
+            list,
         }
     }
 
+    pub fn linux(version: Version) -> Self {
+        Self::new(version, Self::LINUX_NAME, List::LINUX_URL)
+    }
+
     pub fn macosx(version: Version) -> Self {
-        Self {
-            version,
-            target: Self::MACOSX_NAME,
-            list: List::MACOSX_URL,
-        }
+        Self::new(version, Self::MACOSX_NAME, List::MACOSX_URL)
     }
 
     pub fn windows(version: Version) -> Self {
-        Self {
-            version,
-            target: Self::WINDOWS_NAME,
-            list: List::WINDOWS_URL,
-        }
+        Self::new(version, Self::WINDOWS_NAME, List::WINDOWS_URL)
     }
 
     pub fn wasm(version: Version) -> Self {
-        Self {
-            version,
-            target: Self::WASM_NAME,
-            list: List::WASM_URL,
-        }
+        Self::new(version, Self::WASM_NAME, List::WASM_URL)
+    }
+
+    /// Returns the download link.
+    pub fn url(&self) -> String {
+        format!("{}/v{}/{}", Self::BASE_URL, &self.version, &self.target)
     }
 
     /// Download the solc binary.
@@ -92,8 +89,7 @@ impl GHDownloader {
             .ok_or_else(|| anyhow::anyhow!("solc v{} not found builds", self.version))
             .map(|b| b.sha256.strip_prefix("0x").unwrap_or(&b.sha256).to_string())?;
 
-        let url = format!("{}/v{}/{}", Self::BASE_URL, self.version, self.target);
-        let file = reqwest::blocking::get(&url)?.bytes()?.to_vec();
+        let file = reqwest::blocking::get(self.url())?.bytes()?.to_vec();
 
         if hex::encode(Sha256::digest(&file)) != expected_digest {
             anyhow::bail!("sha256 mismatch for solc version {}", self.version);

crates/solc-binaries/src/lib.rs

@@ -1,8 +1,39 @@
-//! This crates provides serializable Rust type definitions for the [solc binary lists][0].
-//! The `download` feature enables helpers to download and cache solc binaries.
+//! This crates provides serializable Rust type definitions for the [solc binary lists][0]
+//! and download helpers.
 //!
 //! [0]: https://binaries.soliditylang.org
 
-#[cfg(feature = "download")]
+use std::path::{Path, PathBuf};
+
+use cache::get_or_download;
+use download::GHDownloader;
+use semver::Version;
+
+pub mod cache;
 pub mod download;
 pub mod list;
+
+/// Downloads the solc binary for Wasm is `wasm` is set, otherwise for
+/// the target platform.
+///
+/// Subsequent calls for the same version will use a cached artifact
+/// and not download it again.
+pub fn download_solc(
+    working_directory: &Path,
+    version: Version,
+    wasm: bool,
+) -> anyhow::Result<PathBuf> {
+    let downloader = if wasm {
+        GHDownloader::wasm(version)
+    } else if cfg!(target_os = "linux") {
+        GHDownloader::linux(version)
+    } else if cfg!(target_os = "macos") {
+        GHDownloader::macosx(version)
+    } else if cfg!(target_os = "windows") {
+        GHDownloader::windows(version)
+    } else {
+        unimplemented!()
+    };
+
+    get_or_download(working_directory, &downloader)
+}