Separate compilation and linker phases (#376)

Separate between compilation and linker phases to allow deploy time
linking and back-porting era compiler changes to fix #91. Unlinked
contract binaries (caused by missing libraries or missing factory
dependencies in turn) are emitted as raw ELF object.

Few drive by fixes:
- #98
- A compiler panic on missing libraries definitions.
- Fixes some incosistent type forwarding in JSON output (empty string
vs. null object).
- Remove the unused fallback for size optimization setting.
- Remove the broken `--lvm-ir`  mode.
- CI workflow fixes.

---------

Signed-off-by: Cyrill Leutwiler <bigcyrill@hotmail.com>
Signed-off-by: xermicus <bigcyrill@hotmail.com>
Signed-off-by: xermicus <cyrill@parity.io>

crates/resolc/src/process/input.rs

@@ -1,51 +1,69 @@
 //! Process for compiling a single compilation unit.
 //! The input data.
 
+use std::collections::BTreeMap;
+use std::collections::BTreeSet;
+
+use revive_common::MetadataHash;
+use revive_llvm_context::DebugConfig;
+use revive_llvm_context::OptimizerSettings;
 use revive_solc_json_interface::SolcStandardJsonInputSettingsPolkaVMMemory;
 use serde::Deserialize;
 use serde::Serialize;
 
 use crate::project::contract::Contract;
-use crate::project::Project;
+use crate::SolcVersion;
 
 /// The input data.
 #[derive(Debug, Serialize, Deserialize)]
 pub struct Input {
     /// The contract representation.
     pub contract: Contract,
-    /// The project representation.
-    pub project: Project,
+    /// The `solc` compiler version.
+    pub solc_version: Option<SolcVersion>,
     /// Whether to append the metadata hash.
-    pub include_metadata_hash: bool,
+    pub metadata_hash: MetadataHash,
     /// The optimizer settings.
-    pub optimizer_settings: revive_llvm_context::OptimizerSettings,
+    pub optimizer_settings: OptimizerSettings,
     /// The debug output config.
-    pub debug_config: revive_llvm_context::DebugConfig,
+    pub debug_config: DebugConfig,
     /// The extra LLVM arguments give used for manual control.
     pub llvm_arguments: Vec<String>,
     /// The PVM memory configuration.
     pub memory_config: SolcStandardJsonInputSettingsPolkaVMMemory,
+    /// Missing unlinked libraries.
+    pub missing_libraries: BTreeSet<String>,
+    /// Factory dependencies.
+    pub factory_dependencies: BTreeSet<String>,
+    /// The mapping of auxiliary identifiers, e.g. Yul object names, to full contract paths.
+    pub identifier_paths: BTreeMap<String, String>,
 }
 
 impl Input {
     /// A shortcut constructor.
     pub fn new(
         contract: Contract,
-        project: Project,
-        include_metadata_hash: bool,
-        optimizer_settings: revive_llvm_context::OptimizerSettings,
-        debug_config: revive_llvm_context::DebugConfig,
+        solc_version: Option<SolcVersion>,
+        metadata_hash: MetadataHash,
+        optimizer_settings: OptimizerSettings,
+        debug_config: DebugConfig,
         llvm_arguments: Vec<String>,
         memory_config: SolcStandardJsonInputSettingsPolkaVMMemory,
+        missing_libraries: BTreeSet<String>,
+        factory_dependencies: BTreeSet<String>,
+        identifier_paths: BTreeMap<String, String>,
     ) -> Self {
         Self {
             contract,
-            project,
-            include_metadata_hash,
+            solc_version,
+            metadata_hash,
             optimizer_settings,
             debug_config,
             llvm_arguments,
             memory_config,
+            missing_libraries,
+            factory_dependencies,
+            identifier_paths,
         }
     }
 }

crates/resolc/src/process/mod.rs

@@ -1,5 +1,12 @@
 //! Process for compiling a single compilation unit.
 
+use revive_solc_json_interface::SolcStandardJsonOutputError;
+use serde::de::DeserializeOwned;
+use serde::Serialize;
+
+use self::input::Input;
+use self::output::Output;
+
 pub mod input;
 #[cfg(not(target_os = "emscripten"))]
 pub mod native_process;
@@ -7,71 +14,13 @@ pub mod output;
 #[cfg(target_os = "emscripten")]
 pub mod worker_process;
 
-use std::io::{Read, Write};
-
-use self::input::Input;
-use self::output::Output;
-
 pub trait Process {
     /// Read input from `stdin`, compile a contract, and write the output to `stdout`.
-    fn run(input_file: Option<&mut std::fs::File>) -> anyhow::Result<()> {
-        let mut stdin = std::io::stdin();
-        let mut stdout = std::io::stdout();
-        let mut stderr = std::io::stderr();
-
-        let mut buffer = Vec::with_capacity(16384);
-        match input_file {
-            Some(ins) => {
-                if let Err(error) = ins.read_to_end(&mut buffer) {
-                    anyhow::bail!("Failed to read recursive process input file: {:?}", error);
-                }
-            }
-            None => {
-                if let Err(error) = stdin.read_to_end(&mut buffer) {
-                    anyhow::bail!(
-                        "Failed to read recursive process input from stdin: {:?}",
-                        error
-                    )
-                }
-            }
-        }
-
-        let input: Input = revive_common::deserialize_from_slice(buffer.as_slice())?;
-
-        revive_llvm_context::initialize_llvm(
-            revive_llvm_context::Target::PVM,
-            crate::DEFAULT_EXECUTABLE_NAME,
-            &input.llvm_arguments,
-        );
-
-        let result = input.contract.compile(
-            input.project,
-            input.optimizer_settings,
-            input.include_metadata_hash,
-            input.debug_config,
-            &input.llvm_arguments,
-            input.memory_config,
-        );
-
-        match result {
-            Ok(build) => {
-                let output = Output::new(build);
-                let json = serde_json::to_vec(&output).expect("Always valid");
-                stdout
-                    .write_all(json.as_slice())
-                    .expect("Stdout writing error");
-                Ok(())
-            }
-            Err(error) => {
-                let message = error.to_string();
-                stderr
-                    .write_all(message.as_bytes())
-                    .expect("Stderr writing error");
-                Err(error)
-            }
-        }
-    }
+    fn run(input: Input) -> anyhow::Result<()>;
 
     /// Runs this process recursively to compile a single contract.
-    fn call(input: Input) -> anyhow::Result<Output>;
+    fn call<I: Serialize, O: DeserializeOwned>(
+        path: &str,
+        input: I,
+    ) -> Result<O, SolcStandardJsonOutputError>;
 }

crates/resolc/src/process/native_process.rs

@@ -5,6 +5,12 @@ use std::path::PathBuf;
 use std::process::Command;
 
 use once_cell::sync::OnceCell;
+use revive_common::deserialize_from_slice;
+use revive_common::EXIT_CODE_SUCCESS;
+use revive_solc_json_interface::standard_json::output::error::source_location::SourceLocation;
+use revive_solc_json_interface::SolcStandardJsonOutputError;
+use serde::de::DeserializeOwned;
+use serde::Serialize;
 
 use super::Input;
 use super::Output;
@@ -16,61 +22,95 @@ pub static EXECUTABLE: OnceCell<PathBuf> = OnceCell::new();
 pub struct NativeProcess;
 
 impl Process for NativeProcess {
-    fn call(input: Input) -> anyhow::Result<Output> {
-        let input_json = serde_json::to_vec(&input).expect("Always valid");
+    fn run(input: Input) -> anyhow::Result<()> {
+        let source_location = SourceLocation::new(input.contract.identifier.path.to_owned());
 
-        let executable = match EXECUTABLE.get() {
-            Some(executable) => executable.to_owned(),
-            None => std::env::current_exe()?,
-        };
+        let result = std::thread::Builder::new()
+            .stack_size(crate::RAYON_WORKER_STACK_SIZE)
+            .spawn(move || {
+                input
+                    .contract
+                    .compile(
+                        input.solc_version,
+                        input.optimizer_settings,
+                        input.metadata_hash,
+                        input.debug_config,
+                        &input.llvm_arguments,
+                        input.memory_config,
+                        input.missing_libraries,
+                        input.factory_dependencies,
+                        input.identifier_paths,
+                    )
+                    .map(Output::new)
+                    .map_err(|error| {
+                        SolcStandardJsonOutputError::new_error(error, Some(source_location), None)
+                    })
+            })
+            .expect("Threading error")
+            .join()
+            .expect("Threading error");
 
+        serde_json::to_writer(std::io::stdout(), &result)
+            .map_err(|error| anyhow::anyhow!("Stdout writing error: {error}"))?;
+
+        Ok(())
+    }
+
+    fn call<I, O>(path: &str, input: I) -> Result<O, SolcStandardJsonOutputError>
+    where
+        I: Serialize,
+        O: DeserializeOwned,
+    {
+        let executable = EXECUTABLE
+            .get()
+            .cloned()
+            .unwrap_or_else(|| std::env::current_exe().expect("Should have an executable"));
         let mut command = Command::new(executable.as_path());
         command.stdin(std::process::Stdio::piped());
         command.stdout(std::process::Stdio::piped());
         command.stderr(std::process::Stdio::piped());
         command.arg("--recursive-process");
-        let process = command.spawn().map_err(|error| {
-            anyhow::anyhow!("{:?} subprocess spawning error: {:?}", executable, error)
-        })?;
+        command.arg(path);
 
-        #[cfg(debug_assertions)]
-        input
-            .debug_config
-            .dump_stage_output(&input.contract.path, Some("stage"), &input_json)
-            .map_err(|error| {
-                anyhow::anyhow!(
-                    "{:?} failed to log the recursive process output: {:?}",
-                    executable,
-                    error,
-                )
-            })?;
-
-        process
+        let mut process = command
+            .spawn()
+            .unwrap_or_else(|error| panic!("{executable:?} subprocess spawning: {error:?}"));
+        let stdin = process
             .stdin
-            .as_ref()
-            .ok_or_else(|| anyhow::anyhow!("{:?} stdin getting error", executable))?
-            .write_all(input_json.as_slice())
-            .map_err(|error| {
-                anyhow::anyhow!("{:?} stdin writing error: {:?}", executable, error)
-            })?;
-        let output = process.wait_with_output().map_err(|error| {
-            anyhow::anyhow!("{:?} subprocess output error: {:?}", executable, error)
-        })?;
-        if !output.status.success() {
-            anyhow::bail!(
-                "{}",
-                String::from_utf8_lossy(output.stderr.as_slice()).to_string(),
+            .as_mut()
+            .unwrap_or_else(|| panic!("{executable:?} subprocess stdin getting error"));
+        let stdin_input = serde_json::to_vec(&input).expect("Always valid");
+        stdin
+            .write_all(stdin_input.as_slice())
+            .unwrap_or_else(|error| panic!("{executable:?} subprocess stdin writing: {error:?}"));
+
+        let result = process
+            .wait_with_output()
+            .unwrap_or_else(|error| panic!("{executable:?} subprocess output reading: {error:?}"));
+
+        if result.status.code() != Some(EXIT_CODE_SUCCESS) {
+            let message = format!(
+                "{executable:?} subprocess failed with exit code {:?}:\n{}\n{}",
+                result.status.code(),
+                String::from_utf8_lossy(result.stdout.as_slice()),
+                String::from_utf8_lossy(result.stderr.as_slice()),
             );
+            return Err(SolcStandardJsonOutputError::new_error(
+                message,
+                Some(SourceLocation::new(path.to_owned())),
+                None,
+            ));
         }
 
-        let output: Output = revive_common::deserialize_from_slice(output.stdout.as_slice())
-            .map_err(|error| {
-                anyhow::anyhow!(
-                    "{:?} subprocess output parsing error: {}",
-                    executable,
-                    error,
-                )
-            })?;
-        Ok(output)
+        match deserialize_from_slice(result.stdout.as_slice()) {
+            Ok(output) => output,
+            Err(error) => {
+                panic!(
+                    "{executable:?} subprocess stdout parsing error: {error:?}\n{}\n{}",
+                    String::from_utf8_lossy(result.stdout.as_slice()),
+                    String::from_utf8_lossy(result.stderr.as_slice()),
+                );
+            }
+        }
     }
 }

crates/resolc/src/process/worker_process.rs

@@ -2,13 +2,18 @@
 
 use std::ffi::{c_char, c_void, CStr, CString};
 
+use serde::de::DeserializeOwned;
+use serde::Deserialize;
+use serde::Serialize;
+
+use revive_common::deserialize_from_slice;
+use revive_solc_json_interface::standard_json::output::error::source_location::SourceLocation;
+use revive_solc_json_interface::SolcStandardJsonOutputError;
+
 use super::Input;
 use super::Output;
 use super::Process;
 
-use anyhow::Context;
-use serde::Deserialize;
-
 #[derive(Deserialize)]
 struct Error {
     message: String,
@@ -29,10 +34,40 @@ enum Response {
 pub struct WorkerProcess;
 
 impl Process for WorkerProcess {
-    fn call(input: Input) -> anyhow::Result<Output> {
+    fn run(input: Input) -> anyhow::Result<()> {
+        let source_location = SourceLocation::new(input.contract.identifier.path.to_owned());
+
+        let result = input
+            .contract
+            .compile(
+                None,
+                input.optimizer_settings,
+                input.metadata_hash,
+                input.debug_config,
+                &input.llvm_arguments,
+                input.memory_config,
+                input.missing_libraries,
+                input.factory_dependencies,
+                input.identifier_paths,
+            )
+            .map(Output::new)
+            .map_err(|error| {
+                SolcStandardJsonOutputError::new_error(error, Some(source_location), None)
+            });
+
+        serde_json::to_writer(std::io::stdout(), &result)
+            .map_err(|error| anyhow::anyhow!("Stdout writing error: {error}"))?;
+
+        Ok(())
+    }
+
+    fn call<I, O>(_path: &str, input: I) -> Result<O, SolcStandardJsonOutputError>
+    where
+        I: Serialize,
+        O: DeserializeOwned,
+    {
         let input_json = serde_json::to_vec(&input).expect("Always valid");
         let input_str = String::from_utf8(input_json).expect("Input shall be valid");
-        // Prepare the input string for the Emscripten function
         let input_cstring = CString::new(input_str).expect("CString allocation failed");
 
         // Call the Emscripten function
@@ -40,26 +75,20 @@ impl Process for WorkerProcess {
             unsafe { resolc_compile(input_cstring.as_ptr(), input_cstring.as_bytes().len()) };
 
         // Convert the output pointer back to a Rust string
-        let output_str = unsafe {
-            CStr::from_ptr(output_ptr)
-                .to_str()
-                .with_context(|| "Failed to convert C string to Rust string")
-                .map(str::to_owned)
-        };
+        let output_str = unsafe { CStr::from_ptr(output_ptr).to_str().map(str::to_owned) };
         unsafe { libc::free(output_ptr as *mut c_void) };
-        let output_str = output_str?;
-        let response: Response = serde_json::from_str(&output_str)
-            .map_err(|error| anyhow::anyhow!("Worker output parsing error: {}", error,))?;
-        match response {
-            Response::Success(out) => {
-                let output: Output = revive_common::deserialize_from_slice(out.data.as_bytes())
-                    .map_err(|error| {
-                        anyhow::anyhow!("resolc.js subprocess output parsing error: {}", error,)
-                    })?;
 
-                Ok(output)
-            }
-            Response::Error(err) => anyhow::bail!("Worker error: {}", err.message,),
+        let output_str = output_str.unwrap_or_else(|error| panic!("resolc.js output: {error:?}"));
+        let response = serde_json::from_str(&output_str)
+            .unwrap_or_else(|error| panic!("Worker output parsing error: {error}"));
+        match response {
+            Response::Success(out) => match deserialize_from_slice(out.data.as_bytes()) {
+                Ok(output) => output,
+                Err(error) => {
+                    panic!("resolc.js subprocess output parsing error: {error}")
+                }
+            },
+            Response::Error(err) => panic!("Worker error: {}", err.message),
         }
     }
 }