Separate compilation and linker phases (#376)

Separate between compilation and linker phases to allow deploy time
linking and back-porting era compiler changes to fix #91. Unlinked
contract binaries (caused by missing libraries or missing factory
dependencies in turn) are emitted as raw ELF object.

Few drive by fixes:
- #98
- A compiler panic on missing libraries definitions.
- Fixes some incosistent type forwarding in JSON output (empty string
vs. null object).
- Remove the unused fallback for size optimization setting.
- Remove the broken `--lvm-ir`  mode.
- CI workflow fixes.

---------

Signed-off-by: Cyrill Leutwiler <bigcyrill@hotmail.com>
Signed-off-by: xermicus <bigcyrill@hotmail.com>
Signed-off-by: xermicus <cyrill@parity.io>

crates/resolc/src/build/contract.rs

@@ -1,10 +1,19 @@
 //! The Solidity contract build.
 
-use std::collections::HashSet;
+use std::collections::BTreeMap;
+use std::collections::BTreeSet;
 use std::fs::File;
 use std::io::Write;
 use std::path::Path;
+use std::path::PathBuf;
 
+use revive_common::ContractIdentifier;
+use revive_common::ObjectFormat;
+use revive_common::BYTE_LENGTH_WORD;
+use revive_common::EXTENSION_JSON;
+use revive_common::EXTENSION_POLKAVM_ASSEMBLY;
+use revive_common::EXTENSION_POLKAVM_BINARY;
+use revive_llvm_context::PolkaVMBuild;
 use revive_solc_json_interface::CombinedJsonContract;
 use revive_solc_json_interface::SolcStandardJsonOutputContract;
 use serde::Deserialize;
@@ -13,92 +22,137 @@ use serde::Serialize;
 /// The Solidity contract build.
 #[derive(Debug, Serialize, Deserialize)]
 pub struct Contract {
-    /// The contract path.
-    pub path: String,
-    /// The auxiliary identifier. Used to identify Yul objects.
-    pub identifier: String,
+    /// The contract identifier.
+    pub identifier: ContractIdentifier,
     /// The LLVM module build.
-    pub build: revive_llvm_context::PolkaVMBuild,
+    pub build: PolkaVMBuild,
     /// The metadata JSON.
     pub metadata_json: serde_json::Value,
-    /// The factory dependencies.
-    pub factory_dependencies: HashSet<String>,
+    /// The unlinked missing libraries.
+    pub missing_libraries: BTreeSet<String>,
+    /// The unresolved factory dependencies.
+    pub factory_dependencies: BTreeSet<String>,
+    /// The resolved factory dependencies.
+    pub factory_dependencies_resolved: BTreeMap<[u8; BYTE_LENGTH_WORD], String>,
+    /// The binary object format.
+    pub object_format: ObjectFormat,
 }
 
 impl Contract {
     /// A shortcut constructor.
     pub fn new(
-        path: String,
-        identifier: String,
-        build: revive_llvm_context::PolkaVMBuild,
+        identifier: ContractIdentifier,
+        build: PolkaVMBuild,
         metadata_json: serde_json::Value,
-        factory_dependencies: HashSet<String>,
+        missing_libraries: BTreeSet<String>,
+        factory_dependencies: BTreeSet<String>,
+        object_format: ObjectFormat,
     ) -> Self {
         Self {
-            path,
             identifier,
             build,
             metadata_json,
+            missing_libraries,
             factory_dependencies,
+            factory_dependencies_resolved: BTreeMap::new(),
+            object_format,
         }
     }
 
+    /// Writes the contract text assembly and bytecode to terminal.
+    pub fn write_to_terminal(
+        self,
+        path: String,
+        output_metadata: bool,
+        output_assembly: bool,
+        output_binary: bool,
+    ) -> anyhow::Result<()> {
+        writeln!(std::io::stdout(), "\n======= {path} =======")?;
+        if output_assembly {
+            writeln!(
+                std::io::stdout(),
+                "Assembly:\n{}",
+                self.build.assembly_text.unwrap_or_default(),
+            )?;
+        }
+        if output_metadata {
+            writeln!(std::io::stdout(), "Metadata:\n{}", self.metadata_json)?;
+        }
+        if output_binary {
+            writeln!(
+                std::io::stdout(),
+                "Binary:\n{}",
+                hex::encode(self.build.bytecode)
+            )?;
+        }
+
+        Ok(())
+    }
+
     /// Writes the contract text assembly and bytecode to files.
     pub fn write_to_directory(
         self,
         path: &Path,
+        output_metadata: bool,
         output_assembly: bool,
         output_binary: bool,
         overwrite: bool,
     ) -> anyhow::Result<()> {
-        let file_name = Self::short_path(self.path.as_str());
-
-        if output_assembly {
-            let file_name = format!(
-                "{}.{}",
-                file_name,
-                revive_common::EXTENSION_POLKAVM_ASSEMBLY
-            );
-            let mut file_path = path.to_owned();
-            file_path.push(file_name);
+        let file_path = PathBuf::from(self.identifier.path);
+        let file_name = file_path
+            .file_name()
+            .expect("Always exists")
+            .to_str()
+            .expect("Always valid");
+        let output_path = path.to_owned();
+        std::fs::create_dir_all(output_path.as_path())?;
 
+        if output_metadata {
+            let file_path = output_path.join(format!(
+                "{file_name}:{}.{EXTENSION_JSON}",
+                self.identifier.name.as_deref().unwrap_or(file_name),
+            ));
             if file_path.exists() && !overwrite {
                 anyhow::bail!(
                     "Refusing to overwrite an existing file {file_path:?} (use --overwrite to force)."
                 );
-            } else {
-                let assembly_text = self.build.assembly_text;
-
-                File::create(&file_path)
-                    .map_err(|error| {
-                        anyhow::anyhow!("File {:?} creating error: {}", file_path, error)
-                    })?
-                    .write_all(assembly_text.as_bytes())
-                    .map_err(|error| {
-                        anyhow::anyhow!("File {:?} writing error: {}", file_path, error)
-                    })?;
             }
+            std::fs::write(
+                file_path.as_path(),
+                self.metadata_json.to_string().as_bytes(),
+            )
+            .map_err(|error| anyhow::anyhow!("File {file_path:?} writing: {error}"))?;
+        }
+        if output_assembly {
+            let file_path = output_path.join(format!(
+                "{file_name}:{}.{EXTENSION_POLKAVM_ASSEMBLY}",
+                self.identifier.name.as_deref().unwrap_or(file_name),
+            ));
+            if file_path.exists() && !overwrite {
+                anyhow::bail!(
+                    "Refusing to overwrite an existing file {file_path:?} (use --overwrite to force)."
+                );
+            }
+            File::create(&file_path)
+                .map_err(|error| anyhow::anyhow!("File {file_path:?} creating error: {error}"))?
+                .write_all(self.build.assembly_text.unwrap_or_default().as_bytes())
+                .map_err(|error| anyhow::anyhow!("File {file_path:?} writing error: {error}"))?;
         }
 
         if output_binary {
-            let file_name = format!("{}.{}", file_name, revive_common::EXTENSION_POLKAVM_BINARY);
-            let mut file_path = path.to_owned();
-            file_path.push(file_name);
-
+            let file_path = output_path.join(format!(
+                "{file_name}:{}.{EXTENSION_POLKAVM_BINARY}",
+                self.identifier.name.as_deref().unwrap_or(file_name),
+            ));
             if file_path.exists() && !overwrite {
                 anyhow::bail!(
                     "Refusing to overwrite an existing file {file_path:?} (use --overwrite to force)."
                 );
-            } else {
-                File::create(&file_path)
-                    .map_err(|error| {
-                        anyhow::anyhow!("File {:?} creating error: {}", file_path, error)
-                    })?
-                    .write_all(self.build.bytecode.as_slice())
-                    .map_err(|error| {
-                        anyhow::anyhow!("File {:?} writing error: {}", file_path, error)
-                    })?;
             }
+            File::create(&file_path)
+                .map_err(|error| anyhow::anyhow!("File {file_path:?} creating error: {error}"))?
+                .write_all(self.build.bytecode.as_slice())
+                .map_err(|error| anyhow::anyhow!("File {file_path:?} writing error: {error}"))?;
         }
 
         Ok(())
@@ -109,20 +163,30 @@ impl Contract {
         self,
         combined_json_contract: &mut CombinedJsonContract,
     ) -> anyhow::Result<()> {
+        let hexadecimal_bytecode = hex::encode(self.build.bytecode);
+
         if let Some(metadata) = combined_json_contract.metadata.as_mut() {
             *metadata = self.metadata_json.to_string();
         }
 
-        if let Some(asm) = combined_json_contract.asm.as_mut() {
-            *asm = serde_json::Value::String(self.build.assembly_text);
-        }
-        let hexadecimal_bytecode = hex::encode(self.build.bytecode);
+        combined_json_contract.assembly = self.build.assembly_text;
         combined_json_contract.bin = Some(hexadecimal_bytecode);
         combined_json_contract
             .bin_runtime
             .clone_from(&combined_json_contract.bin);
 
-        combined_json_contract.factory_deps = Some(self.build.factory_dependencies);
+        combined_json_contract
+            .missing_libraries
+            .extend(self.missing_libraries);
+        combined_json_contract
+            .factory_deps_unlinked
+            .extend(self.factory_dependencies);
+        combined_json_contract.factory_deps.extend(
+            self.factory_dependencies_resolved
+                .into_iter()
+                .map(|(hash, path)| (hex::encode(hash), path)),
+        );
+        combined_json_contract.object_format = Some(self.object_format);
 
         Ok(())
     }
@@ -132,16 +196,27 @@ impl Contract {
         self,
         standard_json_contract: &mut SolcStandardJsonOutputContract,
     ) -> anyhow::Result<()> {
-        standard_json_contract.metadata = Some(self.metadata_json);
-
-        let assembly_text = self.build.assembly_text;
         let bytecode = hex::encode(self.build.bytecode.as_slice());
-        if let Some(evm) = standard_json_contract.evm.as_mut() {
-            evm.modify(assembly_text, bytecode);
-        }
+        let assembly_text = self.build.assembly_text.unwrap_or_default();
 
-        standard_json_contract.factory_dependencies = Some(self.build.factory_dependencies);
-        standard_json_contract.hash = Some(self.build.bytecode_hash);
+        standard_json_contract.metadata = self.metadata_json;
+        standard_json_contract
+            .evm
+            .get_or_insert_with(Default::default)
+            .modify(assembly_text, bytecode);
+        standard_json_contract.hash = self.build.bytecode_hash.map(hex::encode);
+        standard_json_contract
+            .missing_libraries
+            .extend(self.missing_libraries);
+        standard_json_contract
+            .factory_dependencies_unlinked
+            .extend(self.factory_dependencies);
+        standard_json_contract.factory_dependencies.extend(
+            self.factory_dependencies_resolved
+                .into_iter()
+                .map(|(hash, path)| (hex::encode(hash), path)),
+        );
+        standard_json_contract.object_format = Some(self.object_format);
 
         Ok(())
     }

crates/resolc/src/build/mod.rs

@@ -1,94 +1,352 @@
 //! The Solidity project build.
 
+use std::collections::BTreeMap;
+use std::io::Write;
+use std::path::Path;
+use std::path::PathBuf;
+
+use normpath::PathExt;
+
+use revive_common::ObjectFormat;
+use revive_common::BYTE_LENGTH_ETH_ADDRESS;
+use revive_llvm_context::polkavm_disassemble;
+use revive_llvm_context::polkavm_hash;
+use revive_llvm_context::polkavm_link;
+use revive_llvm_context::DebugConfig;
+use revive_solc_json_interface::combined_json::CombinedJson;
+use revive_solc_json_interface::CombinedJsonContract;
+use revive_solc_json_interface::SolcStandardJsonOutput;
+use revive_solc_json_interface::SolcStandardJsonOutputContract;
+use revive_solc_json_interface::SolcStandardJsonOutputError;
+use revive_solc_json_interface::SolcStandardJsonOutputErrorHandler;
+
+use crate::build::contract::Contract;
+use crate::solc::version::Version as SolcVersion;
+
 pub mod contract;
 
-use std::collections::BTreeMap;
-use std::path::Path;
-
-use revive_solc_json_interface::combined_json::CombinedJson;
-use revive_solc_json_interface::SolcStandardJsonOutput;
-
-use crate::solc::version::Version as SolcVersion;
-use crate::ResolcVersion;
-
-use self::contract::Contract;
-
-/// The Solidity project build.
+/// The Solidity project PVM build.
 #[derive(Debug, Default)]
 pub struct Build {
     /// The contract data,
-    pub contracts: BTreeMap<String, Contract>,
+    pub results: BTreeMap<String, Result<Contract, SolcStandardJsonOutputError>>,
+    /// The additional message to output (added by the revive compiler).
+    pub messages: Vec<SolcStandardJsonOutputError>,
 }
 
 impl Build {
-    /// Writes all contracts to the specified directory.
-    pub fn write_to_directory(
-        self,
-        output_directory: &Path,
+    /// A shorthand constructor.
+    ///
+    /// Note: Takes the supplied `messages`, leaving an empty vec.
+    pub fn new(
+        results: BTreeMap<String, Result<Contract, SolcStandardJsonOutputError>>,
+        messages: &mut Vec<SolcStandardJsonOutputError>,
+    ) -> Self {
+        Self {
+            results,
+            messages: std::mem::take(messages),
+        }
+    }
+
+    /// Links the PVM build.
+    pub fn link(
+        mut self,
+        linker_symbols: BTreeMap<String, [u8; BYTE_LENGTH_ETH_ADDRESS]>,
+        debug_config: &DebugConfig,
+    ) -> Self {
+        let mut contracts: BTreeMap<String, Contract> = self
+            .results
+            .into_iter()
+            .map(|(path, result)| (path, result.expect("Cannot link a project with errors")))
+            .collect();
+
+        loop {
+            let mut linkage_data = BTreeMap::new();
+            for (path, contract) in contracts
+                .iter()
+                .filter(|(_path, contract)| contract.object_format == ObjectFormat::ELF)
+            {
+                match polkavm_link(
+                    &contract.build.bytecode,
+                    &linker_symbols,
+                    &contract
+                        .factory_dependencies
+                        .iter()
+                        .filter_map(|dependency| {
+                            let bytecode_hash = contracts
+                                .get(dependency)
+                                .as_ref()?
+                                .build
+                                .bytecode_hash
+                                .as_ref()?
+                                .to_owned();
+                            Some((dependency.to_owned(), bytecode_hash))
+                        })
+                        .collect(),
+                    !debug_config.emit_debug_info,
+                ) {
+                    Ok((memory_buffer_linked, ObjectFormat::PVM)) => {
+                        let bytecode_hash = polkavm_hash(&memory_buffer_linked);
+                        let assembly_text =
+                            polkavm_disassemble(path, &memory_buffer_linked, debug_config)
+                                .unwrap_or_else(|error| {
+                                    panic!("ICE: The PVM disassembler failed: {error}")
+                                });
+                        linkage_data.insert(
+                            path.to_owned(),
+                            (memory_buffer_linked, bytecode_hash, assembly_text),
+                        );
+                    }
+                    Ok((_memory_buffer_linked, ObjectFormat::ELF)) => {}
+                    Err(error) => self
+                        .messages
+                        .push(SolcStandardJsonOutputError::new_error(error, None, None)),
+                }
+            }
+            if linkage_data.is_empty() {
+                break;
+            }
+
+            for (path, (memory_buffer_linked, bytecode_hash, assembly_text)) in
+                linkage_data.into_iter()
+            {
+                let contract = contracts.get(path.as_str()).expect("Always exists");
+                let factory_dependencies_resolved = contract
+                    .factory_dependencies
+                    .iter()
+                    .filter_map(|dependency| {
+                        Some((
+                            contracts
+                                .get(dependency)
+                                .as_ref()?
+                                .build
+                                .bytecode_hash
+                                .as_ref()?
+                                .to_owned(),
+                            dependency.to_owned(),
+                        ))
+                    })
+                    .collect();
+                let contract = contracts.get_mut(path.as_str()).expect("Always exists");
+                contract.build.bytecode = memory_buffer_linked.as_slice().to_vec();
+                contract.build.bytecode_hash = Some(bytecode_hash);
+                contract.build.assembly_text = Some(assembly_text);
+                contract.factory_dependencies_resolved = factory_dependencies_resolved;
+                contract.object_format = ObjectFormat::PVM;
+            }
+        }
+
+        let results = contracts
+            .into_iter()
+            .map(|(path, contract)| {
+                if contract.object_format == ObjectFormat::ELF {
+                    self.messages.push(SolcStandardJsonOutputError::new_warning(
+                        format!("{path} is unlinked. Consider providing missing libraries."),
+                        None,
+                        None,
+                    ));
+                }
+                (path, Ok(contract))
+            })
+            .collect();
+        Self::new(results, &mut self.messages)
+    }
+
+    /// Writes all contracts to the terminal.
+    pub fn write_to_terminal(
+        mut self,
+        output_metadata: bool,
         output_assembly: bool,
         output_binary: bool,
-        overwrite: bool,
     ) -> anyhow::Result<()> {
-        for (_path, contract) in self.contracts.into_iter() {
-            contract.write_to_directory(
-                output_directory,
+        self.take_and_write_warnings();
+        self.exit_on_error();
+
+        if !output_metadata && !output_assembly && !output_binary {
+            writeln!(
+                std::io::stderr(),
+                "Compiler run successful. No output requested. Use flags --metadata, --asm, --bin."
+            )?;
+            return Ok(());
+        }
+
+        for (path, build) in self.results.into_iter() {
+            build.expect("Always valid").write_to_terminal(
+                path,
+                output_metadata,
                 output_assembly,
                 output_binary,
-                overwrite,
             )?;
         }
 
         Ok(())
     }
 
-    /// Writes all contracts assembly and bytecode to the combined JSON.
-    pub fn write_to_combined_json(self, combined_json: &mut CombinedJson) -> anyhow::Result<()> {
-        for (path, contract) in self.contracts.into_iter() {
-            let combined_json_contract = combined_json
-                .contracts
-                .iter_mut()
-                .find_map(|(json_path, contract)| {
-                    if path.ends_with(json_path) {
-                        Some(contract)
-                    } else {
-                        None
-                    }
-                })
-                .ok_or_else(|| anyhow::anyhow!("Contract `{}` not found in the project", path))?;
+    /// Writes all contracts to the specified directory.
+    pub fn write_to_directory(
+        mut self,
+        output_directory: &Path,
+        output_metadata: bool,
+        output_assembly: bool,
+        output_binary: bool,
+        overwrite: bool,
+    ) -> anyhow::Result<()> {
+        self.take_and_write_warnings();
+        self.exit_on_error();
 
-            contract.write_to_combined_json(combined_json_contract)?;
+        std::fs::create_dir_all(output_directory)?;
+
+        for build in self.results.into_values() {
+            build.expect("Always valid").write_to_directory(
+                output_directory,
+                output_metadata,
+                output_assembly,
+                output_binary,
+                overwrite,
+            )?;
         }
 
-        combined_json.revive_version = Some(ResolcVersion::default().long);
+        writeln!(
+            std::io::stderr(),
+            "Compiler run successful. Artifact(s) can be found in directory {output_directory:?}."
+        )?;
+        Ok(())
+    }
+
+    /// Writes all contracts assembly and bytecode to the combined JSON.
+    pub fn write_to_combined_json(
+        mut self,
+        combined_json: &mut CombinedJson,
+    ) -> anyhow::Result<()> {
+        self.take_and_write_warnings();
+        self.exit_on_error();
+
+        for result in self.results.into_values() {
+            let build = result.expect("Exits on an error above");
+            let identifier = build.identifier.clone();
+
+            let combined_json_contract =
+                match combined_json
+                    .contracts
+                    .iter_mut()
+                    .find_map(|(json_path, contract)| {
+                        if Self::normalize_full_path(identifier.full_path.as_str())
+                            .ends_with(Self::normalize_full_path(json_path).as_str())
+                        {
+                            Some(contract)
+                        } else {
+                            None
+                        }
+                    }) {
+                    Some(contract) => contract,
+                    None => {
+                        combined_json.contracts.insert(
+                            identifier.full_path.clone(),
+                            CombinedJsonContract::default(),
+                        );
+                        combined_json
+                            .contracts
+                            .get_mut(identifier.full_path.as_str())
+                            .expect("Always exists")
+                    }
+                };
+
+            build.write_to_combined_json(combined_json_contract)?;
+        }
 
         Ok(())
     }
 
     /// Writes all contracts assembly and bytecode to the standard JSON.
     pub fn write_to_standard_json(
-        mut self,
+        self,
         standard_json: &mut SolcStandardJsonOutput,
         solc_version: &SolcVersion,
     ) -> anyhow::Result<()> {
-        let contracts = match standard_json.contracts.as_mut() {
-            Some(contracts) => contracts,
-            None => return Ok(()),
-        };
+        let mut errors = Vec::with_capacity(self.results.len());
+        for result in self.results.into_values() {
+            let build = match result {
+                Ok(build) => build,
+                Err(error) => {
+                    errors.push(error);
+                    continue;
+                }
+            };
+            let identifier = build.identifier.clone();
 
-        for (path, contracts) in contracts.iter_mut() {
-            for (name, contract) in contracts.iter_mut() {
-                let full_name = format!("{path}:{name}");
-
-                if let Some(contract_data) = self.contracts.remove(full_name.as_str()) {
-                    contract_data.write_to_standard_json(contract)?;
+            match standard_json
+                .contracts
+                .get_mut(identifier.path.as_str())
+                .and_then(|contracts| {
+                    contracts.get_mut(
+                        identifier
+                            .name
+                            .as_deref()
+                            .unwrap_or(identifier.path.as_str()),
+                    )
+                }) {
+                Some(contract) => {
+                    build.write_to_standard_json(contract)?;
+                }
+                None => {
+                    let contracts = standard_json
+                        .contracts
+                        .entry(identifier.path.clone())
+                        .or_default();
+                    let mut contract = SolcStandardJsonOutputContract::default();
+                    build.write_to_standard_json(&mut contract)?;
+                    contracts.insert(identifier.name.unwrap_or(identifier.path), contract);
                 }
             }
         }
 
+        standard_json.errors.extend(errors);
         standard_json.version = Some(solc_version.default.to_string());
         standard_json.long_version = Some(solc_version.long.to_owned());
-        standard_json.revive_version = Some(ResolcVersion::default().long);
 
         Ok(())
     }
+
+    /// Normalizes the full contract path.
+    ///
+    /// # Panics
+    /// If the path does not contain a colon.
+    fn normalize_full_path(path: &str) -> String {
+        let mut iterator = path.split(':');
+        let path = iterator.next().expect("Always exists");
+        let name = iterator.next().expect("Always exists");
+
+        let mut full_path = PathBuf::from(path)
+            .normalize()
+            .expect("Path normalization error")
+            .as_os_str()
+            .to_string_lossy()
+            .into_owned();
+        full_path.push(':');
+        full_path.push_str(name);
+        full_path
+    }
+}
+
+impl SolcStandardJsonOutputErrorHandler for Build {
+    fn errors(&self) -> Vec<&SolcStandardJsonOutputError> {
+        let mut errors: Vec<&SolcStandardJsonOutputError> = self
+            .results
+            .values()
+            .filter_map(|build| build.as_ref().err())
+            .collect();
+        errors.extend(self.messages.iter().filter(|message| message.is_error()));
+        errors
+    }
+
+    fn take_warnings(&mut self) -> Vec<SolcStandardJsonOutputError> {
+        let warnings = self
+            .messages
+            .iter()
+            .filter(|message| message.is_warning())
+            .cloned()
+            .collect();
+        self.messages.retain(|message| !message.is_warning());
+        warnings
+    }
 }