Separate compilation and linker phases (#376)

Separate between compilation and linker phases to allow deploy time
linking and back-porting era compiler changes to fix #91. Unlinked
contract binaries (caused by missing libraries or missing factory
dependencies in turn) are emitted as raw ELF object.

Few drive by fixes:
- #98
- A compiler panic on missing libraries definitions.
- Fixes some incosistent type forwarding in JSON output (empty string
vs. null object).
- Remove the unused fallback for size optimization setting.
- Remove the broken `--lvm-ir`  mode.
- CI workflow fixes.

---------

Signed-off-by: Cyrill Leutwiler <bigcyrill@hotmail.com>
Signed-off-by: xermicus <bigcyrill@hotmail.com>
Signed-off-by: xermicus <cyrill@parity.io>

crates/common/src/contract_identifier.rs

@@ -0,0 +1,33 @@
+//! The contract identifier helper library.
+
+use serde::{Deserialize, Serialize};
+
+/// This structure simplifies passing the contract identifiers through the compilation pipeline.
+#[derive(Debug, Clone, Serialize, Deserialize)]
+pub struct ContractIdentifier {
+    /// The absolute file path.
+    pub path: String,
+    /// The contract name.
+    /// Is set for Solidity contracts only. Otherwise it would be equal to the file name.
+    pub name: Option<String>,
+    /// The full contract identifier.
+    /// For Solidity, The format is `<absolute file path>:<contract name>`.
+    /// For other languages, `<absolute file path>`.
+    pub full_path: String,
+}
+
+impl ContractIdentifier {
+    /// A shortcut constructor.
+    pub fn new(path: String, name: Option<String>) -> Self {
+        let full_path = match name {
+            Some(ref name) => format!("{path}:{name}"),
+            None => path.clone(),
+        };
+
+        Self {
+            path,
+            name,
+            full_path,
+        }
+    }
+}

crates/common/src/extension.rs

@@ -37,4 +37,4 @@ pub static EXTENSION_POLKAVM_ASSEMBLY: &str = "pvmasm";
 pub static EXTENSION_POLKAVM_BINARY: &str = "pvm";
 
 /// The ELF shared object file extension.
-pub static EXTENSION_SHARED_OBJECT: &str = "so";
+pub static EXTENSION_OBJECT: &str = "o";

crates/common/src/keccak256.rs

@@ -0,0 +1,68 @@
+//! Keccak-256 hash utilities.
+
+use serde::{Deserialize, Serialize};
+use sha3::digest::FixedOutput;
+use sha3::Digest;
+
+pub const DIGEST_BYTES: usize = 32;
+
+/// Keccak-256 hash utilities.
+#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
+pub struct Keccak256 {
+    /// Binary representation.
+    bytes: [u8; DIGEST_BYTES],
+    /// Hexadecimal string representation.
+    string: String,
+}
+
+impl Keccak256 {
+    /// Computes the `keccak256` hash for `preimage`.
+    pub fn from_slice(preimage: &[u8]) -> Self {
+        let bytes = sha3::Keccak256::digest(preimage).into();
+        let string = format!("0x{}", hex::encode(bytes));
+        Self { bytes, string }
+    }
+
+    /// Computes the `keccak256` hash for an array of `preimages`.
+    pub fn from_slices<R: AsRef<[u8]>>(preimages: &[R]) -> Self {
+        let mut hasher = sha3::Keccak256::new();
+        for preimage in preimages.iter() {
+            hasher.update(preimage);
+        }
+        let bytes: [u8; DIGEST_BYTES] = hasher.finalize_fixed().into();
+        let string = format!("0x{}", hex::encode(bytes));
+        Self { bytes, string }
+    }
+
+    /// Returns a reference to the 32-byte SHA-3 hash.
+    pub fn as_bytes(&self) -> &[u8] {
+        self.bytes.as_slice()
+    }
+
+    /// Returns a reference to the hexadecimal string representation.
+    pub fn as_str(&self) -> &str {
+        self.string.as_str()
+    }
+
+    /// Extracts the binary representation.
+    pub fn to_vec(&self) -> Vec<u8> {
+        self.bytes.to_vec()
+    }
+}
+
+impl std::fmt::Display for Keccak256 {
+    fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result {
+        write!(f, "{}", self.as_str())
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    #[test]
+    fn hash_and_stringify_works() {
+        assert_eq!(
+            super::Keccak256::from_slices(&["foo".as_bytes(), "bar".as_bytes(),]).as_str(),
+            "0x38d18acb67d25c8bb9942764b62f18e17054f66a817bd4295423adf9ed98873e"
+        );
+    }
+}

crates/common/src/lib.rs

@@ -3,9 +3,13 @@
 pub(crate) mod base;
 pub(crate) mod bit_length;
 pub(crate) mod byte_length;
+pub(crate) mod contract_identifier;
 pub(crate) mod evm_version;
 pub(crate) mod exit_code;
 pub(crate) mod extension;
+pub(crate) mod keccak256;
+pub(crate) mod metadata;
+pub(crate) mod object;
 pub(crate) mod utils;
 
 pub use self::base::*;
@@ -14,4 +18,8 @@ pub use self::byte_length::*;
 pub use self::evm_version::EVMVersion;
 pub use self::exit_code::*;
 pub use self::extension::*;
+pub use self::keccak256::*;
+pub use self::metadata::*;
+pub use self::object::*;
 pub use self::utils::*;
+pub use contract_identifier::*;

crates/common/src/metadata.rs

@@ -0,0 +1,42 @@
+//! The metadata hash type.
+
+use std::str::FromStr;
+
+use serde::{Deserialize, Serialize};
+
+/// The metadata hash type.
+#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, Serialize, Deserialize)]
+pub enum MetadataHash {
+    /// Do not include bytecode hash.
+    #[serde(rename = "none")]
+    None,
+    /// Include the `ipfs` hash.
+    #[serde(rename = "ipfs")]
+    IPFS,
+    /// Include the `keccak256`` hash.
+    #[serde(rename = "keccak256")]
+    Keccak256,
+}
+
+impl FromStr for MetadataHash {
+    type Err = anyhow::Error;
+
+    fn from_str(string: &str) -> Result<Self, Self::Err> {
+        match string {
+            "none" => Ok(Self::None),
+            "ipfs" => Ok(Self::IPFS),
+            "keccak256" => Ok(Self::Keccak256),
+            string => anyhow::bail!("unknown bytecode hash mode: `{string}`"),
+        }
+    }
+}
+
+impl std::fmt::Display for MetadataHash {
+    fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result {
+        match self {
+            Self::None => write!(f, "none"),
+            Self::IPFS => write!(f, "ipfs"),
+            Self::Keccak256 => write!(f, "keccak256"),
+        }
+    }
+}

crates/common/src/object.rs

@@ -0,0 +1,61 @@
+//! The revive binary object helper module.
+
+use std::str::FromStr;
+
+use serde::{Deserialize, Serialize};
+
+/// The binary object format.
+///
+/// Unlinked contracts are stored in a different object format
+/// than final (linked) contract blobs.
+#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, Serialize, Deserialize)]
+pub enum ObjectFormat {
+    /// The unlinked ELF object format.
+    ELF,
+    /// The fully linked PVM format.
+    PVM,
+}
+
+impl ObjectFormat {
+    pub const PVM_MAGIC: [u8; 4] = [b'P', b'V', b'M', b'\0'];
+    pub const ELF_MAGIC: [u8; 4] = [0x7f, b'E', b'L', b'F'];
+}
+
+impl FromStr for ObjectFormat {
+    type Err = anyhow::Error;
+
+    fn from_str(value: &str) -> Result<Self, Self::Err> {
+        match value {
+            "ELF" => Ok(Self::ELF),
+            "PVM" => Ok(Self::PVM),
+            _ => anyhow::bail!(
+                "Unknown object format: {value}. Supported formats: {}, {}",
+                Self::ELF.to_string(),
+                Self::PVM.to_string()
+            ),
+        }
+    }
+}
+
+impl TryFrom<&[u8]> for ObjectFormat {
+    type Error = &'static str;
+
+    fn try_from(value: &[u8]) -> Result<Self, Self::Error> {
+        if value.starts_with(&Self::PVM_MAGIC) {
+            return Ok(Self::PVM);
+        }
+        if value.starts_with(&Self::ELF_MAGIC) {
+            return Ok(Self::ELF);
+        }
+        Err("expected a contract object")
+    }
+}
+
+impl std::fmt::Display for ObjectFormat {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        match self {
+            Self::ELF => write!(f, "ELF"),
+            Self::PVM => write!(f, "PVM"),
+        }
+    }
+}

crates/common/src/utils.rs

@@ -1,25 +1,45 @@
 //! The compiler common utils.
 
 /// Deserializes a `serde_json` object from slice with the recursion limit disabled.
+///
 /// Must be used for all JSON I/O to avoid crashes due to the aforementioned limit.
 pub fn deserialize_from_slice<O>(input: &[u8]) -> anyhow::Result<O>
 where
     O: serde::de::DeserializeOwned,
 {
-    let mut deserializer = serde_json::Deserializer::from_slice(input);
-    deserializer.disable_recursion_limit();
-    let deserializer = serde_stacker::Deserializer::new(&mut deserializer);
-    let result = O::deserialize(deserializer)?;
-    Ok(result)
+    let deserializer = serde_json::Deserializer::from_slice(input);
+    deserialize(deserializer)
 }
 
 /// Deserializes a `serde_json` object from string with the recursion limit disabled.
+///
 /// Must be used for all JSON I/O to avoid crashes due to the aforementioned limit.
 pub fn deserialize_from_str<O>(input: &str) -> anyhow::Result<O>
 where
     O: serde::de::DeserializeOwned,
 {
-    let mut deserializer = serde_json::Deserializer::from_str(input);
+    let deserializer = serde_json::Deserializer::from_str(input);
+    deserialize(deserializer)
+}
+
+/// Deserializes a `serde_json` object from reader with the recursion limit disabled.
+///
+/// Must be used for all JSON I/O to avoid crashes due to the aforementioned limit.
+pub fn deserialize_from_reader<R, O>(reader: R) -> anyhow::Result<O>
+where
+    R: std::io::Read,
+    O: serde::de::DeserializeOwned,
+{
+    let deserializer = serde_json::Deserializer::from_reader(reader);
+    deserialize(deserializer)
+}
+
+/// Runs the generic deserializer.
+pub fn deserialize<'de, R, O>(mut deserializer: serde_json::Deserializer<R>) -> anyhow::Result<O>
+where
+    R: serde_json::de::Read<'de>,
+    O: serde::de::DeserializeOwned,
+{
     deserializer.disable_recursion_limit();
     let deserializer = serde_stacker::Deserializer::new(&mut deserializer);
     let result = O::deserialize(deserializer)?;