Emerge Yul recompiler (#1)

Provide a modified (and incomplete) version of ZKSync zksolc that can compile the most basic contracts
This commit is contained in:
Cyrill Leutwiler
2024-03-12 12:06:02 +01:00
committed by GitHub
parent d238d8f39e
commit cffa14a4d2
247 changed files with 35357 additions and 4905 deletions
@@ -0,0 +1,58 @@
//!
//! The contract EVM legacy assembly source code.
//!
use std::collections::HashSet;
use serde::Deserialize;
use serde::Serialize;
use crate::evmla::assembly::Assembly;
use crate::solc::standard_json::output::contract::evm::extra_metadata::ExtraMetadata;
///
/// The contract EVM legacy assembly source code.
///
#[derive(Debug, Serialize, Deserialize, Clone)]
#[allow(non_camel_case_types)]
#[allow(clippy::upper_case_acronyms)]
pub struct EVMLA {
/// The EVM legacy assembly source code.
pub assembly: Assembly,
}
impl EVMLA {
///
/// A shortcut constructor.
///
pub fn new(mut assembly: Assembly, extra_metadata: ExtraMetadata) -> Self {
assembly.extra_metadata = Some(extra_metadata);
Self { assembly }
}
///
/// Get the list of missing deployable libraries.
///
pub fn get_missing_libraries(&self) -> HashSet<String> {
self.assembly.get_missing_libraries()
}
}
impl<D> era_compiler_llvm_context::EraVMWriteLLVM<D> for EVMLA
where
D: era_compiler_llvm_context::EraVMDependency + Clone,
{
fn declare(
&mut self,
context: &mut era_compiler_llvm_context::EraVMContext<D>,
) -> anyhow::Result<()> {
self.assembly.declare(context)
}
fn into_llvm(
self,
context: &mut era_compiler_llvm_context::EraVMContext<D>,
) -> anyhow::Result<()> {
self.assembly.into_llvm(context)
}
}
@@ -0,0 +1,27 @@
//!
//! The contract LLVM IR source code.
//!
use serde::Deserialize;
use serde::Serialize;
///
/// The contract LLVM IR source code.
///
#[derive(Debug, Serialize, Deserialize, Clone)]
#[allow(clippy::upper_case_acronyms)]
pub struct LLVMIR {
/// The LLVM IR file path.
pub path: String,
/// The LLVM IR source code.
pub source: String,
}
impl LLVMIR {
///
/// A shortcut constructor.
///
pub fn new(path: String, source: String) -> Self {
Self { path, source }
}
}
@@ -0,0 +1,111 @@
//!
//! The contract source code.
//!
pub mod evmla;
pub mod llvm_ir;
pub mod yul;
pub mod zkasm;
use std::collections::HashSet;
use serde::Deserialize;
use serde::Serialize;
use crate::evmla::assembly::Assembly;
use crate::solc::standard_json::output::contract::evm::extra_metadata::ExtraMetadata;
use crate::yul::parser::statement::object::Object;
use self::evmla::EVMLA;
use self::llvm_ir::LLVMIR;
use self::yul::Yul;
use self::zkasm::ZKASM;
///
/// The contract source code.
///
#[derive(Debug, Serialize, Deserialize, Clone)]
#[allow(non_camel_case_types)]
#[allow(clippy::upper_case_acronyms)]
#[allow(clippy::enum_variant_names)]
pub enum IR {
/// The Yul source code.
Yul(Yul),
/// The EVM legacy assembly source code.
EVMLA(EVMLA),
/// The LLVM IR source code.
LLVMIR(LLVMIR),
/// The EraVM assembly source code.
ZKASM(ZKASM),
}
impl IR {
///
/// A shortcut constructor.
///
pub fn new_yul(source_code: String, object: Object) -> Self {
Self::Yul(Yul::new(source_code, object))
}
///
/// A shortcut constructor.
///
pub fn new_evmla(assembly: Assembly, extra_metadata: ExtraMetadata) -> Self {
Self::EVMLA(EVMLA::new(assembly, extra_metadata))
}
///
/// A shortcut constructor.
///
pub fn new_llvm_ir(path: String, source: String) -> Self {
Self::LLVMIR(LLVMIR::new(path, source))
}
///
/// A shortcut constructor.
///
pub fn new_zkasm(path: String, source: String) -> Self {
Self::ZKASM(ZKASM::new(path, source))
}
///
/// Get the list of missing deployable libraries.
///
pub fn get_missing_libraries(&self) -> HashSet<String> {
match self {
Self::Yul(inner) => inner.get_missing_libraries(),
Self::EVMLA(inner) => inner.get_missing_libraries(),
Self::LLVMIR(_inner) => HashSet::new(),
Self::ZKASM(_inner) => HashSet::new(),
}
}
}
impl<D> era_compiler_llvm_context::EraVMWriteLLVM<D> for IR
where
D: era_compiler_llvm_context::EraVMDependency + Clone,
{
fn declare(
&mut self,
context: &mut era_compiler_llvm_context::EraVMContext<D>,
) -> anyhow::Result<()> {
match self {
Self::Yul(inner) => inner.declare(context),
Self::EVMLA(inner) => inner.declare(context),
Self::LLVMIR(_inner) => Ok(()),
Self::ZKASM(_inner) => Ok(()),
}
}
fn into_llvm(
self,
context: &mut era_compiler_llvm_context::EraVMContext<D>,
) -> anyhow::Result<()> {
match self {
Self::Yul(inner) => inner.into_llvm(context),
Self::EVMLA(inner) => inner.into_llvm(context),
Self::LLVMIR(_inner) => Ok(()),
Self::ZKASM(_inner) => Ok(()),
}
}
}
@@ -0,0 +1,59 @@
//!
//! The contract Yul source code.
//!
use std::collections::HashSet;
use serde::Deserialize;
use serde::Serialize;
use crate::yul::parser::statement::object::Object;
///
/// The contract Yul source code.
///
#[derive(Debug, Serialize, Deserialize, Clone)]
pub struct Yul {
/// The Yul source code.
pub source_code: String,
/// The Yul AST object.
pub object: Object,
}
impl Yul {
///
/// A shortcut constructor.
///
pub fn new(source_code: String, object: Object) -> Self {
Self {
source_code,
object,
}
}
///
/// Get the list of missing deployable libraries.
///
pub fn get_missing_libraries(&self) -> HashSet<String> {
self.object.get_missing_libraries()
}
}
impl<D> era_compiler_llvm_context::EraVMWriteLLVM<D> for Yul
where
D: era_compiler_llvm_context::EraVMDependency + Clone,
{
fn declare(
&mut self,
context: &mut era_compiler_llvm_context::EraVMContext<D>,
) -> anyhow::Result<()> {
self.object.declare(context)
}
fn into_llvm(
self,
context: &mut era_compiler_llvm_context::EraVMContext<D>,
) -> anyhow::Result<()> {
self.object.into_llvm(context)
}
}
@@ -0,0 +1,27 @@
//!
//! The contract EraVM assembly source code.
//!
use serde::Deserialize;
use serde::Serialize;
///
/// The contract EraVM assembly source code.
///
#[derive(Debug, Serialize, Deserialize, Clone)]
#[allow(clippy::upper_case_acronyms)]
pub struct ZKASM {
/// The EraVM assembly file path.
pub path: String,
/// The EraVM assembly source code.
pub source: String,
}
impl ZKASM {
///
/// A shortcut constructor.
///
pub fn new(path: String, source: String) -> Self {
Self { path, source }
}
}
@@ -0,0 +1,45 @@
//!
//! The Solidity contract metadata.
//!
use serde::Serialize;
///
/// The Solidity contract metadata.
///
/// Is used to append the metadata hash to the contract bytecode.
///
#[derive(Debug, Serialize)]
pub struct Metadata {
/// The `solc` metadata.
pub solc_metadata: serde_json::Value,
/// The `solc` version.
pub solc_version: semver::Version,
/// The zkVM `solc` edition.
pub solc_zkvm_edition: Option<semver::Version>,
/// The EraVM compiler version.
pub zk_version: semver::Version,
/// The EraVM compiler optimizer settings.
pub optimizer_settings: era_compiler_llvm_context::OptimizerSettings,
}
impl Metadata {
///
/// A shortcut constructor.
///
pub fn new(
solc_metadata: serde_json::Value,
solc_version: semver::Version,
solc_zkvm_edition: Option<semver::Version>,
zk_version: semver::Version,
optimizer_settings: era_compiler_llvm_context::OptimizerSettings,
) -> Self {
Self {
solc_metadata,
solc_version,
solc_zkvm_edition,
zk_version,
optimizer_settings,
}
}
}
+224
View File
@@ -0,0 +1,224 @@
//!
//! The contract data.
//!
pub mod ir;
pub mod metadata;
use std::collections::HashSet;
use serde::Deserialize;
use serde::Serialize;
use sha3::Digest;
use era_compiler_llvm_context::EraVMWriteLLVM;
use crate::build::contract::Contract as ContractBuild;
use crate::project::Project;
use crate::solc::version::Version as SolcVersion;
use self::ir::IR;
use self::metadata::Metadata;
///
/// The contract data.
///
#[derive(Debug, Serialize, Deserialize, Clone)]
pub struct Contract {
/// The absolute file path.
pub path: String,
/// The IR source code data.
pub ir: IR,
/// The metadata JSON.
pub metadata_json: serde_json::Value,
}
impl Contract {
///
/// A shortcut constructor.
///
pub fn new(
path: String,
source_hash: [u8; era_compiler_common::BYTE_LENGTH_FIELD],
source_version: SolcVersion,
ir: IR,
metadata_json: Option<serde_json::Value>,
) -> Self {
let metadata_json = metadata_json.unwrap_or_else(|| {
serde_json::json!({
"source_hash": hex::encode(source_hash.as_slice()),
"source_version": serde_json::to_value(&source_version).expect("Always valid"),
})
});
Self {
path,
ir,
metadata_json,
}
}
///
/// Returns the contract identifier, which is:
/// - the Yul object identifier for Yul
/// - the full contract path for EVM legacy assembly
/// - the module name for LLVM IR
///
pub fn identifier(&self) -> &str {
match self.ir {
IR::Yul(ref yul) => yul.object.identifier.as_str(),
IR::EVMLA(ref evm) => evm.assembly.full_path(),
IR::LLVMIR(ref llvm_ir) => llvm_ir.path.as_str(),
IR::ZKASM(ref zkasm) => zkasm.path.as_str(),
}
}
///
/// Extract factory dependencies.
///
pub fn drain_factory_dependencies(&mut self) -> HashSet<String> {
match self.ir {
IR::Yul(ref mut yul) => yul.object.factory_dependencies.drain().collect(),
IR::EVMLA(ref mut evm) => evm.assembly.factory_dependencies.drain().collect(),
IR::LLVMIR(_) => HashSet::new(),
IR::ZKASM(_) => HashSet::new(),
}
}
///
/// Compiles the specified contract, setting its build artifacts.
///
pub fn compile(
mut self,
project: Project,
optimizer_settings: era_compiler_llvm_context::OptimizerSettings,
is_system_mode: bool,
include_metadata_hash: bool,
debug_config: Option<era_compiler_llvm_context::DebugConfig>,
) -> anyhow::Result<ContractBuild> {
let llvm = inkwell::context::Context::create();
let optimizer = era_compiler_llvm_context::Optimizer::new(optimizer_settings);
let version = project.version.clone();
let identifier = self.identifier().to_owned();
let metadata = Metadata::new(
self.metadata_json.take(),
version.default.clone(),
version.l2_revision.clone(),
semver::Version::parse(env!("CARGO_PKG_VERSION")).expect("Always valid"),
optimizer.settings().to_owned(),
);
let metadata_json = serde_json::to_value(&metadata).expect("Always valid");
let metadata_hash: Option<[u8; era_compiler_common::BYTE_LENGTH_FIELD]> =
if include_metadata_hash {
let metadata_string = serde_json::to_string(&metadata).expect("Always valid");
Some(sha3::Keccak256::digest(metadata_string.as_bytes()).into())
} else {
None
};
let module = match self.ir {
IR::LLVMIR(ref llvm_ir) => {
let memory_buffer =
inkwell::memory_buffer::MemoryBuffer::create_from_memory_range_copy(
llvm_ir.source.as_bytes(),
self.path.as_str(),
);
llvm.create_module_from_ir(memory_buffer)
.map_err(|error| anyhow::anyhow!(error.to_string()))?
}
IR::ZKASM(ref zkasm) => {
let build = era_compiler_llvm_context::eravm_build_assembly_text(
self.path.as_str(),
zkasm.source.as_str(),
metadata_hash,
debug_config.as_ref(),
)?;
return Ok(ContractBuild::new(
self.path,
identifier,
build,
metadata_json,
HashSet::new(),
));
}
_ => llvm.create_module(self.path.as_str()),
};
let mut context = era_compiler_llvm_context::EraVMContext::new(
&llvm,
module,
optimizer,
Some(project),
include_metadata_hash,
debug_config,
);
context.set_solidity_data(era_compiler_llvm_context::EraVMContextSolidityData::default());
match self.ir {
IR::Yul(_) => {
let yul_data = era_compiler_llvm_context::EraVMContextYulData::new(is_system_mode);
context.set_yul_data(yul_data);
}
IR::EVMLA(_) => {
let evmla_data =
era_compiler_llvm_context::EraVMContextEVMLAData::new(version.default);
context.set_evmla_data(evmla_data);
}
IR::LLVMIR(_) => {}
IR::ZKASM(_) => {}
}
let factory_dependencies = self.drain_factory_dependencies();
self.ir.declare(&mut context).map_err(|error| {
anyhow::anyhow!(
"The contract `{}` LLVM IR generator declaration pass error: {}",
self.path,
error
)
})?;
self.ir.into_llvm(&mut context).map_err(|error| {
anyhow::anyhow!(
"The contract `{}` LLVM IR generator definition pass error: {}",
self.path,
error
)
})?;
let build = context.build(self.path.as_str(), metadata_hash)?;
Ok(ContractBuild::new(
self.path,
identifier,
build,
metadata_json,
factory_dependencies,
))
}
///
/// Get the list of missing deployable libraries.
///
pub fn get_missing_libraries(&self) -> HashSet<String> {
self.ir.get_missing_libraries()
}
}
impl<D> EraVMWriteLLVM<D> for Contract
where
D: era_compiler_llvm_context::EraVMDependency + Clone,
{
fn declare(
&mut self,
context: &mut era_compiler_llvm_context::EraVMContext<D>,
) -> anyhow::Result<()> {
self.ir.declare(context)
}
fn into_llvm(
self,
context: &mut era_compiler_llvm_context::EraVMContext<D>,
) -> anyhow::Result<()> {
self.ir.into_llvm(context)
}
}
+351
View File
@@ -0,0 +1,351 @@
//!
//! The processed input data.
//!
pub mod contract;
use std::collections::BTreeMap;
use std::collections::HashMap;
use std::collections::HashSet;
use std::path::Path;
use rayon::iter::IntoParallelIterator;
use rayon::iter::ParallelIterator;
use serde::Deserialize;
use serde::Serialize;
use sha3::Digest;
use crate::build::contract::Contract as ContractBuild;
use crate::build::Build;
use crate::missing_libraries::MissingLibraries;
use crate::process::input::Input as ProcessInput;
use crate::project::contract::ir::IR;
use crate::solc::version::Version as SolcVersion;
use crate::solc::Compiler as SolcCompiler;
use crate::yul::lexer::Lexer;
use crate::yul::parser::statement::object::Object;
use self::contract::Contract;
///
/// The processes input data.
///
#[derive(Debug, Serialize, Deserialize, Clone)]
pub struct Project {
/// The source code version.
pub version: SolcVersion,
/// The project contracts,
pub contracts: BTreeMap<String, Contract>,
/// The mapping of auxiliary identifiers, e.g. Yul object names, to full contract paths.
pub identifier_paths: BTreeMap<String, String>,
/// The library addresses.
pub libraries: BTreeMap<String, BTreeMap<String, String>>,
}
impl Project {
///
/// A shortcut constructor.
///
pub fn new(
version: SolcVersion,
contracts: BTreeMap<String, Contract>,
libraries: BTreeMap<String, BTreeMap<String, String>>,
) -> Self {
let mut identifier_paths = BTreeMap::new();
for (path, contract) in contracts.iter() {
identifier_paths.insert(contract.identifier().to_owned(), path.to_owned());
}
Self {
version,
contracts,
identifier_paths,
libraries,
}
}
///
/// Compiles all contracts, returning their build artifacts.
///
pub fn compile(
self,
optimizer_settings: era_compiler_llvm_context::OptimizerSettings,
is_system_mode: bool,
include_metadata_hash: bool,
bytecode_encoding_testing: bool,
debug_config: Option<era_compiler_llvm_context::DebugConfig>,
) -> anyhow::Result<Build> {
let project = self.clone();
let results: BTreeMap<String, anyhow::Result<ContractBuild>> = self
.contracts
.into_par_iter()
.map(|(full_path, contract)| {
let process_output = crate::process::call(ProcessInput::new(
contract,
project.clone(),
is_system_mode,
include_metadata_hash,
bytecode_encoding_testing,
optimizer_settings.clone(),
debug_config.clone(),
));
(full_path, process_output.map(|output| output.build))
})
.collect();
let mut build = Build::default();
let mut hashes = HashMap::with_capacity(results.len());
for (path, result) in results.iter() {
match result {
Ok(contract) => {
hashes.insert(path.to_owned(), contract.build.bytecode_hash.to_owned());
}
Err(error) => {
anyhow::bail!("Contract `{}` compiling error: {:?}", path, error);
}
}
}
for (path, result) in results.into_iter() {
match result {
Ok(mut contract) => {
for dependency in contract.factory_dependencies.drain() {
let dependency_path = project
.identifier_paths
.get(dependency.as_str())
.cloned()
.unwrap_or_else(|| {
panic!("Dependency `{dependency}` full path not found")
});
let hash = match hashes.get(dependency_path.as_str()) {
Some(hash) => hash.to_owned(),
None => anyhow::bail!(
"Dependency contract `{}` not found in the project",
dependency_path
),
};
contract
.build
.factory_dependencies
.insert(hash, dependency_path);
}
build.contracts.insert(path, contract);
}
Err(error) => {
anyhow::bail!("Contract `{}` compiling error: {:?}", path, error);
}
}
}
Ok(build)
}
///
/// Get the list of missing deployable libraries.
///
pub fn get_missing_libraries(&self) -> MissingLibraries {
let deployed_libraries = self
.libraries
.iter()
.flat_map(|(file, names)| {
names
.iter()
.map(|(name, _address)| format!("{file}:{name}"))
.collect::<HashSet<String>>()
})
.collect::<HashSet<String>>();
let mut missing_deployable_libraries = BTreeMap::new();
for (contract_path, contract) in self.contracts.iter() {
let missing_libraries = contract
.get_missing_libraries()
.into_iter()
.filter(|library| !deployed_libraries.contains(library))
.collect::<HashSet<String>>();
missing_deployable_libraries.insert(contract_path.to_owned(), missing_libraries);
}
MissingLibraries::new(missing_deployable_libraries)
}
///
/// Parses the Yul source code file and returns the source data.
///
pub fn try_from_yul_path(
path: &Path,
solc_validator: Option<&SolcCompiler>,
) -> anyhow::Result<Self> {
let source_code = std::fs::read_to_string(path)
.map_err(|error| anyhow::anyhow!("Yul file {:?} reading error: {}", path, error))?;
Self::try_from_yul_string(path, source_code.as_str(), solc_validator)
}
///
/// Parses the test Yul source code string and returns the source data.
///
/// Only for integration testing purposes.
///
pub fn try_from_yul_string(
path: &Path,
source_code: &str,
solc_validator: Option<&SolcCompiler>,
) -> anyhow::Result<Self> {
if let Some(solc) = solc_validator {
solc.validate_yul(path)?;
}
let source_version = SolcVersion::new_simple(SolcCompiler::LAST_SUPPORTED_VERSION);
let path = path.to_string_lossy().to_string();
let source_hash = sha3::Keccak256::digest(source_code.as_bytes()).into();
let mut lexer = Lexer::new(source_code.to_owned());
let object = Object::parse(&mut lexer, None)
.map_err(|error| anyhow::anyhow!("Yul object `{}` parsing error: {}", path, error))?;
let mut project_contracts = BTreeMap::new();
project_contracts.insert(
path.to_owned(),
Contract::new(
path,
source_hash,
source_version.clone(),
IR::new_yul(source_code.to_owned(), object),
None,
),
);
Ok(Self::new(
source_version,
project_contracts,
BTreeMap::new(),
))
}
///
/// Parses the LLVM IR source code file and returns the source data.
///
pub fn try_from_llvm_ir_path(path: &Path) -> anyhow::Result<Self> {
let source_code = std::fs::read_to_string(path)
.map_err(|error| anyhow::anyhow!("LLVM IR file {:?} reading error: {}", path, error))?;
let source_hash = sha3::Keccak256::digest(source_code.as_bytes()).into();
let source_version =
SolcVersion::new_simple(era_compiler_llvm_context::eravm_const::LLVM_VERSION);
let path = path.to_string_lossy().to_string();
let mut project_contracts = BTreeMap::new();
project_contracts.insert(
path.clone(),
Contract::new(
path.clone(),
source_hash,
source_version.clone(),
IR::new_llvm_ir(path, source_code),
None,
),
);
Ok(Self::new(
source_version,
project_contracts,
BTreeMap::new(),
))
}
///
/// Parses the EraVM assembly source code file and returns the source data.
///
pub fn try_from_zkasm_path(path: &Path) -> anyhow::Result<Self> {
let source_code = std::fs::read_to_string(path).map_err(|error| {
anyhow::anyhow!("EraVM assembly file {:?} reading error: {}", path, error)
})?;
let source_hash = sha3::Keccak256::digest(source_code.as_bytes()).into();
let source_version =
SolcVersion::new_simple(era_compiler_llvm_context::eravm_const::ZKEVM_VERSION);
let path = path.to_string_lossy().to_string();
let mut project_contracts = BTreeMap::new();
project_contracts.insert(
path.clone(),
Contract::new(
path.clone(),
source_hash,
source_version.clone(),
IR::new_zkasm(path, source_code),
None,
),
);
Ok(Self::new(
source_version,
project_contracts,
BTreeMap::new(),
))
}
}
impl era_compiler_llvm_context::EraVMDependency for Project {
fn compile(
project: Self,
identifier: &str,
optimizer_settings: era_compiler_llvm_context::OptimizerSettings,
is_system_mode: bool,
include_metadata_hash: bool,
debug_config: Option<era_compiler_llvm_context::DebugConfig>,
) -> anyhow::Result<String> {
let contract_path = project.resolve_path(identifier)?;
let contract = project
.contracts
.get(contract_path.as_str())
.cloned()
.ok_or_else(|| {
anyhow::anyhow!(
"Dependency contract `{}` not found in the project",
contract_path
)
})?;
contract
.compile(
project,
optimizer_settings,
is_system_mode,
include_metadata_hash,
debug_config,
)
.map_err(|error| {
anyhow::anyhow!(
"Dependency contract `{}` compiling error: {}",
identifier,
error
)
})
.map(|contract| contract.build.bytecode_hash)
}
fn resolve_path(&self, identifier: &str) -> anyhow::Result<String> {
self.identifier_paths
.get(identifier.strip_suffix("_deployed").unwrap_or(identifier))
.cloned()
.ok_or_else(|| {
anyhow::anyhow!(
"Contract with identifier `{}` not found in the project",
identifier
)
})
}
fn resolve_library(&self, path: &str) -> anyhow::Result<String> {
for (file_path, contracts) in self.libraries.iter() {
for (contract_name, address) in contracts.iter() {
let key = format!("{file_path}:{contract_name}");
if key.as_str() == path {
return Ok(address["0x".len()..].to_owned());
}
}
}
anyhow::bail!("Library `{}` not found in the project", path);
}
}