diff --git a/.github/workflows/release-llvm.yml b/.github/workflows/release-llvm.yml index 504dfb9..80e0b82 100644 --- a/.github/workflows/release-llvm.yml +++ b/.github/workflows/release-llvm.yml @@ -96,10 +96,6 @@ jobs: run: | cargo install --locked --force --path crates/llvm-builder - - name: Clone LLVM - run: | - revive-llvm --target-env ${{ matrix.builder-arg }} clone - - name: Build LLVM if: ${{ matrix.target != 'wasm32-unknown-emscripten' }} run: | diff --git a/.github/workflows/test-llvm-builder.yml b/.github/workflows/test-llvm-builder.yml index 66c9079..b25f7e4 100644 --- a/.github/workflows/test-llvm-builder.yml +++ b/.github/workflows/test-llvm-builder.yml @@ -4,11 +4,8 @@ on: branches: ["main"] types: [opened, synchronize] paths: - - 'LLVM.lock' - 'crates/llvm-builder/**' - '.github/workflows/test-llvm-builder.yml' - paths-ignore: - - "**.md" concurrency: group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} @@ -25,6 +22,8 @@ jobs: runs-on: ${{ matrix.runner }} steps: - uses: actions/checkout@v4 + with: + submodules: true - uses: actions-rust-lang/setup-rust-toolchain@v1 with: # without this it will override our rust flags diff --git a/.gitignore b/.gitignore index b856ff9..4f35c58 100644 --- a/.gitignore +++ b/.gitignore @@ -8,7 +8,9 @@ target-llvm /*.yul /*.ll /*.s -/llvm* +/llvm-* +# Allow llvm submodule directory +!/llvm node_modules artifacts tmp diff --git a/.gitmodules b/.gitmodules new file mode 100644 index 0000000..8d51bb7 --- /dev/null +++ b/.gitmodules @@ -0,0 +1,4 @@ +[submodule "llvm"] + path = llvm + url = https://github.com/llvm/llvm-project.git + branch = release/18.x diff --git a/LLVM.lock b/LLVM.lock deleted file mode 100644 index 302ea0a..0000000 --- a/LLVM.lock +++ /dev/null @@ -1,3 +0,0 @@ -url = "https://github.com/llvm/llvm-project.git" -branch = "release/18.x" -ref = "3b5b5c1ec4a3095ab096dd780e84d7ab81f3d7ff" \ No newline at end of file diff --git a/Makefile b/Makefile index 3f1bf35..64223c6 100644 --- a/Makefile +++ b/Makefile @@ -38,7 +38,7 @@ install-llvm-builder: cargo install --force --locked --path crates/llvm-builder install-llvm: install-llvm-builder - revive-llvm clone + git submodule update --init --recursive --depth 1 revive-llvm build --llvm-projects lld --llvm-projects clang install-revive-runner: diff --git a/crates/llvm-builder/Cargo.toml b/crates/llvm-builder/Cargo.toml index 4abb8e3..ae68419 100644 --- a/crates/llvm-builder/Cargo.toml +++ b/crates/llvm-builder/Cargo.toml @@ -21,8 +21,6 @@ doctest = false [dependencies] clap = { workspace = true, features = ["help", "std", "derive"] } anyhow = { workspace = true } -serde = { workspace = true, features = [ "derive" ] } -toml = { workspace = true } num_cpus = { workspace = true } fs_extra = { workspace = true } path-slash = { workspace = true } diff --git a/crates/llvm-builder/README.md b/crates/llvm-builder/README.md index ffa11ef..d772739 100644 --- a/crates/llvm-builder/README.md +++ b/crates/llvm-builder/README.md @@ -67,12 +67,6 @@ Obtain a compatible build for your host platform from the release section of thi
-4. (Optional) Create the `LLVM.lock` file. - - * The `LLVM.lock` dictates the LLVM source tree being used. - A default `./LLVM.lock` pointing to the release used for development is already provided. - -
5. Build LLVM. diff --git a/crates/llvm-builder/src/builtins.rs b/crates/llvm-builder/src/builtins.rs index 8970f85..b4b0020 100644 --- a/crates/llvm-builder/src/builtins.rs +++ b/crates/llvm-builder/src/builtins.rs @@ -4,13 +4,14 @@ use crate::utils::path_windows_to_unix as to_unix; use std::{env::consts::EXE_EXTENSION, process::Command}; /// Static CFLAGS variable passed to the compiler building the compiler-rt builtins. -const C_FLAGS: [&str; 6] = [ +const C_FLAGS: [&str; 7] = [ "--target=riscv64", "-march=rv64emac", "-mabi=lp64e", "-mcpu=generic-rv64", "-nostdlib", "-nodefaultlibs", + "-fuse-ld=lld", ]; /// Static CMAKE arguments for building the compiler-rt builtins. diff --git a/crates/llvm-builder/src/lib.rs b/crates/llvm-builder/src/lib.rs index 94b1955..fd22a2b 100644 --- a/crates/llvm-builder/src/lib.rs +++ b/crates/llvm-builder/src/lib.rs @@ -5,7 +5,6 @@ pub mod builtins; pub mod ccache_variant; pub mod llvm_path; pub mod llvm_project; -pub mod lock; pub mod platforms; pub mod sanitizer; pub mod target_env; @@ -14,7 +13,6 @@ pub mod utils; pub use self::build_type::BuildType; pub use self::llvm_path::LLVMPath; -pub use self::lock::Lock; pub use self::platforms::Platform; use std::collections::HashSet; @@ -23,87 +21,25 @@ use std::process::Command; pub use target_env::TargetEnv; pub use target_triple::TargetTriple; -/// Executes the LLVM repository cloning. -pub fn clone(lock: Lock, deep: bool, target_env: TargetEnv) -> anyhow::Result<()> { +/// Initializes the LLVM submodule if not already done. +pub fn init(init_emscripten: bool) -> anyhow::Result<()> { utils::check_presence("git")?; - if target_env == TargetEnv::Emscripten { + if init_emscripten { utils::install_emsdk()?; } let destination_path = PathBuf::from(LLVMPath::DIRECTORY_LLVM_SOURCE); - if destination_path.exists() { - log::warn!( - "LLVM repository directory {} already exists, falling back to checkout", - destination_path.display() - ); - return checkout(lock, false); - } - - let mut clone_args = vec!["clone", "--branch", lock.branch.as_str()]; - if !deep { - clone_args.push("--depth"); - clone_args.push("1"); + if destination_path.join(".git").exists() { + log::info!("LLVM submodule already initialized"); + return Ok(()); } utils::command( - Command::new("git") - .args(clone_args) - .arg(lock.url.as_str()) - .arg(destination_path.to_string_lossy().as_ref()), - "LLVM repository cloning", + Command::new("git").args(["submodule", "update", "--init", "--recursive"]), + "LLVM submodule initialization", )?; - if let Some(r#ref) = lock.r#ref { - utils::command( - Command::new("git") - .args(["checkout", r#ref.as_str()]) - .current_dir(destination_path.to_string_lossy().as_ref()), - "LLVM repository commit checking out", - )?; - } - - Ok(()) -} - -/// Executes the checkout of the specified branch. -pub fn checkout(lock: Lock, force: bool) -> anyhow::Result<()> { - let destination_path = PathBuf::from(LLVMPath::DIRECTORY_LLVM_SOURCE); - - utils::command( - Command::new("git") - .current_dir(destination_path.as_path()) - .args(["fetch", "--all", "--tags"]), - "LLVM repository data fetching", - )?; - - if force { - utils::command( - Command::new("git") - .current_dir(destination_path.as_path()) - .args(["clean", "-d", "-x", "--force"]), - "LLVM repository cleaning", - )?; - } - - utils::command( - Command::new("git") - .current_dir(destination_path.as_path()) - .args(["checkout", "--force", lock.branch.as_str()]), - "LLVM repository data pulling", - )?; - - if let Some(r#ref) = lock.r#ref { - let mut checkout_command = Command::new("git"); - checkout_command.current_dir(destination_path.as_path()); - checkout_command.arg("checkout"); - if force { - checkout_command.arg("--force"); - } - checkout_command.arg(r#ref); - utils::command(&mut checkout_command, "LLVM repository checking out")?; - } - Ok(()) } @@ -332,8 +268,6 @@ pub fn clean() -> anyhow::Result<()> { .expect("target_env parent directory is target-llvm"), )?; remove_if_exists(&PathBuf::from(LLVMPath::DIRECTORY_EMSDK_SOURCE))?; - remove_if_exists(&PathBuf::from(LLVMPath::DIRECTORY_LLVM_SOURCE))?; - remove_if_exists(&PathBuf::from(LLVMPath::DIRECTORY_LLVM_HOST_SOURCE))?; Ok(()) } diff --git a/crates/llvm-builder/src/lock.rs b/crates/llvm-builder/src/lock.rs deleted file mode 100644 index 2932262..0000000 --- a/crates/llvm-builder/src/lock.rs +++ /dev/null @@ -1,37 +0,0 @@ -//! The revive LLVM builder lock file. - -use anyhow::Context; -use std::fs::File; -use std::io::Read; -use std::path::PathBuf; - -use serde::Deserialize; -use serde::Serialize; - -/// The default lock file location. -pub const LLVM_LOCK_DEFAULT_PATH: &str = "LLVM.lock"; - -/// The lock file data. -/// -/// This file describes the exact reference of the LLVM framework. -#[derive(Debug, Deserialize, Serialize)] -pub struct Lock { - /// The LLVM repository URL. - pub url: String, - /// The LLVM repository branch. - pub branch: String, - /// The LLVM repository commit reference. - pub r#ref: Option, -} - -impl TryFrom<&PathBuf> for Lock { - type Error = anyhow::Error; - - fn try_from(path: &PathBuf) -> Result { - let mut config_str = String::new(); - let mut config_file = - File::open(path).with_context(|| format!("Error opening {path:?} file"))?; - config_file.read_to_string(&mut config_str)?; - Ok(toml::from_str(&config_str)?) - } -} diff --git a/crates/llvm-builder/src/revive_llvm/arguments.rs b/crates/llvm-builder/src/revive_llvm/arguments.rs index 93a0122..b4fe0e5 100644 --- a/crates/llvm-builder/src/revive_llvm/arguments.rs +++ b/crates/llvm-builder/src/revive_llvm/arguments.rs @@ -18,13 +18,6 @@ pub struct Arguments { /// The revive LLVM builder arguments. #[derive(Debug, clap::Subcommand)] pub enum Subcommand { - /// Clone the branch specified in `LLVM.lock`. - Clone { - /// Clone with full commits history. - #[arg(long)] - deep: bool, - }, - /// Build the LLVM framework. Build { /// LLVM build type (`Debug`, `Release`, `RelWithDebInfo`, or `MinSizeRel`). @@ -77,12 +70,8 @@ pub enum Subcommand { enable_valgrind: bool, }, - /// Checkout the branch specified in `LLVM.lock`. - Checkout { - /// Remove all artifacts preventing the checkout (removes all local changes!). - #[arg(long)] - force: bool, - }, + /// Install emsdk + Emsdk, /// Clean the build artifacts. Clean, diff --git a/crates/llvm-builder/src/revive_llvm/main.rs b/crates/llvm-builder/src/revive_llvm/main.rs index 25850d8..90901ef 100644 --- a/crates/llvm-builder/src/revive_llvm/main.rs +++ b/crates/llvm-builder/src/revive_llvm/main.rs @@ -3,7 +3,6 @@ pub(crate) mod arguments; use std::collections::HashSet; -use std::path::PathBuf; use std::str::FromStr; use anyhow::Context; @@ -29,13 +28,6 @@ fn main_inner() -> anyhow::Result<()> { revive_llvm_builder::utils::directory_target_llvm(arguments.target_env); match arguments.subcommand { - Subcommand::Clone { deep } => { - let lock = revive_llvm_builder::Lock::try_from(&PathBuf::from( - revive_llvm_builder::lock::LLVM_LOCK_DEFAULT_PATH, - ))?; - revive_llvm_builder::clone(lock, deep, arguments.target_env)?; - } - Subcommand::Build { build_type, targets, @@ -50,6 +42,8 @@ fn main_inner() -> anyhow::Result<()> { sanitizer, enable_valgrind, } => { + revive_llvm_builder::init(false)?; + let mut targets = targets .into_iter() .map(|target| revive_llvm_builder::Platform::from_str(target.as_str())) @@ -107,11 +101,8 @@ fn main_inner() -> anyhow::Result<()> { )?; } - Subcommand::Checkout { force } => { - let lock = revive_llvm_builder::Lock::try_from(&PathBuf::from( - revive_llvm_builder::lock::LLVM_LOCK_DEFAULT_PATH, - ))?; - revive_llvm_builder::checkout(lock, force)?; + Subcommand::Emsdk => { + revive_llvm_builder::init(true)?; } Subcommand::Clean => { diff --git a/crates/llvm-builder/tests/build.rs b/crates/llvm-builder/tests/build.rs index d5abc66..0dbb5a0 100644 --- a/crates/llvm-builder/tests/build.rs +++ b/crates/llvm-builder/tests/build.rs @@ -4,16 +4,10 @@ use std::process::Command; use assert_cmd::{cargo, prelude::*}; -/// This test verifies that the LLVM repository can be successfully cloned, built, and cleaned. +/// This test verifies that the LLVM repository can be successfully built and cleaned. #[test] -fn clone_build_and_clean() -> anyhow::Result<()> { - let test_dir = common::TestDir::with_lockfile(None)?; - - Command::new(cargo::cargo_bin!("revive-llvm")) - .current_dir(test_dir.path()) - .arg("clone") - .assert() - .success(); +fn build_and_clean() -> anyhow::Result<()> { + let test_dir = common::TestDir::new()?; Command::new(cargo::cargo_bin!("revive-llvm")) .current_dir(test_dir.path()) @@ -40,18 +34,12 @@ fn clone_build_and_clean() -> anyhow::Result<()> { Ok(()) } -/// This test verifies that the LLVM repository can be successfully cloned, built, and cleaned +/// This test verifies that the LLVM repository can be successfully built and cleaned /// with 2-staged build using MUSL as sysroot. #[test] #[cfg(target_os = "linux")] -fn clone_build_and_clean_musl() -> anyhow::Result<()> { - let test_dir = common::TestDir::with_lockfile(None)?; - - Command::new(cargo::cargo_bin!("revive-llvm")) - .arg("clone") - .current_dir(test_dir.path()) - .assert() - .success(); +fn build_and_clean_musl() -> anyhow::Result<()> { + let test_dir = common::TestDir::new()?; Command::new(cargo::cargo_bin!("revive-llvm")) .current_dir(test_dir.path()) @@ -84,18 +72,12 @@ fn clone_build_and_clean_musl() -> anyhow::Result<()> { Ok(()) } -/// This test verifies that the LLVM repository can be successfully cloned and built in debug mode +/// This test verifies that the LLVM repository can be successfully built in debug mode /// with tests and coverage enabled. #[test] #[cfg(target_os = "linux")] fn debug_build_with_tests_coverage() -> anyhow::Result<()> { - let test_dir = common::TestDir::with_lockfile(None)?; - - Command::new(cargo::cargo_bin!("revive-llvm")) - .current_dir(test_dir.path()) - .arg("clone") - .assert() - .success(); + let test_dir = common::TestDir::new()?; Command::new(cargo::cargo_bin!("revive-llvm")) .current_dir(test_dir.path()) @@ -118,13 +100,7 @@ fn debug_build_with_tests_coverage() -> anyhow::Result<()> { #[test] #[cfg(target_os = "linux")] fn build_with_sanitizers() -> anyhow::Result<()> { - let test_dir = common::TestDir::with_lockfile(None)?; - - Command::new(cargo::cargo_bin!("revive-llvm")) - .current_dir(test_dir.path()) - .arg("clone") - .assert() - .success(); + let test_dir = common::TestDir::new()?; Command::new(cargo::cargo_bin!("revive-llvm")) .current_dir(test_dir.path()) @@ -141,27 +117,28 @@ fn build_with_sanitizers() -> anyhow::Result<()> { Ok(()) } -/// Tests the clone, build, and clean process of the LLVM repository for the emscripten target. +/// Tests the build and clean process of the LLVM repository for the emscripten target. #[test] #[cfg(target_os = "linux")] -fn clone_build_and_clean_emscripten() -> anyhow::Result<()> { - let test_dir = common::TestDir::with_lockfile(None)?; +fn build_and_clean_emscripten() -> anyhow::Result<()> { + let test_dir = common::TestDir::new()?; let command = Command::new(cargo::cargo_bin!("revive-llvm")); let program = command.get_program().to_string_lossy(); + let path = test_dir.path(); Command::new(cargo::cargo_bin!("revive-llvm")) - .current_dir(test_dir.path()) - .arg("clone") + .current_dir(path) + .arg("build") + .arg("--llvm-projects") + .arg("clang") + .arg("--llvm-projects") + .arg("lld") .assert() .success(); Command::new(cargo::cargo_bin!("revive-llvm")) - .current_dir(test_dir.path()) - .arg("build") - .arg("--llvm-projects") - .arg("lld") - .arg("--llvm-projects") - .arg("clang") + .current_dir(path) + .arg("emsdk") .assert() .success(); @@ -170,22 +147,20 @@ fn clone_build_and_clean_emscripten() -> anyhow::Result<()> { // `cd {} && . ./emsdk_env.sh && cd ..` helps the script to locate `emsdk.py` // @see https://github.com/emscripten-core/emsdk/blob/9dbdc4b3437750b85d16931c7c801bb71a782122/emsdk_env.sh#L61-L69 let emsdk_wrapped_build_command = format!( - "{program} --target-env emscripten clone && \ - cd {} && . ./emsdk_env.sh && cd .. && \ + "cd {} && . ./emsdk_env.sh && cd .. && \ {program} --target-env emscripten build --llvm-projects lld", revive_llvm_builder::LLVMPath::DIRECTORY_EMSDK_SOURCE, ); - Command::new("sh") .arg("-c") .arg(emsdk_wrapped_build_command) - .current_dir(test_dir.path()) + .current_dir(path) .assert() .success(); Command::new(cargo::cargo_bin!("revive-llvm")) .arg("clean") - .current_dir(test_dir.path()) + .current_dir(path) .assert() .success(); diff --git a/crates/llvm-builder/tests/checkout.rs b/crates/llvm-builder/tests/checkout.rs deleted file mode 100644 index 1f4c923..0000000 --- a/crates/llvm-builder/tests/checkout.rs +++ /dev/null @@ -1,48 +0,0 @@ -pub mod common; - -use std::process::Command; - -use assert_cmd::{cargo, prelude::*}; - -/// This test verifies that after cloning the LLVM repository, checking out a specific branch -/// or reference works as expected. -#[test] -fn checkout_after_clone() -> anyhow::Result<()> { - let test_dir = common::TestDir::with_lockfile(None)?; - - Command::new(cargo::cargo_bin!("revive-llvm")) - .current_dir(test_dir.path()) - .arg("clone") - .assert() - .success(); - - Command::new(cargo::cargo_bin!("revive-llvm")) - .current_dir(test_dir.path()) - .arg("checkout") - .assert() - .success(); - - Ok(()) -} - -/// This test verifies that after cloning the LLVM repository, checking out a specific branch -/// or reference with the `--force` option works as expected. -#[test] -fn force_checkout() -> anyhow::Result<()> { - let test_dir = common::TestDir::with_lockfile(None)?; - - Command::new(cargo::cargo_bin!("revive-llvm")) - .current_dir(test_dir.path()) - .arg("clone") - .assert() - .success(); - - Command::new(cargo::cargo_bin!("revive-llvm")) - .current_dir(test_dir.path()) - .arg("checkout") - .arg("--force") - .assert() - .success(); - - Ok(()) -} diff --git a/crates/llvm-builder/tests/clone.rs b/crates/llvm-builder/tests/clone.rs deleted file mode 100644 index 61d04f5..0000000 --- a/crates/llvm-builder/tests/clone.rs +++ /dev/null @@ -1,36 +0,0 @@ -pub mod common; - -use std::process::Command; - -use assert_cmd::{cargo, prelude::*}; - -/// This test verifies that the LLVM repository can be successfully cloned using a specific branch -/// and reference. -#[test] -fn clone() -> anyhow::Result<()> { - let test_dir = common::TestDir::with_lockfile(None)?; - - Command::new(cargo::cargo_bin!("revive-llvm")) - .current_dir(test_dir.path()) - .arg("clone") - .assert() - .success(); - - Ok(()) -} - -/// This test verifies that the LLVM repository can be successfully cloned using a specific branch -/// and reference with --deep option. -#[test] -fn clone_deep() -> anyhow::Result<()> { - let test_dir = common::TestDir::with_lockfile(None)?; - - Command::new(cargo::cargo_bin!("revive-llvm")) - .current_dir(test_dir.path()) - .arg("clone") - .arg("--deep") - .assert() - .success(); - - Ok(()) -} diff --git a/crates/llvm-builder/tests/common.rs b/crates/llvm-builder/tests/common.rs index 78465fc..22793b0 100644 --- a/crates/llvm-builder/tests/common.rs +++ b/crates/llvm-builder/tests/common.rs @@ -1,32 +1,51 @@ -use assert_fs::fixture::FileWriteStr; +use assert_fs::TempDir; pub const REVIVE_LLVM: &str = "revive-llvm"; -pub const REVIVE_LLVM_REPO_URL: &str = "https://github.com/llvm/llvm-project"; -pub const REVIVE_LLVM_REPO_TEST_BRANCH: &str = "release/18.x"; pub struct TestDir { - _lockfile: assert_fs::NamedTempFile, + _tempdir: TempDir, path: std::path::PathBuf, } -/// Creates a temporary lock file for testing. +/// Creates a temporary directory for testing with submodule setup. impl TestDir { - pub fn with_lockfile(reference: Option) -> anyhow::Result { - let file = - assert_fs::NamedTempFile::new(revive_llvm_builder::lock::LLVM_LOCK_DEFAULT_PATH)?; - let lock = revive_llvm_builder::Lock { - url: REVIVE_LLVM_REPO_URL.to_string(), - branch: REVIVE_LLVM_REPO_TEST_BRANCH.to_string(), - r#ref: reference, - }; - file.write_str(toml::to_string(&lock)?.as_str())?; + pub fn new() -> anyhow::Result { + let tempdir = TempDir::new()?; + let tmppath = tempdir.path(); + + // Initialize a git repo and add the LLVM submodule + std::process::Command::new("git") + .args(["init"]) + .current_dir(tmppath) + .output()?; + + std::process::Command::new("git") + .args([ + "submodule", + "add", + "-b", + "release/18.x", + "https://github.com/llvm/llvm-project.git", + "llvm", + ]) + .current_dir(tmppath) + .output()?; + + std::process::Command::new("git") + .args([ + "submodule", + "update", + "--init", + "--recursive", + "--force", + "--depth 1", + ]) + .current_dir(tmppath) + .output()?; Ok(Self { - path: file - .parent() - .expect("lockfile parent dir always exists") - .into(), - _lockfile: file, + path: tmppath.to_path_buf(), + _tempdir: tempdir, }) } diff --git a/llvm b/llvm new file mode 160000 index 0000000..3b5b5c1 --- /dev/null +++ b/llvm @@ -0,0 +1 @@ +Subproject commit 3b5b5c1ec4a3095ab096dd780e84d7ab81f3d7ff