add LLVM 18.x as a git submodule (#399)

This changeset is based on https://github.com/paritytech/revive/pull/346

.github/workflows/release-llvm.yml

@@ -96,10 +96,6 @@ jobs:
         run: |
           cargo install --locked --force --path crates/llvm-builder
 
-      - name: Clone LLVM
-        run: |
-          revive-llvm --target-env ${{ matrix.builder-arg }} clone
-
       - name: Build LLVM
         if: ${{ matrix.target != 'wasm32-unknown-emscripten' }}
         run: |

.github/workflows/test-llvm-builder.yml

@@ -4,11 +4,8 @@ on:
     branches: ["main"]
     types: [opened, synchronize]
     paths:
-      - 'LLVM.lock'
       - 'crates/llvm-builder/**'
       - '.github/workflows/test-llvm-builder.yml'
-    paths-ignore:
-      - "**.md"
 
 concurrency:
   group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
@@ -25,6 +22,8 @@ jobs:
     runs-on: ${{ matrix.runner }}
     steps:
       - uses: actions/checkout@v4
+        with:
+          submodules: true
       - uses: actions-rust-lang/setup-rust-toolchain@v1
         with:
           # without this it will override our rust flags

.gitignore

@@ -8,7 +8,9 @@ target-llvm
 /*.yul
 /*.ll
 /*.s
-/llvm*
+/llvm-*
+# Allow llvm submodule directory
+!/llvm
 node_modules
 artifacts
 tmp

.gitmodules

@@ -0,0 +1,4 @@
+[submodule "llvm"]
+	path = llvm
+	url = https://github.com/llvm/llvm-project.git
+	branch = release/18.x

LLVM.lock

@@ -1,3 +0,0 @@
-url = "https://github.com/llvm/llvm-project.git"
-branch = "release/18.x"
-ref = "3b5b5c1ec4a3095ab096dd780e84d7ab81f3d7ff"

Makefile

@@ -38,7 +38,7 @@ install-llvm-builder:
 	cargo install --force --locked --path crates/llvm-builder
 
 install-llvm: install-llvm-builder
-	revive-llvm clone
+	git submodule update --init --recursive --depth 1
 	revive-llvm build --llvm-projects lld --llvm-projects clang
 
 install-revive-runner:

crates/llvm-builder/Cargo.toml

@@ -21,8 +21,6 @@ doctest = false
 [dependencies]
 clap = { workspace = true, features = ["help", "std", "derive"] }
 anyhow = { workspace = true }
-serde = { workspace = true, features = [ "derive" ] }
-toml = { workspace = true }
 num_cpus = { workspace = true }
 fs_extra = { workspace = true }
 path-slash = { workspace = true }

crates/llvm-builder/README.md

@@ -67,12 +67,6 @@ Obtain a compatible build for your host platform from the release section of thi
 </details>
 
 <details>
-<summary>4. (Optional) Create the `LLVM.lock` file.</summary>
-
-   * The `LLVM.lock` dictates the LLVM source tree being used.
-     A default `./LLVM.lock` pointing to the release used for development is already provided.
-
-</details>
 
 <details>
 <summary>5. Build LLVM.</summary>

crates/llvm-builder/src/builtins.rs

@@ -4,13 +4,14 @@ use crate::utils::path_windows_to_unix as to_unix;
 use std::{env::consts::EXE_EXTENSION, process::Command};
 
 /// Static CFLAGS variable passed to the compiler building the compiler-rt builtins.
-const C_FLAGS: [&str; 6] = [
+const C_FLAGS: [&str; 7] = [
     "--target=riscv64",
     "-march=rv64emac",
     "-mabi=lp64e",
     "-mcpu=generic-rv64",
     "-nostdlib",
     "-nodefaultlibs",
+    "-fuse-ld=lld",
 ];
 
 /// Static CMAKE arguments for building the compiler-rt builtins.

crates/llvm-builder/src/lib.rs

@@ -5,7 +5,6 @@ pub mod builtins;
 pub mod ccache_variant;
 pub mod llvm_path;
 pub mod llvm_project;
-pub mod lock;
 pub mod platforms;
 pub mod sanitizer;
 pub mod target_env;
@@ -14,7 +13,6 @@ pub mod utils;
 
 pub use self::build_type::BuildType;
 pub use self::llvm_path::LLVMPath;
-pub use self::lock::Lock;
 pub use self::platforms::Platform;
 
 use std::collections::HashSet;
@@ -23,87 +21,25 @@ use std::process::Command;
 pub use target_env::TargetEnv;
 pub use target_triple::TargetTriple;
 
-/// Executes the LLVM repository cloning.
+/// Initializes the LLVM submodule if not already done.
-pub fn clone(lock: Lock, deep: bool, target_env: TargetEnv) -> anyhow::Result<()> {
+pub fn init(init_emscripten: bool) -> anyhow::Result<()> {
     utils::check_presence("git")?;
 
-    if target_env == TargetEnv::Emscripten {
+    if init_emscripten {
         utils::install_emsdk()?;
     }
 
     let destination_path = PathBuf::from(LLVMPath::DIRECTORY_LLVM_SOURCE);
-    if destination_path.exists() {
+    if destination_path.join(".git").exists() {
-        log::warn!(
+        log::info!("LLVM submodule already initialized");
-            "LLVM repository directory {} already exists, falling back to checkout",
+        return Ok(());
-            destination_path.display()
-        );
-        return checkout(lock, false);
-    }
-
-    let mut clone_args = vec!["clone", "--branch", lock.branch.as_str()];
-    if !deep {
-        clone_args.push("--depth");
-        clone_args.push("1");
     }
 
     utils::command(
-        Command::new("git")
+        Command::new("git").args(["submodule", "update", "--init", "--recursive"]),
-            .args(clone_args)
+        "LLVM submodule initialization",
-            .arg(lock.url.as_str())
-            .arg(destination_path.to_string_lossy().as_ref()),
-        "LLVM repository cloning",
     )?;
 
-    if let Some(r#ref) = lock.r#ref {
-        utils::command(
-            Command::new("git")
-                .args(["checkout", r#ref.as_str()])
-                .current_dir(destination_path.to_string_lossy().as_ref()),
-            "LLVM repository commit checking out",
-        )?;
-    }
-
-    Ok(())
-}
-
-/// Executes the checkout of the specified branch.
-pub fn checkout(lock: Lock, force: bool) -> anyhow::Result<()> {
-    let destination_path = PathBuf::from(LLVMPath::DIRECTORY_LLVM_SOURCE);
-
-    utils::command(
-        Command::new("git")
-            .current_dir(destination_path.as_path())
-            .args(["fetch", "--all", "--tags"]),
-        "LLVM repository data fetching",
-    )?;
-
-    if force {
-        utils::command(
-            Command::new("git")
-                .current_dir(destination_path.as_path())
-                .args(["clean", "-d", "-x", "--force"]),
-            "LLVM repository cleaning",
-        )?;
-    }
-
-    utils::command(
-        Command::new("git")
-            .current_dir(destination_path.as_path())
-            .args(["checkout", "--force", lock.branch.as_str()]),
-        "LLVM repository data pulling",
-    )?;
-
-    if let Some(r#ref) = lock.r#ref {
-        let mut checkout_command = Command::new("git");
-        checkout_command.current_dir(destination_path.as_path());
-        checkout_command.arg("checkout");
-        if force {
-            checkout_command.arg("--force");
-        }
-        checkout_command.arg(r#ref);
-        utils::command(&mut checkout_command, "LLVM repository checking out")?;
-    }
-
     Ok(())
 }
 
@@ -332,8 +268,6 @@ pub fn clean() -> anyhow::Result<()> {
             .expect("target_env parent directory is target-llvm"),
     )?;
     remove_if_exists(&PathBuf::from(LLVMPath::DIRECTORY_EMSDK_SOURCE))?;
-    remove_if_exists(&PathBuf::from(LLVMPath::DIRECTORY_LLVM_SOURCE))?;
-    remove_if_exists(&PathBuf::from(LLVMPath::DIRECTORY_LLVM_HOST_SOURCE))?;
 
     Ok(())
 }

crates/llvm-builder/src/lock.rs

@@ -1,37 +0,0 @@
-//! The revive LLVM builder lock file.
-
-use anyhow::Context;
-use std::fs::File;
-use std::io::Read;
-use std::path::PathBuf;
-
-use serde::Deserialize;
-use serde::Serialize;
-
-/// The default lock file location.
-pub const LLVM_LOCK_DEFAULT_PATH: &str = "LLVM.lock";
-
-/// The lock file data.
-///
-/// This file describes the exact reference of the LLVM framework.
-#[derive(Debug, Deserialize, Serialize)]
-pub struct Lock {
-    /// The LLVM repository URL.
-    pub url: String,
-    /// The LLVM repository branch.
-    pub branch: String,
-    /// The LLVM repository commit reference.
-    pub r#ref: Option<String>,
-}
-
-impl TryFrom<&PathBuf> for Lock {
-    type Error = anyhow::Error;
-
-    fn try_from(path: &PathBuf) -> Result<Self, Self::Error> {
-        let mut config_str = String::new();
-        let mut config_file =
-            File::open(path).with_context(|| format!("Error opening {path:?} file"))?;
-        config_file.read_to_string(&mut config_str)?;
-        Ok(toml::from_str(&config_str)?)
-    }
-}

crates/llvm-builder/src/revive_llvm/arguments.rs

@@ -18,13 +18,6 @@ pub struct Arguments {
 /// The revive LLVM builder arguments.
 #[derive(Debug, clap::Subcommand)]
 pub enum Subcommand {
-    /// Clone the branch specified in `LLVM.lock`.
-    Clone {
-        /// Clone with full commits history.
-        #[arg(long)]
-        deep: bool,
-    },
-
     /// Build the LLVM framework.
     Build {
         /// LLVM build type (`Debug`, `Release`, `RelWithDebInfo`, or `MinSizeRel`).
@@ -77,12 +70,8 @@ pub enum Subcommand {
         enable_valgrind: bool,
     },
 
-    /// Checkout the branch specified in `LLVM.lock`.
+    /// Install emsdk
-    Checkout {
+    Emsdk,
-        /// Remove all artifacts preventing the checkout (removes all local changes!).
-        #[arg(long)]
-        force: bool,
-    },
 
     /// Clean the build artifacts.
     Clean,

crates/llvm-builder/src/revive_llvm/main.rs

@@ -3,7 +3,6 @@
 pub(crate) mod arguments;
 
 use std::collections::HashSet;
-use std::path::PathBuf;
 use std::str::FromStr;
 
 use anyhow::Context;
@@ -29,13 +28,6 @@ fn main_inner() -> anyhow::Result<()> {
     revive_llvm_builder::utils::directory_target_llvm(arguments.target_env);
 
     match arguments.subcommand {
-        Subcommand::Clone { deep } => {
-            let lock = revive_llvm_builder::Lock::try_from(&PathBuf::from(
-                revive_llvm_builder::lock::LLVM_LOCK_DEFAULT_PATH,
-            ))?;
-            revive_llvm_builder::clone(lock, deep, arguments.target_env)?;
-        }
-
         Subcommand::Build {
             build_type,
             targets,
@@ -50,6 +42,8 @@ fn main_inner() -> anyhow::Result<()> {
             sanitizer,
             enable_valgrind,
         } => {
+            revive_llvm_builder::init(false)?;
+
             let mut targets = targets
                 .into_iter()
                 .map(|target| revive_llvm_builder::Platform::from_str(target.as_str()))
@@ -107,11 +101,8 @@ fn main_inner() -> anyhow::Result<()> {
             )?;
         }
 
-        Subcommand::Checkout { force } => {
+        Subcommand::Emsdk => {
-            let lock = revive_llvm_builder::Lock::try_from(&PathBuf::from(
+            revive_llvm_builder::init(true)?;
-                revive_llvm_builder::lock::LLVM_LOCK_DEFAULT_PATH,
-            ))?;
-            revive_llvm_builder::checkout(lock, force)?;
         }
 
         Subcommand::Clean => {

crates/llvm-builder/tests/build.rs

@@ -4,16 +4,10 @@ use std::process::Command;
 
 use assert_cmd::{cargo, prelude::*};
 
-/// This test verifies that the LLVM repository can be successfully cloned, built, and cleaned.
+/// This test verifies that the LLVM repository can be successfully built and cleaned.
 #[test]
-fn clone_build_and_clean() -> anyhow::Result<()> {
+fn build_and_clean() -> anyhow::Result<()> {
-    let test_dir = common::TestDir::with_lockfile(None)?;
+    let test_dir = common::TestDir::new()?;
-
-    Command::new(cargo::cargo_bin!("revive-llvm"))
-        .current_dir(test_dir.path())
-        .arg("clone")
-        .assert()
-        .success();
 
     Command::new(cargo::cargo_bin!("revive-llvm"))
         .current_dir(test_dir.path())
@@ -40,18 +34,12 @@ fn clone_build_and_clean() -> anyhow::Result<()> {
     Ok(())
 }
 
-/// This test verifies that the LLVM repository can be successfully cloned, built, and cleaned
+/// This test verifies that the LLVM repository can be successfully built and cleaned
 /// with 2-staged build using MUSL as sysroot.
 #[test]
 #[cfg(target_os = "linux")]
-fn clone_build_and_clean_musl() -> anyhow::Result<()> {
+fn build_and_clean_musl() -> anyhow::Result<()> {
-    let test_dir = common::TestDir::with_lockfile(None)?;
+    let test_dir = common::TestDir::new()?;
-
-    Command::new(cargo::cargo_bin!("revive-llvm"))
-        .arg("clone")
-        .current_dir(test_dir.path())
-        .assert()
-        .success();
 
     Command::new(cargo::cargo_bin!("revive-llvm"))
         .current_dir(test_dir.path())
@@ -84,18 +72,12 @@ fn clone_build_and_clean_musl() -> anyhow::Result<()> {
     Ok(())
 }
 
-/// This test verifies that the LLVM repository can be successfully cloned and built in debug mode
+/// This test verifies that the LLVM repository can be successfully built in debug mode
 /// with tests and coverage enabled.
 #[test]
 #[cfg(target_os = "linux")]
 fn debug_build_with_tests_coverage() -> anyhow::Result<()> {
-    let test_dir = common::TestDir::with_lockfile(None)?;
+    let test_dir = common::TestDir::new()?;
-
-    Command::new(cargo::cargo_bin!("revive-llvm"))
-        .current_dir(test_dir.path())
-        .arg("clone")
-        .assert()
-        .success();
 
     Command::new(cargo::cargo_bin!("revive-llvm"))
         .current_dir(test_dir.path())
@@ -118,13 +100,7 @@ fn debug_build_with_tests_coverage() -> anyhow::Result<()> {
 #[test]
 #[cfg(target_os = "linux")]
 fn build_with_sanitizers() -> anyhow::Result<()> {
-    let test_dir = common::TestDir::with_lockfile(None)?;
+    let test_dir = common::TestDir::new()?;
-
-    Command::new(cargo::cargo_bin!("revive-llvm"))
-        .current_dir(test_dir.path())
-        .arg("clone")
-        .assert()
-        .success();
 
     Command::new(cargo::cargo_bin!("revive-llvm"))
         .current_dir(test_dir.path())
@@ -141,27 +117,28 @@ fn build_with_sanitizers() -> anyhow::Result<()> {
     Ok(())
 }
 
-/// Tests the clone, build, and clean process of the LLVM repository for the emscripten target.
+/// Tests the build and clean process of the LLVM repository for the emscripten target.
 #[test]
 #[cfg(target_os = "linux")]
-fn clone_build_and_clean_emscripten() -> anyhow::Result<()> {
+fn build_and_clean_emscripten() -> anyhow::Result<()> {
-    let test_dir = common::TestDir::with_lockfile(None)?;
+    let test_dir = common::TestDir::new()?;
     let command = Command::new(cargo::cargo_bin!("revive-llvm"));
     let program = command.get_program().to_string_lossy();
+    let path = test_dir.path();
 
     Command::new(cargo::cargo_bin!("revive-llvm"))
-        .current_dir(test_dir.path())
+        .current_dir(path)
-        .arg("clone")
+        .arg("build")
+        .arg("--llvm-projects")
+        .arg("clang")
+        .arg("--llvm-projects")
+        .arg("lld")
         .assert()
         .success();
 
     Command::new(cargo::cargo_bin!("revive-llvm"))
-        .current_dir(test_dir.path())
+        .current_dir(path)
-        .arg("build")
+        .arg("emsdk")
-        .arg("--llvm-projects")
-        .arg("lld")
-        .arg("--llvm-projects")
-        .arg("clang")
         .assert()
         .success();
 
@@ -170,22 +147,20 @@ fn clone_build_and_clean_emscripten() -> anyhow::Result<()> {
     // `cd {} && . ./emsdk_env.sh && cd ..` helps the script to locate `emsdk.py`
     // @see https://github.com/emscripten-core/emsdk/blob/9dbdc4b3437750b85d16931c7c801bb71a782122/emsdk_env.sh#L61-L69
     let emsdk_wrapped_build_command = format!(
-        "{program} --target-env emscripten clone && \
+        "cd {} && . ./emsdk_env.sh && cd .. && \
-        cd {} && . ./emsdk_env.sh && cd .. && \
         {program} --target-env emscripten build --llvm-projects lld",
         revive_llvm_builder::LLVMPath::DIRECTORY_EMSDK_SOURCE,
     );
-
     Command::new("sh")
         .arg("-c")
         .arg(emsdk_wrapped_build_command)
-        .current_dir(test_dir.path())
+        .current_dir(path)
         .assert()
         .success();
 
     Command::new(cargo::cargo_bin!("revive-llvm"))
         .arg("clean")
-        .current_dir(test_dir.path())
+        .current_dir(path)
         .assert()
         .success();
 

crates/llvm-builder/tests/checkout.rs

@@ -1,48 +0,0 @@
-pub mod common;
-
-use std::process::Command;
-
-use assert_cmd::{cargo, prelude::*};
-
-/// This test verifies that after cloning the LLVM repository, checking out a specific branch
-/// or reference works as expected.
-#[test]
-fn checkout_after_clone() -> anyhow::Result<()> {
-    let test_dir = common::TestDir::with_lockfile(None)?;
-
-    Command::new(cargo::cargo_bin!("revive-llvm"))
-        .current_dir(test_dir.path())
-        .arg("clone")
-        .assert()
-        .success();
-
-    Command::new(cargo::cargo_bin!("revive-llvm"))
-        .current_dir(test_dir.path())
-        .arg("checkout")
-        .assert()
-        .success();
-
-    Ok(())
-}
-
-/// This test verifies that after cloning the LLVM repository, checking out a specific branch
-/// or reference with the `--force` option works as expected.
-#[test]
-fn force_checkout() -> anyhow::Result<()> {
-    let test_dir = common::TestDir::with_lockfile(None)?;
-
-    Command::new(cargo::cargo_bin!("revive-llvm"))
-        .current_dir(test_dir.path())
-        .arg("clone")
-        .assert()
-        .success();
-
-    Command::new(cargo::cargo_bin!("revive-llvm"))
-        .current_dir(test_dir.path())
-        .arg("checkout")
-        .arg("--force")
-        .assert()
-        .success();
-
-    Ok(())
-}

crates/llvm-builder/tests/clone.rs

@@ -1,36 +0,0 @@
-pub mod common;
-
-use std::process::Command;
-
-use assert_cmd::{cargo, prelude::*};
-
-/// This test verifies that the LLVM repository can be successfully cloned using a specific branch
-/// and reference.
-#[test]
-fn clone() -> anyhow::Result<()> {
-    let test_dir = common::TestDir::with_lockfile(None)?;
-
-    Command::new(cargo::cargo_bin!("revive-llvm"))
-        .current_dir(test_dir.path())
-        .arg("clone")
-        .assert()
-        .success();
-
-    Ok(())
-}
-
-/// This test verifies that the LLVM repository can be successfully cloned using a specific branch
-/// and reference with --deep option.
-#[test]
-fn clone_deep() -> anyhow::Result<()> {
-    let test_dir = common::TestDir::with_lockfile(None)?;
-
-    Command::new(cargo::cargo_bin!("revive-llvm"))
-        .current_dir(test_dir.path())
-        .arg("clone")
-        .arg("--deep")
-        .assert()
-        .success();
-
-    Ok(())
-}

crates/llvm-builder/tests/common.rs

@@ -1,32 +1,51 @@
-use assert_fs::fixture::FileWriteStr;
+use assert_fs::TempDir;
 
 pub const REVIVE_LLVM: &str = "revive-llvm";
-pub const REVIVE_LLVM_REPO_URL: &str = "https://github.com/llvm/llvm-project";
-pub const REVIVE_LLVM_REPO_TEST_BRANCH: &str = "release/18.x";
 
 pub struct TestDir {
-    _lockfile: assert_fs::NamedTempFile,
+    _tempdir: TempDir,
     path: std::path::PathBuf,
 }
 
-/// Creates a temporary lock file for testing.
+/// Creates a temporary directory for testing with submodule setup.
 impl TestDir {
-    pub fn with_lockfile(reference: Option<String>) -> anyhow::Result<Self> {
+    pub fn new() -> anyhow::Result<Self> {
-        let file =
+        let tempdir = TempDir::new()?;
-            assert_fs::NamedTempFile::new(revive_llvm_builder::lock::LLVM_LOCK_DEFAULT_PATH)?;
+        let tmppath = tempdir.path();
-        let lock = revive_llvm_builder::Lock {
+
-            url: REVIVE_LLVM_REPO_URL.to_string(),
+        // Initialize a git repo and add the LLVM submodule
-            branch: REVIVE_LLVM_REPO_TEST_BRANCH.to_string(),
+        std::process::Command::new("git")
-            r#ref: reference,
+            .args(["init"])
-        };
+            .current_dir(tmppath)
-        file.write_str(toml::to_string(&lock)?.as_str())?;
+            .output()?;
+
+        std::process::Command::new("git")
+            .args([
+                "submodule",
+                "add",
+                "-b",
+                "release/18.x",
+                "https://github.com/llvm/llvm-project.git",
+                "llvm",
+            ])
+            .current_dir(tmppath)
+            .output()?;
+
+        std::process::Command::new("git")
+            .args([
+                "submodule",
+                "update",
+                "--init",
+                "--recursive",
+                "--force",
+                "--depth 1",
+            ])
+            .current_dir(tmppath)
+            .output()?;
 
         Ok(Self {
-            path: file
+            path: tmppath.to_path_buf(),
-                .parent()
+            _tempdir: tempdir,
-                .expect("lockfile parent dir always exists")
-                .into(),
-            _lockfile: file,
         })
     }