From 066c6d26f0dea83410498c412e560757da44e355 Mon Sep 17 00:00:00 2001 From: LJ <81748770+elle-j@users.noreply.github.com> Date: Wed, 8 Jul 2026 14:31:39 +0200 Subject: [PATCH] Fix rate-limited cloning of LLVM submodule (#569) ## Description We started seeing rate-limit `429` errors when cloning LLVM (as seen in e.g. [this failed run](https://github.com/paritytech/revive/actions/runs/28924583524/job/85809154518) and [this failed run](https://github.com/paritytech/revive/actions/runs/28926259175/job/85821336465?pr=564)). `llvm/llvm-project` now seems to be heavily rate-limited by GitHub. (Authenticating the clone did not seem to have much effect.) Even though our resolc build steps only do a treeless clone, all build steps still did the same clone. ### Changes * I've instead introduced a `.llvm-version` file that the `get-llvm` action reads from, eliminating the network request all together. * `.llvm-version` is meant to be kept in sync with the LLVM submodule version we use. This is enforced by: * Introduced a `check-llvm-version.yml` workflow * Runs once to check that the recorded version in `.llvm-version` coincides with the version detected from the submodule. * The submodule version detection *does* perform a network request as before, but this is only run once now (when relevant files change), rather than for all our resolc build jobs. * Since the differential test jobs require checking out our revive-differential-tests (rdt) submodule, those checkout steps cloned not only rdt, but also LLVM. I've updated this to not clone LLVM at all. --- .github/actions/get-llvm/action.yml | 38 +++++----------- .github/workflows/check-llvm-version.yml | 57 ++++++++++++++++++++++++ .github/workflows/differential-tests.yml | 20 +++++++-- .llvm-version | 1 + 4 files changed, 87 insertions(+), 29 deletions(-) create mode 100644 .github/workflows/check-llvm-version.yml create mode 100644 .llvm-version diff --git a/.github/actions/get-llvm/action.yml b/.github/actions/get-llvm/action.yml index 3eb02ed..d4e42f4 100644 --- a/.github/actions/get-llvm/action.yml +++ b/.github/actions/get-llvm/action.yml @@ -14,48 +14,34 @@ inputs: runs: using: "composite" steps: - - name: detect llvm version from submodule - id: detect-version + - name: check override version + id: override shell: bash env: # To test a draft LLVM release across every CI workflow, set this to the full # draft name (e.g. "llvm-21.1.8-revive.0f80912") and revert to "" before merging. OVERRIDE_VERSION: "" + run: echo "version=${OVERRIDE_VERSION}" | tee -a "$GITHUB_OUTPUT" + + - name: resolve llvm version + id: resolve-version + if: ${{ steps.override.outputs.version == '' }} + shell: bash run: | set -euo pipefail - if [ -n "${OVERRIDE_VERSION}" ]; then - echo "Using override version: ${OVERRIDE_VERSION}" - echo "version_prefix=${OVERRIDE_VERSION}" >> "$GITHUB_OUTPUT" - exit 0 - fi - - LLVM_SHA=$(git ls-tree HEAD llvm | awk '{print $3}') - echo "LLVM_SHA: $LLVM_SHA" - LLVM_URL=$(git config -f .gitmodules submodule.llvm.url) - echo "LLVM_URL: $LLVM_URL" - if [ -z "$LLVM_SHA" ] || [ -z "$LLVM_URL" ]; then - echo "::error::Could not resolve LLVM submodule SHA or URL from this repo." >&2 + if [ ! -s .llvm-version ]; then + echo "::error file=.llvm-version::Missing or empty .llvm-version. It must contain the version matching the llvm submodule pin." exit 1 fi - - # Do a treeless clone to avoid unnecessarily fetching source files. - git clone --filter=tree:0 --no-checkout "$LLVM_URL" llvm-tmp - # Try to get the most recent tag (e.g., "llvmorg-18.1.8"). - LLVM_TAG=$(git -C llvm-tmp describe --tags --abbrev=0 "$LLVM_SHA") - rm -rf llvm-tmp - - # Convert "llvmorg-x.y.z" to "llvm-x.y.z". - LLVM_VERSION=$(echo "$LLVM_TAG" | sed 's/^llvmorg-/llvm-/') - echo "Detected LLVM version from submodule pin: $LLVM_VERSION" - echo "version_prefix=$LLVM_VERSION" >> "$GITHUB_OUTPUT" + echo "version_prefix=$(cat .llvm-version)" | tee -a "$GITHUB_OUTPUT" - name: find asset id: find uses: actions/github-script@v7 env: target: ${{ inputs.target }} - version_prefix: ${{ steps.detect-version.outputs.version_prefix }} + version_prefix: ${{ steps.override.outputs.version || steps.resolve-version.outputs.version_prefix }} with: result-encoding: string script: | diff --git a/.github/workflows/check-llvm-version.yml b/.github/workflows/check-llvm-version.yml new file mode 100644 index 0000000..ce3c5ec --- /dev/null +++ b/.github/workflows/check-llvm-version.yml @@ -0,0 +1,57 @@ +# Guards the checked-in .llvm-version file, which CI reads (via the get-llvm +# action) to select the prebuilt LLVM release to build against. Whenever the +# llvm submodule pin or the file changes, this derives the version from the +# pinned commit and fails if the file was not updated to match. + +name: Check LLVM Version + +on: + push: + branches: ["main"] + paths: + - "llvm" + - ".llvm-version" + - ".gitmodules" + - ".github/workflows/check-llvm-version.yml" + - ".github/actions/get-llvm/action.yml" + pull_request: + types: [opened, synchronize] + paths: + - "llvm" + - ".llvm-version" + - ".gitmodules" + - ".github/workflows/check-llvm-version.yml" + - ".github/actions/get-llvm/action.yml" + +jobs: + check-llvm-version: + runs-on: ubuntu-latest + steps: + - name: Checkout revive + uses: actions/checkout@v6 + + - name: Verify .llvm-version Matches the Submodule Pin + env: + GH_TOKEN: ${{ github.token }} + run: | + set -euo pipefail + + LLVM_SHA=$(git ls-tree HEAD llvm | awk '{print $3}') + LLVM_URL=$(git config -f .gitmodules submodule.llvm.url) + + # Do a treeless clone to avoid unnecessarily fetching source files. + git clone --filter=tree:0 --no-checkout \ + "https://x-access-token:${GH_TOKEN}@${LLVM_URL#https://}" llvm-tmp + + # Try to get the most recent tag (e.g., "llvmorg-18.1.8"). + DETECTED_LLVM_TAG=$(git -C llvm-tmp describe --tags --abbrev=0 "$LLVM_SHA") + + # Convert "llvmorg-x.y.z" to "llvm-x.y.z". + DETECTED_LLVM_VERSION=$(echo "$DETECTED_LLVM_TAG" | sed 's/^llvmorg-/llvm-/') + RECORDED_LLVM_VERSION=$(cat .llvm-version) + + if [ "$DETECTED_LLVM_VERSION" != "$RECORDED_LLVM_VERSION" ]; then + echo "::error file=.llvm-version::.llvm-version contains '$RECORDED_LLVM_VERSION' but the submodule pin describes to '$DETECTED_LLVM_VERSION'." + exit 1 + fi + echo "OK: .llvm-version ($RECORDED_LLVM_VERSION) matches the submodule pin." diff --git a/.github/workflows/differential-tests.yml b/.github/workflows/differential-tests.yml index 3914214..15de82d 100644 --- a/.github/workflows/differential-tests.yml +++ b/.github/workflows/differential-tests.yml @@ -85,7 +85,12 @@ jobs: uses: actions/checkout@v6 with: path: revive - submodules: recursive + # Submodules are initialized selectively below to avoid cloning + # LLVM which is both large and prone to GitHub rate limiting. + submodules: false + + - name: Checkout revive-differential-tests + run: git -C revive submodule update --init --recursive --depth=1 revive-differential-tests - name: Download Binary uses: actions/download-artifact@v7 @@ -124,7 +129,12 @@ jobs: uses: actions/checkout@v6 with: path: revive - submodules: true + # Submodules are initialized selectively below to avoid cloning + # LLVM which is both large and prone to GitHub rate limiting. + submodules: false + + - name: Checkout revive-differential-tests + run: git -C revive submodule update --init --depth=1 revive-differential-tests - name: Compare Bytecode Hashes uses: ./revive/revive-differential-tests/.github/actions/compare-bytecode-hashes @@ -154,7 +164,11 @@ jobs: uses: actions/checkout@v6 with: path: revive - submodules: recursive + # Submodules are initialized selectively below to avoid cloning LLVM. + submodules: false + + - name: Checkout revive-differential-tests + run: git -C revive submodule update --init --recursive --depth=1 revive-differential-tests - name: Download Binary uses: actions/download-artifact@v7 diff --git a/.llvm-version b/.llvm-version new file mode 100644 index 0000000..0302608 --- /dev/null +++ b/.llvm-version @@ -0,0 +1 @@ +llvm-22.1.4