From 687cec31efab24087ba354b97b6c6c8be46d65a9 Mon Sep 17 00:00:00 2001 From: Yuri Volkov <0@mcornholio.ru> Date: Fri, 21 Feb 2025 16:30:26 +0100 Subject: [PATCH] Using released LLVM in revive build (#220) --- .github/workflows/get-release-artifact.js | 36 ++ .github/workflows/release.yml | 713 +++++++++++----------- 2 files changed, 402 insertions(+), 347 deletions(-) create mode 100644 .github/workflows/get-release-artifact.js diff --git a/.github/workflows/get-release-artifact.js b/.github/workflows/get-release-artifact.js new file mode 100644 index 0000000..b04921b --- /dev/null +++ b/.github/workflows/get-release-artifact.js @@ -0,0 +1,36 @@ +module.exports = async ({ + octokit, + context, + releasePrefix, + artifactSuffix, +}) => { + let page = 1; + + while (true) { + const res = await octokit.rest.repos.listReleases({ + owner: context.repo.owner, + repo: context.repo.repo, + per_page: 100, + page, + }); + if (res.data.length === 0) { + throw new Error( + `No LLVM releases with '${artifactSuffix}' atifacts found! Please release LLVM before running this workflow.`, + ); + } + + for (let release of res.data) { + if (release.tag_name.startsWith(releasePrefix)) { + for (let asset of release.assets) { + if (asset.name.includes(artifactSuffix)) { + return asset.browser_download_url; + } + } + console.warn( + `LLVM release ${release.tag_name} doesn't have a '${artifactSuffix}' artifact; searching for older releases...`, + ); + } + } + page++; + } +}; diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 4e93a4b..f199b00 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -1,347 +1,366 @@ -name: Release -run-name: Release ${{ github.ref_name }} -on: - push: - branches: - - 'main' - -concurrency: - group: ${{ github.ref }}-${{ github.workflow }} - cancel-in-progress: true - -env: - #rust-musl-cross:x86_64-musl - RUST_MUSL_CROSS_IMAGE: messense/rust-musl-cross@sha256:68b86bc7cb2867259e6b233415a665ff4469c28b57763e78c3bfea1c68091561 - -jobs: - - # - # - # - tag: - runs-on: ubuntu-24.04 - permissions: - contents: write - outputs: - TAG: ${{ steps.versions.outputs.TAG }} - PKG_VER: ${{ steps.versions.outputs.PKG_VER }} - steps: - - name: Checkout - uses: actions/checkout@v4 - with: - fetch-tags: 'true' - fetch-depth: 0 - - - name: Versions - id: versions - run: | - export CURRENT_TAG=$(git describe --tags --abbrev=0 --exclude "llvm-*") - export PKG_VER=v$(cat Cargo.toml | grep -A 5 package] | grep version | cut -d '=' -f 2 | tr -d '"' | tr -d " ") - echo "Current tag $CURRENT_TAG" - echo "Package version $PKG_VER" - # - echo "PKG_VER=$PKG_VER" >> $GITHUB_OUTPUT - if [ $CURRENT_TAG == $PKG_VER ]; - then - echo "Tag is up to date. Nothing to do."; - export TAG=old; - else - echo "Tag was updated."; - export TAG=new; - fi - echo "TAG=$TAG" >> $GITHUB_OUTPUT - - # - # - # - build-linux-all: - if: ${{ needs.tag.outputs.TAG == 'new' }} - runs-on: parity-large - needs: [tag] - steps: - - uses: actions/checkout@v4 - - - name: install linux deps - run: | - sudo apt-get update && sudo apt-get install -y cmake ninja-build \ - curl git libssl-dev pkg-config clang lld musl - - - name: Install Rust stable toolchain - uses: actions-rs/toolchain@v1 - with: - profile: minimal - toolchain: stable - components: rust-src - target: wasm32-unknown-emscripten - - - name: versions - run: | - rustup show - cargo --version - cmake --version - echo "bash:" && bash --version - echo "ninja:" && ninja --version - echo "clang:" && clang --version - - - name: build revive-llvm - run: make install-llvm-builder - - # musl LLVM - - - name: llvm-musl-cache restore - id: llvm-musl-cache - uses: actions/cache/restore@v4 - with: - path: target-llvm/musl/target-final - key: llvm-linux-musl-${{ hashFiles('crates/solidity/**') }} - - - name: Build musl LLVM - if: steps.llvm-musl-cache.outputs.cache-hit != 'true' - run: | - revive-llvm --target-env musl clone - revive-llvm --target-env musl build --llvm-projects lld --llvm-projects clang - - - name: llvm-musl-cache save - if: steps.llvm-musl-cache.outputs.cache-hit != 'true' - uses: actions/cache/save@v4 - with: - path: target-llvm/musl/target-final - key: llvm-linux-musl-${{ hashFiles('crates/solidity/**') }} - - # emscripten LLVM - - - name: llvm-emscripten-cache restore - id: llvm-emscripten-cache - uses: actions/cache/restore@v4 - with: - path: | - target-llvm/emscripten/target-final - emsdk - key: llvm-linux-emscripten-${{ hashFiles('crates/solidity/**') }} - - - name: Build emscripten LLVM - if: steps.llvm-emscripten-cache.outputs.cache-hit != 'true' - run: | - revive-llvm --target-env emscripten clone - source emsdk/emsdk_env.sh - revive-llvm --target-env emscripten build --llvm-projects lld - - - name: llvm-emscripten-cache save - if: steps.llvm-emscripten-cache.outputs.cache-hit != 'true' - uses: actions/cache/save@v4 - with: - path: | - target-llvm/emscripten/target-final - emsdk - key: llvm-linux-emscripten-${{ hashFiles('crates/solidity/**') }} - - # Build revive - - - name: build musl - run: | - mkdir resolc-out - docker run -v $PWD:/opt/revive $RUST_MUSL_CROSS_IMAGE /bin/bash -c " - cd /opt/revive - apt update && apt upgrade -y && apt install -y pkg-config - export LLVM_SYS_181_PREFIX=/opt/revive/target-llvm/musl/target-final - make install-bin - cp /root/.cargo/bin/resolc /opt/revive/resolc-out/resolc - " - - - name: check musl - run: | - mkdir solc - curl -sSLo solc/solc https://github.com/ethereum/solidity/releases/download/v0.8.28/solc-static-linux - chmod +x solc/solc - PATH=$PWD/solc:$PATH - result=$(./resolc-out/resolc --bin crates/integration/contracts/flipper.sol) - echo $result - if [[ $result == *'0x50564d'* ]]; then exit 0; else exit 1; fi - - - name: Set Up Node.js - uses: actions/setup-node@v3 - with: - node-version: "20" - - - name: build wasm - run: | - export LLVM_SYS_181_PREFIX=$PWD/target-llvm/musl/target-final - export REVIVE_LLVM_TARGET_PREFIX=$PWD/target-llvm/emscripten/target-final - source emsdk/emsdk_env.sh - rustup target add wasm32-unknown-emscripten - make install-wasm - - - name: check wasm - run: | - curl -sSLo solc/soljson.js https://github.com/ethereum/solidity/releases/download/v0.8.28/soljson.js - node -e " - const soljson = require('solc/soljson'); - const createRevive = require('./target/wasm32-unknown-emscripten/release/resolc.js'); - - const compiler = createRevive(); - compiler.soljson = soljson; - - const standardJsonInput = - { - language: 'Solidity', - sources: { - 'MyContract.sol': { - content: 'pragma solidity ^0.8.0; contract MyContract { function greet() public pure returns (string memory) { return \'Hello\'; } }', - }, - }, - settings: { optimizer: { enabled: false } } - }; - - compiler.writeToStdin(JSON.stringify(standardJsonInput)); - compiler.callMain(['--standard-json']); - - // Collect output - const stdout = compiler.readFromStdout(); - const stderr = compiler.readFromStderr(); - - if (stderr) { console.error(stderr); process.exit(1); } - - let out = JSON.parse(stdout); - let bytecode = out.contracts['MyContract.sol']['MyContract'].evm.bytecode.object - console.log(bytecode); - - if(!bytecode.startsWith('50564d')) { process.exit(1); } - " - - - uses: actions/upload-artifact@v4 - with: - name: revive-wasm - path: | - ./target/wasm32-unknown-emscripten/release/resolc.js - ./target/wasm32-unknown-emscripten/release/resolc.wasm - ./target/wasm32-unknown-emscripten/release/resolc_web.js - retention-days: 1 - - - uses: actions/upload-artifact@v4 - with: - name: revive-linux - path: | - ./resolc-out/resolc - retention-days: 1 - - # - # - # - create-release: - needs: [tag, build-linux-all] - runs-on: ubuntu-24.04 - permissions: - contents: write - outputs: - upload_url: ${{ steps.create_release.outputs.result }} - steps: - - uses: actions/checkout@v4 - - - name: Create/update tag - id: tag - uses: actions/github-script@v7 - with: - result-encoding: string - script: | - await github.rest.git.createRef({ - owner: context.repo.owner, - repo: context.repo.repo, - ref: 'refs/tags/${{ needs.tag.outputs.PKG_VER }}', - sha: context.sha - }) - - - name: get relese notes - id: get-notes - run: | - { - echo 'releaseNotes<> "$GITHUB_OUTPUT" - - - - name: Create release - id: create_release - env: - releaseNotes: ${{ steps.get-notes.outputs.releaseNotes }} - version: ${{ needs.tag.outputs.PKG_VER }} - uses: actions/github-script@v7 - with: - result-encoding: string - script: | - let response = await github.rest.repos.createRelease({ - owner: context.repo.owner, - repo: context.repo.repo, - tag_name: process.env.version, - name: process.env.version, - body: process.env.releaseNotes, - draft: true, - prerelease: true - }); - console.log(response); - return response.data.upload_url; - - - name: Log - run: | - echo "tag result: ${{ needs.tag.outputs.TAG }}" - echo "pkg version: ${{ needs.tag.outputs.PKG_VER }}" - - # - # - # - upload-assets: - runs-on: ubuntu-24.04 - needs: [create-release] - steps: - - - name: Download Artifact - uses: actions/download-artifact@v4 - with: - name: revive-wasm - path: resolc/ - - - name: Download Artifact - uses: actions/download-artifact@v4 - with: - name: revive-linux - path: resolc/ - - - name: upload resolc - uses: actions/upload-release-asset@v1 - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - with: - upload_url: ${{ needs.create-release.outputs.upload_url }} - asset_path: ./resolc/resolc - asset_name: resolc-static-linux - asset_content_type: application/octet-stream - - - name: upload resolc.js - uses: actions/upload-release-asset@v1 - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - with: - upload_url: ${{ needs.create-release.outputs.upload_url }} - asset_path: ./resolc/resolc.js - asset_name: resolc.js - asset_content_type: application/octet-stream - - - name: upload resolc.wasm - uses: actions/upload-release-asset@v1 - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - with: - upload_url: ${{ needs.create-release.outputs.upload_url }} - asset_path: ./resolc/resolc.wasm - asset_name: resolc.wasm - asset_content_type: application/octet-stream - - - name: upload resolc_web.js - uses: actions/upload-release-asset@v1 - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - with: - upload_url: ${{ needs.create-release.outputs.upload_url }} - asset_path: ./resolc/resolc_web.js - asset_name: resolc_web.js - asset_content_type: application/octet-stream \ No newline at end of file +name: Release +run-name: Release ${{ github.ref_name }} +on: + push: + branches: + - "main" + +concurrency: + group: ${{ github.ref }}-${{ github.workflow }} + cancel-in-progress: true + +env: + #rust-musl-cross:x86_64-musl + RUST_MUSL_CROSS_IMAGE: messense/rust-musl-cross@sha256:68b86bc7cb2867259e6b233415a665ff4469c28b57763e78c3bfea1c68091561 + RUST_LOG: trace + +jobs: + tag: + runs-on: ubuntu-24.04 + permissions: + contents: write + outputs: + TAG: ${{ steps.versions.outputs.TAG }} + PKG_VER: ${{ steps.versions.outputs.PKG_VER }} + RELEASE_NOTES: ${{ steps.versions.outputs.RELEASE_NOTES }} + steps: + - name: Checkout + uses: actions/checkout@v4 + with: + fetch-tags: "true" + fetch-depth: 0 + + - name: Versions + id: versions + run: | + export CURRENT_TAG=$(git describe --tags --abbrev=0 --exclude "llvm-*") + export PKG_VER=v$(cat Cargo.toml | grep -A 5 package] | grep version | cut -d '=' -f 2 | tr -d '"' | tr -d " ") + echo "Current tag $CURRENT_TAG" + echo "Package version $PKG_VER" + # + echo "PKG_VER=$PKG_VER" >> $GITHUB_OUTPUT + if [[ $CURRENT_TAG == $PKG_VER ]]; + then + echo "Tag is up to date. Nothing to do."; + export TAG=old; + else + echo "Tag was updated."; + export TAG=new; + fi + echo "TAG=$TAG" >> $GITHUB_OUTPUT + + # Generating release notes early, in order to avoid checkout at the last step + export RELEASE_NOTES="$(sed '/^## '${PKG_VER}'/,/^## v/!d' CHANGELOG.md | sed -e '1d' -e '$d')" + + echo "Release notes:" + echo "$RELEASE_NOTES" + + echo 'RELEASE_NOTES<> $GITHUB_OUTPUT + echo "$RELEASE_NOTES" >> $GITHUB_OUTPUT + echo 'EOF' >> $GITHUB_OUTPUT + + build-macos: + strategy: + matrix: + os: [macos-14, macos-13] + include: + - os: macos-13 + arch: x64 + - os: macos-14 + arch: arm64 + if: ${{ needs.tag.outputs.TAG == 'new' }} + runs-on: ${{ matrix.os }} + name: "build-macos-${{ matrix.arch }}" + needs: [tag] + steps: + - uses: actions/checkout@v4 + + - name: Get latest macos ${{ matrix.arch }} LLVM release artifact + id: get-llvm-artifact + uses: actions/github-script@v7 + with: + result-encoding: string + script: | + const getReleaseArtifact = require("./.github/workflows/get-release-artifact.js"); + return await getReleaseArtifact({ + octokit: github, + context, + releasePrefix: "llvm-", + artifactSuffix: "-macos-${{ matrix.arch }}" + }); + + - uses: actions-rust-lang/setup-rust-toolchain@v1 + with: + toolchain: stable + components: rust-src + target: wasm32-unknown-emscripten + rustflags: "" + + - name: install macos deps + run: | + brew install ninja + + - name: versions + run: | + rustup show + cargo --version + cmake --version + echo "bash:" && bash --version + echo "ninja:" && ninja --version + echo "clang:" && clang --version + + - name: download llvm + run: | + curl -L -o llvm.tar.gz "${{ steps.get-llvm-artifact.outputs.result }}" + tar -xvf llvm.tar.gz + + - name: build revive + run: | + export LLVM_SYS_181_PREFIX=$PWD/target-llvm/gnu/target-final + make install-bin + cp ./target/release/resolc ./target/release/resolc-${{ matrix.arch }} + + - name: check revive + run: | + mkdir solc + curl -sSLo solc/solc https://github.com/ethereum/solidity/releases/download/v0.8.28/solc-macos + chmod +x solc/solc + PATH=$PWD/solc:$PATH + result=$(./target/release/resolc-${{ matrix.arch }} --bin crates/integration/contracts/flipper.sol) + echo $result + if [[ $result == *'0x50564d'* ]]; then exit 0; else exit 1; fi + + - uses: actions/upload-artifact@v4 + with: + name: "revive-macos-${{ matrix.arch }}" + path: | + ./target/release/resolc-${{ matrix.arch }} + retention-days: 1 + + macos-universal-binary: + runs-on: macos-14 + needs: [build-macos] + steps: + - uses: actions/download-artifact@v4 + with: + pattern: revive-macos-* + path: revive-macos + + - name: run lipo + run: | + lipo revive-macos/revive-macos-arm64/resolc-arm64 revive-macos/revive-macos-x64/resolc-x64 -create -output resolc-macos + + - name: compress macos artifact + run: | + tar -czf resolc-macos.tar.gz ./resolc-macos + + - uses: actions/upload-artifact@v4 + with: + name: revive-macos + path: | + resolc-macos.tar.gz + retention-days: 1 + + build-linux-all: + if: ${{ needs.tag.outputs.TAG == 'new' }} + runs-on: parity-large + needs: [tag] + steps: + - uses: actions/checkout@v4 + + - name: Get latest linux LLVM release artifact + id: get-llvm-musl-artifact + uses: actions/github-script@v7 + with: + result-encoding: string + script: | + const getReleaseArtifact = require("./.github/workflows/get-release-artifact.js") + return await getReleaseArtifact({ + octokit: github, + context, + releasePrefix: "llvm-", + artifactSuffix: "-x86_64-linux-musl" + }) + + - name: install linux deps + run: | + sudo apt-get update && sudo apt-get install -y cmake ninja-build \ + curl git libssl-dev pkg-config clang lld musl + + - uses: actions-rust-lang/setup-rust-toolchain@v1 + with: + toolchain: stable + components: rust-src + target: wasm32-unknown-emscripten + rustflags: "" + + - name: versions + run: | + rustup show + cargo --version + cmake --version + echo "bash:" && bash --version + echo "ninja:" && ninja --version + echo "clang:" && clang --version + + - name: download llvm + run: | + curl -L -o llvm.tar.gz "${{ steps.get-llvm-musl-artifact.outputs.result }}" + tar -xvf llvm.tar.gz + + # Build revive + + - name: build musl + run: | + mkdir resolc-out + docker run -v $PWD:/opt/revive $RUST_MUSL_CROSS_IMAGE /bin/bash -c " + cd /opt/revive + apt update && apt upgrade -y && apt install -y pkg-config + export LLVM_SYS_181_PREFIX=/opt/revive/target-llvm/musl/target-final + make install-bin + cp /root/.cargo/bin/resolc /opt/revive/resolc-out/resolc-static-linux + " + + - name: check musl + run: | + mkdir solc + curl -sSLo solc/solc https://github.com/ethereum/solidity/releases/download/v0.8.28/solc-static-linux + chmod +x solc/solc + PATH=$PWD/solc:$PATH + result=$(./resolc-out/resolc-static-linux --bin crates/integration/contracts/flipper.sol) + echo $result + if [[ $result == *'0x50564d'* ]]; then exit 0; else exit 1; fi + + - name: compress musl artifact + run: | + tar --strip-components 1 -czf resolc-static-linux.tar.gz ./resolc-out/resolc-static-linux + + - uses: actions/upload-artifact@v4 + with: + name: revive-linux + path: | + ./resolc-static-linux.tar.gz + retention-days: 1 + + - name: Set Up Node.js + uses: actions/setup-node@v3 + with: + node-version: "20" + + - name: Get latest emscripten LLVM release artifact + id: get-llvm-emscripten-artifact + uses: actions/github-script@v7 + with: + result-encoding: string + script: | + const getReleaseArtifact = require("./.github/workflows/get-release-artifact.js") + return await getReleaseArtifact({ + octokit: github, + context, + releasePrefix: "llvm-", + artifactSuffix: "-wasm32-unknown-emscripten" + }) + + - name: download llvm + run: | + curl -L -o llvm.tar.gz "${{ steps.get-llvm-emscripten-artifact.outputs.result }}" + tar -xvf llvm.tar.gz + + - name: build wasm + run: | + make install-llvm-builder + revive-llvm --target-env emscripten clone + export LLVM_SYS_181_PREFIX=$PWD/target-llvm/musl/target-final + export REVIVE_LLVM_TARGET_PREFIX=$PWD/target-llvm/emscripten/target-final + source emsdk/emsdk_env.sh + rustup target add wasm32-unknown-emscripten + make install-wasm + + - name: check wasm + run: | + curl -sSLo solc/soljson.js https://github.com/ethereum/solidity/releases/download/v0.8.28/soljson.js + node -e " + const soljson = require('solc/soljson'); + const createRevive = require('./target/wasm32-unknown-emscripten/release/resolc.js'); + + const compiler = createRevive(); + compiler.soljson = soljson; + + const standardJsonInput = + { + language: 'Solidity', + sources: { + 'MyContract.sol': { + content: 'pragma solidity ^0.8.0; contract MyContract { function greet() public pure returns (string memory) { return \'Hello\'; } }', + }, + }, + settings: { optimizer: { enabled: false } } + }; + + compiler.writeToStdin(JSON.stringify(standardJsonInput)); + compiler.callMain(['--standard-json']); + + // Collect output + const stdout = compiler.readFromStdout(); + const stderr = compiler.readFromStderr(); + + if (stderr) { console.error(stderr); process.exit(1); } + + let out = JSON.parse(stdout); + let bytecode = out.contracts['MyContract.sol']['MyContract'].evm.bytecode.object + console.log(bytecode); + + if(!bytecode.startsWith('50564d')) { process.exit(1); } + " + + - name: compress wasm artifact + run: | + tar --strip-components 3 -czf resolc-wasm.tar.gz \ + ./target/wasm32-unknown-emscripten/release/resolc.js \ + ./target/wasm32-unknown-emscripten/release/resolc.wasm \ + ./target/wasm32-unknown-emscripten/release/resolc_web.js + + - uses: actions/upload-artifact@v4 + with: + name: revive-wasm + path: | + resolc-wasm.tar.gz + retention-days: 1 + + create-release: + needs: [tag, build-linux-all, macos-universal-binary] + runs-on: ubuntu-24.04 + permissions: + contents: write + outputs: + upload_url: ${{ steps.create_release.outputs.result }} + steps: + - name: Download revive-wasm + uses: actions/download-artifact@v4 + with: + name: revive-wasm + path: resolc-wasm/ + + - name: Download revive-linux + uses: actions/download-artifact@v4 + with: + name: revive-linux + path: resolc-linux/ + + - name: Download revive-macos + uses: actions/download-artifact@v4 + with: + name: revive-macos + path: resolc-macos/ + + - name: create-release + uses: softprops/action-gh-release@v2 + with: + body: ${{ needs.tag.outputs.RELEASE_NOTES }} + tag_name: ${{ needs.tag.outputs.PKG_VER }} + name: ${{ needs.tag.outputs.PKG_VER }} + draft: true + files: | + ./resolc-linux/resolc-static-linux.tar.gz + ./resolc-macos/resolc-macos.tar.gz + ./resolc-wasm/resolc-wasm.tar.gz