From 7c932f719b50cb122593a7fd1f25e93ced7d3d79 Mon Sep 17 00:00:00 2001 From: Alexander Samusev <41779041+alvicsam@users.noreply.github.com> Date: Mon, 7 Jul 2025 17:46:55 +0200 Subject: [PATCH] Generate nightly builds and json files (#357) PR adds a nightly release pipeline. Since the logic for the nightly build and release differs significantly from the usual pipeline, I decided to put it in a separate file. The same applies to the Python script that generates JSON files. The nightly pipeline runs at 01:00 UTC every day, producing artefacts and retaining them for 40 days if commits are made the day before it runs. cc https://github.com/paritytech/revive/issues/355 cc https://github.com/paritytech/devops/issues/4141 --- .github/scripts/json_generator.py | 10 + .github/scripts/json_generator_nightly.py | 155 +++++++++ .github/workflows/release-nightly.yml | 385 ++++++++++++++++++++++ .github/workflows/release.yml | 1 + 4 files changed, 551 insertions(+) create mode 100644 .github/scripts/json_generator_nightly.py create mode 100644 .github/workflows/release-nightly.yml diff --git a/.github/scripts/json_generator.py b/.github/scripts/json_generator.py index 6c0aa24..b6b0c41 100644 --- a/.github/scripts/json_generator.py +++ b/.github/scripts/json_generator.py @@ -1,3 +1,13 @@ +#!/usr/bin/env python3 +""" +This script generates JSON files for different platforms based on GitHub release data. +It fetches release information from a specified GitHub repository and tag, +parses the release assets, and generates JSON files for each platform with relevant metadata. +It also handles checksum files and updates a list.json file for each platform. +It requires the GITHUB_TOKEN environment variable to be set for authentication. +Usage: + python json_generator.py +""" import os import sys import json diff --git a/.github/scripts/json_generator_nightly.py b/.github/scripts/json_generator_nightly.py new file mode 100644 index 0000000..a8e64b7 --- /dev/null +++ b/.github/scripts/json_generator_nightly.py @@ -0,0 +1,155 @@ +#!/usr/bin/env python3 +""" +This script generates JSON files for different platforms based on GitHub data. +Requires the GITHUB_SHA, FIRST_SOLC_VERSION, LAST_SOLC_VERSION, TAG and FILEPATH environment variables to be set. +Usage: + python json_generator_nightly.py +""" +import os +import sys +import json +from datetime import datetime + +def validate_env_variables(): + """Validate that environment variables are set.""" + if "GITHUB_SHA" not in os.environ: + print("Error: GITHUB_SHA environment variable is not set.") + sys.exit(1) + if "FIRST_SOLC_VERSION" not in os.environ: + print("Error: FIRST_SOLC_VERSION environment variable is not set.") + sys.exit(1) + if "LAST_SOLC_VERSION" not in os.environ: + print("Error: LAST_SOLC_VERSION environment variable is not set.") + sys.exit(1) + if "TAG" not in os.environ: + print("Error: TAG environment variable is not set.") + sys.exit(1) + if "FILEPATH" not in os.environ: + print("Error: FILEPATH environment variable is not set.") + sys.exit(1) + + +def fetch_data_file(): + """ + Fetch the data.json file with artifacts urls and sha256 checksums + and parse it into a single dictionary mapping artifact names to their URLs and SHAs. + """ + # read data.json file + artifacts_data = {} + data_file_path = os.environ["FILEPATH"] + if not os.path.exists(data_file_path): + print("Error: data.json file not found.") + sys.exit(1) + with open(data_file_path, 'r') as f: + try: + artifacts_data = json.load(f) + except json.JSONDecodeError: + print("Error: data.json file is not a valid JSON.") + sys.exit(1) + + result = {} + + for item in artifacts_data: + for key, value in item.items(): + if key.endswith('_url'): + base_key = key.rsplit('_url', 1)[0] + if base_key not in result: + result[base_key] = {} + result[base_key]['url'] = value + elif key.endswith('_sha'): + base_key = key.rsplit('_sha', 1)[0] + if base_key not in result: + result[base_key] = {} + result[base_key]['sha'] = value + + return result + + + + + +def extract_build_hash(): + """Extract the first 8 characters of the commit hash.""" + sha = os.environ.get("GITHUB_SHA") + return f"commit.{sha[:8]}" + +def generate_asset_json_nightly(name, url, checksum): + """Generate JSON for a specific asset.""" + # Date in format YYYY-MM-DD + date = datetime.now().strftime("%Y.%m.%d") + last_version = os.environ.get("TAG").replace('v','') + version = f"{last_version}-nightly.{date}" + SHA = os.environ.get("GITHUB_SHA", "")[:8] + build = f"commit.{SHA}" + long_version = f"{version}+{build}" + + return { + "name": name, + "version": version, + "build": build, + "longVersion": long_version, + "url": url, + "sha256": checksum, + "firstSolcVersion": os.environ.get("FIRST_SOLC_VERSION"), + "lastSolcVersion": os.environ.get("LAST_SOLC_VERSION") + } + +def save_platform_json(platform_folder, asset_json): + """Save asset JSON and update list.json for a specific platform.""" + # Create platform folder if it doesn't exist + os.makedirs(platform_folder, exist_ok=True) + + # Update or create list.json + list_file_path = os.path.join(platform_folder, "list.json") + + if os.path.exists(list_file_path): + with open(list_file_path, 'r') as f: + try: + list_data = json.load(f) + except json.JSONDecodeError: + list_data = {"builds": [], "releases": {}, "latestRelease": ""} + else: + list_data = {"builds": [], "releases": {}, "latestRelease": ""} + + # Remove any existing entry with the same path + list_data['builds'] = [ + build for build in list_data['builds'] + if build['version'] != asset_json['version'] + ] + # Add the new build + list_data['builds'].append(asset_json) + + # Update releases + version = asset_json['version'] + list_data['releases'][version] = f"{asset_json['name']}+{asset_json['longVersion']}" + + # Update latest release + list_data['latestRelease'] = version + + with open(list_file_path, 'w') as f: + json.dump(list_data, f, indent=4) + +def main(): + + validate_env_variables() + data = fetch_data_file() + + # Mapping of asset names to platform folders + platform_mapping = { + 'resolc-x86_64-unknown-linux-musl': 'linux', + 'resolc-universal-apple-darwin': 'macos', + 'resolc-x86_64-pc-windows-msvc': 'windows', + 'resolc-web.js': 'wasm' + } + + # Process each asset + for asset in data.keys(): + platform_name = platform_mapping.get(asset) + if platform_name: + platform_folder = os.path.join(platform_name) + asset_json = generate_asset_json_nightly(asset, data[asset]['url'], data[asset]['sha']) + save_platform_json(platform_folder, asset_json) + print(f"Processed {asset} for {platform_name}") + +if __name__ == "__main__": + main() \ No newline at end of file diff --git a/.github/workflows/release-nightly.yml b/.github/workflows/release-nightly.yml new file mode 100644 index 0000000..d27f697 --- /dev/null +++ b/.github/workflows/release-nightly.yml @@ -0,0 +1,385 @@ +name: Nightly Release +on: + schedule: + # Run every day at 01:00 UTC + - cron: "0 1 * * *" + +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true + +env: + CARGO_TERM_COLOR: always + RUST_MUSL_CROSS_IMAGE: messense/rust-musl-cross@sha256:c0154e992adb791c3b848dd008939d19862549204f8cb26f5ca7a00f629e6067 + +jobs: + # check if there were commits yesterday + check_commits: + runs-on: ubuntu-latest + outputs: + has_commits: ${{ steps.check_commits.outputs.has_commits }} + steps: + - name: Checkout repository + uses: actions/checkout@v4 + with: + fetch-depth: 0 # Fetch full history to check previous commits + ref: "main" + + - name: Check for commits from yesterday + id: check_commits + run: | + # Get yesterday's date in YYYY-MM-DD format + YESTERDAY=$(date -d "yesterday" +%Y-%m-%d) + echo "Checking for commits from: $YESTERDAY" + + # Check if there were any commits yesterday + COMMIT_COUNT=$(git log --oneline --since="$YESTERDAY 00:00:00" --until="$YESTERDAY 23:59:59" | wc -l) + + echo "Found $COMMIT_COUNT commits from yesterday" + + if [ $COMMIT_COUNT -gt 0 ]; then + echo "has_commits=true" >> $GITHUB_OUTPUT + echo "✅ Found $COMMIT_COUNT commits from yesterday - continuing workflow" + else + echo "has_commits=false" >> $GITHUB_OUTPUT + echo "❌ No commits found from yesterday - skipping remaining steps" + echo "::notice::❌ No commits found from yesterday - skipping remaining steps" + fi + + build: + # github actions matrix jobs don't support multiple outputs + # ugly workaround from https://github.com/orgs/community/discussions/17245#discussioncomment-11222880 + if: ${{ needs.check_commits.outputs.has_commits == 'true' }} + outputs: + resolc-x86_64-unknown-linux-musl_url: ${{ steps.set-output.outputs.resolc-x86_64-unknown-linux-musl_url }} + resolc-x86_64-unknown-linux-musl_sha: ${{ steps.set-output.outputs.resolc-x86_64-unknown-linux-musl_sha }} + resolc-aarch64-apple-darwin_url: ${{ steps.set-output.outputs.resolc-aarch64-apple-darwin_url }} + resolc-aarch64-apple-darwin_sha: ${{ steps.set-output.outputs.resolc-aarch64-apple-darwin_sha }} + resolc-x86_64-apple-darwin_url: ${{ steps.set-output.outputs.resolc-x86_64-apple-darwin_url }} + resolc-x86_64-apple-darwin_sha: ${{ steps.set-output.outputs.resolc-x86_64-apple-darwin_sha }} + resolc-x86_64-pc-windows-msvc_url: ${{ steps.set-output.outputs.resolc-x86_64-pc-windows-msvc_url }} + resolc-x86_64-pc-windows-msvc_sha: ${{ steps.set-output.outputs.resolc-x86_64-pc-windows-msvc_sha }} + strategy: + matrix: + target: + [ + x86_64-unknown-linux-musl, + aarch64-apple-darwin, + x86_64-apple-darwin, + x86_64-pc-windows-msvc, + ] + include: + - target: x86_64-unknown-linux-musl + type: musl + runner: ubuntu-24.04 + - target: aarch64-apple-darwin + type: native + runner: macos-14 + - target: x86_64-apple-darwin + type: native + runner: macos-13 + - target: x86_64-pc-windows-msvc + type: native + runner: windows-2022 + runs-on: ${{ matrix.runner }} + needs: [check_commits] + steps: + - uses: actions/checkout@v4 + - uses: actions-rust-lang/setup-rust-toolchain@v1 + with: + # without this it will override our rust flags + rustflags: "" + cache-key: ${{ matrix.target }} + + - name: Download LLVM + uses: ./.github/actions/get-llvm + with: + target: ${{ matrix.target }} + + - name: Build + if: ${{ matrix.type == 'native' }} + shell: bash + run: | + export LLVM_SYS_181_PREFIX=$PWD/llvm-${{ matrix.target }} + make install-bin + mv target/release/resolc resolc-${{ matrix.target }} || mv target/release/resolc.exe resolc-${{ matrix.target }}.exe + + - name: Build + if: ${{ matrix.type == 'musl' }} + run: | + docker run -v $PWD:/opt/revive $RUST_MUSL_CROSS_IMAGE /bin/bash -c " + cd /opt/revive + chown -R root:root . + apt update && apt upgrade -y && apt install -y pkg-config + export LLVM_SYS_181_PREFIX=/opt/revive/llvm-${{ matrix.target }} + make install-bin + mv target/${{ matrix.target }}/release/resolc resolc-${{ matrix.target }} + " + sudo chown -R $(id -u):$(id -g) . + + - name: Install Solc + uses: ./.github/actions/get-solc + + - name: Basic Sanity Check + shell: bash + run: | + result=$(./resolc-${{ matrix.target }} --bin crates/integration/contracts/flipper.sol) + echo $result + if [[ $result == *'0x50564d'* ]]; then exit 0; else exit 1; fi + + - name: Upload artifacts (nightly) + uses: actions/upload-artifact@v4 + id: artifact-upload-step + with: + name: resolc-${{ matrix.target }} + path: resolc-${{ matrix.target }}* + retention-days: 40 + + - name: Set output variables (nightly) + id: set-output + shell: bash + run: | + echo "Artifact URL is ${{ steps.artifact-upload-step.outputs.artifact-url }}" + echo "Artifact SHA is ${{ steps.artifact-upload-step.outputs.artifact-digest }}" + echo "resolc-${{ matrix.target }}_url=${{ steps.artifact-upload-step.outputs.artifact-url }}" + echo "resolc-${{ matrix.target }}_url=${{ steps.artifact-upload-step.outputs.artifact-url }}" >> "$GITHUB_OUTPUT" + echo "resolc-${{ matrix.target }}_sha=${{ steps.artifact-upload-step.outputs.artifact-digest }}" + echo "resolc-${{ matrix.target }}_sha=${{ steps.artifact-upload-step.outputs.artifact-digest }}" >> "$GITHUB_OUTPUT" + + build-wasm: + runs-on: ubuntu-24.04 + needs: [check_commits] + if: ${{ needs.check_commits.outputs.has_commits == 'true' }} + outputs: + resolc-web.js_url: ${{ steps.set-output.outputs.resolc_web_js_url }} + resolc-web.js_sha: ${{ steps.set-output.outputs.resolc_web_js_sha }} + env: + RELEASE_RESOLC_WASM_URI: https://github.com/paritytech/revive/releases/download/${{ github.ref_name }}/resolc.wasm + steps: + - uses: actions/checkout@v4 + - uses: actions-rust-lang/setup-rust-toolchain@v1 + with: + target: wasm32-unknown-emscripten + # without this it will override our rust flags + rustflags: "" + + - name: Download Host LLVM + uses: ./.github/actions/get-llvm + with: + target: x86_64-unknown-linux-gnu + + - name: Download Wasm LLVM + uses: ./.github/actions/get-llvm + with: + target: wasm32-unknown-emscripten + + - name: Download EMSDK + uses: ./.github/actions/get-emsdk + + - name: Build + run: | + export LLVM_SYS_181_PREFIX=$PWD/llvm-x86_64-unknown-linux-gnu + export REVIVE_LLVM_TARGET_PREFIX=$PWD/llvm-wasm32-unknown-emscripten + source emsdk/emsdk_env.sh + make install-wasm + chmod -x ./target/wasm32-unknown-emscripten/release/resolc.wasm + + - name: Set Up Node.js + uses: actions/setup-node@v3 + with: + node-version: "20" + + - name: Basic Sanity Check + run: | + mkdir -p solc + curl -sSLo solc/soljson.js https://github.com/ethereum/solidity/releases/download/v0.8.30/soljson.js + node -e " + const soljson = require('solc/soljson'); + const createRevive = require('./target/wasm32-unknown-emscripten/release/resolc.js'); + + const compiler = createRevive(); + compiler.soljson = soljson; + + const standardJsonInput = + { + language: 'Solidity', + sources: { + 'MyContract.sol': { + content: 'pragma solidity ^0.8.0; contract MyContract { function greet() public pure returns (string memory) { return \'Hello\'; } }', + }, + }, + settings: { optimizer: { enabled: false } } + }; + + compiler.writeToStdin(JSON.stringify(standardJsonInput)); + compiler.callMain(['--standard-json']); + + // Collect output + const stdout = compiler.readFromStdout(); + const stderr = compiler.readFromStderr(); + + if (stderr) { console.error(stderr); process.exit(1); } + + let out = JSON.parse(stdout); + let bytecode = out.contracts['MyContract.sol']['MyContract'].evm.bytecode.object + console.log(bytecode); + + if(!bytecode.startsWith('50564d')) { process.exit(1); } + " + + - name: Compress Artifact + run: | + mkdir -p resolc-wasm32-unknown-emscripten + mv ./target/wasm32-unknown-emscripten/release/resolc.js ./resolc-wasm32-unknown-emscripten/ + mv ./target/wasm32-unknown-emscripten/release/resolc.wasm ./resolc-wasm32-unknown-emscripten/ + mv ./target/wasm32-unknown-emscripten/release/resolc_web.js ./resolc-wasm32-unknown-emscripten/ + + # There is no way to upload several files as several artifacts with a single upload-artifact step + # It's needed to have resolc_web.js separately for night builds for resolc-bin repo + # https://github.com/actions/upload-artifact/issues/331 + - name: Upload artifact resolc.js (nightly) + uses: actions/upload-artifact@v4 + with: + name: resolc.js + path: resolc-wasm32-unknown-emscripten/resolc.js + retention-days: 40 + + - name: Upload artifacts resolc.wasm (nightly) + uses: actions/upload-artifact@v4 + with: + name: resolc.wasm + path: resolc-wasm32-unknown-emscripten/resolc.wasm + retention-days: 40 + + - name: Upload artifacts resolc_web.js (nightly) + uses: actions/upload-artifact@v4 + id: artifact-upload-step + with: + name: resolc_web.js + path: resolc-wasm32-unknown-emscripten/resolc_web.js + retention-days: 40 + + - name: Set output variables + id: set-output + env: + TARGET: resolc_web_js + run: | + echo "Artifact URL is ${{ steps.artifact-upload-step.outputs.artifact-url }}" + echo "Artifact SHA is ${{ steps.artifact-upload-step.outputs.artifact-digest }}" + echo "${TARGET}_url=${{ steps.artifact-upload-step.outputs.artifact-url }}" + echo "${TARGET}_url=${{ steps.artifact-upload-step.outputs.artifact-url }}" >> "$GITHUB_OUTPUT" + echo "${TARGET}_sha=${{ steps.artifact-upload-step.outputs.artifact-digest }}"" + echo "${TARGET}_sha=${{ steps.artifact-upload-step.outputs.artifact-digest }}"" >> "$GITHUB_OUTPUT" + + create-macos-fat-binary: + if: ${{ needs.check_commits.outputs.has_commits == 'true' }} + needs: [build] + outputs: + resolc-universal-apple-darwin_url: ${{ steps.set-output.outputs.resolc-universal-apple-darwin_url }} + resolc-universal-apple-darwin_sha: ${{ steps.set-output.outputs.resolc-universal-apple-darwin_sha }} + runs-on: macos-14 + steps: + - uses: actions/download-artifact@v4 + with: + merge-multiple: true + + - name: Create macOS Fat Binary + run: | + lipo resolc-aarch64-apple-darwin resolc-x86_64-apple-darwin -create -output resolc-universal-apple-darwin + + - name: Make Executable + run: | + chmod +x resolc-universal-apple-darwin + + - uses: actions/upload-artifact@v4 + id: artifact-upload-step + with: + name: resolc-universal-apple-darwin + path: resolc-universal-apple-darwin + retention-days: 40 + + - name: Set output variables + id: set-output + env: + TARGET: resolc-universal-apple-darwin + run: | + echo "Artifact URL is ${{ steps.artifact-upload-step.outputs.artifact-url }}" + echo "Artifact SHA is ${{ steps.artifact-upload-step.outputs.artifact-digest }}" + echo "${TARGET}_url=${{ steps.artifact-upload-step.outputs.artifact-url }}" + echo "${TARGET}_url=${{ steps.artifact-upload-step.outputs.artifact-url }}" >> "$GITHUB_OUTPUT" + echo "${TARGET}_sha=${{ steps.artifact-upload-step.outputs.artifact-digest }}"" + echo "${TARGET}_sha=${{ steps.artifact-upload-step.outputs.artifact-digest }}"" >> "$GITHUB_OUTPUT" + + generate-nightly-json: + runs-on: ubuntu-24.04 + if: ${{ needs.check_commits.outputs.has_commits == 'true' }} + environment: tags + needs: [build-wasm, build, create-macos-fat-binary, check_commits] + steps: + - name: Checkout revive + uses: actions/checkout@v4 + with: + path: revive + + - name: Checkout resolc-bin + uses: actions/checkout@v4 + with: + repository: paritytech/resolc-bin + path: resolc-bin + + - name: Download Artifacts + uses: actions/download-artifact@v4 + with: + merge-multiple: true + path: bins + + - uses: actions/create-github-app-token@v1 + id: app-token + with: + app-id: ${{ secrets.REVIVE_JSON_APP_ID }} + private-key: ${{ secrets.REVIVE_JSON_APP_KEY }} + owner: paritytech + repositories: resolc-bin + + - name: Generate JSON + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + TOKEN: ${{ steps.app-token.outputs.token }} + APP_NAME: "paritytech-revive-json" + Green: "\e[32m" + NC: "\e[0m" + run: | + echo '[' > data.json + echo '${{ toJSON(needs.build.outputs) }}' >> data.json + echo ',' >> data.json + echo '${{ toJSON(needs.build-wasm.outputs) }}' >> data.json + echo ',' >> data.json + echo '${{ toJSON(needs.create-macos-fat-binary.outputs) }}' >> data.json + echo ']' >> data.json + chmod +x bins/resolc-x86_64-unknown-linux-musl + export FIRST_SOLC_VERSION=$(./bins/resolc-x86_64-unknown-linux-musl --supported-solc-versions | cut -f 1 -d "," | tr -d ">=") + export LAST_SOLC_VERSION=$(./bins/resolc-x86_64-unknown-linux-musl --supported-solc-versions | cut -f 2 -d "," | tr -d "<=") + export FILEPATH=$(readlink -f data.json) + export TAG=$(cd revive;gh release list --json name,isLatest --jq '.[] | select(.isLatest)|.name') + cd resolc-bin + mkdir -p nightly + cd nightly + python3 ../../revive/.github/scripts/json_generator_nightly.py + cd .. + git status + + echo "${Green}Add new remote with gh app token${NC}" + git remote set-url origin $(git config remote.origin.url | sed "s/github.com/${APP_NAME}:${TOKEN}@github.com/g") + + echo "${Green}Remove http section that causes issues with gh app auth token${NC}" + sed -i.bak '/\[http/d' ./.git/config + sed -i.bak '/extraheader/d' ./.git/config + + git config user.email "ci@parity.io" + git config user.name "${APP_NAME}" + + git add nightly/ + git commit -m "Update nightly json" + git push origin main + + echo "::notice::nightly info.list files were successfully published to https://github.com/paritytech/resolc-bin" diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 5aa63da..4c33506 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -14,6 +14,7 @@ concurrency: env: CARGO_TERM_COLOR: always + # if changed, dont forget to update the env var in release-nightly.yml RUST_MUSL_CROSS_IMAGE: messense/rust-musl-cross@sha256:c0154e992adb791c3b848dd008939d19862549204f8cb26f5ca7a00f629e6067 jobs: