GHW for building and publishing docker images (#1391)

* add ghw and scripts for docker image deployment * debug * add permissions for content * fix path to the bin folder * add tags * rename env * fix path to docker file * make polkadot-parachain executable * fix typo * fix more typos * test * revert back use of working directory * mke bin executable in the artifacts folder * use cd instead of working directory * change path to cash * fix path to cash * change cache key * delete old flows * addressed PR comments * fix path * reorg docker files
2026-04-25 23:27:56 +00:00 · 2023-09-06 16:11:10 +02:00
parent 4c077b209b
commit eeb368ed9c
55 changed files with 783 additions and 676 deletions
@@ -0,0 +1,266 @@
+#!/bin/sh
+
+api_base="https://api.github.com/repos"
+
+# Function to take 2 git tags/commits and get any lines from commit messages
+# that contain something that looks like a PR reference: e.g., (#1234)
+sanitised_git_logs(){
+  git --no-pager log --pretty=format:"%s" "$1...$2" |
+  # Only find messages referencing a PR
+  grep -E '\(#[0-9]+\)' |
+  # Strip any asterisks
+  sed 's/^* //g'
+}
+
+# Checks whether a tag on github has been verified
+# repo: 'organization/repo'
+# tagver: 'v1.2.3'
+# Usage: check_tag $repo $tagver
+check_tag () {
+  repo=$1
+  tagver=$2
+  if [ -n "$GITHUB_RELEASE_TOKEN" ]; then
+    echo '[+] Fetching tag using privileged token'
+    tag_out=$(curl -H "Authorization: token $GITHUB_RELEASE_TOKEN" -s "$api_base/$repo/git/refs/tags/$tagver")
+  else
+    echo '[+] Fetching tag using unprivileged token'
+    tag_out=$(curl -H "Authorization: token $GITHUB_PR_TOKEN" -s "$api_base/$repo/git/refs/tags/$tagver")
+  fi
+  tag_sha=$(echo "$tag_out" | jq -r .object.sha)
+  object_url=$(echo "$tag_out" | jq -r .object.url)
+  if [ "$tag_sha" = "null" ]; then
+    return 2
+  fi
+  echo "[+] Tag object SHA: $tag_sha"
+  verified_str=$(curl -H "Authorization: token $GITHUB_RELEASE_TOKEN" -s "$object_url" | jq -r .verification.verified)
+  if [ "$verified_str" = "true" ]; then
+    # Verified, everything is good
+    return 0
+  else
+    # Not verified. Bad juju.
+    return 1
+  fi
+}
+
+# Checks whether a given PR has a given label.
+# repo: 'organization/repo'
+# pr_id: 12345
+# label: B1-silent
+# Usage: has_label $repo $pr_id $label
+has_label(){
+  repo="$1"
+  pr_id="$2"
+  label="$3"
+
+  # These will exist if the function is called in Gitlab.
+  # If the function's called in Github, we should have GITHUB_ACCESS_TOKEN set
+  # already.
+  if [ -n "$GITHUB_RELEASE_TOKEN" ]; then
+    GITHUB_TOKEN="$GITHUB_RELEASE_TOKEN"
+  elif [ -n "$GITHUB_PR_TOKEN" ]; then
+    GITHUB_TOKEN="$GITHUB_PR_TOKEN"
+  fi
+
+  out=$(curl -H "Authorization: token $GITHUB_TOKEN" -s "$api_base/$repo/pulls/$pr_id")
+  [ -n "$(echo "$out" | tr -d '\r\n' | jq ".labels | .[] | select(.name==\"$label\")")" ]
+}
+
+github_label () {
+  echo
+  echo "# run github-api job for labeling it ${1}"
+  curl -sS -X POST \
+    -F "token=${CI_JOB_TOKEN}" \
+    -F "ref=master" \
+    -F "variables[LABEL]=${1}" \
+    -F "variables[PRNO]=${CI_COMMIT_REF_NAME}" \
+    -F "variables[PROJECT]=paritytech/polkadot" \
+    "${GITLAB_API}/projects/${GITHUB_API_PROJECT}/trigger/pipeline"
+}
+
+# Formats a message into a JSON string for posting to Matrix
+# message: 'any plaintext message'
+# formatted_message: '<strong>optional message formatted in <em>html</em></strong>'
+# Usage: structure_message $content $formatted_content (optional)
+structure_message() {
+  if [ -z "$2" ]; then
+    body=$(jq -Rs --arg body "$1" '{"msgtype": "m.text", $body}' < /dev/null)
+  else
+    body=$(jq -Rs --arg body "$1" --arg formatted_body "$2" '{"msgtype": "m.text", $body, "format": "org.matrix.custom.html", $formatted_body}' < /dev/null)
+  fi
+  echo "$body"
+}
+
+# Post a message to a matrix room
+# body: '{body: "JSON string produced by structure_message"}'
+# room_id: !fsfSRjgjBWEWffws:matrix.parity.io
+# access_token: see https://matrix.org/docs/guides/client-server-api/
+# Usage: send_message $body (json formatted) $room_id $access_token
+send_message() {
+  curl -XPOST -d "$1" "https://m.parity.io/_matrix/client/r0/rooms/$2/send/m.room.message?access_token=$3"
+}
+
+# Pretty-printing functions
+boldprint () { printf "|\n| \033[1m%s\033[0m\n|\n" "${@}"; }
+boldcat () { printf "|\n"; while read -r l; do printf "| \033[1m%s\033[0m\n" "${l}"; done; printf "|\n" ; }
+
+skip_if_companion_pr() {
+  url="https://api.github.com/repos/paritytech/polkadot/pulls/${CI_COMMIT_REF_NAME}"
+  echo "[+] API URL: $url"
+
+  pr_title=$(curl -sSL -H "Authorization: token ${GITHUB_PR_TOKEN}" "$url" | jq -r .title)
+  echo "[+] PR title: $pr_title"
+
+  if echo "$pr_title" | grep -qi '^companion'; then
+    echo "[!] PR is a companion PR. Build is already done in substrate"
+    exit 0
+  else
+    echo "[+] PR is not a companion PR. Proceeding test"
+  fi
+}
+
+# Fetches the tag name of the latest release from a repository
+# repo: 'organisation/repo'
+# Usage: latest_release 'paritytech/polkadot'
+latest_release() {
+  curl -s "$api_base/$1/releases/latest" | jq -r '.tag_name'
+}
+
+# Check for runtime changes between two commits. This is defined as any changes
+# to /primitives/src/* and any *production* chains under /runtime
+has_runtime_changes() {
+  from=$1
+  to=$2
+
+  if git diff --name-only "${from}...${to}" \
+    | grep -q -e '^runtime/polkadot' -e '^runtime/kusama' -e '^primitives/src/' -e '^runtime/common'
+  then
+    return 0
+  else
+    return 1
+  fi
+}
+
+# given a bootnode and the path to a chainspec file, this function will create a new chainspec file
+# with only the bootnode specified and test whether that bootnode provides peers
+# The optional third argument is the index of the bootnode in the list of bootnodes, this is just used to pick an ephemeral
+# port for the node to run on. If you're only testing one, it'll just use the first ephemeral port
+# BOOTNODE: /dns/polkadot-connect-0.parity.io/tcp/443/wss/p2p/12D3KooWEPmjoRpDSUuiTjvyNDd8fejZ9eNWH5bE965nyBMDrB4o
+# CHAINSPEC_FILE: /path/to/polkadot.json
+check_bootnode(){
+    BOOTNODE=$1
+    BASE_CHAINSPEC=$2
+    RUNTIME=$(basename "$BASE_CHAINSPEC" | cut -d '.' -f 1)
+    MIN_PEERS=1
+
+    # Generate a temporary chainspec file containing only the bootnode we care about
+    TMP_CHAINSPEC_FILE="$RUNTIME.$(echo "$BOOTNODE" | tr '/' '_').tmp.json"
+    jq ".bootNodes = [\"$BOOTNODE\"] " < "$CHAINSPEC_FILE" > "$TMP_CHAINSPEC_FILE"
+
+    # Grab an unused port by binding to port 0 and then immediately closing the socket
+    # This is a bit of a hack, but it's the only way to do it in the shell
+    RPC_PORT=$(python -c "import socket; s=socket.socket(); s.bind(('', 0)); print(s.getsockname()[1]); s.close()")
+
+    echo "[+] Checking bootnode $BOOTNODE"
+    polkadot --chain "$TMP_CHAINSPEC_FILE" --no-mdns --rpc-port="$RPC_PORT" --tmp > /dev/null 2>&1 &
+    # Wait a few seconds for the node to start up
+    sleep 5
+    POLKADOT_PID=$!
+
+    MAX_POLLS=10
+    TIME_BETWEEN_POLLS=3
+    for _ in $(seq 1 "$MAX_POLLS"); do
+    # Check the health endpoint of the RPC node
+      PEERS="$(curl -s -X POST -H "Content-Type: application/json" --data '{"jsonrpc":"2.0","method":"system_health","params":[],"id":1}' http://localhost:"$RPC_PORT" | jq -r '.result.peers')"
+      # Sometimes due to machine load or other reasons, we don't get a response from the RPC node
+      # If $PEERS is an empty variable, make it 0 so we can still do the comparison
+      if [ -z "$PEERS" ]; then
+        PEERS=0
+      fi
+      if [ "$PEERS" -ge $MIN_PEERS ]; then
+        echo "[+] $PEERS peers found for $BOOTNODE"
+        echo "    Bootnode appears contactable"
+        kill $POLKADOT_PID
+        # Delete the temporary chainspec file now we're done running the node
+        rm "$TMP_CHAINSPEC_FILE"
+        return 0
+      fi
+      sleep "$TIME_BETWEEN_POLLS"
+    done
+    kill $POLKADOT_PID
+    # Delete the temporary chainspec file now we're done running the node
+    rm "$TMP_CHAINSPEC_FILE"
+    echo "[!] No peers found for $BOOTNODE"
+    echo "    Bootnode appears unreachable"
+    return 1
+}
+
+# Assumes the ENV are set:
+# - RELEASE_ID
+# - GITHUB_TOKEN
+# - REPO in the form paritytech/polkadot
+fetch_release_artifacts() {
+  echo "Release ID : $RELEASE_ID"
+  echo "Repo       : $REPO"
+  echo "Binary     : $BINARY"
+
+  curl -L -s \
+    -H "Accept: application/vnd.github+json" \
+    -H "Authorization: Bearer ${GITHUB_TOKEN}" \
+    -H "X-GitHub-Api-Version: 2022-11-28" \
+    https://api.github.com/repos/${REPO}/releases/${RELEASE_ID} > release.json
+
+  # Get Asset ids
+  ids=($(jq -r '.assets[].id' < release.json ))
+  count=$(jq '.assets|length' < release.json )
+
+  # Fetch artifacts
+  mkdir -p "./release-artifacts/${BINARY}"
+  pushd "./release-artifacts/${BINARY}" > /dev/null
+
+  iter=1
+  for id in "${ids[@]}"
+  do
+      echo " - $iter/$count: downloading asset id: $id..."
+      curl -s -OJ -L -H "Accept: application/octet-stream" \
+          -H "Authorization: Token ${GITHUB_TOKEN}" \
+          "https://api.github.com/repos/${REPO}/releases/assets/$id"
+      iter=$((iter + 1))
+  done
+
+  pwd
+  ls -al --color
+  popd > /dev/null
+}
+
+# Check the checksum for a given binary
+function check_sha256() {
+    echo "Checking SHA256 for $1"
+    shasum -qc $1.sha256
+}
+
+# Import GPG keys of the release team members
+# This is done in parallel as it can take a while sometimes
+function import_gpg_keys() {
+  GPG_KEYSERVER=${GPG_KEYSERVER:-"keyserver.ubuntu.com"}
+  SEC="9D4B2B6EB8F97156D19669A9FF0812D491B96798"
+  WILL="2835EAF92072BC01D188AF2C4A092B93E97CE1E2"
+  EGOR="E6FC4D4782EB0FA64A4903CCDB7D3555DD3932D3"
+  MARA="533C920F40E73A21EEB7E9EBF27AEA7E7594C9CF"
+  MORGAN="2E92A9D8B15D7891363D1AE8AF9E6C43F7F8C4CF"
+
+  echo "Importing GPG keys from $GPG_KEYSERVER in parallel"
+  for key in $SEC $WILL $EGOR $MARA $MORGAN; do
+    (
+      echo "Importing GPG key $key"
+      gpg --no-tty --quiet --keyserver $GPG_KEYSERVER --recv-keys $key
+      echo -e "5\ny\n" | gpg --no-tty --command-fd 0 --expert --edit-key $key trust;
+    ) &
+  done
+  wait
+}
+
+# Check the GPG signature for a given binary
+function check_gpg() {
+    echo "Checking GPG Signature for $1"
+    gpg --no-tty --verify -q $1.asc $1
+}
@@ -0,0 +1,206 @@
+name: Release - Publish Docker Image
+
+# This workflow listens to pubished releases or can be triggered manually.
+# It includes releases and rc candidates.
+# It fetches the binaries, checks sha256 and GPG
+# signatures, then builds an injected docker
+# image and publishes it.
+
+on:
+  #TODO: activate automated run later
+  # release:
+  #  types:
+  #    - published
+  workflow_dispatch:
+    inputs:
+      release_id:
+        description: |
+          Release ID.
+          You can find it using the command:
+          curl -s \
+            -H "Authorization: Bearer ${GITHUB_TOKEN}" https://api.github.com/repos/$OWNER/$REPO/releases | \
+            jq '.[] | { name: .name, id: .id }'
+        required: true
+        type: string
+      image_type:
+        description: Type of the image to be published
+        required: true
+        default: rc
+        type: choice
+        options:
+          - rc
+          - release
+      registry:
+        description: Container registry
+        required: true
+        type: string
+        default: docker.io
+      owner:
+        description: Owner of the container image repo
+        required: true
+        type: string
+        default: parity
+      binary:
+        description: Binary to be published
+        required: true
+        default: polkadot
+        type: choice
+        options:
+          - polkadot
+          - staking-miner
+          - polkadot-parachain
+
+permissions:
+  contents: write
+
+env:
+  RELEASE_ID: ${{ inputs.release_id }}
+  ENGINE: docker
+  REGISTRY: ${{ inputs.registry }}
+  GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+  DOCKER_OWNER: ${{ inputs.owner || github.repository_owner }}
+  REPO: ${{ github.repository }}
+  BINARY: ${{ inputs.binary }}
+  # EVENT_ACTION: ${{ github.event.action }}
+  EVENT_NAME: ${{ github.event_name }}
+  IMAGE_TYPE: ${{ inputs.image_type }}
+
+jobs:
+  fetch-artifacts:
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Checkout sources
+        uses: actions/checkout@3df4ab11eba7bda6032a0b82a6bb43b11571feac # v4.0.0
+
+      #TODO: this step will be needed when automated triggering will work
+        #this step runs only if the workflow is triggered automatically when new release is published
+        # if: ${{ env.EVENT_NAME == 'release' && env.EVENT_ACTION != '' && env.EVENT_ACTION == 'published' }}
+        # run: |
+        #   mkdir -p release-artifacts && cd release-artifacts
+
+        #   for f in $BINARY $BINARY.asc $BINARY.sha256; do
+        #     URL="https://github.com/${{ github.event.repository.full_name }}/releases/download/${{ github.event.release.tag_name }}/$f"
+        #     echo " - Fetching $f from $URL"
+        #     wget "$URL" -O "$f"
+        #   done
+        #   chmod a+x $BINARY
+        #   ls -al
+
+      - name: Fetch rc artifacts or release artifacts based on release id
+        #this step runs only if the workflow is triggered manually
+        if: ${{ env.EVENT_NAME  == 'workflow_dispatch' }}
+        run: |
+          . ./.github/scripts/common/lib.sh
+
+          fetch_release_artifacts
+
+      - name: Cache the artifacts
+        uses: actions/cache@88522ab9f39a2ea568f7027eddc7d8d8bc9d59c8 # v3.3.1
+        with:
+          key: artifacts-${{ env.BINARY }}-${{ github.sha }}
+          path: |
+            ./release-artifacts/${{ env.BINARY }}/**/*
+
+  build-container:
+    runs-on: ubuntu-latest
+    needs: fetch-artifacts
+
+    steps:
+      - name: Checkout sources
+        uses: actions/checkout@3df4ab11eba7bda6032a0b82a6bb43b11571feac # v4.0.0
+
+      - name: Get artifacts from cache
+        uses: actions/cache@88522ab9f39a2ea568f7027eddc7d8d8bc9d59c8 # v3.3.1
+        with:
+          key: artifacts-${{ env.BINARY }}-${{ github.sha }}
+          fail-on-cache-miss: true
+          path: |
+            ./release-artifacts/${{ env.BINARY }}/**/*
+
+      - name: Check sha256 ${{ env.BINARY }}
+        working-directory: ./release-artifacts/${{ env.BINARY }}
+        run: |
+          . ../../.github/scripts/common/lib.sh
+
+          echo "Checking binary $BINARY"
+          check_sha256 $BINARY && echo "OK" || echo "ERR"
+
+      - name: Check GPG ${{ env.BINARY }}
+        working-directory: ./release-artifacts/${{ env.BINARY }}
+        run: |
+          . ../../.github/scripts/common/lib.sh
+          import_gpg_keys
+          check_gpg $BINARY
+
+      - name: Fetch rc commit and tag
+        if: ${{ env.IMAGE_TYPE == 'rc' }}
+        id: fetch_rc_refs
+        run: |
+          release=release-${{ inputs.release_id }} && \
+          echo "release=${release}" >> $GITHUB_OUTPUT
+
+          commit=$(git rev-parse --short HEAD) && \
+          echo "commit=${commit}" >> $GITHUB_OUTPUT
+
+          tag=$(git name-rev --tags --name-only $(git rev-parse HEAD)) && \
+          [ "${tag}" != "undefined" ] && echo "tag=${tag}" >> $GITHUB_OUTPUT || \
+          echo "No tag, doing without"
+
+      - name: Fetch release tags
+        working-directory: ./release-artifacts/${{ env.BINARY }}
+        if: ${{ env.IMAGE_TYPE == 'release'}}
+        id: fetch_release_refs
+        run: |
+          chmod a+rx $BINARY
+          VERSION=$(./$BINARY --version | awk '{ print $2 }' )
+          release=$( echo $VERSION | cut -f1 -d- )
+          echo "tag=latest" >> $GITHUB_OUTPUT
+          echo "release=${release}" >> $GITHUB_OUTPUT
+
+      - name: Build Injected Container image for polkadot/staking-miner
+        if: ${{ env.BINARY == 'polkadot' || env.BINARY == 'staking-miner' }}
+        env:
+          ARTIFACTS_FOLDER: ./release-artifacts
+          IMAGE_NAME: ${{ env.BINARY }}
+          OWNER: ${{ env.DOCKER_OWNER }}
+          TAGS: ${{ join(steps.fetch_rc_refs.outputs.*, ',') || join(steps.fetch_release_refs.outputs.*, ',') }}
+        run: |
+          ls -al
+          echo "Building container for $BINARY"
+          ./docker/scripts/build-injected.sh
+
+      - name: Build Injected Container image for polkadot-parachain
+        if: ${{ env.BINARY == 'polkadot-parachain' }}
+        env:
+          ARTIFACTS_FOLDER: ./release-artifacts
+          IMAGE_NAME: ${{ env.BINARY }}
+          OWNER: ${{ env.DOCKER_OWNER }}
+          DOCKERFILE: docker/dockerfiles/polkadot-parachain/polkadot-parachain_injected.Dockerfile
+          TAGS: ${{ join(steps.fetch_rc_refs.outputs.*, ',') || join(steps.fetch_release_refs.outputs.*, ',') }}
+        run: |
+          ls -al
+          mkdir -p $ARTIFACTS_FOLDER/specs
+          cp cumulus/parachains/chain-specs/*.json $ARTIFACTS_FOLDER/specs
+
+          echo "Building container for $BINARY"
+          ./docker/scripts/build-injected.sh
+
+      - name: Login to Dockerhub
+        uses: docker/login-action@v2
+        with:
+          username: ${{ secrets.DOCKERHUB_USERNAME }}
+          password: ${{ secrets.DOCKERHUB_TOKEN }}
+
+      - name: Push Container image for ${{ env.BINARY }}
+        id: docker_push
+        run: |
+          $ENGINE images | grep ${BINARY}
+          $ENGINE push --all-tags ${REGISTRY}/${DOCKER_OWNER}/${BINARY}
+
+      - name: Check version for the published image for ${{ env.BINARY }}
+        env:
+          RELEASE_TAG: ${{ steps.fetch_rc_refs.outputs.release || steps.fetch_release_refs.outputs.release  }}
+        run: |
+          echo "Checking tag ${RELEASE_TAG} for image ${REGISTRY}/${DOCKER_OWNER}/${BINARY}"
+          $ENGINE run -i ${REGISTRY}/${DOCKER_OWNER}/${BINARY}:${RELEASE_TAG} --version