GHW for building and publishing docker images (#1391)

* add ghw and scripts for docker image deployment * debug * add permissions for content * fix path to the bin folder * add tags * rename env * fix path to docker file * make polkadot-parachain executable * fix typo * fix more typos * test * revert back use of working directory * mke bin executable in the artifacts folder * use cd instead of working directory * change path to cash * fix path to cash * change cache key * delete old flows * addressed PR comments * fix path * reorg docker files
2026-04-25 22:17:58 +00:00 · 2023-09-06 16:11:10 +02:00
parent 4c077b209b
commit eeb368ed9c
55 changed files with 783 additions and 676 deletions
@@ -0,0 +1,13 @@
+#!/usr/bin/env bash
+
+# Sample call:
+# $0 /path/to/folder_with_binary
+# This script replace the former dedicated Dockerfile
+# and shows how to use the generic binary_injected.dockerfile
+
+PROJECT_ROOT=`git rev-parse --show-toplevel`
+
+export BINARY=adder-collator,undying-collator
+export ARTIFACTS_FOLDER=$1
+
+$PROJECT_ROOT/docker/scripts/build-injected.sh
@@ -0,0 +1,23 @@
+#!/usr/bin/env bash
+
+TMP=$(mktemp -d)
+ENGINE=${ENGINE:-podman}
+
+# TODO: Switch to /bin/bash when the image is built from parity/base-bin
+
+# Fetch some binaries
+$ENGINE run --user root --rm -i \
+  --pull always \
+  -v "$TMP:/export" \
+  --entrypoint /usr/bin/bash \
+  paritypr/colander:master -c \
+  'cp "$(which adder-collator)" /export'
+
+$ENGINE run --user root --rm -i \
+  --pull always \
+  -v "$TMP:/export" \
+  --entrypoint /usr/bin/bash \
+  paritypr/colander:master -c \
+  'cp "$(which undying-collator)" /export'
+
+./build-injected.sh $TMP
@@ -0,0 +1,100 @@
+#!/usr/bin/env bash
+#set -e
+
+# This script allows building a Container Image from a Linux
+# binary that is injected into a base-image.
+
+ENGINE=${ENGINE:-podman}
+
+if [ "$ENGINE" == "podman" ]; then
+  PODMAN_FLAGS="--format docker"
+else
+  PODMAN_FLAGS=""
+fi
+
+CONTEXT=$(mktemp -d)
+REGISTRY=${REGISTRY:-docker.io}
+
+# The following line ensure we know the project root
+PROJECT_ROOT=${PROJECT_ROOT:-$(git rev-parse --show-toplevel)}
+DOCKERFILE=${DOCKERFILE:-docker/dockerfiles/binary_injected.Dockerfile}
+VERSION_TOML=$(grep "^version " $PROJECT_ROOT/Cargo.toml | grep -oE "([0-9\.]+-?[0-9]+)")
+
+#n The following VAR have default that can be overriden
+DOCKER_OWNER=${DOCKER_OWNER:-parity}
+
+# We may get 1..n binaries, comma separated
+BINARY=${BINARY:-polkadot}
+IFS=',' read -r -a BINARIES <<< "$BINARY"
+
+VERSION=${VERSION:-$VERSION_TOML}
+ARTIFACTS_FOLDER=${ARTIFACTS_FOLDER:-.}
+
+IMAGE=${IMAGE:-${REGISTRY}/${DOCKER_OWNER}/${BINARIES[0]}}
+DESCRIPTION_DEFAULT="Injected Container image built for ${BINARY}"
+DESCRIPTION=${DESCRIPTION:-$DESCRIPTION_DEFAULT}
+
+VCS_REF=${VCS_REF:-01234567}
+
+# Build the image
+echo "Using engine: $ENGINE"
+echo "Using Dockerfile: $DOCKERFILE"
+echo "Using context: $CONTEXT"
+echo "Building ${IMAGE}:latest container image for ${BINARY} v${VERSION} from ${ARTIFACTS_FOLDER} hang on!"
+echo "ARTIFACTS_FOLDER=$ARTIFACTS_FOLDER"
+echo "CONTEXT=$CONTEXT"
+
+# We need all binaries and resources available in the Container build "CONTEXT"
+mkdir -p $CONTEXT/bin
+for bin in "${BINARIES[@]}"
+do
+  echo "Copying $ARTIFACTS_FOLDER/$bin to context: $CONTEXT/bin"
+  ls -al "$ARTIFACTS_FOLDER/$bin"
+  cp -r "$ARTIFACTS_FOLDER/$bin" "$CONTEXT/bin"
+done
+
+cp "$PROJECT_ROOT/docker/scripts/entrypoint.sh" "$CONTEXT"
+
+if [[ "$BINARY" == "polkadot-parachain" ]]; then
+  mkdir -p "$CONTEXT/specs"
+  echo "Copying parachains chain-specs from $ARTIFACTS_FOLDER/specs to context: $CONTEXT/specs"
+  ls -al "$ARTIFACTS_FOLDER/specs"
+  cp -r "$ARTIFACTS_FOLDER/specs" "$CONTEXT/specs"
+fi
+
+echo "Building image: ${IMAGE}"
+
+TAGS=${TAGS[@]:-latest}
+IFS=',' read -r -a TAG_ARRAY <<< "$TAGS"
+TAG_ARGS=" "
+
+echo "The image ${IMAGE} will be tagged with ${TAG_ARRAY[*]}"
+for tag in "${TAG_ARRAY[@]}"; do
+  TAG_ARGS+="--tag ${IMAGE}:${tag} "
+done
+
+echo "$TAG_ARGS"
+
+# time \
+$ENGINE build \
+    ${PODMAN_FLAGS} \
+    --build-arg VCS_REF="${VCS_REF}" \
+    --build-arg BUILD_DATE=$(date -u '+%Y-%m-%dT%H:%M:%SZ') \
+    --build-arg IMAGE_NAME="${IMAGE}" \
+    --build-arg BINARY="${BINARY}" \
+    --build-arg ARTIFACTS_FOLDER="${ARTIFACTS_FOLDER}" \
+    --build-arg DESCRIPTION="${DESCRIPTION}" \
+    ${TAG_ARGS} \
+    -f "${PROJECT_ROOT}/${DOCKERFILE}" \
+    ${CONTEXT}
+
+echo "Your Container image for ${IMAGE} is ready"
+$ENGINE images
+
+if [[ -z "${SKIP_IMAGE_VALIDATION}" ]]; then
+  echo "Check the image ${IMAGE}:${TAG_ARRAY[0]}"
+  $ENGINE run --rm -i "${IMAGE}:${TAG_ARRAY[0]}" --version
+
+  echo "Query binaries"
+  $ENGINE run --rm -i --entrypoint /bin/bash "${IMAGE}:${TAG_ARRAY[0]}" -c "echo BINARY: ${BINARY}"
+fi
@@ -0,0 +1,18 @@
+#!/usr/bin/env bash
+
+# Sanity check
+if [ -z "$BINARY" ]
+then
+    echo "BINARY ENV not defined, this should never be the case. Aborting..."
+    exit 1
+fi
+
+# If the user built the image with multiple binaries,
+# we consider the first one to be the canonical one
+# To start with another binary, the user can either:
+#  - use the --entrypoint option
+#  - pass the ENV BINARY with a single binary
+IFS=',' read -r -a BINARIES <<< "$BINARY"
+BIN0=${BINARIES[0]}
+echo "Starting binary $BIN0"
+$BIN0 $@
@@ -0,0 +1,14 @@
+#!/usr/bin/env bash
+
+# Sample call:
+# $0 /path/to/folder_with_binary
+# This script replace the former dedicated Dockerfile
+# and shows how to use the generic binary_injected.dockerfile
+
+PROJECT_ROOT=`git rev-parse --show-toplevel`
+
+export BINARY=malus,polkadot-execute-worker,polkadot-prepare-worker
+export ARTIFACTS_FOLDER=$1
+# export TAGS=...
+
+$PROJECT_ROOT/docker/scripts/build-injected.sh
@@ -0,0 +1,19 @@
+#!/usr/bin/env bash
+
+TMP=$(mktemp -d)
+ENGINE=${ENGINE:-podman}
+
+export TAGS=latest,beta,7777,1.0.2-rc23
+
+# Fetch some binaries
+$ENGINE run --user root --rm -i \
+  --pull always \
+  -v "$TMP:/export" \
+  --entrypoint /bin/bash \
+  paritypr/malus:7217 -c \
+  'cp "$(which malus)" /export'
+
+echo "Checking binaries we got:"
+ls -al $TMP
+
+./build-injected.sh $TMP
@@ -0,0 +1,15 @@
+#!/usr/bin/env bash
+
+# Sample call:
+# $0 /path/to/folder_with_binary
+# This script replace the former dedicated Dockerfile
+# and shows how to use the generic binary_injected.dockerfile
+
+PROJECT_ROOT=`git rev-parse --show-toplevel`
+
+export BINARY=polkadot-parachain
+export ARTIFACTS_FOLDER=$1
+export DOCKERFILE="docker/dockerfiles/polkadot-parachain/polkadot-parachain_injected.Dockerfile"
+# export TAGS=...
+
+$PROJECT_ROOT/docker/scripts/build-injected.sh
@@ -0,0 +1,19 @@
+#!/usr/bin/env bash
+
+TMP=$(mktemp -d)
+ENGINE=${ENGINE:-podman}
+
+export TAGS=latest,beta,7777,1.0.2-rc23
+
+# Fetch some binaries
+$ENGINE run --user root --rm -i \
+  --pull always \
+  -v "$TMP:/export" \
+  --entrypoint /bin/bash \
+  parity/polkadot-parachain:7217 -c \
+  'cp "$(which malus)" /export'
+
+echo "Checking binaries we got:"
+ls -al $TMP
+
+./build-injected.sh $TMP
@@ -0,0 +1,10 @@
+#!/usr/bin/env bash
+
+OWNER=${OWNER:-parity}
+IMAGE_NAME=${IMAGE_NAME:-polkadot-parachain}
+
+docker build --no-cache \
+    --build-arg IMAGE_NAME=$IMAGE_NAME \
+    -t $OWNER/$IMAGE_NAME \
+    -f ./docker/dockerfiles/polkadot-parachain/polkadot-parachain_injected.Dockerfile \
+    . && docker images
@@ -0,0 +1,13 @@
+#!/usr/bin/env bash
+
+# Sample call:
+# $0 /path/to/folder_with_binary
+# This script replace the former dedicated Dockerfile
+# and shows how to use the generic binary_injected.dockerfile
+
+PROJECT_ROOT=`git rev-parse --show-toplevel`
+
+export BINARY=polkadot,polkadot-execute-worker,polkadot-prepare-worker
+export ARTIFACTS_FOLDER=$1
+
+$PROJECT_ROOT/docker/scripts/build-injected.sh
@@ -0,0 +1,18 @@
+#!/usr/bin/env bash
+
+TMP=$(mktemp -d)
+ENGINE=${ENGINE:-podman}
+
+# You need to build an injected image first
+
+# Fetch some binaries
+$ENGINE run --user root --rm -i \
+  -v "$TMP:/export" \
+  --entrypoint /bin/bash \
+  parity/polkadot -c \
+  'cp "$(which polkadot)" /export'
+
+echo "Checking binaries we got:"
+tree $TMP
+
+./build-injected.sh $TMP
@@ -0,0 +1,37 @@
+# staking-miner container image
+
+## Build using the Builder
+
+```
+./build.sh
+```
+
+## Build the injected Image
+
+You first need a valid Linux binary to inject. Let's assume this binary is located in `BIN_FOLDER`.
+
+```
+./build-injected.sh "$BIN_FOLDER"
+```
+
+## Test
+
+Here is how to test the image. We can generate a valid seed but the staking-miner will quickly notice that our
+account is not funded and "does not exist".
+
+You may pass any ENV supported by the binary and must provide at least a few such as `SEED` and `URI`:
+```
+ENV SEED=""
+ENV URI="wss://rpc.polkadot.io:443"
+ENV RUST_LOG="info"
+```
+
+```
+export SEED=$(subkey generate -n polkadot --output-type json  | jq -r .secretSeed)
+podman run --rm -it \
+    -e URI="wss://rpc.polkadot.io:443" \
+    -e RUST_LOG="info" \
+    -e SEED \
+    localhost/parity/staking-miner \
+        dry-run seq-phragmen
+```
@@ -0,0 +1,13 @@
+#!/usr/bin/env bash
+
+# Sample call:
+# $0 /path/to/folder_with_staking-miner_binary
+# This script replace the former dedicated staking-miner "injected" Dockerfile
+# and shows how to use the generic binary_injected.dockerfile
+
+PROJECT_ROOT=`git rev-parse --show-toplevel`
+
+export BINARY=staking-miner
+export ARTIFACTS_FOLDER=$1
+
+$PROJECT_ROOT/docker/scripts/build-injected.sh
@@ -0,0 +1,13 @@
+#!/usr/bin/env bash
+
+# Sample call:
+# $0 /path/to/folder_with_staking-miner_binary
+# This script replace the former dedicated staking-miner "injected" Dockerfile
+# and shows how to use the generic binary_injected.dockerfile
+
+PROJECT_ROOT=`git rev-parse --show-toplevel`
+ENGINE=podman
+
+echo "Building the staking-miner using the Builder image"
+echo "PROJECT_ROOT=$PROJECT_ROOT"
+$ENGINE build -t staking-miner -f "${PROJECT_ROOT}/docker/dockerfiles/staking-miner/staking-miner_builder.Dockerfile" "$PROJECT_ROOT"
@@ -0,0 +1,3 @@
+# Staking-miner Docker image
+
+## [GitHub](https://github.com/paritytech/polkadot/tree/master/utils/staking-miner)
@@ -0,0 +1,43 @@
+FROM paritytech/ci-linux:production as builder
+
+# metadata
+ARG VCS_REF
+ARG BUILD_DATE
+ARG IMAGE_NAME="staking-miner"
+ARG PROFILE=production
+
+LABEL description="This is the build stage. Here we create the binary."
+
+WORKDIR /app
+COPY . /app
+RUN cargo build --locked --profile $PROFILE --package staking-miner
+
+# ===== SECOND STAGE ======
+
+FROM docker.io/parity/base-bin:latest
+LABEL description="This is the 2nd stage: a very small image where we copy the binary."
+LABEL io.parity.image.authors="devops-team@parity.io" \
+	io.parity.image.vendor="Parity Technologies" \
+	io.parity.image.title="${IMAGE_NAME}" \
+	io.parity.image.description="${IMAGE_NAME} for substrate based chains" \
+	io.parity.image.source="https://github.com/paritytech/polkadot/blob/${VCS_REF}/scripts/ci/dockerfiles/${IMAGE_NAME}/${IMAGE_NAME}_builder.Dockerfile" \
+	io.parity.image.revision="${VCS_REF}" \
+	io.parity.image.created="${BUILD_DATE}" \
+	io.parity.image.documentation="https://github.com/paritytech/polkadot/"
+
+ARG PROFILE=release
+COPY --from=builder /app/target/$PROFILE/staking-miner /usr/local/bin
+
+# show backtraces
+ENV RUST_BACKTRACE 1
+
+USER parity
+
+ENV SEED=""
+ENV URI="wss://rpc.polkadot.io"
+ENV RUST_LOG="info"
+
+# check if the binary works in this container
+RUN /usr/local/bin/staking-miner --version
+
+ENTRYPOINT [ "/usr/local/bin/staking-miner" ]
@@ -0,0 +1,18 @@
+#!/usr/bin/env bash
+
+TMP=$(mktemp -d)
+ENGINE=${ENGINE:-podman}
+
+# You need to build an injected image first
+
+# Fetch some binaries
+$ENGINE run --user root --rm -i \
+  -v "$TMP:/export" \
+  --entrypoint /bin/bash \
+  parity/staking-miner -c \
+  'cp "$(which staking-miner)" /export'
+
+echo "Checking binaries we got:"
+tree $TMP
+
+./build-injected.sh $TMP