GHW for building and publishing docker images (#1391)

* add ghw and scripts for docker image deployment

* debug

* add permissions for content

* fix path to the bin folder

* add tags

* rename env

* fix path to docker file

* make polkadot-parachain executable

* fix typo

* fix more typos

* test

* revert back  use of  working directory

* mke bin executable in the artifacts folder

* use cd instead of working directory

* change path to cash

* fix path to cash

* change cache key

* delete old flows

* addressed PR comments

* fix path

* reorg docker files
This commit is contained in:
Egor_P
2023-09-06 16:11:10 +02:00
committed by GitHub
parent 4c077b209b
commit eeb368ed9c
55 changed files with 783 additions and 676 deletions
-206
View File
@@ -1,206 +0,0 @@
name: Release - Publish Docker Image
# This workflow listens to pubished releases or can be triggered manually.
# It includes releases and rc candidates.
# It fetches the binaries, checks sha256 and GPG
# signatures, then builds an injected docker
# image and publishes it.
on:
release:
types:
- published
workflow_dispatch:
inputs:
release_id:
description: |
Release ID.
You can find it using the command:
curl -s \
-H "Authorization: Bearer ${GITHUB_TOKEN}" https://api.github.com/repos/$OWNER/$REPO/releases | \
jq '.[] | { name: .name, id: .id }'
required: true
type: string
image_type:
description: Type of the image to be published
required: true
default: rc
type: choice
options:
- rc
- release
registry:
description: Container registry
required: true
type: string
default: docker.io
owner:
description: Owner of the container image repo
required: true
type: string
default: parity
env:
RELEASE_ID: ${{ inputs.release_id }}
ENGINE: docker
REGISTRY: ${{ inputs.registry }}
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
DOCKER_OWNER: ${{ inputs.owner || github.repository_owner }}
REPO: ${{ github.repository }}
BINARY: polkadot-parachain
EVENT_ACTION: ${{ github.event.action }}
EVENT_NAME: ${{ github.event_name }}
IMAGE_TYPE: ${{ inputs.image_type }}
jobs:
fetch-artifacts:
runs-on: ubuntu-latest
steps:
- name: Checkout sources
uses: actions/checkout@c85c95e3d7251135ab7dc9ce3241c5835cc595a9 # v3.5.3
- name: Prepare temp folder
run: |
TMP=$(mktemp -d)
echo "TMP=$TMP" >> "$GITHUB_ENV"
pwd
ls -al "$TMP"
- name: Fetch lib.sh from polkadot repo
working-directory: ${{ env.TMP }}
run: |
curl -O -L \
-H "Accept: application/vnd.github.v3.raw" \
https://raw.githubusercontent.com/paritytech/polkadot/master/scripts/ci/common/lib.sh
chmod a+x lib.sh
ls -al
- name: Fetch release artifacts based on final release tag
#this step runs only if the workflow is triggered automatically when new release is published
if: ${{ env.EVENT_NAME == 'release' && env.EVENT_ACTION != '' && env.EVENT_ACTION == 'published' }}
run: |
mkdir -p release-artifacts && cd release-artifacts
for f in $BINARY $BINARY.asc $BINARY.sha256; do
URL="https://github.com/${{ github.event.repository.full_name }}/releases/download/${{ github.event.release.tag_name }}/$f"
echo " - Fetching $f from $URL"
wget "$URL" -O "$f"
done
chmod a+x $BINARY
cp -f ${TMP}/lib.sh .
ls -al
- name: Fetch rc artifacts or release artifacts based on release id
#this step runs only if the workflow is triggered manually
if: ${{ env.EVENT_NAME == 'workflow_dispatch' }}
run: |
. ${TMP}/lib.sh
fetch_release_artifacts
chmod a+x release-artifacts/$BINARY
ls -al
cp -f ${TMP}/lib.sh release-artifacts/
- name: Cache the artifacts
uses: actions/cache@88522ab9f39a2ea568f7027eddc7d8d8bc9d59c8 # v3.3.1
with:
key: artifacts-${{ github.sha }}
path: |
./release-artifacts/**/*
build-container:
runs-on: ubuntu-latest
needs: fetch-artifacts
steps:
- name: Checkout sources
uses: actions/checkout@c85c95e3d7251135ab7dc9ce3241c5835cc595a9 # v3.5.3
- name: Get artifacts from cache
uses: actions/cache@88522ab9f39a2ea568f7027eddc7d8d8bc9d59c8 # v3.3.1
with:
key: artifacts-${{ github.sha }}
fail-on-cache-miss: true
path: |
./release-artifacts/**/*
- name: Check sha256 ${{ env.BINARY }}
working-directory: ./release-artifacts
run: |
. ./lib.sh
echo "Checking binary $BINARY"
check_sha256 $BINARY && echo "OK" || echo "ERR"
- name: Check GPG ${{ env.BINARY }}
working-directory: ./release-artifacts
run: |
. ./lib.sh
import_gpg_keys
check_gpg $BINARY
- name: Build Injected Container image for ${{ env.BINARY }}
env:
IMAGE_NAME: ${{ env.BINARY }}
OWNER: ${{ env.DOCKER_OWNER }}
run: |
ls -al
echo "Building container for $BINARY"
./docker/scripts/build-injected-image.sh
- name: Fetch rc commit and tag
if: ${{ env.IMAGE_TYPE == 'rc' }}
id: fetch_rc_refs
run: |
release=release-${{ inputs.release_id }} && \
echo "release=${release}" >> $GITHUB_OUTPUT
commit=$(git rev-parse --short HEAD) && \
echo "commit=${commit}" >> $GITHUB_OUTPUT
tag=$(git name-rev --tags --name-only $(git rev-parse HEAD)) && \
[ "${tag}" != "undefined" ] && echo "tag=${tag}" >> $GITHUB_OUTPUT || \
echo "No tag, doing without"
- name: Fetch release tags
if: ${{ env.IMAGE_TYPE == 'release' || env.EVENT_NAME == 'release' && env.EVENT_ACTION != '' && env.EVENT_ACTION == 'published' }}
id: fetch_release_refs
run: |
VERSION=$(docker run --pull never --rm $DOCKER_OWNER/$BINARY --version | awk '{ print $2 }' )
release=$( echo $VERSION | cut -f1 -d- )
echo "tag=latest" >> $GITHUB_OUTPUT
echo "release=${release}" >> $GITHUB_OUTPUT
- name: Login to Dockerhub
uses: docker/login-action@v2
with:
username: ${{ secrets.DOCKERHUB_USERNAME }}
password: ${{ secrets.DOCKERHUB_TOKEN }}
- name: Tag and Push Container image for ${{ env.BINARY }}
id: docker_push
env:
TAGS: ${{ join(steps.fetch_rc_refs.outputs.*, ',') || join(steps.fetch_release_refs.outputs.*, ',') }}
run: |
TAGS=${TAGS[@]:-latest}
IFS=',' read -r -a TAG_ARRAY <<< "$TAGS"
echo "The image ${BINARY} will be tagged with ${TAG_ARRAY[*]}"
for TAG in "${TAG_ARRAY[@]}"; do
$ENGINE tag ${DOCKER_OWNER}/${BINARY} ${DOCKER_OWNER}/${BINARY}:${TAG}
$ENGINE push ${DOCKER_OWNER}/${BINARY}:${TAG}
done
$ENGINE images | grep ${BINARY}
- name: Check version for the published image for ${{ env.BINARY }}
env:
RELEASE_TAG: ${{ steps.fetch_rc_refs.outputs.release || steps.fetch_release_refs.outputs.release }}
run: |
echo "Checking tag ${RELEASE_TAG} for image ${REGISTRY}/${DOCKER_OWNER}/${BINARY}"
$ENGINE run -i ${REGISTRY}/${DOCKER_OWNER}/${BINARY}:${RELEASE_TAG} --version
-129
View File
@@ -1,129 +0,0 @@
version: '3.7'
services:
node_alice:
image: "polkadot:${BRANCH:-cumulus-branch}"
ports:
- "30333:30333"
- "9933:9933"
- "9944:9944"
volumes:
- "polkadot-data-alice:/data"
- type: bind
source: ./test/parachain/chain-specs/polkadot_chainspec.json
target: /chainspec.json
read_only: true
command: >
polkadot
--chain=/chainspec.json
--base-path=/data
--port 30333
--rpc-port 9933
--ws-port 9944
--rpc-external
--rpc-cors all
--ws-external
--alice
networks:
testing_net:
ipv4_address: 172.28.1.1
aliases:
- alice
node_bob:
image: "polkadot:${BRANCH:-cumulus-branch}"
ports:
- "30344:30333"
- "9935:9933"
- "9945:9944"
volumes:
- "polkadot-data-bob:/data"
- type: bind
source: ./test/parachain/chain-specs/polkadot_chainspec.json
target: /chainspec.json
read_only: true
command: >
polkadot
--chain=/chainspec.json
--base-path=/data
--port 30333
--rpc-port 9933
--ws-port 9944
--rpc-external
--ws-external
--rpc-cors all
--bob
networks:
testing_net:
ipv4_address: 172.28.1.2
aliases:
- bob
genesis_state:
build:
context: .
dockerfile: ./docker/test-parachain-collator.dockerfile
image: "ctpc:latest"
volumes:
- "genesis-state:/data"
command: >
polkadot-parachain
export-genesis-state
/data/genesis-state
collator:
build:
context: .
dockerfile: ./docker/test-parachain-collator.dockerfile
target: collator
image: "ctpc:collator"
volumes:
- "collator-data:/data"
depends_on:
- node_alice
- node_bob
command: >
inject_bootnodes.sh
--base-path=/data
networks:
testing_net:
runtime:
build:
context: .
dockerfile: ./docker/test-parachain-collator.dockerfile
target: runtime
image: "ctpc:runtime"
volumes:
- "parachain-runtime:/runtime"
registrar:
build:
context: .
dockerfile: ./docker/parachain-registrar.dockerfile
image: para-reg:latest
volumes:
- "genesis-state:/genesis"
- "parachain-runtime:/runtime"
depends_on:
- node_alice
- runtime
- genesis_state
networks:
testing_net:
volumes:
polkadot-data-alice:
polkadot-data-bob:
collator-data:
genesis-state:
parachain-runtime:
networks:
testing_net:
ipam:
driver: default
config:
- subnet: 172.28.0.0/16
-36
View File
@@ -1,36 +0,0 @@
FROM docker.io/parity/base-bin
# metadata
ARG VCS_REF
ARG BUILD_DATE
ARG IMAGE_NAME
LABEL io.parity.image.authors="devops-team@parity.io" \
io.parity.image.vendor="Parity Technologies" \
io.parity.image.title="${IMAGE_NAME}" \
io.parity.image.description="Cumulus, the Polkadot collator." \
io.parity.image.source="https://github.com/paritytech/polkadot/blob/${VCS_REF}/scripts/docker/Dockerfile" \
io.parity.image.revision="${VCS_REF}" \
io.parity.image.created="${BUILD_DATE}" \
io.parity.image.documentation="https://github.com/paritytech/cumulus/"
# show backtraces
ENV RUST_BACKTRACE 1
USER root
RUN mkdir -p /specs
# add polkadot-parachain binary to the docker image
COPY ./release-artifacts/* /usr/local/bin
COPY ./parachains/chain-specs/*.json /specs/
USER parity
# check if executable works in this container
RUN /usr/local/bin/polkadot-parachain --version
EXPOSE 30333 9933 9944 9615
VOLUME ["/polkadot", "/specs"]
ENTRYPOINT ["/usr/local/bin/polkadot-parachain"]
@@ -1,27 +0,0 @@
FROM node:latest AS pjs
# It would be great to depend on a more stable tag, but we need some
# as-yet-unreleased features.
RUN yarn global add @polkadot/api-cli@0.10.0-beta.14
ENTRYPOINT [ "polkadot-js-api" ]
CMD [ "--version" ]
# To use the pjs build stage to access the blockchain from the host machine:
#
# docker build -f docker/parachain-registrar.dockerfile --target pjs -t parachain-registrar:pjs .
# alias pjs='docker run --rm --net cumulus_testing_net parachain-registrar:pjs --ws ws://172.28.1.1:9944'
#
# Then, as long as the chain is running, you can use the polkadot-js-api CLI like:
#
# pjs query.sudo.key
FROM pjs
RUN apt-get update && apt-get install curl netcat -y && \
curl -sSo /wait-for-it.sh https://raw.githubusercontent.com/vishnubob/wait-for-it/master/wait-for-it.sh && \
chmod +x /wait-for-it.sh
# the only thing left to do is to actually run the transaction.
COPY ./docker/scripts/register_para.sh /usr/bin
# unset the previous stage's entrypoint
ENTRYPOINT []
CMD [ "/usr/bin/register_para.sh" ]
@@ -1,49 +0,0 @@
FROM docker.io/library/ubuntu:20.04
# metadata
ARG VCS_REF
ARG BUILD_DATE
ARG IMAGE_NAME
LABEL io.parity.image.authors="devops-team@parity.io" \
io.parity.image.vendor="Parity Technologies" \
io.parity.image.title="${IMAGE_NAME}" \
io.parity.image.description="Cumulus, the Polkadot collator." \
io.parity.image.source="https://github.com/paritytech/cumulus/blob/${VCS_REF}/scripts/docker/polkadot-parachain-debug_unsigned_injected.Dockerfile" \
io.parity.image.revision="${VCS_REF}" \
io.parity.image.created="${BUILD_DATE}" \
io.parity.image.documentation="https://github.com/paritytech/cumulus/"
# show backtraces
ENV RUST_BACKTRACE 1
# install tools and dependencies
RUN apt-get update && \
DEBIAN_FRONTEND=noninteractive apt-get install -y \
libssl1.1 \
ca-certificates \
curl && \
# apt cleanup
apt-get autoremove -y && \
apt-get clean && \
find /var/lib/apt/lists/ -type f -not -name lock -delete; \
# add user and link ~/.local/share/polkadot-parachain to /data
useradd -m -u 10000 -U -s /bin/sh -d /polkadot-parachain polkadot-parachain && \
mkdir -p /data /polkadot-parachain/.local/share && \
chown -R polkadot-parachain:polkadot-parachain /data && \
ln -s /data /polkadot-parachain/.local/share/polkadot-parachain && \
mkdir -p /specs
# add polkadot-parachain binary to the docker image
COPY ./artifacts/polkadot-parachain /usr/local/bin
COPY ./parachains/chain-specs/*.json /specs/
USER polkadot-parachain
# check if executable works in this container
RUN /usr/local/bin/polkadot-parachain --version
EXPOSE 30333 9933 9944
VOLUME ["/polkadot-parachain"]
ENTRYPOINT ["/usr/local/bin/polkadot-parachain"]
@@ -1,36 +0,0 @@
# This file is sourced from https://github.com/paritytech/polkadot/blob/master/scripts/ci/dockerfiles/polkadot/polkadot_builder.Dockerfile
# This is the build stage for polkadot-parachain. Here we create the binary in a temporary image.
FROM docker.io/paritytech/ci-linux:production as builder
WORKDIR /cumulus
COPY . /cumulus
RUN cargo build --release --locked -p polkadot-parachain
# This is the 2nd stage: a very small image where we copy the Polkadot binary."
FROM docker.io/library/ubuntu:20.04
LABEL io.parity.image.type="builder" \
io.parity.image.authors="devops-team@parity.io" \
io.parity.image.vendor="Parity Technologies" \
io.parity.image.description="Multistage Docker image for polkadot-parachain" \
io.parity.image.source="https://github.com/paritytech/polkadot/blob/${VCS_REF}/scripts/ci/dockerfiles/polkadot/polkadot-parachain_builder.Dockerfile" \
io.parity.image.documentation="https://github.com/paritytech/cumulus"
COPY --from=builder /cumulus/target/release/polkadot-parachain /usr/local/bin
RUN useradd -m -u 1000 -U -s /bin/sh -d /cumulus polkadot-parachain && \
mkdir -p /data /cumulus/.local/share && \
chown -R polkadot-parachain:polkadot-parachain /data && \
ln -s /data /cumulus/.local/share/polkadot-parachain && \
# unclutter and minimize the attack surface
rm -rf /usr/bin /usr/sbin && \
# check if executable works in this container
/usr/local/bin/polkadot-parachain --version
USER polkadot-parachain
EXPOSE 30333 9933 9944 9615
VOLUME ["/data"]
ENTRYPOINT ["/usr/local/bin/polkadot-parachain"]
@@ -1,10 +0,0 @@
#!/usr/bin/env bash
OWNER=${OWNER:-parity}
IMAGE_NAME=${IMAGE_NAME:-polkadot-parachain}
docker build --no-cache \
--build-arg IMAGE_NAME=$IMAGE_NAME \
-t $OWNER/$IMAGE_NAME \
-f ./docker/injected.Dockerfile \
. && docker images
@@ -1,46 +0,0 @@
# This file is sourced from https://github.com/paritytech/polkadot/blob/master/scripts/ci/dockerfiles/polkadot/polkadot_builder.Dockerfile
FROM docker.io/paritytech/ci-linux:production as builder
WORKDIR /cumulus
COPY . /cumulus
RUN cargo build --release --locked -p polkadot-parachain
# the collator stage is normally built once, cached, and then ignored, but can
# be specified with the --target build flag. This adds some extra tooling to the
# image, which is required for a launcher script. The script simply adds two
# arguments to the list passed in:
#
# --bootnodes /ip4/127.0.0.1/tcp/30333/p2p/PEER_ID
#
# with the appropriate ip and ID for both Alice and Bob
FROM debian:buster-slim as collator
RUN apt-get update && apt-get install jq curl bash -y && \
curl -sSo /wait-for-it.sh https://raw.githubusercontent.com/vishnubob/wait-for-it/master/wait-for-it.sh && \
chmod +x /wait-for-it.sh && \
curl -sL https://deb.nodesource.com/setup_12.x | bash - && \
apt-get install -y nodejs && \
npm install --global yarn && \
yarn global add @polkadot/api-cli@0.10.0-beta.14
COPY --from=builder \
/paritytech/cumulus/target/release/polkadot-parachain /usr/bin
COPY ./docker/scripts/inject_bootnodes.sh /usr/bin
CMD ["/usr/bin/inject_bootnodes.sh"]
COPY ./docker/scripts/healthcheck.sh /usr/bin/
HEALTHCHECK --interval=300s --timeout=75s --start-period=30s --retries=3 \
CMD ["/usr/bin/healthcheck.sh"]
# the runtime stage is normally built once, cached, and ignored, but can be
# specified with the --target build flag. This just preserves one of the builder's
# outputs, which can then be moved into a volume at runtime
FROM debian:buster-slim as runtime
COPY --from=builder \
/paritytech/cumulus/target/release/wbuild/cumulus-test-parachain-runtime/cumulus_test_parachain_runtime.compact.wasm \
/var/opt/
CMD ["cp", "-v", "/var/opt/cumulus_test_parachain_runtime.compact.wasm", "/runtime/"]
FROM debian:buster-slim
COPY --from=builder \
/paritytech/cumulus/target/release/polkadot-parachain /usr/bin
CMD ["/usr/bin/polkadot-parachain"]
@@ -1,49 +0,0 @@
FROM docker.io/library/ubuntu:20.04
# metadata
ARG VCS_REF
ARG BUILD_DATE
ARG IMAGE_NAME
LABEL io.parity.image.authors="devops-team@parity.io" \
io.parity.image.vendor="Parity Technologies" \
io.parity.image.title="${IMAGE_NAME}" \
io.parity.image.description="Test parachain for Zombienet" \
io.parity.image.source="https://github.com/paritytech/cumulus/blob/${VCS_REF}/docker/test-parachain_injected.Dockerfile" \
io.parity.image.revision="${VCS_REF}" \
io.parity.image.created="${BUILD_DATE}" \
io.parity.image.documentation="https://github.com/paritytech/cumulus/"
# show backtraces
ENV RUST_BACKTRACE 1
# install tools and dependencies
RUN apt-get update && \
DEBIAN_FRONTEND=noninteractive apt-get install -y \
libssl1.1 \
ca-certificates \
curl && \
# apt cleanup
apt-get autoremove -y && \
apt-get clean && \
find /var/lib/apt/lists/ -type f -not -name lock -delete; \
# add user and link ~/.local/share/test-parachain to /data
useradd -m -u 10000 -U -s /bin/sh -d /test-parachain test-parachain && \
mkdir -p /data /test-parachain/.local/share && \
chown -R test-parachain:test-parachain /data && \
ln -s /data /test-parachain/.local/share/test-parachain && \
mkdir -p /specs
# add test-parachain binary to the docker image
COPY ./artifacts/test-parachain /usr/local/bin
COPY ./parachains/chain-specs/*.json /specs/
USER test-parachain
# check if executable works in this container
RUN /usr/local/bin/test-parachain --version
EXPOSE 30333 9933 9944
VOLUME ["/test-parachain"]
ENTRYPOINT ["/usr/local/bin/test-parachain"]
-66
View File
@@ -1,66 +0,0 @@
# Using Containers
Using containers via **Podman** or **Docker** brings benefit, whether it is to build a container image or run a node
while keeping a minimum footprint on your local system.
This document mentions using `podman` or `docker`. Those are usually interchangeable and it is encouraged using
preferably **Podman**. If you have podman installed and want to use all the commands mentioned below, you can simply
create an alias with `alias docker=podman`.
There are a few options to build a node within a container and inject a binary inside an image.
## Parity built container image
Parity builds and publishes a container image that can be found as `docker.io/parity/polkadot-parachain`.
## Parity CI image
Parity maintains and uses internally a generic "CI" image that can be used as a base to build binaries: [Parity CI
container image](https://github.com/paritytech/scripts/tree/master/dockerfiles/ci-linux):
The command below allows building a Linux binary without having to even install Rust or any dependency locally:
```bash
docker run --rm -it \
-w /shellhere/cumulus \
-v $(pwd):/shellhere/cumulus \
paritytech/ci-linux:production
cargo build --release --locked --bin polkadot-parachain
sudo chown -R $(id -u):$(id -g) target/
```
If you want to reproduce other steps of CI process you can use the following
[guide](https://github.com/paritytech/scripts#gitlab-ci-for-building-docker-images).
## Injected image
Injecting a binary inside a base image is the quickest option to get a working container image. This only works if you
were able to build a Linux binary, either locally, or using a container as described above.
After building a Linux binary ()`polkadot-parachain`) with cargo or with Parity CI image as documented above, the
following command allows producing a new container image where the compiled binary is injected:
```bash
./docker/scripts/build-injected-image.sh
```
## Container build
Alternatively, you can build an image with a builder pattern. This options takes a while but offers a simple method for
anyone to get a working container image without requiring any of the Rust toolchain installed locally.
```bash
docker build \
--tag $OWNER/$IMAGE_NAME \
--file ./docker/polkadot-parachain_builder.Containerfile .
```
You may then run your new container:
```bash
docker run --rm -it \
$OWNER/$IMAGE_NAME \
--collator --tmp \
--execution wasm \
--chain /specs/westmint.json
```