diff --git a/.gitlab/common/lib.sh b/.github/scripts/common/lib.sh similarity index 76% rename from .gitlab/common/lib.sh rename to .github/scripts/common/lib.sh index ba5b171487..b0f9cb3206 100755 --- a/.gitlab/common/lib.sh +++ b/.github/scripts/common/lib.sh @@ -96,7 +96,7 @@ structure_message() { # access_token: see https://matrix.org/docs/guides/client-server-api/ # Usage: send_message $body (json formatted) $room_id $access_token send_message() { -curl -XPOST -d "$1" "https://matrix.parity.io/_matrix/client/r0/rooms/$2/send/m.room.message?access_token=$3" + curl -XPOST -d "$1" "https://m.parity.io/_matrix/client/r0/rooms/$2/send/m.room.message?access_token=$3" } # Pretty-printing functions @@ -193,3 +193,74 @@ check_bootnode(){ echo " Bootnode appears unreachable" return 1 } + +# Assumes the ENV are set: +# - RELEASE_ID +# - GITHUB_TOKEN +# - REPO in the form paritytech/polkadot +fetch_release_artifacts() { + echo "Release ID : $RELEASE_ID" + echo "Repo : $REPO" + echo "Binary : $BINARY" + + curl -L -s \ + -H "Accept: application/vnd.github+json" \ + -H "Authorization: Bearer ${GITHUB_TOKEN}" \ + -H "X-GitHub-Api-Version: 2022-11-28" \ + https://api.github.com/repos/${REPO}/releases/${RELEASE_ID} > release.json + + # Get Asset ids + ids=($(jq -r '.assets[].id' < release.json )) + count=$(jq '.assets|length' < release.json ) + + # Fetch artifacts + mkdir -p "./release-artifacts/${BINARY}" + pushd "./release-artifacts/${BINARY}" > /dev/null + + iter=1 + for id in "${ids[@]}" + do + echo " - $iter/$count: downloading asset id: $id..." + curl -s -OJ -L -H "Accept: application/octet-stream" \ + -H "Authorization: Token ${GITHUB_TOKEN}" \ + "https://api.github.com/repos/${REPO}/releases/assets/$id" + iter=$((iter + 1)) + done + + pwd + ls -al --color + popd > /dev/null +} + +# Check the checksum for a given binary +function check_sha256() { + echo "Checking SHA256 for $1" + shasum -qc $1.sha256 +} + +# Import GPG keys of the release team members +# This is done in parallel as it can take a while sometimes +function import_gpg_keys() { + GPG_KEYSERVER=${GPG_KEYSERVER:-"keyserver.ubuntu.com"} + SEC="9D4B2B6EB8F97156D19669A9FF0812D491B96798" + WILL="2835EAF92072BC01D188AF2C4A092B93E97CE1E2" + EGOR="E6FC4D4782EB0FA64A4903CCDB7D3555DD3932D3" + MARA="533C920F40E73A21EEB7E9EBF27AEA7E7594C9CF" + MORGAN="2E92A9D8B15D7891363D1AE8AF9E6C43F7F8C4CF" + + echo "Importing GPG keys from $GPG_KEYSERVER in parallel" + for key in $SEC $WILL $EGOR $MARA $MORGAN; do + ( + echo "Importing GPG key $key" + gpg --no-tty --quiet --keyserver $GPG_KEYSERVER --recv-keys $key + echo -e "5\ny\n" | gpg --no-tty --command-fd 0 --expert --edit-key $key trust; + ) & + done + wait +} + +# Check the GPG signature for a given binary +function check_gpg() { + echo "Checking GPG Signature for $1" + gpg --no-tty --verify -q $1.asc $1 +} diff --git a/cumulus/.github/workflows/release-50_publish-docker.yml b/.github/workflows/release-50_publish-docker.yml similarity index 59% rename from cumulus/.github/workflows/release-50_publish-docker.yml rename to .github/workflows/release-50_publish-docker.yml index 6ad943c390..0e466f2689 100644 --- a/cumulus/.github/workflows/release-50_publish-docker.yml +++ b/.github/workflows/release-50_publish-docker.yml @@ -7,9 +7,10 @@ name: Release - Publish Docker Image # image and publishes it. on: - release: - types: - - published + #TODO: activate automated run later + # release: + # types: + # - published workflow_dispatch: inputs: release_id: @@ -39,6 +40,18 @@ on: required: true type: string default: parity + binary: + description: Binary to be published + required: true + default: polkadot + type: choice + options: + - polkadot + - staking-miner + - polkadot-parachain + +permissions: + contents: write env: RELEASE_ID: ${{ inputs.release_id }} @@ -47,8 +60,8 @@ env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} DOCKER_OWNER: ${{ inputs.owner || github.repository_owner }} REPO: ${{ github.repository }} - BINARY: polkadot-parachain - EVENT_ACTION: ${{ github.event.action }} + BINARY: ${{ inputs.binary }} + # EVENT_ACTION: ${{ github.event.action }} EVENT_NAME: ${{ github.event_name }} IMAGE_TYPE: ${{ inputs.image_type }} @@ -58,59 +71,36 @@ jobs: steps: - name: Checkout sources - uses: actions/checkout@c85c95e3d7251135ab7dc9ce3241c5835cc595a9 # v3.5.3 + uses: actions/checkout@3df4ab11eba7bda6032a0b82a6bb43b11571feac # v4.0.0 - - name: Prepare temp folder - run: | - TMP=$(mktemp -d) - echo "TMP=$TMP" >> "$GITHUB_ENV" - pwd - ls -al "$TMP" - - - name: Fetch lib.sh from polkadot repo - working-directory: ${{ env.TMP }} - run: | - curl -O -L \ - -H "Accept: application/vnd.github.v3.raw" \ - https://raw.githubusercontent.com/paritytech/polkadot/master/scripts/ci/common/lib.sh - - chmod a+x lib.sh - ls -al - - - name: Fetch release artifacts based on final release tag + #TODO: this step will be needed when automated triggering will work #this step runs only if the workflow is triggered automatically when new release is published - if: ${{ env.EVENT_NAME == 'release' && env.EVENT_ACTION != '' && env.EVENT_ACTION == 'published' }} - run: | - mkdir -p release-artifacts && cd release-artifacts + # if: ${{ env.EVENT_NAME == 'release' && env.EVENT_ACTION != '' && env.EVENT_ACTION == 'published' }} + # run: | + # mkdir -p release-artifacts && cd release-artifacts - for f in $BINARY $BINARY.asc $BINARY.sha256; do - URL="https://github.com/${{ github.event.repository.full_name }}/releases/download/${{ github.event.release.tag_name }}/$f" - echo " - Fetching $f from $URL" - wget "$URL" -O "$f" - done - chmod a+x $BINARY - cp -f ${TMP}/lib.sh . - ls -al + # for f in $BINARY $BINARY.asc $BINARY.sha256; do + # URL="https://github.com/${{ github.event.repository.full_name }}/releases/download/${{ github.event.release.tag_name }}/$f" + # echo " - Fetching $f from $URL" + # wget "$URL" -O "$f" + # done + # chmod a+x $BINARY + # ls -al - name: Fetch rc artifacts or release artifacts based on release id #this step runs only if the workflow is triggered manually if: ${{ env.EVENT_NAME == 'workflow_dispatch' }} run: | - . ${TMP}/lib.sh + . ./.github/scripts/common/lib.sh fetch_release_artifacts - chmod a+x release-artifacts/$BINARY - ls -al - - cp -f ${TMP}/lib.sh release-artifacts/ - - name: Cache the artifacts uses: actions/cache@88522ab9f39a2ea568f7027eddc7d8d8bc9d59c8 # v3.3.1 with: - key: artifacts-${{ github.sha }} + key: artifacts-${{ env.BINARY }}-${{ github.sha }} path: | - ./release-artifacts/**/* + ./release-artifacts/${{ env.BINARY }}/**/* build-container: runs-on: ubuntu-latest @@ -118,40 +108,31 @@ jobs: steps: - name: Checkout sources - uses: actions/checkout@c85c95e3d7251135ab7dc9ce3241c5835cc595a9 # v3.5.3 + uses: actions/checkout@3df4ab11eba7bda6032a0b82a6bb43b11571feac # v4.0.0 - name: Get artifacts from cache uses: actions/cache@88522ab9f39a2ea568f7027eddc7d8d8bc9d59c8 # v3.3.1 with: - key: artifacts-${{ github.sha }} + key: artifacts-${{ env.BINARY }}-${{ github.sha }} fail-on-cache-miss: true path: | - ./release-artifacts/**/* + ./release-artifacts/${{ env.BINARY }}/**/* - name: Check sha256 ${{ env.BINARY }} - working-directory: ./release-artifacts + working-directory: ./release-artifacts/${{ env.BINARY }} run: | - . ./lib.sh + . ../../.github/scripts/common/lib.sh echo "Checking binary $BINARY" check_sha256 $BINARY && echo "OK" || echo "ERR" - name: Check GPG ${{ env.BINARY }} - working-directory: ./release-artifacts + working-directory: ./release-artifacts/${{ env.BINARY }} run: | - . ./lib.sh + . ../../.github/scripts/common/lib.sh import_gpg_keys check_gpg $BINARY - - name: Build Injected Container image for ${{ env.BINARY }} - env: - IMAGE_NAME: ${{ env.BINARY }} - OWNER: ${{ env.DOCKER_OWNER }} - run: | - ls -al - echo "Building container for $BINARY" - ./docker/scripts/build-injected-image.sh - - name: Fetch rc commit and tag if: ${{ env.IMAGE_TYPE == 'rc' }} id: fetch_rc_refs @@ -167,14 +148,43 @@ jobs: echo "No tag, doing without" - name: Fetch release tags - if: ${{ env.IMAGE_TYPE == 'release' || env.EVENT_NAME == 'release' && env.EVENT_ACTION != '' && env.EVENT_ACTION == 'published' }} + working-directory: ./release-artifacts/${{ env.BINARY }} + if: ${{ env.IMAGE_TYPE == 'release'}} id: fetch_release_refs run: | - VERSION=$(docker run --pull never --rm $DOCKER_OWNER/$BINARY --version | awk '{ print $2 }' ) + chmod a+rx $BINARY + VERSION=$(./$BINARY --version | awk '{ print $2 }' ) release=$( echo $VERSION | cut -f1 -d- ) echo "tag=latest" >> $GITHUB_OUTPUT echo "release=${release}" >> $GITHUB_OUTPUT + - name: Build Injected Container image for polkadot/staking-miner + if: ${{ env.BINARY == 'polkadot' || env.BINARY == 'staking-miner' }} + env: + ARTIFACTS_FOLDER: ./release-artifacts + IMAGE_NAME: ${{ env.BINARY }} + OWNER: ${{ env.DOCKER_OWNER }} + TAGS: ${{ join(steps.fetch_rc_refs.outputs.*, ',') || join(steps.fetch_release_refs.outputs.*, ',') }} + run: | + ls -al + echo "Building container for $BINARY" + ./docker/scripts/build-injected.sh + + - name: Build Injected Container image for polkadot-parachain + if: ${{ env.BINARY == 'polkadot-parachain' }} + env: + ARTIFACTS_FOLDER: ./release-artifacts + IMAGE_NAME: ${{ env.BINARY }} + OWNER: ${{ env.DOCKER_OWNER }} + DOCKERFILE: docker/dockerfiles/polkadot-parachain/polkadot-parachain_injected.Dockerfile + TAGS: ${{ join(steps.fetch_rc_refs.outputs.*, ',') || join(steps.fetch_release_refs.outputs.*, ',') }} + run: | + ls -al + mkdir -p $ARTIFACTS_FOLDER/specs + cp cumulus/parachains/chain-specs/*.json $ARTIFACTS_FOLDER/specs + + echo "Building container for $BINARY" + ./docker/scripts/build-injected.sh - name: Login to Dockerhub uses: docker/login-action@v2 @@ -182,21 +192,11 @@ jobs: username: ${{ secrets.DOCKERHUB_USERNAME }} password: ${{ secrets.DOCKERHUB_TOKEN }} - - name: Tag and Push Container image for ${{ env.BINARY }} + - name: Push Container image for ${{ env.BINARY }} id: docker_push - env: - TAGS: ${{ join(steps.fetch_rc_refs.outputs.*, ',') || join(steps.fetch_release_refs.outputs.*, ',') }} run: | - TAGS=${TAGS[@]:-latest} - IFS=',' read -r -a TAG_ARRAY <<< "$TAGS" - - echo "The image ${BINARY} will be tagged with ${TAG_ARRAY[*]}" - for TAG in "${TAG_ARRAY[@]}"; do - $ENGINE tag ${DOCKER_OWNER}/${BINARY} ${DOCKER_OWNER}/${BINARY}:${TAG} - $ENGINE push ${DOCKER_OWNER}/${BINARY}:${TAG} - done - $ENGINE images | grep ${BINARY} + $ENGINE push --all-tags ${REGISTRY}/${DOCKER_OWNER}/${BINARY} - name: Check version for the published image for ${{ env.BINARY }} env: diff --git a/.gitlab/pipeline/build.yml b/.gitlab/pipeline/build.yml index 20fed5df3d..2d74187cad 100644 --- a/.gitlab/pipeline/build.yml +++ b/.gitlab/pipeline/build.yml @@ -328,7 +328,7 @@ build-linux-substrate: cut -d ' ' -f 2 | tee ./artifacts/substrate/VERSION; fi - sha256sum ./artifacts/substrate/substrate | tee ./artifacts/substrate/substrate.sha256 - - cp -r ./docker/substrate_injected.Dockerfile ./artifacts/substrate/ + - cp -r ./docker/dockerfiles/substrate_injected.Dockerfile ./artifacts/substrate/ # - printf '\n# building node-template\n\n' # - ./scripts/ci/node-template-release.sh ./artifacts/substrate/substrate-node-template.tar.gz diff --git a/.gitlab/pipeline/publish.yml b/.gitlab/pipeline/publish.yml index ed18082344..341d3ac2a8 100644 --- a/.gitlab/pipeline/publish.yml +++ b/.gitlab/pipeline/publish.yml @@ -35,7 +35,7 @@ build-push-image-polkadot-parachain-debug: - job: build-linux-stable-cumulus artifacts: true variables: - DOCKERFILE: "docker/polkadot-parachain-debug_unsigned_injected.Dockerfile" + DOCKERFILE: "docker/dockerfiles/polkadot-parachain/polkadot-parachain-debug_unsigned_injected.Dockerfile" IMAGE_NAME: "docker.io/paritypr/polkadot-parachain-debug" build-push-image-test-parachain: @@ -48,7 +48,7 @@ build-push-image-test-parachain: - job: build-test-parachain artifacts: true variables: - DOCKERFILE: "docker/test-parachain_injected.Dockerfile" + DOCKERFILE: "docker/dockerfiles/test-parachain_injected.Dockerfile" IMAGE_NAME: "docker.io/paritypr/test-parachain" # publish-s3: # stage: publish @@ -114,7 +114,7 @@ build-push-image-polkadot-debug: - job: build-linux-stable artifacts: true variables: - DOCKERFILE: "docker/polkadot_injected_debug.Dockerfile" + DOCKERFILE: "docker/dockerfiles/polkadot/polkadot_injected_debug.Dockerfile" IMAGE_NAME: "docker.io/paritypr/polkadot-debug" build-push-image-colander: @@ -127,7 +127,7 @@ build-push-image-colander: - job: build-test-collators artifacts: true variables: - DOCKERFILE: "docker/collator_injected.Dockerfile" + DOCKERFILE: "docker/dockerfiles/collator_injected.Dockerfile" IMAGE_NAME: "docker.io/paritypr/colander" build-push-image-malus: @@ -140,7 +140,7 @@ build-push-image-malus: - job: build-malus artifacts: true variables: - DOCKERFILE: "docker/malus_injected.Dockerfile" + DOCKERFILE: "docker/dockerfiles/malus_injected.Dockerfile" IMAGE_NAME: "docker.io/paritypr/malus" build-push-image-substrate-pr: @@ -153,7 +153,7 @@ build-push-image-substrate-pr: - job: build-linux-substrate artifacts: true variables: - DOCKERFILE: "docker/substrate_injected.Dockerfile" + DOCKERFILE: "docker/dockerfiles/substrate_injected.Dockerfile" IMAGE_NAME: "docker.io/paritypr/substrate" # old way @@ -201,7 +201,7 @@ build-push-image-substrate-pr: # GIT_STRATEGY: none # DOCKER_USER: ${PARITYPR_USER} # DOCKER_PASS: ${PARITYPR_PASS} -# # scripts/ci/dockerfiles/polkadot_injected_debug.Dockerfile +# # docker/dockerfiles/polkadot/polkadot_injected_debug.Dockerfile # DOCKERFILE: polkadot_injected_debug.Dockerfile # IMAGE_NAME: docker.io/paritypr/polkadot-debug # needs: @@ -230,7 +230,7 @@ build-push-image-substrate-pr: # GIT_STRATEGY: none # DOCKER_USER: ${PARITYPR_USER} # DOCKER_PASS: ${PARITYPR_PASS} -# # scripts/ci/dockerfiles/collator_injected.Dockerfile +# # docker/dockerfiles/collator_injected.Dockerfile # DOCKERFILE: collator_injected.Dockerfile # IMAGE_NAME: docker.io/paritypr/colander # needs: @@ -258,7 +258,7 @@ build-push-image-substrate-pr: # GIT_STRATEGY: none # DOCKER_USER: ${PARITYPR_USER} # DOCKER_PASS: ${PARITYPR_PASS} -# # scripts/ci/dockerfiles/malus_injected.Dockerfile +# # docker/dockerfiles/malus_injected.Dockerfile # DOCKERFILE: malus_injected.Dockerfile # IMAGE_NAME: docker.io/paritypr/malus # needs: diff --git a/.gitlab/test_deterministic_wasm.sh b/.gitlab/test_deterministic_wasm.sh index 4f1d2981ff..fac28fce1f 100755 --- a/.gitlab/test_deterministic_wasm.sh +++ b/.gitlab/test_deterministic_wasm.sh @@ -2,7 +2,7 @@ set -e #shellcheck source=../common/lib.sh -source "$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )/common/lib.sh" +source "$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )/../.github/scripts/common/lib.sh" # build runtime WASM_BUILD_NO_COLOR=1 cargo build -q --locked --release -p staging-kusama-runtime -p polkadot-runtime -p westend-runtime diff --git a/cumulus/docker/parachain-registrar.dockerfile b/cumulus/docker/parachain-registrar.dockerfile deleted file mode 100644 index f7d77454a2..0000000000 --- a/cumulus/docker/parachain-registrar.dockerfile +++ /dev/null @@ -1,27 +0,0 @@ -FROM node:latest AS pjs - -# It would be great to depend on a more stable tag, but we need some -# as-yet-unreleased features. -RUN yarn global add @polkadot/api-cli@0.10.0-beta.14 - -ENTRYPOINT [ "polkadot-js-api" ] -CMD [ "--version" ] - -# To use the pjs build stage to access the blockchain from the host machine: -# -# docker build -f docker/parachain-registrar.dockerfile --target pjs -t parachain-registrar:pjs . -# alias pjs='docker run --rm --net cumulus_testing_net parachain-registrar:pjs --ws ws://172.28.1.1:9944' -# -# Then, as long as the chain is running, you can use the polkadot-js-api CLI like: -# -# pjs query.sudo.key - -FROM pjs -RUN apt-get update && apt-get install curl netcat -y && \ - curl -sSo /wait-for-it.sh https://raw.githubusercontent.com/vishnubob/wait-for-it/master/wait-for-it.sh && \ - chmod +x /wait-for-it.sh -# the only thing left to do is to actually run the transaction. -COPY ./docker/scripts/register_para.sh /usr/bin -# unset the previous stage's entrypoint -ENTRYPOINT [] -CMD [ "/usr/bin/register_para.sh" ] diff --git a/cumulus/docker/polkadot-parachain-debug_unsigned_injected.Dockerfile b/cumulus/docker/polkadot-parachain-debug_unsigned_injected.Dockerfile deleted file mode 100644 index a2e32049f5..0000000000 --- a/cumulus/docker/polkadot-parachain-debug_unsigned_injected.Dockerfile +++ /dev/null @@ -1,49 +0,0 @@ -FROM docker.io/library/ubuntu:20.04 - -# metadata -ARG VCS_REF -ARG BUILD_DATE -ARG IMAGE_NAME - -LABEL io.parity.image.authors="devops-team@parity.io" \ - io.parity.image.vendor="Parity Technologies" \ - io.parity.image.title="${IMAGE_NAME}" \ - io.parity.image.description="Cumulus, the Polkadot collator." \ - io.parity.image.source="https://github.com/paritytech/cumulus/blob/${VCS_REF}/scripts/docker/polkadot-parachain-debug_unsigned_injected.Dockerfile" \ - io.parity.image.revision="${VCS_REF}" \ - io.parity.image.created="${BUILD_DATE}" \ - io.parity.image.documentation="https://github.com/paritytech/cumulus/" - -# show backtraces -ENV RUST_BACKTRACE 1 - -# install tools and dependencies -RUN apt-get update && \ - DEBIAN_FRONTEND=noninteractive apt-get install -y \ - libssl1.1 \ - ca-certificates \ - curl && \ - # apt cleanup - apt-get autoremove -y && \ - apt-get clean && \ - find /var/lib/apt/lists/ -type f -not -name lock -delete; \ - # add user and link ~/.local/share/polkadot-parachain to /data - useradd -m -u 10000 -U -s /bin/sh -d /polkadot-parachain polkadot-parachain && \ - mkdir -p /data /polkadot-parachain/.local/share && \ - chown -R polkadot-parachain:polkadot-parachain /data && \ - ln -s /data /polkadot-parachain/.local/share/polkadot-parachain && \ - mkdir -p /specs - -# add polkadot-parachain binary to the docker image -COPY ./artifacts/polkadot-parachain /usr/local/bin -COPY ./parachains/chain-specs/*.json /specs/ - -USER polkadot-parachain - -# check if executable works in this container -RUN /usr/local/bin/polkadot-parachain --version - -EXPOSE 30333 9933 9944 -VOLUME ["/polkadot-parachain"] - -ENTRYPOINT ["/usr/local/bin/polkadot-parachain"] diff --git a/cumulus/docker/polkadot-parachain_builder.Containerfile b/cumulus/docker/polkadot-parachain_builder.Containerfile deleted file mode 100644 index 159bcb3236..0000000000 --- a/cumulus/docker/polkadot-parachain_builder.Containerfile +++ /dev/null @@ -1,36 +0,0 @@ -# This file is sourced from https://github.com/paritytech/polkadot/blob/master/scripts/ci/dockerfiles/polkadot/polkadot_builder.Dockerfile -# This is the build stage for polkadot-parachain. Here we create the binary in a temporary image. -FROM docker.io/paritytech/ci-linux:production as builder - -WORKDIR /cumulus -COPY . /cumulus - -RUN cargo build --release --locked -p polkadot-parachain - -# This is the 2nd stage: a very small image where we copy the Polkadot binary." -FROM docker.io/library/ubuntu:20.04 - -LABEL io.parity.image.type="builder" \ - io.parity.image.authors="devops-team@parity.io" \ - io.parity.image.vendor="Parity Technologies" \ - io.parity.image.description="Multistage Docker image for polkadot-parachain" \ - io.parity.image.source="https://github.com/paritytech/polkadot/blob/${VCS_REF}/scripts/ci/dockerfiles/polkadot/polkadot-parachain_builder.Dockerfile" \ - io.parity.image.documentation="https://github.com/paritytech/cumulus" - -COPY --from=builder /cumulus/target/release/polkadot-parachain /usr/local/bin - -RUN useradd -m -u 1000 -U -s /bin/sh -d /cumulus polkadot-parachain && \ - mkdir -p /data /cumulus/.local/share && \ - chown -R polkadot-parachain:polkadot-parachain /data && \ - ln -s /data /cumulus/.local/share/polkadot-parachain && \ -# unclutter and minimize the attack surface - rm -rf /usr/bin /usr/sbin && \ -# check if executable works in this container - /usr/local/bin/polkadot-parachain --version - -USER polkadot-parachain - -EXPOSE 30333 9933 9944 9615 -VOLUME ["/data"] - -ENTRYPOINT ["/usr/local/bin/polkadot-parachain"] diff --git a/cumulus/docker/test-parachain-collator.dockerfile b/cumulus/docker/test-parachain-collator.dockerfile deleted file mode 100644 index 9c2d8fbe58..0000000000 --- a/cumulus/docker/test-parachain-collator.dockerfile +++ /dev/null @@ -1,46 +0,0 @@ -# This file is sourced from https://github.com/paritytech/polkadot/blob/master/scripts/ci/dockerfiles/polkadot/polkadot_builder.Dockerfile -FROM docker.io/paritytech/ci-linux:production as builder - -WORKDIR /cumulus -COPY . /cumulus - -RUN cargo build --release --locked -p polkadot-parachain - -# the collator stage is normally built once, cached, and then ignored, but can -# be specified with the --target build flag. This adds some extra tooling to the -# image, which is required for a launcher script. The script simply adds two -# arguments to the list passed in: -# -# --bootnodes /ip4/127.0.0.1/tcp/30333/p2p/PEER_ID -# -# with the appropriate ip and ID for both Alice and Bob -FROM debian:buster-slim as collator -RUN apt-get update && apt-get install jq curl bash -y && \ - curl -sSo /wait-for-it.sh https://raw.githubusercontent.com/vishnubob/wait-for-it/master/wait-for-it.sh && \ - chmod +x /wait-for-it.sh && \ - curl -sL https://deb.nodesource.com/setup_12.x | bash - && \ - apt-get install -y nodejs && \ - npm install --global yarn && \ - yarn global add @polkadot/api-cli@0.10.0-beta.14 -COPY --from=builder \ - /paritytech/cumulus/target/release/polkadot-parachain /usr/bin -COPY ./docker/scripts/inject_bootnodes.sh /usr/bin -CMD ["/usr/bin/inject_bootnodes.sh"] -COPY ./docker/scripts/healthcheck.sh /usr/bin/ -HEALTHCHECK --interval=300s --timeout=75s --start-period=30s --retries=3 \ - CMD ["/usr/bin/healthcheck.sh"] - -# the runtime stage is normally built once, cached, and ignored, but can be -# specified with the --target build flag. This just preserves one of the builder's -# outputs, which can then be moved into a volume at runtime -FROM debian:buster-slim as runtime -COPY --from=builder \ - /paritytech/cumulus/target/release/wbuild/cumulus-test-parachain-runtime/cumulus_test_parachain_runtime.compact.wasm \ - /var/opt/ -CMD ["cp", "-v", "/var/opt/cumulus_test_parachain_runtime.compact.wasm", "/runtime/"] - -FROM debian:buster-slim -COPY --from=builder \ - /paritytech/cumulus/target/release/polkadot-parachain /usr/bin - -CMD ["/usr/bin/polkadot-parachain"] diff --git a/cumulus/docker/test-parachain_injected.Dockerfile b/cumulus/docker/test-parachain_injected.Dockerfile deleted file mode 100644 index 6056c50460..0000000000 --- a/cumulus/docker/test-parachain_injected.Dockerfile +++ /dev/null @@ -1,49 +0,0 @@ -FROM docker.io/library/ubuntu:20.04 - -# metadata -ARG VCS_REF -ARG BUILD_DATE -ARG IMAGE_NAME - -LABEL io.parity.image.authors="devops-team@parity.io" \ - io.parity.image.vendor="Parity Technologies" \ - io.parity.image.title="${IMAGE_NAME}" \ - io.parity.image.description="Test parachain for Zombienet" \ - io.parity.image.source="https://github.com/paritytech/cumulus/blob/${VCS_REF}/docker/test-parachain_injected.Dockerfile" \ - io.parity.image.revision="${VCS_REF}" \ - io.parity.image.created="${BUILD_DATE}" \ - io.parity.image.documentation="https://github.com/paritytech/cumulus/" - -# show backtraces -ENV RUST_BACKTRACE 1 - -# install tools and dependencies -RUN apt-get update && \ - DEBIAN_FRONTEND=noninteractive apt-get install -y \ - libssl1.1 \ - ca-certificates \ - curl && \ - # apt cleanup - apt-get autoremove -y && \ - apt-get clean && \ - find /var/lib/apt/lists/ -type f -not -name lock -delete; \ - # add user and link ~/.local/share/test-parachain to /data - useradd -m -u 10000 -U -s /bin/sh -d /test-parachain test-parachain && \ - mkdir -p /data /test-parachain/.local/share && \ - chown -R test-parachain:test-parachain /data && \ - ln -s /data /test-parachain/.local/share/test-parachain && \ - mkdir -p /specs - -# add test-parachain binary to the docker image -COPY ./artifacts/test-parachain /usr/local/bin -COPY ./parachains/chain-specs/*.json /specs/ - -USER test-parachain - -# check if executable works in this container -RUN /usr/local/bin/test-parachain --version - -EXPOSE 30333 9933 9944 -VOLUME ["/test-parachain"] - -ENTRYPOINT ["/usr/local/bin/test-parachain"] diff --git a/docker/docker-compose.yml b/docker/docker-compose.yml deleted file mode 100644 index 8344ad43bb..0000000000 --- a/docker/docker-compose.yml +++ /dev/null @@ -1,129 +0,0 @@ -version: '3.7' -services: - node_alice: - image: "polkadot:${BRANCH:-cumulus-branch}" - ports: - - "30333:30333" - - "9933:9933" - - "9944:9944" - volumes: - - "polkadot-data-alice:/data" - - type: bind - source: ./test/parachain/chain-specs/polkadot_chainspec.json - target: /chainspec.json - read_only: true - command: > - polkadot - --chain=/chainspec.json - --base-path=/data - --port 30333 - --rpc-port 9933 - --ws-port 9944 - --rpc-external - --rpc-cors all - --ws-external - --alice - networks: - testing_net: - ipv4_address: 172.28.1.1 - aliases: - - alice - - node_bob: - image: "polkadot:${BRANCH:-cumulus-branch}" - ports: - - "30344:30333" - - "9935:9933" - - "9945:9944" - volumes: - - "polkadot-data-bob:/data" - - type: bind - source: ./test/parachain/chain-specs/polkadot_chainspec.json - target: /chainspec.json - read_only: true - command: > - polkadot - --chain=/chainspec.json - --base-path=/data - --port 30333 - --rpc-port 9933 - --ws-port 9944 - --rpc-external - --ws-external - --rpc-cors all - --bob - networks: - testing_net: - ipv4_address: 172.28.1.2 - aliases: - - bob - - genesis_state: - build: - context: . - dockerfile: ./docker/test-parachain-collator.dockerfile - image: "ctpc:latest" - volumes: - - "genesis-state:/data" - command: > - polkadot-parachain - export-genesis-state - /data/genesis-state - - collator: - build: - context: . - dockerfile: ./docker/test-parachain-collator.dockerfile - target: collator - image: "ctpc:collator" - volumes: - - "collator-data:/data" - depends_on: - - node_alice - - node_bob - command: > - inject_bootnodes.sh - --base-path=/data - networks: - testing_net: - - runtime: - build: - context: . - dockerfile: ./docker/test-parachain-collator.dockerfile - target: runtime - image: "ctpc:runtime" - volumes: - - "parachain-runtime:/runtime" - - - registrar: - build: - context: . - dockerfile: ./docker/parachain-registrar.dockerfile - image: para-reg:latest - volumes: - - "genesis-state:/genesis" - - "parachain-runtime:/runtime" - depends_on: - - node_alice - - runtime - - genesis_state - networks: - testing_net: - - -volumes: - polkadot-data-alice: - polkadot-data-bob: - collator-data: - genesis-state: - parachain-runtime: - - -networks: - testing_net: - ipam: - driver: default - config: - - subnet: 172.28.0.0/16 diff --git a/docker/dockerfiles/binary_injected.Dockerfile b/docker/dockerfiles/binary_injected.Dockerfile new file mode 100644 index 0000000000..ac1fd5317c --- /dev/null +++ b/docker/dockerfiles/binary_injected.Dockerfile @@ -0,0 +1,48 @@ +FROM docker.io/parity/base-bin + +# This file allows building a Generic container image +# based on one or multiple pre-built Linux binaries. +# Some defaults are set to polkadot but all can be overriden. + +SHELL ["/bin/bash", "-c"] + +# metadata +ARG VCS_REF +ARG BUILD_DATE +ARG IMAGE_NAME + +# That can be a single one or a comma separated list +ARG BINARY=polkadot + +ARG BIN_FOLDER=. +ARG DOC_URL=https://github.com/paritytech/polkadot-sdk +ARG DESCRIPTION="Polkadot: a platform for web3" +ARG AUTHORS="devops-team@parity.io" +ARG VENDOR="Parity Technologies" + +LABEL io.parity.image.authors=${AUTHORS} \ + io.parity.image.vendor="${VENDOR}" \ + io.parity.image.revision="${VCS_REF}" \ + io.parity.image.title="${IMAGE_NAME}" \ + io.parity.image.created="${BUILD_DATE}" \ + io.parity.image.documentation="${DOC_URL}" \ + io.parity.image.description="${DESCRIPTION}" \ + io.parity.image.source="https://github.com/paritytech/polkadot-sdk/blob/${VCS_REF}/docker/dockerfiles/binary_injected.Dockerfile" + +USER root +WORKDIR /app + +# add polkadot binary to docker image +# sample for polkadot: COPY ./polkadot ./polkadot-*-worker /usr/local/bin/ +COPY entrypoint.sh . +COPY "bin/*" "/usr/local/bin/" +RUN chmod -R a+rx "/usr/local/bin" + +USER parity +ENV BINARY=${BINARY} + +# ENTRYPOINT +ENTRYPOINT ["/app/entrypoint.sh"] + +# We call the help by default +CMD ["--help"] diff --git a/docker/collator_injected.Dockerfile b/docker/dockerfiles/collator_injected.Dockerfile similarity index 95% rename from docker/collator_injected.Dockerfile rename to docker/dockerfiles/collator_injected.Dockerfile index 6472c240f3..0c9ea1e0ca 100644 --- a/docker/collator_injected.Dockerfile +++ b/docker/dockerfiles/collator_injected.Dockerfile @@ -10,7 +10,7 @@ LABEL io.parity.image.authors="devops-team@parity.io" \ io.parity.image.vendor="Parity Technologies" \ io.parity.image.title="${IMAGE_NAME}" \ io.parity.image.description="Injected adder-collator Docker image" \ - io.parity.image.source="https://github.com/paritytech/polkadot/blob/${VCS_REF}/scripts/ci/dockerfiles/collator_injected.Dockerfile" \ + io.parity.image.source="https://github.com/paritytech/polkadot/blob/${VCS_REF}/docker/dockerfiles/collator_injected.Dockerfile" \ io.parity.image.revision="${VCS_REF}" \ io.parity.image.created="${BUILD_DATE}" \ io.parity.image.documentation="https://github.com/paritytech/polkadot/" diff --git a/cumulus/docker/docker-compose.yml b/docker/dockerfiles/docker-compose.yml similarity index 89% rename from cumulus/docker/docker-compose.yml rename to docker/dockerfiles/docker-compose.yml index 8344ad43bb..8dc8540353 100644 --- a/cumulus/docker/docker-compose.yml +++ b/docker/dockerfiles/docker-compose.yml @@ -61,7 +61,7 @@ services: genesis_state: build: context: . - dockerfile: ./docker/test-parachain-collator.dockerfile + dockerfile: ./docker/dockerfiles/test-parachain-collator.dockerfile image: "ctpc:latest" volumes: - "genesis-state:/data" @@ -73,7 +73,7 @@ services: collator: build: context: . - dockerfile: ./docker/test-parachain-collator.dockerfile + dockerfile: ./docker/dockerfiles/test-parachain-collator.dockerfile target: collator image: "ctpc:collator" volumes: @@ -90,7 +90,7 @@ services: runtime: build: context: . - dockerfile: ./docker/test-parachain-collator.dockerfile + dockerfile: ./docker/dockerfiles/test-parachain-collator.dockerfile target: runtime image: "ctpc:runtime" volumes: @@ -100,7 +100,7 @@ services: registrar: build: context: . - dockerfile: ./docker/parachain-registrar.dockerfile + dockerfile: ./docker/dockerfiles/parachain-registrar.dockerfile image: para-reg:latest volumes: - "genesis-state:/genesis" diff --git a/docker/malus_injected.Dockerfile b/docker/dockerfiles/malus_injected.Dockerfile similarity index 100% rename from docker/malus_injected.Dockerfile rename to docker/dockerfiles/malus_injected.Dockerfile diff --git a/docker/parachain-registrar.dockerfile b/docker/dockerfiles/parachain-registrar.dockerfile similarity index 89% rename from docker/parachain-registrar.dockerfile rename to docker/dockerfiles/parachain-registrar.dockerfile index f7d77454a2..0090839510 100644 --- a/docker/parachain-registrar.dockerfile +++ b/docker/dockerfiles/parachain-registrar.dockerfile @@ -9,7 +9,7 @@ CMD [ "--version" ] # To use the pjs build stage to access the blockchain from the host machine: # -# docker build -f docker/parachain-registrar.dockerfile --target pjs -t parachain-registrar:pjs . +# docker build -f docker/dockerfiles/parachain-registrar.dockerfile --target pjs -t parachain-registrar:pjs . # alias pjs='docker run --rm --net cumulus_testing_net parachain-registrar:pjs --ws ws://172.28.1.1:9944' # # Then, as long as the chain is running, you can use the polkadot-js-api CLI like: diff --git a/docker/polkadot-parachain-debug_unsigned_injected.Dockerfile b/docker/dockerfiles/polkadot-parachain/polkadot-parachain-debug_unsigned_injected.Dockerfile similarity index 93% rename from docker/polkadot-parachain-debug_unsigned_injected.Dockerfile rename to docker/dockerfiles/polkadot-parachain/polkadot-parachain-debug_unsigned_injected.Dockerfile index e77563b8eb..7a2202d9c5 100644 --- a/docker/polkadot-parachain-debug_unsigned_injected.Dockerfile +++ b/docker/dockerfiles/polkadot-parachain/polkadot-parachain-debug_unsigned_injected.Dockerfile @@ -9,7 +9,7 @@ LABEL io.parity.image.authors="devops-team@parity.io" \ io.parity.image.vendor="Parity Technologies" \ io.parity.image.title="${IMAGE_NAME}" \ io.parity.image.description="Cumulus, the Polkadot collator." \ - io.parity.image.source="https://github.com/paritytech/cumulus/blob/${VCS_REF}/scripts/docker/polkadot-parachain-debug_unsigned_injected.Dockerfile" \ + io.parity.image.source="https://github.com/paritytech/cumulus/blob/${VCS_REF}/docker/dockerfiles/polkadot-parachain/polkadot-parachain-debug_unsigned_injected.Dockerfile" \ io.parity.image.revision="${VCS_REF}" \ io.parity.image.created="${BUILD_DATE}" \ io.parity.image.documentation="https://github.com/paritytech/cumulus/" diff --git a/docker/polkadot-parachain_builder.Containerfile b/docker/dockerfiles/polkadot-parachain/polkadot-parachain_builder.Containerfile similarity index 89% rename from docker/polkadot-parachain_builder.Containerfile rename to docker/dockerfiles/polkadot-parachain/polkadot-parachain_builder.Containerfile index 159bcb3236..4d110d6af4 100644 --- a/docker/polkadot-parachain_builder.Containerfile +++ b/docker/dockerfiles/polkadot-parachain/polkadot-parachain_builder.Containerfile @@ -1,4 +1,4 @@ -# This file is sourced from https://github.com/paritytech/polkadot/blob/master/scripts/ci/dockerfiles/polkadot/polkadot_builder.Dockerfile +# This file is sourced from https://github.com/paritytech/polkadot/blob/master/docker/dockerfiles/polkadot/polkadot_builder.Dockerfile # This is the build stage for polkadot-parachain. Here we create the binary in a temporary image. FROM docker.io/paritytech/ci-linux:production as builder @@ -14,7 +14,7 @@ LABEL io.parity.image.type="builder" \ io.parity.image.authors="devops-team@parity.io" \ io.parity.image.vendor="Parity Technologies" \ io.parity.image.description="Multistage Docker image for polkadot-parachain" \ - io.parity.image.source="https://github.com/paritytech/polkadot/blob/${VCS_REF}/scripts/ci/dockerfiles/polkadot/polkadot-parachain_builder.Dockerfile" \ + io.parity.image.source="https://github.com/paritytech/polkadot/blob/${VCS_REF}/docker/dockerfiles/polkadot-parachain/polkadot-parachain_builder.Dockerfile" \ io.parity.image.documentation="https://github.com/paritytech/cumulus" COPY --from=builder /cumulus/target/release/polkadot-parachain /usr/local/bin diff --git a/cumulus/docker/injected.Dockerfile b/docker/dockerfiles/polkadot-parachain/polkadot-parachain_injected.Dockerfile similarity index 68% rename from cumulus/docker/injected.Dockerfile rename to docker/dockerfiles/polkadot-parachain/polkadot-parachain_injected.Dockerfile index f9b11f022e..16bd0f4cf3 100644 --- a/cumulus/docker/injected.Dockerfile +++ b/docker/dockerfiles/polkadot-parachain/polkadot-parachain_injected.Dockerfile @@ -9,10 +9,10 @@ LABEL io.parity.image.authors="devops-team@parity.io" \ io.parity.image.vendor="Parity Technologies" \ io.parity.image.title="${IMAGE_NAME}" \ io.parity.image.description="Cumulus, the Polkadot collator." \ - io.parity.image.source="https://github.com/paritytech/polkadot/blob/${VCS_REF}/scripts/docker/Dockerfile" \ + io.parity.image.source="https://github.com/paritytech/polkadot-sdk/blob/${VCS_REF}/docker/dockerfiles/polkadot-parachain/polkadot-parachain_injected.Dockerfile" \ io.parity.image.revision="${VCS_REF}" \ io.parity.image.created="${BUILD_DATE}" \ - io.parity.image.documentation="https://github.com/paritytech/cumulus/" + io.parity.image.documentation="https://github.com/paritytech/polkadot-sdk/" # show backtraces ENV RUST_BACKTRACE 1 @@ -22,8 +22,10 @@ USER root RUN mkdir -p /specs # add polkadot-parachain binary to the docker image -COPY ./release-artifacts/* /usr/local/bin -COPY ./parachains/chain-specs/*.json /specs/ +COPY bin/* /usr/local/bin/ +COPY specs/* /specs/ + +RUN chmod -R a+rx "/usr/local/bin" USER parity diff --git a/docker/dockerfiles/polkadot/README.md b/docker/dockerfiles/polkadot/README.md new file mode 100644 index 0000000000..e331d8984c --- /dev/null +++ b/docker/dockerfiles/polkadot/README.md @@ -0,0 +1,9 @@ +# Self built Docker image + +The Polkadot repo contains several options to build Docker images for Polkadot. + +This folder contains a self-contained image that does not require a Linux pre-built binary. + +Instead, building the image is possible on any host having docker installed and will +build Polkadot inside Docker. That also means that no Rust toolchain is required on the host +machine for the build to succeed. diff --git a/docker/dockerfiles/polkadot/docker-compose-local.yml b/docker/dockerfiles/polkadot/docker-compose-local.yml new file mode 100644 index 0000000000..1ff3a1ccaa --- /dev/null +++ b/docker/dockerfiles/polkadot/docker-compose-local.yml @@ -0,0 +1,50 @@ +version: '3' +services: + node_alice: + ports: + - "30333:30333" + - "9933:9933" + - "9944:9944" + - "9615:9615" + image: parity/polkadot:latest + volumes: + - "polkadot-data-alice:/data" + command: | + --chain=polkadot-local + --alice + -d /data + --node-key 0000000000000000000000000000000000000000000000000000000000000001 + networks: + testing_net: + ipv4_address: 172.28.1.1 + + node_bob: + ports: + - "30344:30333" + - "9935:9933" + - "9945:9944" + - "29615:9615" + image: parity/polkadot:latest + volumes: + - "polkadot-data-bob:/data" + links: + - "node_alice:alice" + command: | + --chain=polkadot-local + --bob + -d /data + --bootnodes '/ip4/172.28.1.1/tcp/30333/p2p/QmRpheLN4JWdAnY7HGJfWFNbfkQCb6tFf4vvA6hgjMZKrR' + networks: + testing_net: + ipv4_address: 172.28.1.2 + +volumes: + polkadot-data-alice: + polkadot-data-bob: + +networks: + testing_net: + ipam: + driver: default + config: + - subnet: 172.28.0.0/16 diff --git a/docker/dockerfiles/polkadot/docker-compose.yml b/docker/dockerfiles/polkadot/docker-compose.yml new file mode 100644 index 0000000000..524b116479 --- /dev/null +++ b/docker/dockerfiles/polkadot/docker-compose.yml @@ -0,0 +1,22 @@ +version: '3' +services: + polkadot: + image: parity/polkadot:latest + + ports: + - "127.0.0.1:30333:30333/tcp" + - "127.0.0.1:9933:9933/tcp" + - "127.0.0.1:9944:9944/tcp" + - "127.0.0.1:9615:9615/tcp" + + volumes: + - "polkadot-data:/data" + + command: | + --unsafe-rpc-external + --unsafe-ws-external + --rpc-cors all + --prometheus-external + +volumes: + polkadot-data: diff --git a/docker/dockerfiles/polkadot/polkadot_Dockerfile.README.md b/docker/dockerfiles/polkadot/polkadot_Dockerfile.README.md new file mode 100644 index 0000000000..7e89cb55f3 --- /dev/null +++ b/docker/dockerfiles/polkadot/polkadot_Dockerfile.README.md @@ -0,0 +1,7 @@ +# Polkadot official Docker image + +## [Polkadot](https://polkadot.network/) + +## [GitHub](https://github.com/paritytech/polkadot) + +## [Polkadot Wiki](https://wiki.polkadot.network/) diff --git a/docker/dockerfiles/polkadot/polkadot_builder.Dockerfile b/docker/dockerfiles/polkadot/polkadot_builder.Dockerfile new file mode 100644 index 0000000000..f8dc374a14 --- /dev/null +++ b/docker/dockerfiles/polkadot/polkadot_builder.Dockerfile @@ -0,0 +1,36 @@ +# This is the build stage for Polkadot. Here we create the binary in a temporary image. +FROM docker.io/paritytech/ci-linux:production as builder + +WORKDIR /polkadot +COPY . /polkadot + +RUN cargo build --locked --release + +# This is the 2nd stage: a very small image where we copy the Polkadot binary." +FROM docker.io/parity/base-bin:latest + +LABEL description="Multistage Docker image for Polkadot: a platform for web3" \ + io.parity.image.type="builder" \ + io.parity.image.authors="chevdor@gmail.com, devops-team@parity.io" \ + io.parity.image.vendor="Parity Technologies" \ + io.parity.image.description="Polkadot: a platform for web3" \ + io.parity.image.source="https://github.com/paritytech/polkadot/blob/${VCS_REF}/docker/dockerfiles/polkadot/polkadot_builder.Dockerfile" \ + io.parity.image.documentation="https://github.com/paritytech/polkadot/" + +COPY --from=builder /polkadot/target/release/polkadot /usr/local/bin + +RUN useradd -m -u 1000 -U -s /bin/sh -d /polkadot polkadot && \ + mkdir -p /data /polkadot/.local/share && \ + chown -R polkadot:polkadot /data && \ + ln -s /data /polkadot/.local/share/polkadot && \ +# unclutter and minimize the attack surface + rm -rf /usr/bin /usr/sbin && \ +# check if executable works in this container + /usr/local/bin/polkadot --version + +USER polkadot + +EXPOSE 30333 9933 9944 9615 +VOLUME ["/data"] + +ENTRYPOINT ["/usr/local/bin/polkadot"] diff --git a/docker/dockerfiles/polkadot/polkadot_injected_debian.Dockerfile b/docker/dockerfiles/polkadot/polkadot_injected_debian.Dockerfile new file mode 100644 index 0000000000..e2c72dcfe2 --- /dev/null +++ b/docker/dockerfiles/polkadot/polkadot_injected_debian.Dockerfile @@ -0,0 +1,53 @@ +FROM docker.io/library/ubuntu:20.04 + +# metadata +ARG VCS_REF +ARG BUILD_DATE +ARG POLKADOT_VERSION +ARG POLKADOT_GPGKEY=9D4B2B6EB8F97156D19669A9FF0812D491B96798 +ARG GPG_KEYSERVER="keyserver.ubuntu.com" + +LABEL io.parity.image.authors="devops-team@parity.io" \ + io.parity.image.vendor="Parity Technologies" \ + io.parity.image.title="parity/polkadot" \ + io.parity.image.description="Polkadot: a platform for web3. This is the official Parity image with an injected binary." \ + io.parity.image.source="https://github.com/paritytech/polkadot/blob/${VCS_REF}/scripts/ci/dockerfiles/polkadot/polkadot_injected_debian.Dockerfile" \ + io.parity.image.revision="${VCS_REF}" \ + io.parity.image.created="${BUILD_DATE}" \ + io.parity.image.documentation="https://github.com/paritytech/polkadot/" + +# show backtraces +ENV RUST_BACKTRACE 1 + +# install tools and dependencies +RUN apt-get update && \ + DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \ + libssl1.1 \ + ca-certificates \ + gnupg && \ + useradd -m -u 1000 -U -s /bin/sh -d /polkadot polkadot && \ +# add repo's gpg keys and install the published polkadot binary + gpg --keyserver ${GPG_KEYSERVER} --recv-keys ${POLKADOT_GPGKEY} && \ + gpg --export ${POLKADOT_GPGKEY} > /usr/share/keyrings/parity.gpg && \ + echo 'deb [signed-by=/usr/share/keyrings/parity.gpg] https://releases.parity.io/deb release main' > /etc/apt/sources.list.d/parity.list && \ + apt-get update && \ + apt-get install -y --no-install-recommends polkadot=${POLKADOT_VERSION#?} && \ +# apt cleanup + apt-get autoremove -y && \ + apt-get clean && \ + rm -rf /var/lib/apt/lists/* ; \ + mkdir -p /data /polkadot/.local/share && \ + chown -R polkadot:polkadot /data && \ + ln -s /data /polkadot/.local/share/polkadot + +USER polkadot + +# check if executable works in this container +RUN /usr/bin/polkadot --version +RUN /usr/bin/polkadot-execute-worker --version +RUN /usr/bin/polkadot-prepare-worker --version + +EXPOSE 30333 9933 9944 +VOLUME ["/polkadot"] + +ENTRYPOINT ["/usr/bin/polkadot"] diff --git a/docker/polkadot_injected_debug.Dockerfile b/docker/dockerfiles/polkadot/polkadot_injected_debug.Dockerfile similarity index 94% rename from docker/polkadot_injected_debug.Dockerfile rename to docker/dockerfiles/polkadot/polkadot_injected_debug.Dockerfile index f7f764d335..80ce825898 100644 --- a/docker/polkadot_injected_debug.Dockerfile +++ b/docker/dockerfiles/polkadot/polkadot_injected_debug.Dockerfile @@ -9,7 +9,7 @@ LABEL io.parity.image.authors="devops-team@parity.io" \ io.parity.image.vendor="Parity Technologies" \ io.parity.image.title="${IMAGE_NAME}" \ io.parity.image.description="Polkadot: a platform for web3" \ - io.parity.image.source="https://github.com/paritytech/polkadot/blob/${VCS_REF}/scripts/ci/dockerfiles/polkadot_injected_debug.Dockerfile" \ + io.parity.image.source="https://github.com/paritytech/polkadot/blob/${VCS_REF}/docker/dockerfiles/polkadot/polkadot_injected_debug.Dockerfile" \ io.parity.image.revision="${VCS_REF}" \ io.parity.image.created="${BUILD_DATE}" \ io.parity.image.documentation="https://github.com/paritytech/polkadot/" diff --git a/docker/polkadot_injected_release.Dockerfile b/docker/dockerfiles/polkadot/polkadot_injected_release.Dockerfile similarity index 95% rename from docker/polkadot_injected_release.Dockerfile rename to docker/dockerfiles/polkadot/polkadot_injected_release.Dockerfile index 87ae7ac27d..c13f2db982 100644 --- a/docker/polkadot_injected_release.Dockerfile +++ b/docker/dockerfiles/polkadot/polkadot_injected_release.Dockerfile @@ -11,7 +11,7 @@ LABEL io.parity.image.authors="devops-team@parity.io" \ io.parity.image.vendor="Parity Technologies" \ io.parity.image.title="parity/polkadot" \ io.parity.image.description="Polkadot: a platform for web3. This is the official Parity image with an injected binary." \ - io.parity.image.source="https://github.com/paritytech/polkadot/blob/${VCS_REF}/scripts/ci/dockerfiles/polkadot_injected_release.Dockerfile" \ + io.parity.image.source="https://github.com/paritytech/polkadot/blob/${VCS_REF}/docker/dockerfiles/polkadot/polkadot_injected_release.Dockerfile" \ io.parity.image.revision="${VCS_REF}" \ io.parity.image.created="${BUILD_DATE}" \ io.parity.image.documentation="https://github.com/paritytech/polkadot/" diff --git a/docker/staking-miner/staking-miner_builder.Dockerfile b/docker/dockerfiles/staking-miner/staking-miner_builder.Dockerfile similarity index 100% rename from docker/staking-miner/staking-miner_builder.Dockerfile rename to docker/dockerfiles/staking-miner/staking-miner_builder.Dockerfile diff --git a/docker/staking-miner/staking-miner_injected.Dockerfile b/docker/dockerfiles/staking-miner/staking-miner_injected.Dockerfile similarity index 100% rename from docker/staking-miner/staking-miner_injected.Dockerfile rename to docker/dockerfiles/staking-miner/staking-miner_injected.Dockerfile diff --git a/docker/substrate_injected.Dockerfile b/docker/dockerfiles/substrate_injected.Dockerfile similarity index 100% rename from docker/substrate_injected.Dockerfile rename to docker/dockerfiles/substrate_injected.Dockerfile diff --git a/docker/test-parachain-collator.dockerfile b/docker/dockerfiles/test-parachain-collator.dockerfile similarity index 96% rename from docker/test-parachain-collator.dockerfile rename to docker/dockerfiles/test-parachain-collator.dockerfile index 9c2d8fbe58..0d56949152 100644 --- a/docker/test-parachain-collator.dockerfile +++ b/docker/dockerfiles/test-parachain-collator.dockerfile @@ -1,4 +1,4 @@ -# This file is sourced from https://github.com/paritytech/polkadot/blob/master/scripts/ci/dockerfiles/polkadot/polkadot_builder.Dockerfile +# This file is sourced from https://github.com/paritytech/polkadot/blob/master/docker/dockerfiles/polkadot/polkadot_builder.Dockerfile FROM docker.io/paritytech/ci-linux:production as builder WORKDIR /cumulus diff --git a/docker/test-parachain_injected.Dockerfile b/docker/dockerfiles/test-parachain_injected.Dockerfile similarity index 95% rename from docker/test-parachain_injected.Dockerfile rename to docker/dockerfiles/test-parachain_injected.Dockerfile index 0b345e16e4..e5d0df7aad 100644 --- a/docker/test-parachain_injected.Dockerfile +++ b/docker/dockerfiles/test-parachain_injected.Dockerfile @@ -9,7 +9,7 @@ LABEL io.parity.image.authors="devops-team@parity.io" \ io.parity.image.vendor="Parity Technologies" \ io.parity.image.title="${IMAGE_NAME}" \ io.parity.image.description="Test parachain for Zombienet" \ - io.parity.image.source="https://github.com/paritytech/cumulus/blob/${VCS_REF}/docker/test-parachain_injected.Dockerfile" \ + io.parity.image.source="https://github.com/paritytech/cumulus/blob/${VCS_REF}/docker/dockerfiles/test-parachain_injected.Dockerfile" \ io.parity.image.revision="${VCS_REF}" \ io.parity.image.created="${BUILD_DATE}" \ io.parity.image.documentation="https://github.com/paritytech/cumulus/" diff --git a/docker/injected.Dockerfile b/docker/injected.Dockerfile deleted file mode 100644 index 93d0561ca8..0000000000 --- a/docker/injected.Dockerfile +++ /dev/null @@ -1,51 +0,0 @@ -FROM docker.io/library/ubuntu:20.04 - -# metadata -ARG VCS_REF -ARG BUILD_DATE -ARG IMAGE_NAME - -LABEL io.parity.image.authors="devops-team@parity.io" \ - io.parity.image.vendor="Parity Technologies" \ - io.parity.image.title="${IMAGE_NAME}" \ - io.parity.image.description="Cumulus, the Polkadot collator." \ - io.parity.image.source="https://github.com/paritytech/polkadot/blob/${VCS_REF}/scripts/docker/Dockerfile" \ - io.parity.image.revision="${VCS_REF}" \ - io.parity.image.created="${BUILD_DATE}" \ - io.parity.image.documentation="https://github.com/paritytech/cumulus/" - -# show backtraces -ENV RUST_BACKTRACE 1 - -# install tools and dependencies -RUN apt-get update && \ - DEBIAN_FRONTEND=noninteractive apt-get install -y \ - libssl1.1 \ - ca-certificates \ - curl && \ -# apt cleanup - apt-get autoremove -y && \ - apt-get clean && \ - find /var/lib/apt/lists/ -type f -not -name lock -delete; \ -# add user and link ~/.local/share/polkadot to /data - useradd -m -u 1000 -U -s /bin/sh -d /polkadot polkadot && \ - mkdir -p /data /polkadot/.local/share && \ - chown -R polkadot:polkadot /data && \ - ln -s /data /polkadot/.local/share/polkadot && \ - mkdir -p /specs - -# add polkadot-parachain binary to the docker image -COPY ./target/release/polkadot-parachain /usr/local/bin -COPY ./target/release/polkadot-parachain.asc /usr/local/bin -COPY ./target/release/polkadot-parachain.sha256 /usr/local/bin -COPY ./parachains/chain-specs/*.json /specs/ - -USER polkadot - -# check if executable works in this container -RUN /usr/local/bin/polkadot-parachain --version - -EXPOSE 30333 9933 9944 -VOLUME ["/polkadot"] - -ENTRYPOINT ["/usr/local/bin/polkadot-parachain"] diff --git a/docker/scripts/adder-collator/build-injected.sh b/docker/scripts/adder-collator/build-injected.sh new file mode 100755 index 0000000000..3a2d497413 --- /dev/null +++ b/docker/scripts/adder-collator/build-injected.sh @@ -0,0 +1,13 @@ +#!/usr/bin/env bash + +# Sample call: +# $0 /path/to/folder_with_binary +# This script replace the former dedicated Dockerfile +# and shows how to use the generic binary_injected.dockerfile + +PROJECT_ROOT=`git rev-parse --show-toplevel` + +export BINARY=adder-collator,undying-collator +export ARTIFACTS_FOLDER=$1 + +$PROJECT_ROOT/docker/scripts/build-injected.sh diff --git a/docker/scripts/adder-collator/test-build.sh b/docker/scripts/adder-collator/test-build.sh new file mode 100755 index 0000000000..171e0309f8 --- /dev/null +++ b/docker/scripts/adder-collator/test-build.sh @@ -0,0 +1,23 @@ +#!/usr/bin/env bash + +TMP=$(mktemp -d) +ENGINE=${ENGINE:-podman} + +# TODO: Switch to /bin/bash when the image is built from parity/base-bin + +# Fetch some binaries +$ENGINE run --user root --rm -i \ + --pull always \ + -v "$TMP:/export" \ + --entrypoint /usr/bin/bash \ + paritypr/colander:master -c \ + 'cp "$(which adder-collator)" /export' + +$ENGINE run --user root --rm -i \ + --pull always \ + -v "$TMP:/export" \ + --entrypoint /usr/bin/bash \ + paritypr/colander:master -c \ + 'cp "$(which undying-collator)" /export' + +./build-injected.sh $TMP diff --git a/docker/scripts/build-injected.sh b/docker/scripts/build-injected.sh new file mode 100755 index 0000000000..f415cf43c0 --- /dev/null +++ b/docker/scripts/build-injected.sh @@ -0,0 +1,100 @@ +#!/usr/bin/env bash +#set -e + +# This script allows building a Container Image from a Linux +# binary that is injected into a base-image. + +ENGINE=${ENGINE:-podman} + +if [ "$ENGINE" == "podman" ]; then + PODMAN_FLAGS="--format docker" +else + PODMAN_FLAGS="" +fi + +CONTEXT=$(mktemp -d) +REGISTRY=${REGISTRY:-docker.io} + +# The following line ensure we know the project root +PROJECT_ROOT=${PROJECT_ROOT:-$(git rev-parse --show-toplevel)} +DOCKERFILE=${DOCKERFILE:-docker/dockerfiles/binary_injected.Dockerfile} +VERSION_TOML=$(grep "^version " $PROJECT_ROOT/Cargo.toml | grep -oE "([0-9\.]+-?[0-9]+)") + +#n The following VAR have default that can be overriden +DOCKER_OWNER=${DOCKER_OWNER:-parity} + +# We may get 1..n binaries, comma separated +BINARY=${BINARY:-polkadot} +IFS=',' read -r -a BINARIES <<< "$BINARY" + +VERSION=${VERSION:-$VERSION_TOML} +ARTIFACTS_FOLDER=${ARTIFACTS_FOLDER:-.} + +IMAGE=${IMAGE:-${REGISTRY}/${DOCKER_OWNER}/${BINARIES[0]}} +DESCRIPTION_DEFAULT="Injected Container image built for ${BINARY}" +DESCRIPTION=${DESCRIPTION:-$DESCRIPTION_DEFAULT} + +VCS_REF=${VCS_REF:-01234567} + +# Build the image +echo "Using engine: $ENGINE" +echo "Using Dockerfile: $DOCKERFILE" +echo "Using context: $CONTEXT" +echo "Building ${IMAGE}:latest container image for ${BINARY} v${VERSION} from ${ARTIFACTS_FOLDER} hang on!" +echo "ARTIFACTS_FOLDER=$ARTIFACTS_FOLDER" +echo "CONTEXT=$CONTEXT" + +# We need all binaries and resources available in the Container build "CONTEXT" +mkdir -p $CONTEXT/bin +for bin in "${BINARIES[@]}" +do + echo "Copying $ARTIFACTS_FOLDER/$bin to context: $CONTEXT/bin" + ls -al "$ARTIFACTS_FOLDER/$bin" + cp -r "$ARTIFACTS_FOLDER/$bin" "$CONTEXT/bin" +done + +cp "$PROJECT_ROOT/docker/scripts/entrypoint.sh" "$CONTEXT" + +if [[ "$BINARY" == "polkadot-parachain" ]]; then + mkdir -p "$CONTEXT/specs" + echo "Copying parachains chain-specs from $ARTIFACTS_FOLDER/specs to context: $CONTEXT/specs" + ls -al "$ARTIFACTS_FOLDER/specs" + cp -r "$ARTIFACTS_FOLDER/specs" "$CONTEXT/specs" +fi + +echo "Building image: ${IMAGE}" + +TAGS=${TAGS[@]:-latest} +IFS=',' read -r -a TAG_ARRAY <<< "$TAGS" +TAG_ARGS=" " + +echo "The image ${IMAGE} will be tagged with ${TAG_ARRAY[*]}" +for tag in "${TAG_ARRAY[@]}"; do + TAG_ARGS+="--tag ${IMAGE}:${tag} " +done + +echo "$TAG_ARGS" + +# time \ +$ENGINE build \ + ${PODMAN_FLAGS} \ + --build-arg VCS_REF="${VCS_REF}" \ + --build-arg BUILD_DATE=$(date -u '+%Y-%m-%dT%H:%M:%SZ') \ + --build-arg IMAGE_NAME="${IMAGE}" \ + --build-arg BINARY="${BINARY}" \ + --build-arg ARTIFACTS_FOLDER="${ARTIFACTS_FOLDER}" \ + --build-arg DESCRIPTION="${DESCRIPTION}" \ + ${TAG_ARGS} \ + -f "${PROJECT_ROOT}/${DOCKERFILE}" \ + ${CONTEXT} + +echo "Your Container image for ${IMAGE} is ready" +$ENGINE images + +if [[ -z "${SKIP_IMAGE_VALIDATION}" ]]; then + echo "Check the image ${IMAGE}:${TAG_ARRAY[0]}" + $ENGINE run --rm -i "${IMAGE}:${TAG_ARRAY[0]}" --version + + echo "Query binaries" + $ENGINE run --rm -i --entrypoint /bin/bash "${IMAGE}:${TAG_ARRAY[0]}" -c "echo BINARY: ${BINARY}" +fi diff --git a/docker/scripts/entrypoint.sh b/docker/scripts/entrypoint.sh new file mode 100755 index 0000000000..eaa815faf6 --- /dev/null +++ b/docker/scripts/entrypoint.sh @@ -0,0 +1,18 @@ +#!/usr/bin/env bash + +# Sanity check +if [ -z "$BINARY" ] +then + echo "BINARY ENV not defined, this should never be the case. Aborting..." + exit 1 +fi + +# If the user built the image with multiple binaries, +# we consider the first one to be the canonical one +# To start with another binary, the user can either: +# - use the --entrypoint option +# - pass the ENV BINARY with a single binary +IFS=',' read -r -a BINARIES <<< "$BINARY" +BIN0=${BINARIES[0]} +echo "Starting binary $BIN0" +$BIN0 $@ diff --git a/docker/scripts/malus/build-injected.sh b/docker/scripts/malus/build-injected.sh new file mode 100755 index 0000000000..83e30e1785 --- /dev/null +++ b/docker/scripts/malus/build-injected.sh @@ -0,0 +1,14 @@ +#!/usr/bin/env bash + +# Sample call: +# $0 /path/to/folder_with_binary +# This script replace the former dedicated Dockerfile +# and shows how to use the generic binary_injected.dockerfile + +PROJECT_ROOT=`git rev-parse --show-toplevel` + +export BINARY=malus,polkadot-execute-worker,polkadot-prepare-worker +export ARTIFACTS_FOLDER=$1 +# export TAGS=... + +$PROJECT_ROOT/docker/scripts/build-injected.sh diff --git a/docker/scripts/malus/test-build.sh b/docker/scripts/malus/test-build.sh new file mode 100755 index 0000000000..3114e9e2ad --- /dev/null +++ b/docker/scripts/malus/test-build.sh @@ -0,0 +1,19 @@ +#!/usr/bin/env bash + +TMP=$(mktemp -d) +ENGINE=${ENGINE:-podman} + +export TAGS=latest,beta,7777,1.0.2-rc23 + +# Fetch some binaries +$ENGINE run --user root --rm -i \ + --pull always \ + -v "$TMP:/export" \ + --entrypoint /bin/bash \ + paritypr/malus:7217 -c \ + 'cp "$(which malus)" /export' + +echo "Checking binaries we got:" +ls -al $TMP + +./build-injected.sh $TMP diff --git a/docker/scripts/polkadot-parachain/build-injected.sh b/docker/scripts/polkadot-parachain/build-injected.sh new file mode 100755 index 0000000000..f5c86a0351 --- /dev/null +++ b/docker/scripts/polkadot-parachain/build-injected.sh @@ -0,0 +1,15 @@ +#!/usr/bin/env bash + +# Sample call: +# $0 /path/to/folder_with_binary +# This script replace the former dedicated Dockerfile +# and shows how to use the generic binary_injected.dockerfile + +PROJECT_ROOT=`git rev-parse --show-toplevel` + +export BINARY=polkadot-parachain +export ARTIFACTS_FOLDER=$1 +export DOCKERFILE="docker/dockerfiles/polkadot-parachain/polkadot-parachain_injected.Dockerfile" +# export TAGS=... + +$PROJECT_ROOT/docker/scripts/build-injected.sh diff --git a/docker/scripts/polkadot-parachain/test-build.sh b/docker/scripts/polkadot-parachain/test-build.sh new file mode 100755 index 0000000000..1dc53bd0d0 --- /dev/null +++ b/docker/scripts/polkadot-parachain/test-build.sh @@ -0,0 +1,19 @@ +#!/usr/bin/env bash + +TMP=$(mktemp -d) +ENGINE=${ENGINE:-podman} + +export TAGS=latest,beta,7777,1.0.2-rc23 + +# Fetch some binaries +$ENGINE run --user root --rm -i \ + --pull always \ + -v "$TMP:/export" \ + --entrypoint /bin/bash \ + parity/polkadot-parachain:7217 -c \ + 'cp "$(which malus)" /export' + +echo "Checking binaries we got:" +ls -al $TMP + +./build-injected.sh $TMP diff --git a/cumulus/docker/scripts/build-injected-image.sh b/docker/scripts/polkadot-parachain_build-injected-image.sh similarity index 70% rename from cumulus/docker/scripts/build-injected-image.sh rename to docker/scripts/polkadot-parachain_build-injected-image.sh index b8bb0dd7dd..bb6909dd3b 100755 --- a/cumulus/docker/scripts/build-injected-image.sh +++ b/docker/scripts/polkadot-parachain_build-injected-image.sh @@ -6,5 +6,5 @@ IMAGE_NAME=${IMAGE_NAME:-polkadot-parachain} docker build --no-cache \ --build-arg IMAGE_NAME=$IMAGE_NAME \ -t $OWNER/$IMAGE_NAME \ - -f ./docker/injected.Dockerfile \ + -f ./docker/dockerfiles/polkadot-parachain/polkadot-parachain_injected.Dockerfile \ . && docker images diff --git a/docker/scripts/polkadot/build-injected.sh b/docker/scripts/polkadot/build-injected.sh new file mode 100755 index 0000000000..7cc6db43a5 --- /dev/null +++ b/docker/scripts/polkadot/build-injected.sh @@ -0,0 +1,13 @@ +#!/usr/bin/env bash + +# Sample call: +# $0 /path/to/folder_with_binary +# This script replace the former dedicated Dockerfile +# and shows how to use the generic binary_injected.dockerfile + +PROJECT_ROOT=`git rev-parse --show-toplevel` + +export BINARY=polkadot,polkadot-execute-worker,polkadot-prepare-worker +export ARTIFACTS_FOLDER=$1 + +$PROJECT_ROOT/docker/scripts/build-injected.sh diff --git a/docker/scripts/polkadot/test-build.sh b/docker/scripts/polkadot/test-build.sh new file mode 100755 index 0000000000..d2d904561c --- /dev/null +++ b/docker/scripts/polkadot/test-build.sh @@ -0,0 +1,18 @@ +#!/usr/bin/env bash + +TMP=$(mktemp -d) +ENGINE=${ENGINE:-podman} + +# You need to build an injected image first + +# Fetch some binaries +$ENGINE run --user root --rm -i \ + -v "$TMP:/export" \ + --entrypoint /bin/bash \ + parity/polkadot -c \ + 'cp "$(which polkadot)" /export' + +echo "Checking binaries we got:" +tree $TMP + +./build-injected.sh $TMP diff --git a/docker/scripts/staking-miner/README.md b/docker/scripts/staking-miner/README.md new file mode 100644 index 0000000000..3610e11303 --- /dev/null +++ b/docker/scripts/staking-miner/README.md @@ -0,0 +1,37 @@ +# staking-miner container image + +## Build using the Builder + +``` +./build.sh +``` + +## Build the injected Image + +You first need a valid Linux binary to inject. Let's assume this binary is located in `BIN_FOLDER`. + +``` +./build-injected.sh "$BIN_FOLDER" +``` + +## Test + +Here is how to test the image. We can generate a valid seed but the staking-miner will quickly notice that our +account is not funded and "does not exist". + +You may pass any ENV supported by the binary and must provide at least a few such as `SEED` and `URI`: +``` +ENV SEED="" +ENV URI="wss://rpc.polkadot.io:443" +ENV RUST_LOG="info" +``` + +``` +export SEED=$(subkey generate -n polkadot --output-type json | jq -r .secretSeed) +podman run --rm -it \ + -e URI="wss://rpc.polkadot.io:443" \ + -e RUST_LOG="info" \ + -e SEED \ + localhost/parity/staking-miner \ + dry-run seq-phragmen +``` diff --git a/docker/scripts/staking-miner/build-injected.sh b/docker/scripts/staking-miner/build-injected.sh new file mode 100755 index 0000000000..efe323b5fe --- /dev/null +++ b/docker/scripts/staking-miner/build-injected.sh @@ -0,0 +1,13 @@ +#!/usr/bin/env bash + +# Sample call: +# $0 /path/to/folder_with_staking-miner_binary +# This script replace the former dedicated staking-miner "injected" Dockerfile +# and shows how to use the generic binary_injected.dockerfile + +PROJECT_ROOT=`git rev-parse --show-toplevel` + +export BINARY=staking-miner +export ARTIFACTS_FOLDER=$1 + +$PROJECT_ROOT/docker/scripts/build-injected.sh diff --git a/docker/scripts/staking-miner/build.sh b/docker/scripts/staking-miner/build.sh new file mode 100755 index 0000000000..c2b6ab77e5 --- /dev/null +++ b/docker/scripts/staking-miner/build.sh @@ -0,0 +1,13 @@ +#!/usr/bin/env bash + +# Sample call: +# $0 /path/to/folder_with_staking-miner_binary +# This script replace the former dedicated staking-miner "injected" Dockerfile +# and shows how to use the generic binary_injected.dockerfile + +PROJECT_ROOT=`git rev-parse --show-toplevel` +ENGINE=podman + +echo "Building the staking-miner using the Builder image" +echo "PROJECT_ROOT=$PROJECT_ROOT" +$ENGINE build -t staking-miner -f "${PROJECT_ROOT}/docker/dockerfiles/staking-miner/staking-miner_builder.Dockerfile" "$PROJECT_ROOT" diff --git a/docker/scripts/staking-miner/staking-miner_Dockerfile.README.md b/docker/scripts/staking-miner/staking-miner_Dockerfile.README.md new file mode 100644 index 0000000000..ce424c42f4 --- /dev/null +++ b/docker/scripts/staking-miner/staking-miner_Dockerfile.README.md @@ -0,0 +1,3 @@ +# Staking-miner Docker image + +## [GitHub](https://github.com/paritytech/polkadot/tree/master/utils/staking-miner) diff --git a/docker/scripts/staking-miner/staking-miner_builder.Dockerfile b/docker/scripts/staking-miner/staking-miner_builder.Dockerfile new file mode 100644 index 0000000000..0ae77f36c7 --- /dev/null +++ b/docker/scripts/staking-miner/staking-miner_builder.Dockerfile @@ -0,0 +1,43 @@ +FROM paritytech/ci-linux:production as builder + +# metadata +ARG VCS_REF +ARG BUILD_DATE +ARG IMAGE_NAME="staking-miner" +ARG PROFILE=production + +LABEL description="This is the build stage. Here we create the binary." + +WORKDIR /app +COPY . /app +RUN cargo build --locked --profile $PROFILE --package staking-miner + +# ===== SECOND STAGE ====== + +FROM docker.io/parity/base-bin:latest +LABEL description="This is the 2nd stage: a very small image where we copy the binary." +LABEL io.parity.image.authors="devops-team@parity.io" \ + io.parity.image.vendor="Parity Technologies" \ + io.parity.image.title="${IMAGE_NAME}" \ + io.parity.image.description="${IMAGE_NAME} for substrate based chains" \ + io.parity.image.source="https://github.com/paritytech/polkadot/blob/${VCS_REF}/scripts/ci/dockerfiles/${IMAGE_NAME}/${IMAGE_NAME}_builder.Dockerfile" \ + io.parity.image.revision="${VCS_REF}" \ + io.parity.image.created="${BUILD_DATE}" \ + io.parity.image.documentation="https://github.com/paritytech/polkadot/" + +ARG PROFILE=release +COPY --from=builder /app/target/$PROFILE/staking-miner /usr/local/bin + +# show backtraces +ENV RUST_BACKTRACE 1 + +USER parity + +ENV SEED="" +ENV URI="wss://rpc.polkadot.io" +ENV RUST_LOG="info" + +# check if the binary works in this container +RUN /usr/local/bin/staking-miner --version + +ENTRYPOINT [ "/usr/local/bin/staking-miner" ] diff --git a/docker/scripts/staking-miner/test-build.sh b/docker/scripts/staking-miner/test-build.sh new file mode 100755 index 0000000000..0ce74e2df2 --- /dev/null +++ b/docker/scripts/staking-miner/test-build.sh @@ -0,0 +1,18 @@ +#!/usr/bin/env bash + +TMP=$(mktemp -d) +ENGINE=${ENGINE:-podman} + +# You need to build an injected image first + +# Fetch some binaries +$ENGINE run --user root --rm -i \ + -v "$TMP:/export" \ + --entrypoint /bin/bash \ + parity/staking-miner -c \ + 'cp "$(which staking-miner)" /export' + +echo "Checking binaries we got:" +tree $TMP + +./build-injected.sh $TMP diff --git a/cumulus/docs/container.md b/docs/container.md similarity index 96% rename from cumulus/docs/container.md rename to docs/container.md index ef7c52a44f..afd3b27957 100644 --- a/cumulus/docs/container.md +++ b/docs/container.md @@ -52,7 +52,7 @@ anyone to get a working container image without requiring any of the Rust toolch ```bash docker build \ --tag $OWNER/$IMAGE_NAME \ - --file ./docker/polkadot-parachain_builder.Containerfile . + --file ./docker/dockerfiles/polkadot-parachain/polkadot-parachain_builder.Containerfile . ``` You may then run your new container: diff --git a/polkadot/doc/docker.md b/docs/docker.md similarity index 98% rename from polkadot/doc/docker.md rename to docs/docker.md index dc679908ec..53619ca1a9 100644 --- a/polkadot/doc/docker.md +++ b/docs/docker.md @@ -144,7 +144,7 @@ There are 3 options to build a Polkadot container image: To get up and running with the smallest footprint on your system, you may use an existing Polkadot Container image. You may also build a Polkadot container image yourself (it takes a while...) using the container specs -`scripts/ci/dockerfiles/polkadot/polkadot_builder.Dockerfile`. +`docker/dockerfiles/polkadot/polkadot_builder.Dockerfile`. ### Debian injected diff --git a/polkadot/.github/workflows/release-40_publish-rc-image.yml b/polkadot/.github/workflows/release-40_publish-rc-image.yml deleted file mode 100644 index 3d91c5b8c6..0000000000 --- a/polkadot/.github/workflows/release-40_publish-rc-image.yml +++ /dev/null @@ -1,132 +0,0 @@ -name: Release - Publish RC Container image -# see https://github.com/paritytech/release-engineering/issues/97#issuecomment-1651372277 - -on: - workflow_dispatch: - inputs: - release_id: - description: | - Release ID. - You can find it using the command: - curl -s \ - -H "Authorization: Bearer ${GITHUB_TOKEN}" https://api.github.com/repos/$OWNER/$REPO/releases | \ - jq '.[] | { name: .name, id: .id }' - required: true - type: string - registry: - description: "Container registry" - required: true - type: string - default: docker.io - owner: - description: Owner of the container image repo - required: true - type: string - default: parity - -env: - RELEASE_ID: ${{ inputs.release_id }} - ENGINE: docker - REGISTRY: ${{ inputs.registry }} - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - DOCKER_OWNER: ${{ inputs.owner || github.repository_owner }} - REPO: ${{ github.repository }} - -jobs: - fetch-artifacts: - runs-on: ubuntu-latest - - steps: - - name: Checkout sources - uses: actions/checkout@v3 - - - name: Fetch all artifacts - run: | - . ./scripts/ci/common/lib.sh - fetch_release_artifacts - - - name: Cache the artifacts - uses: actions/cache@88522ab9f39a2ea568f7027eddc7d8d8bc9d59c8 # v3.3.1 - with: - key: artifacts-${{ github.sha }} - path: | - ./release-artifacts/**/* - - build-container: - runs-on: ubuntu-latest - needs: fetch-artifacts - - strategy: - matrix: - binary: ["polkadot", "staking-miner"] - - steps: - - name: Checkout sources - uses: actions/checkout@v3 - - - name: Get artifacts from cache - uses: actions/cache@88522ab9f39a2ea568f7027eddc7d8d8bc9d59c8 # v3.3.1 - with: - key: artifacts-${{ github.sha }} - fail-on-cache-miss: true - path: | - ./release-artifacts/**/* - - - name: Check sha256 ${{ matrix.binary }} - working-directory: ./release-artifacts - run: | - . ../scripts/ci/common/lib.sh - - echo "Checking binary ${{ matrix.binary }}" - check_sha256 ${{ matrix.binary }} && echo "OK" || echo "ERR" - - - name: Check GPG ${{ matrix.binary }} - working-directory: ./release-artifacts - run: | - . ../scripts/ci/common/lib.sh - import_gpg_keys - check_gpg ${{ matrix.binary }} - - - name: Fetch commit and tag - id: fetch_refs - run: | - release=release-${{ inputs.release_id }} && \ - echo "release=${release}" >> $GITHUB_OUTPUT - - commit=$(git rev-parse --short HEAD) && \ - echo "commit=${commit}" >> $GITHUB_OUTPUT - - tag=$(git name-rev --tags --name-only $(git rev-parse HEAD)) && \ - [ "${tag}" != "undefined" ] && echo "tag=${tag}" >> $GITHUB_OUTPUT || \ - echo "No tag, doing without" - - - name: Build Injected Container image for ${{ matrix.binary }} - env: - BIN_FOLDER: ./release-artifacts - BINARY: ${{ matrix.binary }} - TAGS: ${{join(steps.fetch_refs.outputs.*, ',')}} - run: | - echo "Building container for ${{ matrix.binary }}" - ./scripts/ci/dockerfiles/build-injected.sh - - - name: Login to Dockerhub - uses: docker/login-action@v2 - with: - username: ${{ secrets.DOCKERHUB_USERNAME }} - password: ${{ secrets.DOCKERHUB_TOKEN }} - - - name: Push Container image for ${{ matrix.binary }} - id: docker_push - env: - BINARY: ${{ matrix.binary }} - run: | - $ENGINE images | grep ${BINARY} - $ENGINE push --all-tags ${REGISTRY}/${DOCKER_OWNER}/${BINARY} - - - name: Check version for the published image for ${{ matrix.binary }} - env: - BINARY: ${{ matrix.binary }} - RELEASE_TAG: ${{ steps.fetch_refs.outputs.release }} - run: | - echo "Checking tag ${RELEASE_TAG} for image ${REGISTRY}/${DOCKER_OWNER}/${BINARY}" - $ENGINE run -i ${REGISTRY}/${DOCKER_OWNER}/${BINARY}:${RELEASE_TAG} --version diff --git a/polkadot/.github/workflows/release-51_publish-docker-manual.yml b/polkadot/.github/workflows/release-51_publish-docker-manual.yml deleted file mode 100644 index 919769f870..0000000000 --- a/polkadot/.github/workflows/release-51_publish-docker-manual.yml +++ /dev/null @@ -1,51 +0,0 @@ -name: Release - Publish Docker image (manual dispatch) - -on: - workflow_dispatch: - inputs: - version: - description: version to build/release - default: v0.9.18 - required: true - date: - description: release date of version - default: "2022-02-23T19:11:58Z" - required: true - -jobs: - main: - runs-on: ubuntu-latest - steps: - - name: Checkout sources - uses: actions/checkout@v3 - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@95cb08cb2672c73d4ffd2f422e6d11953d2a9c70 # v2.1.0 - - name: Cache Docker layers - uses: actions/cache@v3 - with: - path: /tmp/.buildx-cache - key: ${{ runner.os }}-buildx-${{ github.sha }} - restore-keys: | - ${{ runner.os }}-buildx- - - name: Login to Dockerhub - uses: docker/login-action@v2 - with: - username: ${{ secrets.DOCKERHUB_USERNAME }} - password: ${{ secrets.DOCKERHUB_TOKEN }} - - name: Build and push - id: docker_build - uses: docker/build-push-action@v4 - with: - push: true - file: scripts/ci/dockerfiles/polkadot/polkadot_injected_debian.Dockerfile - tags: | - parity/polkadot:latest - parity/polkadot:${{ github.event.inputs.version }} - build-args: | - POLKADOT_VERSION=${{ github.event.inputs.version }} - VCS_REF=${{ github.ref }} - BUILD_DATE=${{ github.event.inputs.date }} - cache-from: type=local,src=/tmp/.buildx-cache - cache-to: type=local,dest=/tmp/.buildx-cache - - name: Image digest - run: echo ${{ steps.docker_build.outputs.digest }} diff --git a/polkadot/utils/staking-miner/README.md b/polkadot/utils/staking-miner/README.md index 8fec746e6e..90a00eeac0 100644 --- a/polkadot/utils/staking-miner/README.md +++ b/polkadot/utils/staking-miner/README.md @@ -51,7 +51,7 @@ to be installed. The trade-off however is that it takes a little longer to build tasks. You may build the multi-stage image the root of the Polkadot repository with: ``` TODO: UPDATE THAT -docker build -t staking-miner -f scripts/ci/dockerfiles/staking-miner/staking-miner_builder.Dockerfile . +docker build -t staking-miner -f docker/dockerfiles/staking-miner/staking-miner_builder.Dockerfile . ``` ### Running