diff --git a/polkadot/.gitlab-ci.yml b/polkadot/.gitlab-ci.yml index 5dbda40e52..086cf71074 100644 --- a/polkadot/.gitlab-ci.yml +++ b/polkadot/.gitlab-ci.yml @@ -38,7 +38,7 @@ default: when: on_success expire_in: 28 days paths: - - artifacts/ + - ./artifacts/ .kubernetes-env: &kubernetes-env tags: @@ -160,7 +160,7 @@ check-runtime-benchmarks: sed -n -r 's/^polkadot ([0-9.]+.*-[0-9a-f]{7,13})-.*$/\1/p')"; EXTRATAG="${CI_COMMIT_REF_NAME}-${EXTRATAG}-$(cut -c 1-8 ./artifacts/polkadot.sha256)"; fi - - echo "Polkadot version = ${VERSION} (EXTRATAG ${EXTRATAG})" + - echo "Polkadot version = ${VERSION} (EXTRATAG = ${EXTRATAG})" - echo -n ${VERSION} > ./artifacts/VERSION - echo -n ${EXTRATAG} > ./artifacts/EXTRATAG - cp -r scripts/docker/* ./artifacts @@ -170,22 +170,41 @@ build-linux-release: <<: *collect-artifacts <<: *docker-env <<: *compiler-info - variables: - EXTRA_FLAGS: "" rules: - # extra features when building on `rococo-v1` branch and manual build on PRs - - if: $CI_COMMIT_REF_NAME == "rococo-v1" - variables: - RUSTFLAGS: "-Cdebug-assertions=y" + - if: $CI_PIPELINE_SOURCE == "schedule" + - if: $CI_COMMIT_REF_NAME == "master" + - if: $CI_COMMIT_REF_NAME =~ /^v[0-9]+\.[0-9]+.*$/ # i.e. v1.0, v2.1rc1 - if: $CI_COMMIT_REF_NAME =~ /^[0-9]+$/ # PRs variables: - RUSTFLAGS: "-Cdebug-assertions=y" - - when: always + RUSTFLAGS: "-Cdebug-assertions=y" + - if: $CI_COMMIT_REF_NAME == "rococo-v1" + variables: + RUSTFLAGS: "-Cdebug-assertions=y" script: - - time cargo build --release --verbose ${EXTRA_FLAGS} + - time cargo build --release --verbose - sccache -s - *pack-artifacts +build-adder-collator: + stage: test + <<: *collect-artifacts + <<: *docker-env + <<: *compiler-info + rules: + - if: $CI_PIPELINE_SOURCE == "schedule" + - if: $CI_COMMIT_REF_NAME == "master" + - if: $CI_COMMIT_REF_NAME =~ /^[0-9]+$/ # PRs + script: + - time cargo build --release --verbose -p test-parachain-adder-collator + - sccache -s + # pack artifacts + - mkdir -p ./artifacts + - mv ./target/release/adder-collator ./artifacts/. + - echo -n "${CI_COMMIT_REF_NAME}" > ./artifacts/VERSION + - echo -n "${CI_COMMIT_REF_NAME}-${CI_COMMIT_SHORT_SHA}" > ./artifacts/EXTRATAG + - echo "adder-collator version = $(cat ./artifacts/VERSION) (EXTRATAG = $(cat ./artifacts/EXTRATAG))" + - cp -r scripts/docker/* ./artifacts + #### stage: build check-transaction-versions: @@ -213,12 +232,13 @@ generate-impl-guide: script: - mdbook build roadmap/implementers-guide -.build-push-docker-image: &build-push-docker-image +.build-push-image: &build-push-image <<: *kubernetes-env image: quay.io/buildah/stable variables: GIT_STRATEGY: none - # DOCKERFILE: scripts/docker/Dockerfile + # scripts/docker/Dockerfile + DOCKERFILE: Dockerfile IMAGE_NAME: docker.io/parity/polkadot DOCKER_USER: ${Docker_Hub_User_Parity} DOCKER_PASS: ${Docker_Hub_Pass_Parity} @@ -236,54 +256,79 @@ generate-impl-guide: --format=docker --build-arg VCS_REF="${CI_COMMIT_SHA}" --build-arg BUILD_DATE="$(date -u '+%Y-%m-%dT%H:%M:%SZ')" + --build-arg IMAGE_NAME="${IMAGE_NAME}" --tag "$IMAGE_NAME:$VERSION" - --tag "$IMAGE_NAME:$EXTRATAG" . + --tag "$IMAGE_NAME:$EXTRATAG" + --file ${DOCKERFILE} . # The job will success only on the protected branch - echo "$DOCKER_PASS" | buildah login --username "$DOCKER_USER" --password-stdin docker.io - buildah info - buildah push --format=v2s2 "$IMAGE_NAME:$VERSION" - buildah push --format=v2s2 "$IMAGE_NAME:$EXTRATAG" - # pass artifacts to the trigget-simnet job - # this twist is to match the similar logic in substrate - - echo "IMAGE_NAME=${IMAGE_NAME}" > build.env - - echo "VERSION=${EXTRATAG}" >> build.env after_script: - buildah logout "$IMAGE_NAME" + # pass artifacts to the trigger-simnet job + - echo "IMAGE_NAME=${IMAGE_NAME}" > ./artifacts/build.env + - echo "IMAGE_TAG=$(cat ./artifacts/EXTRATAG)" >> ./artifacts/build.env artifacts: reports: # this artifact is used in trigger-simnet job # https://docs.gitlab.com/ee/ci/multi_project_pipelines.html#with-variable-inheritance - dotenv: artifacts/build.env + dotenv: ./artifacts/build.env - -publish-dockerhub: +publish-polkadot-image: stage: build - <<: *build-push-docker-image + <<: *build-push-image rules: # Don't run on releases - this is handled by the Github Action here: # .github/workflows/publish-docker-release.yml - if: $CI_COMMIT_REF_NAME =~ /^v[0-9]+\.[0-9]+.*$/ # i.e. v1.0, v2.1rc1 when: never + - if: $CI_PIPELINE_SOURCE == "schedule" + - if: $CI_COMMIT_REF_NAME == "master" + - if: $CI_COMMIT_REF_NAME =~ /^[0-9]+$/ # PRs + variables: + IMAGE_NAME: docker.io/paritypr/synth-wave + DOCKER_USER: ${PARITYPR_USER} + DOCKER_PASS: ${PARITYPR_PASS} - if: $CI_COMMIT_REF_NAME == "rococo-v1" variables: IMAGE_NAME: docker.io/parity/rococo - - if: $CI_COMMIT_REF_NAME =~ /^[0-9]+$/ # PRs - variables: - # image to be tested with Simnet - IMAGE_NAME: docker.io/paritypr/synth-wave - DOCKER_USER: ${PARITYPR_USER} - DOCKER_PASS: ${PARITYPR_PASS} - - if: $CI_PIPELINE_SOURCE == "schedule" - - if: $CI_PIPELINE_SOURCE == "web" - - if: $CI_COMMIT_REF_NAME == "master" needs: - job: build-linux-release artifacts: true +publish-adder-collator-image: + # service image for Simnet + stage: build + <<: *build-push-image + variables: + # scripts/docker/collator.Dockerfile + DOCKERFILE: collator.Dockerfile + IMAGE_NAME: docker.io/paritypr/colander + DOCKER_USER: ${PARITYPR_USER} + DOCKER_PASS: ${PARITYPR_PASS} + rules: + - if: $CI_PIPELINE_SOURCE == "schedule" + - if: $CI_COMMIT_REF_NAME == "master" + # FIXME: remove me after merging + - if: $CI_COMMIT_REF_NAME =~ /^[0-9]+$/ # PRs + needs: + - job: build-adder-collator + artifacts: true + after_script: + - buildah logout "$IMAGE_NAME" + # pass artifacts to the trigger-simnet job + - echo "COLLATOR_IMAGE_TAG=$(cat ./artifacts/EXTRATAG)" > ./artifacts/collator.env + artifacts: + reports: + # this artifact is used in trigger-simnet job + dotenv: ./artifacts/collator.env + #### stage: publish -publish-s3-release: +publish-s3-release: &publish-s3 stage: publish needs: - job: build-linux-release @@ -295,10 +340,6 @@ publish-s3-release: BUCKET: "releases.parity.io" PREFIX: "polkadot/${ARCH}-${DOCKER_OS}" rules: - - if: $CI_COMMIT_REF_NAME == "rococo-v1" - variables: - PREFIX: "rococo/${ARCH}-${DOCKER_OS}" - - if: $CI_PIPELINE_SOURCE == "web" - if: $CI_PIPELINE_SOURCE == "schedule" - if: $CI_COMMIT_REF_NAME == "master" - if: $CI_COMMIT_REF_NAME =~ /^v[0-9]+\.[0-9]+.*$/ # i.e. v1.0, v2.1rc1 @@ -327,6 +368,14 @@ publish-s3-release: - aws s3 ls s3://${BUCKET}/${PREFIX}/${EXTRATAG}/ --recursive --human-readable --summarize +publish-s3-adder-collator: + <<: *publish-s3 + variables: + PREFIX: "rococo/${ARCH}-${DOCKER_OS}" + needs: + - job: build-adder-collator + artifacts: true + #### stage: deploy deploy-polkasync-kusama: @@ -335,29 +384,36 @@ deploy-polkasync-kusama: # former .rules-build - if: $CI_COMMIT_REF_NAME == "rococo-v1" when: never - - if: $CI_PIPELINE_SOURCE == "web" - if: $CI_PIPELINE_SOURCE == "schedule" - if: $CI_COMMIT_REF_NAME == "master" - - if: $CI_COMMIT_REF_NAME =~ /^v[0-9]+\.[0-9]+.*$/ # i.e. v1.0, v2.1rc1 variables: POLKADOT_CI_COMMIT_NAME: "${CI_COMMIT_REF_NAME}" - POLKADOT_CI_COMMIT_REF: "${CI_COMMIT_REF}" + POLKADOT_CI_COMMIT_REF: "${CI_COMMIT_SHORT_SHA}" allow_failure: true trigger: "parity/infrastructure/parity-testnet" trigger-simnet: stage: deploy - <<: *rules-test + image: paritytech/tools:latest + rules: + - if: $CI_PIPELINE_SOURCE == "schedule" + - if: $CI_COMMIT_REF_NAME == "master" + - if: $CI_COMMIT_REF_NAME =~ /^[0-9]+$/ # PRs + - if: $CI_COMMIT_REF_NAME == "rococo-v1" needs: - - job: publish-dockerhub - # `build.env` is taken by the triggered job from `publish-dockerhub` job with an - # exact `$VERSION` (`$EXTRATAG` here, i.e. `2643-0.8.29-5f689e0a-6b24dc54`) + - job: publish-polkadot-image + - job: publish-adder-collator-image + # `build.env` brings here `$IMAGE_NAME` and `$IMAGE_TAG` (`$EXTRATAG` here, + # i.e. `2643-0.8.29-5f689e0a-6b24dc54`). + # `collator.env` bears adder-collator unique build tag. In non-triggered builds it + # can be called by `master` tag. # Simnet uses an image published on PRs with this exact version for triggered runs - # on commits. And parity/rococo:rococo-v1 for runs not launched by this job. - trigger: - project: parity/simnet - branch: master - strategy: depend + # on commits. And parity/rococo:rococo-v1 for the runs not launched by this job. variables: TRGR_PROJECT: ${CI_PROJECT_NAME} TRGR_REF: ${CI_COMMIT_REF_NAME} + # simnet project ID + DWNSTRM_ID: 332 + script: + # API trigger for a simnet job + - ./scripts/gitlab/trigger_pipeline.sh diff --git a/polkadot/roadmap/implementers-guide/src/SUMMARY.md b/polkadot/roadmap/implementers-guide/src/SUMMARY.md index 313a765ecd..f0f061fd37 100644 --- a/polkadot/roadmap/implementers-guide/src/SUMMARY.md +++ b/polkadot/roadmap/implementers-guide/src/SUMMARY.md @@ -12,6 +12,7 @@ - [Runtime Architecture](runtime/README.md) - [Initializer Module](runtime/initializer.md) - [Configuration Module](runtime/configuration.md) + - [Shared](runtime/shared.md) - [Disputes Module](runtime/disputes.md) - [Paras Module](runtime/paras.md) - [Scheduler Module](runtime/scheduler.md) diff --git a/polkadot/roadmap/implementers-guide/src/runtime/initializer.md b/polkadot/roadmap/implementers-guide/src/runtime/initializer.md index 76178b3c52..ffeacd5cb3 100644 --- a/polkadot/roadmap/implementers-guide/src/runtime/initializer.md +++ b/polkadot/roadmap/implementers-guide/src/runtime/initializer.md @@ -30,7 +30,7 @@ The other parachains modules are initialized in this order: 1. UMP 1. HRMP -The [Configuration Module](configuration.md) is first, since all other modules need to operate under the same configuration as each other. Then the [Shared][shared.md] module is invoked, which determines the set of active validators. It would lead to inconsistency if, for example, the scheduler ran first and then the configuration was updated before the Inclusion module. +The [Configuration Module](configuration.md) is first, since all other modules need to operate under the same configuration as each other. Then the [Shared](shared.md) module is invoked, which determines the set of active validators. It would lead to inconsistency if, for example, the scheduler ran first and then the configuration was updated before the Inclusion module. Set `HasInitialized` to true. diff --git a/polkadot/scripts/docker/Dockerfile b/polkadot/scripts/docker/Dockerfile index 89b74b7e34..9052892c3f 100644 --- a/polkadot/scripts/docker/Dockerfile +++ b/polkadot/scripts/docker/Dockerfile @@ -3,10 +3,11 @@ FROM debian:buster-slim # metadata ARG VCS_REF ARG BUILD_DATE +ARG IMAGE_NAME LABEL io.parity.image.authors="devops-team@parity.io" \ io.parity.image.vendor="Parity Technologies" \ - io.parity.image.title="parity/polkadot" \ + io.parity.image.title="${IMAGE_NAME}" \ io.parity.image.description="polkadot: a platform for web3" \ io.parity.image.source="https://github.com/paritytech/polkadot/blob/${VCS_REF}/scripts/docker/Dockerfile" \ io.parity.image.revision="${VCS_REF}" \ @@ -18,7 +19,6 @@ ENV RUST_BACKTRACE 1 # install tools and dependencies RUN apt-get update && \ - DEBIAN_FRONTEND=noninteractive apt-get upgrade -y && \ DEBIAN_FRONTEND=noninteractive apt-get install -y \ libssl1.1 \ ca-certificates \ @@ -45,4 +45,3 @@ EXPOSE 30333 9933 9944 VOLUME ["/polkadot"] ENTRYPOINT ["/usr/local/bin/polkadot"] - diff --git a/polkadot/scripts/docker/collator.Dockerfile b/polkadot/scripts/docker/collator.Dockerfile new file mode 100644 index 0000000000..9e25c55df5 --- /dev/null +++ b/polkadot/scripts/docker/collator.Dockerfile @@ -0,0 +1,47 @@ +FROM debian:buster-slim + +# metadata +ARG VCS_REF +ARG BUILD_DATE +ARG IMAGE_NAME + +LABEL io.parity.image.authors="devops-team@parity.io" \ + io.parity.image.vendor="Parity Technologies" \ + io.parity.image.title="${IMAGE_NAME}" \ + io.parity.image.description="polkadot: a platform for web3" \ + io.parity.image.source="https://github.com/paritytech/polkadot/blob/${VCS_REF}/scripts/docker/Dockerfile" \ + io.parity.image.revision="${VCS_REF}" \ + io.parity.image.created="${BUILD_DATE}" \ + io.parity.image.documentation="https://github.com/paritytech/polkadot/" + +# show backtraces +ENV RUST_BACKTRACE 1 + +# install tools and dependencies +RUN apt-get update && \ + DEBIAN_FRONTEND=noninteractive apt-get install -y \ + libssl1.1 \ + ca-certificates \ + curl && \ +# apt cleanup + apt-get autoremove -y && \ + apt-get clean && \ + find /var/lib/apt/lists/ -type f -not -name lock -delete; \ +# add user and link ~/.local/share/adder-collator to /data + useradd -m -u 1000 -U -s /bin/sh -d /adder-collator adder-collator && \ + mkdir -p /data /adder-collator/.local/share && \ + chown -R adder-collator:adder-collator /data && \ + ln -s /data /adder-collator/.local/share/polkadot + +# add adder-collator binary to docker image +COPY ./adder-collator /usr/local/bin + +USER adder-collator + +# check if executable works in this container +RUN /usr/local/bin/adder-collator --version + +EXPOSE 30333 9933 9944 +VOLUME ["/adder-collator"] + +ENTRYPOINT ["/usr/local/bin/adder-collator"] diff --git a/polkadot/scripts/gitlab/trigger_pipeline.sh b/polkadot/scripts/gitlab/trigger_pipeline.sh new file mode 100755 index 0000000000..63ffa71f52 --- /dev/null +++ b/polkadot/scripts/gitlab/trigger_pipeline.sh @@ -0,0 +1,46 @@ +#!/bin/bash + +set -eu +# API trigger another project's pipeline +curl --silent \ + -X POST \ + -F "token=${CI_JOB_TOKEN}" \ + -F "ref=master" \ + -F "variables[TRGR_PROJECT]=${TRGR_PROJECT}" \ + -F "variables[TRGR_REF]=${TRGR_REF}" \ + -F "variables[IMAGE_NAME]=${IMAGE_NAME}" \ + -F "variables[IMAGE_TAG]=${IMAGE_TAG}" \ + -F "variables[COLLATOR_IMAGE_TAG]=${COLLATOR_IMAGE_TAG}" \ + "https://${CI_SERVER_HOST}/api/v4/projects/${DWNSTRM_ID}/trigger/pipeline" | \ + tee pipeline + +PIPELINE_ID=$(cat pipeline | jq ".id") +echo "\nWaiting on ${PIPELINE_ID} status..." + +# This part polls for the triggered pipeline status, the native +# `trigger` job does not return this status via API. +# This is a workaround for a Gitlab bug, waits here until +# https://gitlab.com/gitlab-org/gitlab/-/issues/326137 gets fixed. +# The timeout is 360 curls with 8 sec interval, roughly an hour. + +function get_status() { + curl --silent \ + --header "PRIVATE-TOKEN: ${PIPELINE_TOKEN}" \ + "https://${CI_SERVER_HOST}/api/v4/projects/${DWNSTRM_ID}/pipelines/${PIPELINE_ID}" | \ + jq --raw-output ".status"; +} + +for i in $(seq 1 360); do + STATUS=$(get_status); + echo "Triggered pipeline status is ${STATUS}"; + if [[ ${STATUS} =~ ^(pending|running|created)$ ]]; then + echo "Busy..."; + elif [[ ${STATUS} =~ ^(failed|canceled|skipped|manual)$ ]]; then + exit 1; + elif [[ ${STATUS} =~ ^(success)$ ]]; then + exit 0; + else + exit 1; + fi +sleep 8; +done