From fc8d1765eff920e28b26cc9c755f4a473c6b662d Mon Sep 17 00:00:00 2001 From: Alexander Samusev <41779041+alvicsam@users.noreply.github.com> Date: Mon, 26 Sep 2022 10:50:25 +0200 Subject: [PATCH] [ci] Split gitlab-ci config to separate files (#6031) * [WIP][ci] Split gitlab-ci config to separate files * add stage * move short benches * rename stage short_benchmark * include short-benchmarks * add check * add test stage * quickfix check-no-default-features * add build stage * add weights and publish * rename stage * rename stage * fix publish miner image * fix publish miner image * add dag * fix image * start build-short-benchmark with the pipeline * some more DAGs * empty line to short-benchmarks * add change from master Co-authored-by: parity-processbot <> --- polkadot/.gitlab-ci.yml | 951 +----------------- polkadot/scripts/ci/gitlab/pipeline/build.yml | 193 ++++ polkadot/scripts/ci/gitlab/pipeline/check.yml | 97 ++ .../scripts/ci/gitlab/pipeline/publish.yml | 209 ++++ .../ci/gitlab/pipeline/short-benchmarks.yml | 27 + polkadot/scripts/ci/gitlab/pipeline/test.yml | 74 ++ .../scripts/ci/gitlab/pipeline/weights.yml | 33 + .../scripts/ci/gitlab/pipeline/zombienet.yml | 302 ++++++ 8 files changed, 979 insertions(+), 907 deletions(-) create mode 100644 polkadot/scripts/ci/gitlab/pipeline/build.yml create mode 100644 polkadot/scripts/ci/gitlab/pipeline/check.yml create mode 100644 polkadot/scripts/ci/gitlab/pipeline/publish.yml create mode 100644 polkadot/scripts/ci/gitlab/pipeline/short-benchmarks.yml create mode 100644 polkadot/scripts/ci/gitlab/pipeline/test.yml create mode 100644 polkadot/scripts/ci/gitlab/pipeline/weights.yml create mode 100644 polkadot/scripts/ci/gitlab/pipeline/zombienet.yml diff --git a/polkadot/.gitlab-ci.yml b/polkadot/.gitlab-ci.yml index 798939d006..33ab02c549 100644 --- a/polkadot/.gitlab-ci.yml +++ b/polkadot/.gitlab-ci.yml @@ -15,10 +15,13 @@ # moment of time. stages: - - stage1 - - stage2 - - stage3 - - stage4 + - weights + - check + - test + - build + - publish + - zombienet + - short-benchmarks workflow: rules: @@ -38,7 +41,7 @@ variables: default: cache: {} -.collect-artifacts: &collect-artifacts +.collect-artifacts: artifacts: name: "${CI_JOB_NAME}_${CI_COMMIT_REF_NAME}" when: on_success @@ -46,7 +49,7 @@ default: paths: - ./artifacts/ -.collect-artifacts-short: &collect-artifacts-short +.collect-artifacts-short: artifacts: name: "${CI_JOB_NAME}_${CI_COMMIT_REF_NAME}" when: on_success @@ -56,18 +59,19 @@ default: # collecting vars for pipeline stopper # they will be used if the job fails -.pipeline-stopper-vars: &pipeline-stopper-vars +.pipeline-stopper-vars: + before_script: - echo "FAILED_JOB_URL=${CI_JOB_URL}" > pipeline-stopper.env - echo "FAILED_JOB_NAME=${CI_JOB_NAME}" >> pipeline-stopper.env - echo "FAILED_JOB_NAME=${CI_JOB_NAME}" >> pipeline-stopper.env - echo "PR_NUM=${CI_COMMIT_REF_NAME}" >> pipeline-stopper.env -.pipeline-stopper-artifacts: &pipeline-stopper-artifacts +.pipeline-stopper-artifacts: artifacts: reports: dotenv: pipeline-stopper.env -.kubernetes-env: &kubernetes-env +.kubernetes-env: retry: max: 2 when: @@ -78,7 +82,7 @@ default: tags: - kubernetes-parity-build -.docker-env: &docker-env +.docker-env: image: "${CI_IMAGE}" retry: max: 2 @@ -90,20 +94,20 @@ default: tags: - linux-docker -.compiler-info: &compiler-info +.compiler-info: before_script: - rustup show - cargo --version - sccache -s -.test-refs: &test-refs +.test-refs: rules: - if: $CI_PIPELINE_SOURCE == "web" - if: $CI_PIPELINE_SOURCE == "schedule" - if: $CI_COMMIT_REF_NAME == "master" - if: $CI_COMMIT_REF_NAME =~ /^[0-9]+$/ # PRs -.common-refs: &common-refs +.common-refs: # these jobs run always* rules: - if: $CI_PIPELINE_SOURCE == "web" @@ -113,13 +117,13 @@ default: - if: $CI_COMMIT_REF_NAME =~ /^v[0-9]+\.[0-9]+.*$/ # i.e. v1.0, v2.1rc1 - if: $CI_COMMIT_REF_NAME =~ /^release-v[0-9]+\.[0-9]+.*$/ # i.e. release-v0.9.27 -.test-pr-refs: &test-pr-refs +.test-pr-refs: rules: - if: $CI_PIPELINE_SOURCE == "pipeline" when: never - if: $CI_COMMIT_REF_NAME =~ /^[0-9]+$/ # PRs -.zombienet-refs: &zombienet-refs +.zombienet-refs: rules: - if: $CI_PIPELINE_SOURCE == "pipeline" when: never @@ -128,14 +132,14 @@ default: - if: $CI_COMMIT_REF_NAME == "master" - if: $CI_COMMIT_REF_NAME =~ /^[0-9]+$/ # PRs -.deploy-testnet-refs: &deploy-testnet-refs +.deploy-testnet-refs: rules: - if: $CI_PIPELINE_SOURCE == "pipeline" when: never - if: $CI_PIPELINE_SOURCE == "schedule" - if: $CI_COMMIT_REF_NAME == "master" -.publish-refs: &publish-refs +.publish-refs: rules: - if: $CI_PIPELINE_SOURCE == "pipeline" when: never @@ -145,14 +149,11 @@ default: - if: $CI_COMMIT_REF_NAME == "master" - if: $CI_COMMIT_REF_NAME =~ /^v[0-9]+\.[0-9]+.*$/ # i.e. v1.0, v2.1rc1 -.build-push-image: &build-push-image - <<: *kubernetes-env +.build-push-image: + extends: + - .kubernetes-env image: quay.io/buildah/stable - variables: &image-variables - GIT_STRATEGY: none - DOCKER_USER: ${PARITYPR_USER} - DOCKER_PASS: ${PARITYPR_PASS} - before_script: &check-versions + before_script: - test -s ./artifacts/VERSION || exit 1 - test -s ./artifacts/EXTRATAG || exit 1 - VERSION="$(cat ./artifacts/VERSION)" @@ -179,900 +180,36 @@ default: after_script: - buildah logout --all -#### stage: stage1 - -check-runtime: - stage: stage1 - image: paritytech/tools:latest - <<: *kubernetes-env - rules: - - if: $CI_COMMIT_REF_NAME =~ /^release-v[0-9]+\.[0-9]+.*$/ # i.e. release-v0.9.27 - variables: - GITLAB_API: "https://gitlab.parity.io/api/v4" - GITHUB_API_PROJECT: "parity%2Finfrastructure%2Fgithub-api" - script: - - ./scripts/ci/gitlab/check_runtime.sh - allow_failure: true - -cargo-fmt: - stage: stage1 - <<: *docker-env - <<: *test-refs - script: - - cargo +nightly --version - - cargo +nightly fmt --all -- --check - allow_failure: true - -build-linux-stable: - stage: stage1 - <<: *docker-env - <<: *compiler-info - <<: *collect-artifacts - <<: *common-refs - variables: - RUST_TOOLCHAIN: stable - # Enable debug assertions since we are running optimized builds for testing - # but still want to have debug assertions. - RUSTFLAGS: "-Cdebug-assertions=y -Dwarnings" - # Ensure we run the UI tests. - RUN_UI_TESTS: 1 - script: - - time cargo build --profile testnet --features pyroscope --verbose --bin polkadot - - sccache -s - # pack artifacts - - mkdir -p ./artifacts - - VERSION="${CI_COMMIT_REF_NAME}" # will be tag or branch name - - mv ./target/testnet/polkadot ./artifacts/. - - pushd artifacts - - sha256sum polkadot | tee polkadot.sha256 - - shasum -c polkadot.sha256 - - popd - - EXTRATAG="${CI_COMMIT_REF_NAME}-${CI_COMMIT_SHORT_SHA}" - - echo "Polkadot version = ${VERSION} (EXTRATAG = ${EXTRATAG})" - - echo -n ${VERSION} > ./artifacts/VERSION - - echo -n ${EXTRATAG} > ./artifacts/EXTRATAG - - echo -n ${CI_JOB_ID} > ./artifacts/BUILD_LINUX_JOB_ID - - RELEASE_VERSION=$(./artifacts/polkadot -V | awk '{print $2}'| awk -F "-" '{print $1}') - - echo -n "v${RELEASE_VERSION}" > ./artifacts/BUILD_RELEASE_VERSION - - cp -r scripts/* ./artifacts - -test-linux-stable: - stage: stage1 - <<: *docker-env - <<: *common-refs - <<: *pipeline-stopper-artifacts - before_script: - - rustup show - - cargo --version - - *pipeline-stopper-vars - variables: - RUST_TOOLCHAIN: stable - # Enable debug assertions since we are running optimized builds for testing - # but still want to have debug assertions. - RUSTFLAGS: "-Cdebug-assertions=y -Dwarnings" - script: - - time cargo test --workspace --profile testnet --verbose --locked --features=runtime-benchmarks,runtime-metrics +include: + # weights jobs + - scripts/ci/gitlab/pipeline/weights.yml + # check jobs + - scripts/ci/gitlab/pipeline/check.yml + # test jobs + - scripts/ci/gitlab/pipeline/test.yml + # build jobs + - scripts/ci/gitlab/pipeline/build.yml + # short-benchmarks jobs + - scripts/ci/gitlab/pipeline/short-benchmarks.yml + # publish jobs + - scripts/ci/gitlab/pipeline/publish.yml + # zombienet jobs + - scripts/ci/gitlab/pipeline/zombienet.yml -spellcheck: - stage: stage1 - <<: *docker-env - <<: *test-refs - script: - - cargo spellcheck --version - # compare with the commit parent to the PR, given it's from a default branch - - git fetch origin +${CI_DEFAULT_BRANCH}:${CI_DEFAULT_BRANCH} - - echo "___Spellcheck is going to check your diff___" - - cargo spellcheck list-files -vvv $(git diff --diff-filter=AM --name-only $(git merge-base ${CI_COMMIT_SHA} ${CI_DEFAULT_BRANCH} -- :^bridges)) - - time cargo spellcheck check -vvv --cfg=scripts/ci/gitlab/spellcheck.toml --checkers hunspell --code 1 - $(git diff --diff-filter=AM --name-only $(git merge-base ${CI_COMMIT_SHA} ${CI_DEFAULT_BRANCH} -- :^bridges)) - allow_failure: true - -build-test-collators: - stage: stage1 - <<: *collect-artifacts - <<: *docker-env - <<: *compiler-info - <<: *test-refs - script: - - time cargo build --profile testnet --verbose -p test-parachain-adder-collator - - time cargo build --profile testnet --verbose -p test-parachain-undying-collator - - sccache -s - # pack artifacts - - mkdir -p ./artifacts - - mv ./target/testnet/adder-collator ./artifacts/. - - mv ./target/testnet/undying-collator ./artifacts/. - - echo -n "${CI_COMMIT_REF_NAME}" > ./artifacts/VERSION - - echo -n "${CI_COMMIT_REF_NAME}-${CI_COMMIT_SHORT_SHA}" > ./artifacts/EXTRATAG - - echo "adder-collator version = $(cat ./artifacts/VERSION) (EXTRATAG = $(cat ./artifacts/EXTRATAG))" - - echo "undying-collator version = $(cat ./artifacts/VERSION) (EXTRATAG = $(cat ./artifacts/EXTRATAG))" - - cp -r ./scripts/* ./artifacts - -build-malus: - stage: stage1 - <<: *collect-artifacts - <<: *docker-env - <<: *compiler-info - <<: *test-refs - script: - - time cargo build --profile testnet --verbose -p polkadot-test-malus - - sccache -s - # pack artifacts - - mkdir -p ./artifacts - - mv ./target/testnet/malus ./artifacts/. - - echo -n "${CI_COMMIT_REF_NAME}" > ./artifacts/VERSION - - echo -n "${CI_COMMIT_REF_NAME}-${CI_COMMIT_SHORT_SHA}" > ./artifacts/EXTRATAG - - echo "polkadot-test-malus = $(cat ./artifacts/VERSION) (EXTRATAG = $(cat ./artifacts/EXTRATAG))" - - cp -r ./scripts/* ./artifacts - -build-staking-miner: - stage: stage1 - <<: *collect-artifacts - <<: *docker-env - <<: *compiler-info - <<: *common-refs - script: - - time cargo build --locked --release --package staking-miner - # pack artifacts - - mkdir -p ./artifacts - - mv ./target/release/staking-miner ./artifacts/. - - echo -n "${CI_COMMIT_REF_NAME}" > ./artifacts/VERSION - - echo -n "${CI_COMMIT_REF_NAME}-${CI_COMMIT_SHORT_SHA}" > ./artifacts/EXTRATAG - - echo "staking-miner = $(cat ./artifacts/VERSION) (EXTRATAG = $(cat ./artifacts/EXTRATAG))" - - cp -r ./scripts/* ./artifacts - -#### stage: stage2 - -.check-dependent-project: &check-dependent-project - stage: stage2 - # this is an artificial job dependency, for pipeline optimization using GitLab's DAGs - needs: - - job: cargo-fmt - artifacts: false - <<: *docker-env - <<: *test-pr-refs - script: - - git clone - --depth=1 - "--branch=$PIPELINE_SCRIPTS_TAG" - https://github.com/paritytech/pipeline-scripts - - ./pipeline-scripts/check_dependent_project.sh - --org paritytech - --dependent-repo "$DEPENDENT_REPO" - --github-api-token "$GITHUB_PR_TOKEN" - --extra-dependencies "$EXTRA_DEPENDENCIES" - --companion-overrides "$COMPANION_OVERRIDES" - -check-dependent-cumulus: - <<: *check-dependent-project - variables: - DEPENDENT_REPO: cumulus - EXTRA_DEPENDENCIES: substrate - COMPANION_OVERRIDES: | - polkadot: release-v* - cumulus: polkadot-v* - -test-node-metrics: - stage: stage2 - # this is an artificial job dependency, for pipeline optimization using GitLab's DAGs - needs: - - job: cargo-fmt - artifacts: false - <<: *docker-env - <<: *compiler-info - <<: *test-refs - variables: - RUST_TOOLCHAIN: stable - # Enable debug assertions since we are running optimized builds for testing - # but still want to have debug assertions. - RUSTFLAGS: "-Cdebug-assertions=y -Dwarnings" - script: - - time cargo test --profile testnet --verbose --locked --features=runtime-metrics -p polkadot-node-metrics - -test-deterministic-wasm: - stage: stage2 - # this is an artificial job dependency, for pipeline optimization using GitLab's DAGs - needs: - - job: cargo-fmt - artifacts: false - <<: *test-refs - <<: *docker-env - <<: *compiler-info - script: - - ./scripts/ci/gitlab/test_deterministic_wasm.sh - -check-transaction-versions: - stage: stage2 - <<: *test-refs - <<: *docker-env - image: node:15 - needs: - - job: build-linux-stable - artifacts: true - before_script: - - apt-get -y update; apt-get -y install jq lsof - - npm install --ignore-scripts -g @polkadot/metadata-cmp - # Set git config - - git config remote.origin.url "https://github.com/paritytech/polkadot.git" - - git fetch origin release - script: - - ./scripts/ci/gitlab/check_extrinsics_ordering.sh - - - -# This image is used in testnets -# Release image is handled by the Github Action here: -# .github/workflows/publish-docker-release.yml -publish-polkadot-debug-image: - stage: stage2 - <<: *build-push-image - rules: - # Don't run when triggered from another pipeline - - if: $CI_PIPELINE_SOURCE == "pipeline" - when: never - - if: $CI_PIPELINE_SOURCE == "web" - - if: $CI_PIPELINE_SOURCE == "schedule" - - if: $CI_COMMIT_REF_NAME == "master" - - if: $CI_COMMIT_REF_NAME =~ /^[0-9]+$/ # PRs - - if: $CI_COMMIT_REF_NAME =~ /^v[0-9]+\.[0-9]+.*$/ # i.e. v1.0, v2.1rc1 - variables: - <<: *image-variables - # scripts/ci/dockerfiles/polkadot_injected_debug.Dockerfile - DOCKERFILE: ci/dockerfiles/polkadot_injected_debug.Dockerfile - IMAGE_NAME: docker.io/paritypr/polkadot-debug - needs: - - job: build-linux-stable - artifacts: true - after_script: - # pass artifacts to the zombienet-tests job - # https://docs.gitlab.com/ee/ci/multi_project_pipelines.html#with-variable-inheritance - - echo "PARACHAINS_IMAGE_NAME=${IMAGE_NAME}" > ./artifacts/parachains.env - - echo "PARACHAINS_IMAGE_TAG=$(cat ./artifacts/EXTRATAG)" >> ./artifacts/parachains.env - artifacts: - reports: - # this artifact is used in zombienet-tests job - dotenv: ./artifacts/parachains.env - expire_in: 1 days - -publish-test-collators-image: - # service image for Simnet - stage: stage2 - <<: *build-push-image - <<: *zombienet-refs - variables: - <<: *image-variables - # scripts/ci/dockerfiles/collator_injected.Dockerfile - DOCKERFILE: ci/dockerfiles/collator_injected.Dockerfile - IMAGE_NAME: docker.io/paritypr/colander - needs: - - job: build-test-collators - artifacts: true - after_script: - - buildah logout --all - # pass artifacts to the zombienet-tests job - - echo "COLLATOR_IMAGE_NAME=${IMAGE_NAME}" > ./artifacts/collator.env - - echo "COLLATOR_IMAGE_TAG=$(cat ./artifacts/EXTRATAG)" >> ./artifacts/collator.env - artifacts: - reports: - # this artifact is used in zombienet-tests job - dotenv: ./artifacts/collator.env - -publish-malus-image: - # service image for Simnet - stage: stage2 - <<: *build-push-image - <<: *zombienet-refs - variables: - <<: *image-variables - # scripts/ci/dockerfiles/malus_injected.Dockerfile - DOCKERFILE: ci/dockerfiles/malus_injected.Dockerfile - IMAGE_NAME: docker.io/paritypr/malus - needs: - - job: build-malus - artifacts: true - after_script: - - buildah logout "$IMAGE_NAME" - # pass artifacts to the zombienet-tests job - - echo "MALUS_IMAGE_NAME=${IMAGE_NAME}" > ./artifacts/malus.env - - echo "MALUS_IMAGE_TAG=$(cat ./artifacts/EXTRATAG)" >> ./artifacts/malus.env - artifacts: - reports: - # this artifact is used in zombienet-tests job - dotenv: ./artifacts/malus.env - -publish-staking-miner-image: - stage: stage2 - <<: *build-push-image - <<: *publish-refs - variables: - <<: *image-variables - # scripts/ci/dockerfiles/staking-miner/staking-miner_injected.Dockerfile - DOCKERFILE: ci/dockerfiles/staking-miner/staking-miner_injected.Dockerfile - IMAGE_NAME: docker.io/paritytech/staking-miner - GIT_STRATEGY: none - DOCKER_USER: ${Docker_Hub_User_Parity} - DOCKER_PASS: ${Docker_Hub_Pass_Parity} - needs: - - job: build-staking-miner - artifacts: true - - -publish-s3-release: &publish-s3 - stage: stage3 - needs: - - job: build-linux-stable - artifacts: true - <<: *kubernetes-env - image: paritytech/awscli:latest - variables: - GIT_STRATEGY: none - PREFIX: "builds/polkadot/${ARCH}-${DOCKER_OS}" - rules: - - if: $CI_PIPELINE_SOURCE == "pipeline" - when: never - # publishing binaries nightly - - if: $CI_PIPELINE_SOURCE == "schedule" - before_script: - - *check-versions - script: - - echo "uploading objects to https://releases.parity.io/${PREFIX}/${VERSION}" - - aws s3 sync --acl public-read ./artifacts/ s3://${AWS_BUCKET}/${PREFIX}/${VERSION}/ - - echo "update objects at https://releases.parity.io/${PREFIX}/${EXTRATAG}" - - find ./artifacts -type f | while read file; do - name="${file#./artifacts/}"; - aws s3api copy-object - --copy-source ${AWS_BUCKET}/${PREFIX}/${VERSION}/${name} - --bucket ${AWS_BUCKET} --key ${PREFIX}/${EXTRATAG}/${name}; - done - - | - cat <<-EOM - | - | polkadot binary paths: - | - | - https://releases.parity.io/${PREFIX}/${EXTRATAG}/polkadot - | - https://releases.parity.io/${PREFIX}/${VERSION}/polkadot - | - EOM - after_script: - - aws s3 ls s3://${AWS_BUCKET}/${PREFIX}/${EXTRATAG}/ - --recursive --human-readable --summarize - -update_polkadot_weights: &update-weights - stage: stage2 - timeout: 1d - when: manual - variables: - RUNTIME: polkadot - artifacts: - paths: - - ${RUNTIME}_weights_${CI_COMMIT_SHORT_SHA}.patch - script: - - ./scripts/ci/run_benches_for_runtime.sh $RUNTIME - - git diff -P > ${RUNTIME}_weights_${CI_COMMIT_SHORT_SHA}.patch - # uses the "shell" executors - tags: - - weights - -update_kusama_weights: - <<: *update-weights - variables: - RUNTIME: kusama - -update_westend_weights: - <<: *update-weights - variables: - RUNTIME: westend - -update_rococo_weights: - <<: *update-weights - variables: - RUNTIME: rococo - -#### stage: stage3 - -build-rustdoc: - stage: stage3 - # this is an artificial job dependency, for pipeline optimization using GitLab's DAGs - needs: - - job: test-deterministic-wasm - artifacts: false - <<: *docker-env - <<: *test-refs - variables: - SKIP_WASM_BUILD: 1 - artifacts: - name: "${CI_JOB_NAME}_${CI_COMMIT_REF_NAME}-doc" - when: on_success - expire_in: 1 days - paths: - - ./crate-docs/ - script: - # FIXME: it fails with `RUSTDOCFLAGS="-Dwarnings"` and `--all-features` - # FIXME: return to stable when https://github.com/rust-lang/rust/issues/96937 gets into stable - - time cargo +nightly doc --workspace --verbose --no-deps - - rm -f ./target/doc/.lock - - mv ./target/doc ./crate-docs - # FIXME: remove me after CI image gets nonroot - - chown -R nonroot:nonroot ./crate-docs - - echo "" > ./crate-docs/index.html - -build-implementers-guide: - stage: stage3 - # this is an artificial job dependency, for pipeline optimization using GitLab's DAGs - needs: - - job: test-deterministic-wasm - artifacts: false - <<: *test-refs - <<: *docker-env - <<: *collect-artifacts-short - script: - - cargo install mdbook mdbook-mermaid mdbook-linkcheck - - mdbook build ./roadmap/implementers-guide - - mkdir -p artifacts - - mv roadmap/implementers-guide/book artifacts/ - # FIXME: remove me after CI image gets nonroot - - chown -R nonroot:nonroot artifacts/ - -check-try-runtime: - stage: stage3 - # this is an artificial job dependency, for pipeline optimization using GitLab's DAGs - needs: - - job: test-node-metrics - artifacts: false - <<: *test-refs - <<: *docker-env - <<: *compiler-info - script: - # Check that everything compiles with `try-runtime` feature flag. - - cargo check --features try-runtime --all - -# More info can be found here: https://github.com/paritytech/polkadot/pull/5865 -# Works only in PRs -check-runtime-migration: - stage: stage3 - # this is an artificial job dependency, for pipeline optimization using GitLab's DAGs - needs: - - job: test-node-metrics - artifacts: false - <<: *test-pr-refs - <<: *docker-env - <<: *compiler-info - script: - - | - export has_runtimemigration_label=$(curl -sS -H "Accept: application/vnd.github+json" \ - -H "Authorization: token $GITHUB_PR_TOKEN" \ - https://api.github.com/repos/paritytech/polkadot/issues/$CI_COMMIT_REF_NAME/labels | grep "E1" | wc -l) - - | - if [[ $has_runtimemigration_label != 0 ]]; then - echo "Found label runtimemigration. Running tests" - export RUST_LOG=remote-ext=debug,runtime=debug - time cargo test --release -p westend-runtime -p polkadot-runtime -p kusama-runtime --features try-runtime - else - echo "runtimemigration label not found. Skipping" - fi - allow_failure: true - - -check-no-default-features: - stage: stage3 - # this is an artificial job dependency, for pipeline optimization using GitLab's DAGs - needs: - - job: test-deterministic-wasm - artifacts: false - <<: *test-refs - <<: *docker-env - <<: *compiler-info - script: - # Check that polkadot-cli will compile no default features. - - pushd ./node/service && cargo check --no-default-features && popd - - pushd ./cli && cargo check --no-default-features --features "service" && popd - - sccache -s - -build-short-benchmark: - stage: stage3 - <<: *test-refs - <<: *docker-env - <<: *collect-artifacts - # this is an artificial job dependency, for pipeline optimization using GitLab's DAGs - needs: - - job: test-node-metrics - artifacts: false - script: - - cargo +nightly build --profile release --locked --features=runtime-benchmarks - - mkdir artifacts - - cp ./target/release/polkadot ./artifacts/ +#### stage: .post deploy-parity-testnet: - stage: stage3 - # this is an artificial job dependency, for pipeline optimization using GitLab's DAGs - needs: - - job: test-deterministic-wasm - artifacts: false - <<: *deploy-testnet-refs + stage: .post + extends: + - .deploy-testnet-refs variables: POLKADOT_CI_COMMIT_NAME: "${CI_COMMIT_REF_NAME}" POLKADOT_CI_COMMIT_REF: "${CI_COMMIT_SHORT_SHA}" allow_failure: false trigger: "parity/infrastructure/parity-testnet" -zombienet-tests-parachains-smoke-test: - stage: stage3 - image: "${ZOMBIENET_IMAGE}" - <<: *kubernetes-env - <<: *zombienet-refs - needs: - - job: publish-polkadot-debug-image - - job: publish-malus-image - - job: publish-test-collators-image - variables: - GH_DIR: "https://github.com/paritytech/polkadot/tree/${CI_COMMIT_SHORT_SHA}/zombienet_tests/smoke" - before_script: - - echo "Zombie-net Tests Config" - - echo "${ZOMBIENET_IMAGE}" - - echo "${PARACHAINS_IMAGE_NAME} ${PARACHAINS_IMAGE_TAG}" - - echo "${MALUS_IMAGE_NAME} ${MALUS_IMAGE_TAG}" - - echo "${GH_DIR}" - - export DEBUG=zombie,zombie::network-node - - export ZOMBIENET_INTEGRATION_TEST_IMAGE=${PARACHAINS_IMAGE_NAME}:${PARACHAINS_IMAGE_TAG} - - export MALUS_IMAGE=${MALUS_IMAGE_NAME}:${MALUS_IMAGE_TAG} - - export COL_IMAGE="docker.io/paritypr/colander:4519" # The collator image is fixed - script: - - /home/nonroot/zombie-net/scripts/ci/run-test-env-manager.sh - --github-remote-dir="${GH_DIR}" - --test="0001-parachains-smoke-test.feature" - allow_failure: false - retry: 2 - tags: - - zombienet-polkadot-integration-test - -zombienet-tests-parachains-pvf: - stage: stage3 - image: "${ZOMBIENET_IMAGE}" - <<: *kubernetes-env - <<: *zombienet-refs - needs: - - job: publish-polkadot-debug-image - - job: publish-test-collators-image - variables: - GH_DIR: "https://github.com/paritytech/polkadot/tree/${CI_COMMIT_SHORT_SHA}/zombienet_tests/functional" - before_script: - - echo "Zombie-net Tests Config" - - echo "${ZOMBIENET_IMAGE}" - - echo "${PARACHAINS_IMAGE_NAME} ${PARACHAINS_IMAGE_TAG}" - - echo "COL_IMAGE=${COLLATOR_IMAGE_NAME}:${COLLATOR_IMAGE_TAG}" - - echo "${GH_DIR}" - - export DEBUG=zombie,zombie::network-node - - export ZOMBIENET_INTEGRATION_TEST_IMAGE=${PARACHAINS_IMAGE_NAME}:${PARACHAINS_IMAGE_TAG} - - export MALUS_IMAGE=${MALUS_IMAGE_NAME}:${MALUS_IMAGE_TAG} - - export COL_IMAGE=${COLLATOR_IMAGE_NAME}:${COLLATOR_IMAGE_TAG} - script: - - /home/nonroot/zombie-net/scripts/ci/run-test-env-manager.sh - --github-remote-dir="${GH_DIR}" - --test="0001-parachains-pvf.feature" - allow_failure: false - retry: 2 - tags: - - zombienet-polkadot-integration-test - -zombienet-tests-parachains-disputes: - stage: stage3 - image: "${ZOMBIENET_IMAGE}" - <<: *kubernetes-env - <<: *zombienet-refs - needs: - - job: publish-polkadot-debug-image - - job: publish-test-collators-image - - job: publish-malus-image - variables: - GH_DIR: "https://github.com/paritytech/polkadot/tree/${CI_COMMIT_SHORT_SHA}/zombienet_tests/functional" - before_script: - - echo "Zombie-net Tests Config" - - echo "${ZOMBIENET_IMAGE_NAME}" - - echo "${PARACHAINS_IMAGE_NAME} ${PARACHAINS_IMAGE_TAG}" - - echo "${MALUS_IMAGE_NAME} ${MALUS_IMAGE_TAG}" - - echo "${GH_DIR}" - - export DEBUG=zombie,zombie::network-node - - export ZOMBIENET_INTEGRATION_TEST_IMAGE=${PARACHAINS_IMAGE_NAME}:${PARACHAINS_IMAGE_TAG} - - export MALUS_IMAGE=${MALUS_IMAGE_NAME}:${MALUS_IMAGE_TAG} - - export COL_IMAGE=${COLLATOR_IMAGE_NAME}:${COLLATOR_IMAGE_TAG} - script: - - /home/nonroot/zombie-net/scripts/ci/run-test-env-manager.sh - --github-remote-dir="${GH_DIR}" - --test="0002-parachains-disputes.feature" - allow_failure: false - retry: 2 - tags: - - zombienet-polkadot-integration-test - -zombienet-tests-parachains-disputes-garbage-candidate: - stage: stage3 - image: "${ZOMBIENET_IMAGE}" - <<: *kubernetes-env - <<: *zombienet-refs - needs: - - job: publish-polkadot-debug-image - - job: publish-test-collators-image - - job: publish-malus-image - variables: - GH_DIR: "https://github.com/paritytech/polkadot/tree/${CI_COMMIT_SHORT_SHA}/zombienet_tests/functional" - before_script: - - echo "Zombie-net Tests Config" - - echo "${ZOMBIENET_IMAGE_NAME}" - - echo "${PARACHAINS_IMAGE_NAME} ${PARACHAINS_IMAGE_TAG}" - - echo "${MALUS_IMAGE_NAME} ${MALUS_IMAGE_TAG}" - - echo "${GH_DIR}" - - export DEBUG=zombie,zombie::network-node - - export ZOMBIENET_INTEGRATION_TEST_IMAGE=${PARACHAINS_IMAGE_NAME}:${PARACHAINS_IMAGE_TAG} - - export MALUS_IMAGE=${MALUS_IMAGE_NAME}:${MALUS_IMAGE_TAG} - - export COL_IMAGE=${COLLATOR_IMAGE_NAME}:${COLLATOR_IMAGE_TAG} - script: - - /home/nonroot/zombie-net/scripts/ci/run-test-env-manager.sh - --github-remote-dir="${GH_DIR}" - --test="0003-parachains-garbage-candidate.feature" - allow_failure: false - retry: 2 - tags: - - zombienet-polkadot-integration-test - -zombienet-test-parachains-upgrade-smoke-test: - stage: stage3 - image: "${ZOMBIENET_IMAGE}" - <<: *kubernetes-env - <<: *zombienet-refs - needs: - - job: publish-polkadot-debug-image - - job: publish-malus-image - - job: publish-test-collators-image - variables: - GH_DIR: 'https://github.com/paritytech/polkadot/tree/${CI_COMMIT_SHORT_SHA}/zombienet_tests/smoke' - before_script: - - echo "ZombieNet Tests Config" - - echo "${PARACHAINS_IMAGE_NAME}:${PARACHAINS_IMAGE_TAG}" - - echo "docker.io/parity/polkadot-collator:latest" - - echo "${ZOMBIENET_IMAGE}" - - echo "${GH_DIR}" - - export DEBUG=zombie,zombie::network-node - - export ZOMBIENET_INTEGRATION_TEST_IMAGE=${PARACHAINS_IMAGE_NAME}:${PARACHAINS_IMAGE_TAG} - - export COL_IMAGE="docker.io/parity/polkadot-collator:latest" # Use cumulus lastest image - script: - - /home/nonroot/zombie-net/scripts/ci/run-test-env-manager.sh - --github-remote-dir="${GH_DIR}" - --test="0002-parachains-upgrade-smoke-test.feature" - allow_failure: true - retry: 2 - tags: - - zombienet-polkadot-integration-test - -zombienet-tests-misc-paritydb: - stage: stage3 - image: "${ZOMBIENET_IMAGE}" - <<: *kubernetes-env - <<: *zombienet-refs - needs: - - job: publish-polkadot-debug-image - - job: publish-test-collators-image - artifacts: true - variables: - GH_DIR: "https://github.com/paritytech/polkadot/tree/${CI_COMMIT_SHORT_SHA}/zombienet_tests/misc" - before_script: - - echo "Zombie-net Tests Config" - - echo "${ZOMBIENET_IMAGE_NAME}" - - echo "${PARACHAINS_IMAGE_NAME} ${PARACHAINS_IMAGE_TAG}" - - echo "${GH_DIR}" - - export DEBUG=zombie,zombie::network-node - - export ZOMBIENET_INTEGRATION_TEST_IMAGE=${PARACHAINS_IMAGE_NAME}:${PARACHAINS_IMAGE_TAG} - - export COL_IMAGE=${COLLATOR_IMAGE_NAME}:${COLLATOR_IMAGE_TAG} - script: - - /home/nonroot/zombie-net/scripts/ci/run-test-env-manager.sh - --github-remote-dir="${GH_DIR}" - --test="0001-paritydb.feature" - allow_failure: false - retry: 2 - tags: - - zombienet-polkadot-integration-test - -zombienet-tests-misc-upgrade-node: - stage: stage3 - image: "${ZOMBIENET_IMAGE}" - <<: *kubernetes-env - <<: *zombienet-refs - needs: - - job: publish-polkadot-debug-image - - job: publish-test-collators-image - - job: build-linux-stable - artifacts: true - variables: - GH_DIR: "https://github.com/paritytech/polkadot/tree/${CI_COMMIT_SHORT_SHA}/zombienet_tests/misc" - before_script: - - echo "Zombie-net Tests Config" - - echo "${ZOMBIENET_IMAGE_NAME}" - - echo "${PARACHAINS_IMAGE_NAME} ${PARACHAINS_IMAGE_TAG}" - - echo "${GH_DIR}" - - export DEBUG=zombie,zombie::network-node - - BUILD_RELEASE_VERSION="$(cat ./artifacts/BUILD_RELEASE_VERSION)" - - export ZOMBIENET_INTEGRATION_TEST_IMAGE="docker.io/parity/polkadot:${BUILD_RELEASE_VERSION}" - - export COL_IMAGE=${COLLATOR_IMAGE_NAME}:${COLLATOR_IMAGE_TAG} - - BUILD_LINUX_JOB_ID="$(cat ./artifacts/BUILD_LINUX_JOB_ID)" - - export POLKADOT_PR_BIN_URL="https://gitlab.parity.io/parity/mirrors/polkadot/-/jobs/${BUILD_LINUX_JOB_ID}/artifacts/raw/artifacts/polkadot" - script: - - /home/nonroot/zombie-net/scripts/ci/run-test-env-manager.sh - --github-remote-dir="${GH_DIR}" - --test="0002-upgrade-node.feature" - allow_failure: false - retry: 2 - tags: - - zombienet-polkadot-integration-test - -zombienet-tests-malus-dispute-valid: - stage: stage3 - image: "${ZOMBIENET_IMAGE}" - <<: *kubernetes-env - <<: *zombienet-refs - needs: - - job: publish-polkadot-debug-image - - job: publish-malus-image - - job: publish-test-collators-image - variables: - GH_DIR: "https://github.com/paritytech/polkadot/tree/${CI_COMMIT_SHORT_SHA}/node/malus/integrationtests" - before_script: - - echo "Zombie-net Tests Config" - - echo "${ZOMBIENET_IMAGE_NAME}" - - echo "${PARACHAINS_IMAGE_NAME} ${PARACHAINS_IMAGE_TAG}" - - echo "${MALUS_IMAGE_NAME} ${MALUS_IMAGE_TAG}" - - echo "${GH_DIR}" - - export DEBUG=zombie* - - export ZOMBIENET_INTEGRATION_TEST_IMAGE=${PARACHAINS_IMAGE_NAME}:${PARACHAINS_IMAGE_TAG} - - export MALUS_IMAGE=${MALUS_IMAGE_NAME}:${MALUS_IMAGE_TAG} - - export COL_IMAGE=${COLLATOR_IMAGE_NAME}:${COLLATOR_IMAGE_TAG} - script: - - /home/nonroot/zombie-net/scripts/ci/run-test-env-manager.sh - --github-remote-dir="${GH_DIR}" - --test="0001-dispute-valid-block.feature" - allow_failure: false - retry: 2 - tags: - - zombienet-polkadot-integration-test - -zombienet-tests-deregister-register-validator: - stage: stage3 - image: "${ZOMBIENET_IMAGE}" - <<: *kubernetes-env - <<: *zombienet-refs - needs: - - job: publish-polkadot-debug-image - artifacts: true - variables: - GH_DIR: "https://github.com/paritytech/polkadot/tree/${CI_COMMIT_SHORT_SHA}/zombienet_tests/smoke" - before_script: - - echo "Zombie-net Tests Config" - - echo "${ZOMBIENET_IMAGE_NAME}" - - echo "${PARACHAINS_IMAGE_NAME} ${PARACHAINS_IMAGE_TAG}" - - echo "${GH_DIR}" - - export DEBUG=zombie* - - export ZOMBIENET_INTEGRATION_TEST_IMAGE=${PARACHAINS_IMAGE_NAME}:${PARACHAINS_IMAGE_TAG} - - export MALUS_IMAGE=${MALUS_IMAGE_NAME}:${MALUS_IMAGE_TAG} - script: - - /home/nonroot/zombie-net/scripts/ci/run-test-env-manager.sh - --github-remote-dir="${GH_DIR}" - --test="0003-deregister-register-validator-smoke.feature" - allow_failure: false - retry: 2 - tags: - - zombienet-polkadot-integration-test - -zombienet-tests-beefy-and-mmr: - stage: stage3 - image: "${ZOMBIENET_IMAGE}" - <<: *kubernetes-env - <<: *zombienet-refs - needs: - - job: publish-polkadot-debug-image - variables: - GH_DIR: "https://github.com/paritytech/polkadot/tree/${CI_COMMIT_SHORT_SHA}/zombienet_tests/functional" - before_script: - - echo "Zombie-net Tests Config" - - echo "${ZOMBIENET_IMAGE_NAME}" - - echo "${PARACHAINS_IMAGE_NAME} ${PARACHAINS_IMAGE_TAG}" - - echo "${GH_DIR}" - - export DEBUG=zombie* - - export ZOMBIENET_INTEGRATION_TEST_IMAGE=${PARACHAINS_IMAGE_NAME}:${PARACHAINS_IMAGE_TAG} - script: - - /home/nonroot/zombie-net/scripts/ci/run-test-env-manager.sh - --github-remote-dir="${GH_DIR}" - --test="0003-beefy-and-mmr.feature" - allow_failure: true - retry: 2 - tags: - - zombienet-polkadot-integration-test - -#### stage: stage4 - -publish-rustdoc: - stage: stage4 - <<: *kubernetes-env - image: paritytech/tools:latest - variables: - GIT_DEPTH: 100 - rules: - - if: $CI_PIPELINE_SOURCE == "pipeline" - when: never - - if: $CI_PIPELINE_SOURCE == "web" && $CI_COMMIT_REF_NAME == "master" - - if: $CI_COMMIT_REF_NAME == "master" - # `needs:` can be removed after CI image gets nonroot. In this case `needs:` stops other - # artifacts from being dowloaded by this job. - needs: - - job: build-rustdoc - artifacts: true - - job: build-implementers-guide - artifacts: true - script: - # Save README and docs - - cp -r ./crate-docs/ /tmp/doc/ - - cp -r ./artifacts/book/ /tmp/ - # setup ssh - - eval $(ssh-agent) - - ssh-add - <<< ${GITHUB_SSH_PRIV_KEY} - - mkdir ~/.ssh && touch ~/.ssh/known_hosts - - ssh-keyscan -t rsa github.com >> ~/.ssh/known_hosts - # Set git config - - git config user.email "devops-team@parity.io" - - git config user.name "${GITHUB_USER}" - - git config remote.origin.url "git@github.com:/paritytech/${CI_PROJECT_NAME}.git" - - git config remote.origin.fetch "+refs/heads/*:refs/remotes/origin/*" - - git fetch origin gh-pages - - git checkout gh-pages - # Remove everything and restore generated docs and README - - cp index.html /tmp - - cp README.md /tmp - - rm -rf ./* - # dir for rustdoc - - mkdir -p doc - # dir for implementors guide - - mkdir -p book - - mv /tmp/doc/* doc/ - - mv /tmp/book/html/* book/ - - mv /tmp/index.html . - - mv /tmp/README.md . - # Upload files - - git add --all --force - # `git commit` has an exit code of > 0 if there is nothing to commit. - # This causes GitLab to exit immediately and marks this job failed. - # We don't want to mark the entire job failed if there's nothing to - # publish though, hence the `|| true`. - - git commit -m "Updated docs for ${CI_COMMIT_REF_NAME}" || - echo "___Nothing to commit___" - - git push origin gh-pages --force - - echo "___Rustdoc was successfully published to https://paritytech.github.io/polkadot/___" - after_script: - - rm -rf .git/ ./* - -# Run all pallet benchmarks only once to check if there are any errors -short-benchmark-polkadot: &short-bench - stage: stage4 - <<: *test-pr-refs - <<: *docker-env - # this is an artificial job dependency, for pipeline optimization using GitLab's DAGs - needs: - - job: build-short-benchmark - artifacts: true - variables: - RUNTIME: polkadot - script: - - ./artifacts/polkadot benchmark pallet --execution wasm --wasm-execution compiled --chain $RUNTIME-dev --pallet "*" --extrinsic "*" --steps 2 --repeat 1 - -short-benchmark-kusama: - <<: *short-bench - variables: - RUNTIME: kusama - -short-benchmark-westend: - <<: *short-bench - variables: - RUNTIME: westend - -#### stage: .post - # This job cancels the whole pipeline if any of provided jobs fail. # In a DAG, every jobs chain is executed independently of others. The `fail_fast` principle suggests # to fail the pipeline as soon as possible to shorten the feedback loop. diff --git a/polkadot/scripts/ci/gitlab/pipeline/build.yml b/polkadot/scripts/ci/gitlab/pipeline/build.yml new file mode 100644 index 0000000000..dcc05427f0 --- /dev/null +++ b/polkadot/scripts/ci/gitlab/pipeline/build.yml @@ -0,0 +1,193 @@ +# This file is part of .gitlab-ci.yml +# Here are all jobs that are executed during "build" stage + +build-linux-stable: + stage: build + # this is an artificial job dependency, for pipeline optimization using GitLab's DAGs + # the job can be found in check.yml + needs: + - job: job-starter + artifacts: false + extends: + - .docker-env + - .common-refs + - .compiler-info + - .collect-artifacts + variables: + RUST_TOOLCHAIN: stable + # Enable debug assertions since we are running optimized builds for testing + # but still want to have debug assertions. + RUSTFLAGS: "-Cdebug-assertions=y -Dwarnings" + # Ensure we run the UI tests. + RUN_UI_TESTS: 1 + script: + - time cargo build --profile testnet --features pyroscope --verbose --bin polkadot + - sccache -s + # pack artifacts + - mkdir -p ./artifacts + - VERSION="${CI_COMMIT_REF_NAME}" # will be tag or branch name + - mv ./target/testnet/polkadot ./artifacts/. + - pushd artifacts + - sha256sum polkadot | tee polkadot.sha256 + - shasum -c polkadot.sha256 + - popd + - EXTRATAG="${CI_COMMIT_REF_NAME}-${CI_COMMIT_SHORT_SHA}" + - echo "Polkadot version = ${VERSION} (EXTRATAG = ${EXTRATAG})" + - echo -n ${VERSION} > ./artifacts/VERSION + - echo -n ${EXTRATAG} > ./artifacts/EXTRATAG + - echo -n ${CI_JOB_ID} > ./artifacts/BUILD_LINUX_JOB_ID + - RELEASE_VERSION=$(./artifacts/polkadot -V | awk '{print $2}'| awk -F "-" '{print $1}') + - echo -n "v${RELEASE_VERSION}" > ./artifacts/BUILD_RELEASE_VERSION + - cp -r scripts/* ./artifacts + +check-transaction-versions: + stage: build + extends: + - .docker-env + - .test-refs + image: node:15 + needs: + - job: build-linux-stable + artifacts: true + before_script: + - apt-get -y update; apt-get -y install jq lsof + - npm install --ignore-scripts -g @polkadot/metadata-cmp + # Set git config + - git config remote.origin.url "https://github.com/paritytech/polkadot.git" + - git fetch origin release + script: + - ./scripts/ci/gitlab/check_extrinsics_ordering.sh + +build-test-collators: + stage: build + # this is an artificial job dependency, for pipeline optimization using GitLab's DAGs + # the job can be found in check.yml + needs: + - job: job-starter + artifacts: false + extends: + - .docker-env + - .common-refs + - .compiler-info + - .collect-artifacts + script: + - time cargo build --profile testnet --verbose -p test-parachain-adder-collator + - time cargo build --profile testnet --verbose -p test-parachain-undying-collator + - sccache -s + # pack artifacts + - mkdir -p ./artifacts + - mv ./target/testnet/adder-collator ./artifacts/. + - mv ./target/testnet/undying-collator ./artifacts/. + - echo -n "${CI_COMMIT_REF_NAME}" > ./artifacts/VERSION + - echo -n "${CI_COMMIT_REF_NAME}-${CI_COMMIT_SHORT_SHA}" > ./artifacts/EXTRATAG + - echo "adder-collator version = $(cat ./artifacts/VERSION) (EXTRATAG = $(cat ./artifacts/EXTRATAG))" + - echo "undying-collator version = $(cat ./artifacts/VERSION) (EXTRATAG = $(cat ./artifacts/EXTRATAG))" + - cp -r ./scripts/* ./artifacts + +build-malus: + stage: build + # this is an artificial job dependency, for pipeline optimization using GitLab's DAGs + # the job can be found in check.yml + needs: + - job: job-starter + artifacts: false + extends: + - .docker-env + - .common-refs + - .compiler-info + - .collect-artifacts + script: + - time cargo build --profile testnet --verbose -p polkadot-test-malus + - sccache -s + # pack artifacts + - mkdir -p ./artifacts + - mv ./target/testnet/malus ./artifacts/. + - echo -n "${CI_COMMIT_REF_NAME}" > ./artifacts/VERSION + - echo -n "${CI_COMMIT_REF_NAME}-${CI_COMMIT_SHORT_SHA}" > ./artifacts/EXTRATAG + - echo "polkadot-test-malus = $(cat ./artifacts/VERSION) (EXTRATAG = $(cat ./artifacts/EXTRATAG))" + - cp -r ./scripts/* ./artifacts + +build-staking-miner: + stage: build + # this is an artificial job dependency, for pipeline optimization using GitLab's DAGs + # the job can be found in check.yml + needs: + - job: job-starter + artifacts: false + extends: + - .docker-env + - .common-refs + - .compiler-info + - .collect-artifacts + script: + - time cargo build --locked --release --package staking-miner + # pack artifacts + - mkdir -p ./artifacts + - mv ./target/release/staking-miner ./artifacts/. + - echo -n "${CI_COMMIT_REF_NAME}" > ./artifacts/VERSION + - echo -n "${CI_COMMIT_REF_NAME}-${CI_COMMIT_SHORT_SHA}" > ./artifacts/EXTRATAG + - echo "staking-miner = $(cat ./artifacts/VERSION) (EXTRATAG = $(cat ./artifacts/EXTRATAG))" + - cp -r ./scripts/* ./artifacts + +build-rustdoc: + stage: build + # this is an artificial job dependency, for pipeline optimization using GitLab's DAGs + # the job can be found in test.yml + needs: + - job: test-deterministic-wasm + artifacts: false + extends: + - .docker-env + - .test-refs + variables: + SKIP_WASM_BUILD: 1 + artifacts: + name: "${CI_JOB_NAME}_${CI_COMMIT_REF_NAME}-doc" + when: on_success + expire_in: 1 days + paths: + - ./crate-docs/ + script: + # FIXME: it fails with `RUSTDOCFLAGS="-Dwarnings"` and `--all-features` + # FIXME: return to stable when https://github.com/rust-lang/rust/issues/96937 gets into stable + - time cargo +nightly doc --workspace --verbose --no-deps + - rm -f ./target/doc/.lock + - mv ./target/doc ./crate-docs + # FIXME: remove me after CI image gets nonroot + - chown -R nonroot:nonroot ./crate-docs + - echo "" > ./crate-docs/index.html + +build-implementers-guide: + stage: build + # this is an artificial job dependency, for pipeline optimization using GitLab's DAGs + # the job can be found in test.yml + needs: + - job: test-deterministic-wasm + artifacts: false + extends: + - .docker-env + - .test-refs + - .collect-artifacts-short + script: + - cargo install mdbook mdbook-mermaid mdbook-linkcheck + - mdbook build ./roadmap/implementers-guide + - mkdir -p artifacts + - mv roadmap/implementers-guide/book artifacts/ + # FIXME: remove me after CI image gets nonroot + - chown -R nonroot:nonroot artifacts/ + +build-short-benchmark: + stage: build + # this is an artificial job dependency, for pipeline optimization using GitLab's DAGs + # the job can be found in check.yml + needs: + - job: job-starter + artifacts: false + extends: + - .docker-env + - .test-refs + - .collect-artifacts + script: + - cargo +nightly build --profile release --locked --features=runtime-benchmarks + - mkdir artifacts + - cp ./target/release/polkadot ./artifacts/ diff --git a/polkadot/scripts/ci/gitlab/pipeline/check.yml b/polkadot/scripts/ci/gitlab/pipeline/check.yml new file mode 100644 index 0000000000..0e435d6048 --- /dev/null +++ b/polkadot/scripts/ci/gitlab/pipeline/check.yml @@ -0,0 +1,97 @@ +# This file is part of .gitlab-ci.yml +# Here are all jobs that are executed during "check" stage + +check-runtime: + stage: check + image: paritytech/tools:latest + extends: + - .kubernetes-env + rules: + - if: $CI_COMMIT_REF_NAME =~ /^release-v[0-9]+\.[0-9]+.*$/ # i.e. release-v0.9.27 + variables: + GITLAB_API: "https://gitlab.parity.io/api/v4" + GITHUB_API_PROJECT: "parity%2Finfrastructure%2Fgithub-api" + script: + - ./scripts/ci/gitlab/check_runtime.sh + allow_failure: true + +cargo-fmt: + stage: check + extends: + - .docker-env + - .test-refs + script: + - cargo +nightly --version + - cargo +nightly fmt --all -- --check + allow_failure: true + +spellcheck: + stage: check + extends: + - .docker-env + - .test-refs + script: + - cargo spellcheck --version + # compare with the commit parent to the PR, given it's from a default branch + - git fetch origin +${CI_DEFAULT_BRANCH}:${CI_DEFAULT_BRANCH} + - echo "___Spellcheck is going to check your diff___" + - cargo spellcheck list-files -vvv $(git diff --diff-filter=AM --name-only $(git merge-base ${CI_COMMIT_SHA} ${CI_DEFAULT_BRANCH} -- :^bridges)) + - time cargo spellcheck check -vvv --cfg=scripts/ci/gitlab/spellcheck.toml --checkers hunspell --code 1 + $(git diff --diff-filter=AM --name-only $(git merge-base ${CI_COMMIT_SHA} ${CI_DEFAULT_BRANCH} -- :^bridges)) + allow_failure: true + +check-try-runtime: + stage: check + extends: + - .docker-env + - .test-refs + - .compiler-info + script: + # Check that everything compiles with `try-runtime` feature flag. + - cargo check --features try-runtime --all + +# More info can be found here: https://github.com/paritytech/polkadot/pull/5865 +# Works only in PRs +check-runtime-migration: + stage: check + extends: + - .docker-env + - .test-pr-refs + - .compiler-info + script: + - | + export has_runtimemigration_label=$(curl -sS -H "Accept: application/vnd.github+json" \ + -H "Authorization: token $GITHUB_PR_TOKEN" \ + https://api.github.com/repos/paritytech/polkadot/issues/$CI_COMMIT_REF_NAME/labels | grep "E1" | wc -l) + - | + if [[ $has_runtimemigration_label != 0 ]]; then + echo "Found label runtimemigration. Running tests" + export RUST_LOG=remote-ext=debug,runtime=debug + time cargo test --release -p westend-runtime -p polkadot-runtime -p kusama-runtime --features try-runtime + else + echo "runtimemigration label not found. Skipping" + fi + allow_failure: true + +# is broken, need to fix +check-no-default-features: + stage: check + extends: + - .docker-env + - .test-refs + - .compiler-info + script: + # Check that polkadot-cli will compile no default features. + - pushd ./node/service && cargo check --no-default-features && popd + - pushd ./cli && cargo check --no-default-features --features "service" && popd + - exit 0 + +# this is artificial job to run some build and tests using DAG +job-starter: + stage: check + image: paritytech/tools:latest + extends: + - .kubernetes-env + - .common-refs + script: + - echo ok diff --git a/polkadot/scripts/ci/gitlab/pipeline/publish.yml b/polkadot/scripts/ci/gitlab/pipeline/publish.yml new file mode 100644 index 0000000000..12e087e188 --- /dev/null +++ b/polkadot/scripts/ci/gitlab/pipeline/publish.yml @@ -0,0 +1,209 @@ +# This file is part of .gitlab-ci.yml +# Here are all jobs that are executed during "publish" stage + +# This image is used in testnets +# Release image is handled by the Github Action here: +# .github/workflows/publish-docker-release.yml +publish-polkadot-debug-image: + stage: publish + extends: + - .build-push-image + rules: + # Don't run when triggered from another pipeline + - if: $CI_PIPELINE_SOURCE == "pipeline" + when: never + - if: $CI_PIPELINE_SOURCE == "web" + - if: $CI_PIPELINE_SOURCE == "schedule" + - if: $CI_COMMIT_REF_NAME == "master" + - if: $CI_COMMIT_REF_NAME =~ /^[0-9]+$/ # PRs + - if: $CI_COMMIT_REF_NAME =~ /^v[0-9]+\.[0-9]+.*$/ # i.e. v1.0, v2.1rc1 + variables: + GIT_STRATEGY: none + DOCKER_USER: ${PARITYPR_USER} + DOCKER_PASS: ${PARITYPR_PASS} + # scripts/ci/dockerfiles/polkadot_injected_debug.Dockerfile + DOCKERFILE: ci/dockerfiles/polkadot_injected_debug.Dockerfile + IMAGE_NAME: docker.io/paritypr/polkadot-debug + needs: + - job: build-linux-stable + artifacts: true + after_script: + # pass artifacts to the zombienet-tests job + # https://docs.gitlab.com/ee/ci/multi_project_pipelines.html#with-variable-inheritance + - echo "PARACHAINS_IMAGE_NAME=${IMAGE_NAME}" > ./artifacts/parachains.env + - echo "PARACHAINS_IMAGE_TAG=$(cat ./artifacts/EXTRATAG)" >> ./artifacts/parachains.env + artifacts: + reports: + # this artifact is used in zombienet-tests job + dotenv: ./artifacts/parachains.env + expire_in: 1 days + +publish-test-collators-image: + # service image for Simnet + stage: publish + extends: + - .build-push-image + - .zombienet-refs + variables: + GIT_STRATEGY: none + DOCKER_USER: ${PARITYPR_USER} + DOCKER_PASS: ${PARITYPR_PASS} + # scripts/ci/dockerfiles/collator_injected.Dockerfile + DOCKERFILE: ci/dockerfiles/collator_injected.Dockerfile + IMAGE_NAME: docker.io/paritypr/colander + needs: + - job: build-test-collators + artifacts: true + after_script: + - buildah logout --all + # pass artifacts to the zombienet-tests job + - echo "COLLATOR_IMAGE_NAME=${IMAGE_NAME}" > ./artifacts/collator.env + - echo "COLLATOR_IMAGE_TAG=$(cat ./artifacts/EXTRATAG)" >> ./artifacts/collator.env + artifacts: + reports: + # this artifact is used in zombienet-tests job + dotenv: ./artifacts/collator.env + +publish-malus-image: + # service image for Simnet + stage: publish + extends: + - .build-push-image + - .zombienet-refs + variables: + GIT_STRATEGY: none + DOCKER_USER: ${PARITYPR_USER} + DOCKER_PASS: ${PARITYPR_PASS} + # scripts/ci/dockerfiles/malus_injected.Dockerfile + DOCKERFILE: ci/dockerfiles/malus_injected.Dockerfile + IMAGE_NAME: docker.io/paritypr/malus + needs: + - job: build-malus + artifacts: true + after_script: + - buildah logout "$IMAGE_NAME" + # pass artifacts to the zombienet-tests job + - echo "MALUS_IMAGE_NAME=${IMAGE_NAME}" > ./artifacts/malus.env + - echo "MALUS_IMAGE_TAG=$(cat ./artifacts/EXTRATAG)" >> ./artifacts/malus.env + artifacts: + reports: + # this artifact is used in zombienet-tests job + dotenv: ./artifacts/malus.env + +publish-staking-miner-image: + stage: publish + extends: + - .build-push-image + - .publish-refs + variables: + # scripts/ci/dockerfiles/staking-miner/staking-miner_injected.Dockerfile + DOCKERFILE: ci/dockerfiles/staking-miner/staking-miner_injected.Dockerfile + IMAGE_NAME: docker.io/paritytech/staking-miner + GIT_STRATEGY: none + DOCKER_USER: ${Docker_Hub_User_Parity} + DOCKER_PASS: ${Docker_Hub_Pass_Parity} + needs: + - job: build-staking-miner + artifacts: true + + +publish-s3-release: + stage: publish + extends: + - .kubernetes-env + needs: + - job: build-linux-stable + artifacts: true + image: paritytech/awscli:latest + variables: + GIT_STRATEGY: none + PREFIX: "builds/polkadot/${ARCH}-${DOCKER_OS}" + rules: + - if: $CI_PIPELINE_SOURCE == "pipeline" + when: never + # publishing binaries nightly + - if: $CI_PIPELINE_SOURCE == "schedule" + before_script: + - !reference [.build-push-image, before_script] + script: + - echo "uploading objects to https://releases.parity.io/${PREFIX}/${VERSION}" + - aws s3 sync --acl public-read ./artifacts/ s3://${AWS_BUCKET}/${PREFIX}/${VERSION}/ + - echo "update objects at https://releases.parity.io/${PREFIX}/${EXTRATAG}" + - find ./artifacts -type f | while read file; do + name="${file#./artifacts/}"; + aws s3api copy-object + --copy-source ${AWS_BUCKET}/${PREFIX}/${VERSION}/${name} + --bucket ${AWS_BUCKET} --key ${PREFIX}/${EXTRATAG}/${name}; + done + - | + cat <<-EOM + | + | polkadot binary paths: + | + | - https://releases.parity.io/${PREFIX}/${EXTRATAG}/polkadot + | - https://releases.parity.io/${PREFIX}/${VERSION}/polkadot + | + EOM + after_script: + - aws s3 ls s3://${AWS_BUCKET}/${PREFIX}/${EXTRATAG}/ + --recursive --human-readable --summarize + +publish-rustdoc: + stage: publish + extends: + - .kubernetes-env + image: paritytech/tools:latest + variables: + GIT_DEPTH: 100 + rules: + - if: $CI_PIPELINE_SOURCE == "pipeline" + when: never + - if: $CI_PIPELINE_SOURCE == "web" && $CI_COMMIT_REF_NAME == "master" + - if: $CI_COMMIT_REF_NAME == "master" + # `needs:` can be removed after CI image gets nonroot. In this case `needs:` stops other + # artifacts from being dowloaded by this job. + needs: + - job: build-rustdoc + artifacts: true + - job: build-implementers-guide + artifacts: true + script: + # Save README and docs + - cp -r ./crate-docs/ /tmp/doc/ + - cp -r ./artifacts/book/ /tmp/ + # setup ssh + - eval $(ssh-agent) + - ssh-add - <<< ${GITHUB_SSH_PRIV_KEY} + - mkdir ~/.ssh && touch ~/.ssh/known_hosts + - ssh-keyscan -t rsa github.com >> ~/.ssh/known_hosts + # Set git config + - git config user.email "devops-team@parity.io" + - git config user.name "${GITHUB_USER}" + - git config remote.origin.url "git@github.com:/paritytech/${CI_PROJECT_NAME}.git" + - git config remote.origin.fetch "+refs/heads/*:refs/remotes/origin/*" + - git fetch origin gh-pages + - git checkout gh-pages + # Remove everything and restore generated docs and README + - cp index.html /tmp + - cp README.md /tmp + - rm -rf ./* + # dir for rustdoc + - mkdir -p doc + # dir for implementors guide + - mkdir -p book + - mv /tmp/doc/* doc/ + - mv /tmp/book/html/* book/ + - mv /tmp/index.html . + - mv /tmp/README.md . + # Upload files + - git add --all --force + # `git commit` has an exit code of > 0 if there is nothing to commit. + # This causes GitLab to exit immediately and marks this job failed. + # We don't want to mark the entire job failed if there's nothing to + # publish though, hence the `|| true`. + - git commit -m "Updated docs for ${CI_COMMIT_REF_NAME}" || + echo "___Nothing to commit___" + - git push origin gh-pages --force + - echo "___Rustdoc was successfully published to https://paritytech.github.io/polkadot/___" + after_script: + - rm -rf .git/ ./* diff --git a/polkadot/scripts/ci/gitlab/pipeline/short-benchmarks.yml b/polkadot/scripts/ci/gitlab/pipeline/short-benchmarks.yml new file mode 100644 index 0000000000..52a4074470 --- /dev/null +++ b/polkadot/scripts/ci/gitlab/pipeline/short-benchmarks.yml @@ -0,0 +1,27 @@ +# This file is part of .gitlab-ci.yml +# Here are all jobs that are executed during "short-benchmarks" stage + +# Run all pallet benchmarks only once to check if there are any errors +short-benchmark-polkadot: &short-bench + stage: short-benchmarks + extends: + - .test-pr-refs + - .docker-env + # this is an artificial job dependency, for pipeline optimization using GitLab's DAGs + needs: + - job: build-short-benchmark + artifacts: true + variables: + RUNTIME: polkadot + script: + - ./artifacts/polkadot benchmark pallet --execution wasm --wasm-execution compiled --chain $RUNTIME-dev --pallet "*" --extrinsic "*" --steps 2 --repeat 1 + +short-benchmark-kusama: + <<: *short-bench + variables: + RUNTIME: kusama + +short-benchmark-westend: + <<: *short-bench + variables: + RUNTIME: westend diff --git a/polkadot/scripts/ci/gitlab/pipeline/test.yml b/polkadot/scripts/ci/gitlab/pipeline/test.yml new file mode 100644 index 0000000000..9a3dd0270f --- /dev/null +++ b/polkadot/scripts/ci/gitlab/pipeline/test.yml @@ -0,0 +1,74 @@ +# This file is part of .gitlab-ci.yml +# Here are all jobs that are executed during "test" stage + +test-linux-stable: + stage: test + # this is an artificial job dependency, for pipeline optimization using GitLab's DAGs + # the job can be found in check.yml + needs: + - job: job-starter + artifacts: false + extends: + - .docker-env + - .common-refs + - .pipeline-stopper-artifacts + before_script: + - rustup show + - cargo --version + - !reference [.pipeline-stopper-vars, before_script] + variables: + RUST_TOOLCHAIN: stable + # Enable debug assertions since we are running optimized builds for testing + # but still want to have debug assertions. + RUSTFLAGS: "-Cdebug-assertions=y -Dwarnings" + script: + - time cargo test --workspace --profile testnet --verbose --locked --features=runtime-benchmarks,runtime-metrics + +.check-dependent-project: &check-dependent-project + stage: test + extends: + - .docker-env + - .test-pr-refs + script: + - git clone + --depth=1 + "--branch=$PIPELINE_SCRIPTS_TAG" + https://github.com/paritytech/pipeline-scripts + - ./pipeline-scripts/check_dependent_project.sh + --org paritytech + --dependent-repo "$DEPENDENT_REPO" + --github-api-token "$GITHUB_PR_TOKEN" + --extra-dependencies "$EXTRA_DEPENDENCIES" + --companion-overrides "$COMPANION_OVERRIDES" + +check-dependent-cumulus: + <<: *check-dependent-project + variables: + DEPENDENT_REPO: cumulus + EXTRA_DEPENDENCIES: substrate + COMPANION_OVERRIDES: | + polkadot: release-v* + cumulus: polkadot-v* + +test-node-metrics: + stage: test + extends: + - .docker-env + - .test-refs + - .compiler-info + variables: + RUST_TOOLCHAIN: stable + # Enable debug assertions since we are running optimized builds for testing + # but still want to have debug assertions. + RUSTFLAGS: "-Cdebug-assertions=y -Dwarnings" + script: + - time cargo test --profile testnet --verbose --locked --features=runtime-metrics -p polkadot-node-metrics + +test-deterministic-wasm: + stage: test + extends: + - .docker-env + - .test-refs + - .compiler-info + script: + - ./scripts/ci/gitlab/test_deterministic_wasm.sh diff --git a/polkadot/scripts/ci/gitlab/pipeline/weights.yml b/polkadot/scripts/ci/gitlab/pipeline/weights.yml new file mode 100644 index 0000000000..38b223ce57 --- /dev/null +++ b/polkadot/scripts/ci/gitlab/pipeline/weights.yml @@ -0,0 +1,33 @@ +# This file is part of .gitlab-ci.yml +# Here are all jobs that are executed during "weights" stage + +update_polkadot_weights: &update-weights + stage: weights + timeout: 1d + when: manual + variables: + RUNTIME: polkadot + artifacts: + paths: + - ${RUNTIME}_weights_${CI_COMMIT_SHORT_SHA}.patch + script: + - ./scripts/ci/run_benches_for_runtime.sh $RUNTIME + - git diff -P > ${RUNTIME}_weights_${CI_COMMIT_SHORT_SHA}.patch + # uses the "shell" executors + tags: + - weights + +update_kusama_weights: + <<: *update-weights + variables: + RUNTIME: kusama + +update_westend_weights: + <<: *update-weights + variables: + RUNTIME: westend + +update_rococo_weights: + <<: *update-weights + variables: + RUNTIME: rococo diff --git a/polkadot/scripts/ci/gitlab/pipeline/zombienet.yml b/polkadot/scripts/ci/gitlab/pipeline/zombienet.yml new file mode 100644 index 0000000000..8f7c5ef38a --- /dev/null +++ b/polkadot/scripts/ci/gitlab/pipeline/zombienet.yml @@ -0,0 +1,302 @@ +# This file is part of .gitlab-ci.yml +# Here are all jobs that are executed during "zombienet" stage + +zombienet-tests-parachains-smoke-test: + stage: zombienet + image: "${ZOMBIENET_IMAGE}" + extends: + - .kubernetes-env + - .zombienet-refs + needs: + - job: publish-polkadot-debug-image + - job: publish-malus-image + - job: publish-test-collators-image + variables: + GH_DIR: "https://github.com/paritytech/polkadot/tree/${CI_COMMIT_SHORT_SHA}/zombienet_tests/smoke" + before_script: + - echo "Zombie-net Tests Config" + - echo "${ZOMBIENET_IMAGE}" + - echo "${PARACHAINS_IMAGE_NAME} ${PARACHAINS_IMAGE_TAG}" + - echo "${MALUS_IMAGE_NAME} ${MALUS_IMAGE_TAG}" + - echo "${GH_DIR}" + - export DEBUG=zombie,zombie::network-node + - export ZOMBIENET_INTEGRATION_TEST_IMAGE=${PARACHAINS_IMAGE_NAME}:${PARACHAINS_IMAGE_TAG} + - export MALUS_IMAGE=${MALUS_IMAGE_NAME}:${MALUS_IMAGE_TAG} + - export COL_IMAGE="docker.io/paritypr/colander:4519" # The collator image is fixed + script: + - /home/nonroot/zombie-net/scripts/ci/run-test-env-manager.sh + --github-remote-dir="${GH_DIR}" + --test="0001-parachains-smoke-test.feature" + allow_failure: false + retry: 2 + tags: + - zombienet-polkadot-integration-test + +zombienet-tests-parachains-pvf: + stage: zombienet + image: "${ZOMBIENET_IMAGE}" + extends: + - .kubernetes-env + - .zombienet-refs + needs: + - job: publish-polkadot-debug-image + - job: publish-test-collators-image + variables: + GH_DIR: "https://github.com/paritytech/polkadot/tree/${CI_COMMIT_SHORT_SHA}/zombienet_tests/functional" + before_script: + - echo "Zombie-net Tests Config" + - echo "${ZOMBIENET_IMAGE}" + - echo "${PARACHAINS_IMAGE_NAME} ${PARACHAINS_IMAGE_TAG}" + - echo "COL_IMAGE=${COLLATOR_IMAGE_NAME}:${COLLATOR_IMAGE_TAG}" + - echo "${GH_DIR}" + - export DEBUG=zombie,zombie::network-node + - export ZOMBIENET_INTEGRATION_TEST_IMAGE=${PARACHAINS_IMAGE_NAME}:${PARACHAINS_IMAGE_TAG} + - export MALUS_IMAGE=${MALUS_IMAGE_NAME}:${MALUS_IMAGE_TAG} + - export COL_IMAGE=${COLLATOR_IMAGE_NAME}:${COLLATOR_IMAGE_TAG} + script: + - /home/nonroot/zombie-net/scripts/ci/run-test-env-manager.sh + --github-remote-dir="${GH_DIR}" + --test="0001-parachains-pvf.feature" + allow_failure: false + retry: 2 + tags: + - zombienet-polkadot-integration-test + +zombienet-tests-parachains-disputes: + stage: zombienet + image: "${ZOMBIENET_IMAGE}" + extends: + - .kubernetes-env + - .zombienet-refs + needs: + - job: publish-polkadot-debug-image + - job: publish-test-collators-image + - job: publish-malus-image + variables: + GH_DIR: "https://github.com/paritytech/polkadot/tree/${CI_COMMIT_SHORT_SHA}/zombienet_tests/functional" + before_script: + - echo "Zombie-net Tests Config" + - echo "${ZOMBIENET_IMAGE_NAME}" + - echo "${PARACHAINS_IMAGE_NAME} ${PARACHAINS_IMAGE_TAG}" + - echo "${MALUS_IMAGE_NAME} ${MALUS_IMAGE_TAG}" + - echo "${GH_DIR}" + - export DEBUG=zombie,zombie::network-node + - export ZOMBIENET_INTEGRATION_TEST_IMAGE=${PARACHAINS_IMAGE_NAME}:${PARACHAINS_IMAGE_TAG} + - export MALUS_IMAGE=${MALUS_IMAGE_NAME}:${MALUS_IMAGE_TAG} + - export COL_IMAGE=${COLLATOR_IMAGE_NAME}:${COLLATOR_IMAGE_TAG} + script: + - /home/nonroot/zombie-net/scripts/ci/run-test-env-manager.sh + --github-remote-dir="${GH_DIR}" + --test="0002-parachains-disputes.feature" + allow_failure: false + retry: 2 + tags: + - zombienet-polkadot-integration-test + +zombienet-tests-parachains-disputes-garbage-candidate: + stage: zombienet + image: "${ZOMBIENET_IMAGE}" + extends: + - .kubernetes-env + - .zombienet-refs + needs: + - job: publish-polkadot-debug-image + - job: publish-test-collators-image + - job: publish-malus-image + variables: + GH_DIR: "https://github.com/paritytech/polkadot/tree/${CI_COMMIT_SHORT_SHA}/zombienet_tests/functional" + before_script: + - echo "Zombie-net Tests Config" + - echo "${ZOMBIENET_IMAGE_NAME}" + - echo "${PARACHAINS_IMAGE_NAME} ${PARACHAINS_IMAGE_TAG}" + - echo "${MALUS_IMAGE_NAME} ${MALUS_IMAGE_TAG}" + - echo "${GH_DIR}" + - export DEBUG=zombie,zombie::network-node + - export ZOMBIENET_INTEGRATION_TEST_IMAGE=${PARACHAINS_IMAGE_NAME}:${PARACHAINS_IMAGE_TAG} + - export MALUS_IMAGE=${MALUS_IMAGE_NAME}:${MALUS_IMAGE_TAG} + - export COL_IMAGE=${COLLATOR_IMAGE_NAME}:${COLLATOR_IMAGE_TAG} + script: + - /home/nonroot/zombie-net/scripts/ci/run-test-env-manager.sh + --github-remote-dir="${GH_DIR}" + --test="0003-parachains-garbage-candidate.feature" + allow_failure: false + retry: 2 + tags: + - zombienet-polkadot-integration-test + +zombienet-test-parachains-upgrade-smoke-test: + stage: zombienet + image: "${ZOMBIENET_IMAGE}" + extends: + - .kubernetes-env + - .zombienet-refs + needs: + - job: publish-polkadot-debug-image + - job: publish-malus-image + - job: publish-test-collators-image + variables: + GH_DIR: 'https://github.com/paritytech/polkadot/tree/${CI_COMMIT_SHORT_SHA}/zombienet_tests/smoke' + before_script: + - echo "ZombieNet Tests Config" + - echo "${PARACHAINS_IMAGE_NAME}:${PARACHAINS_IMAGE_TAG}" + - echo "docker.io/parity/polkadot-collator:latest" + - echo "${ZOMBIENET_IMAGE}" + - echo "${GH_DIR}" + - export DEBUG=zombie,zombie::network-node + - export ZOMBIENET_INTEGRATION_TEST_IMAGE=${PARACHAINS_IMAGE_NAME}:${PARACHAINS_IMAGE_TAG} + - export COL_IMAGE="docker.io/parity/polkadot-collator:latest" # Use cumulus lastest image + script: + - /home/nonroot/zombie-net/scripts/ci/run-test-env-manager.sh + --github-remote-dir="${GH_DIR}" + --test="0002-parachains-upgrade-smoke-test.feature" + allow_failure: true + retry: 2 + tags: + - zombienet-polkadot-integration-test + +zombienet-tests-misc-paritydb: + stage: zombienet + image: "${ZOMBIENET_IMAGE}" + extends: + - .kubernetes-env + - .zombienet-refs + needs: + - job: publish-polkadot-debug-image + - job: publish-test-collators-image + artifacts: true + variables: + GH_DIR: "https://github.com/paritytech/polkadot/tree/${CI_COMMIT_SHORT_SHA}/zombienet_tests/misc" + before_script: + - echo "Zombie-net Tests Config" + - echo "${ZOMBIENET_IMAGE_NAME}" + - echo "${PARACHAINS_IMAGE_NAME} ${PARACHAINS_IMAGE_TAG}" + - echo "${GH_DIR}" + - export DEBUG=zombie,zombie::network-node + - export ZOMBIENET_INTEGRATION_TEST_IMAGE=${PARACHAINS_IMAGE_NAME}:${PARACHAINS_IMAGE_TAG} + - export COL_IMAGE=${COLLATOR_IMAGE_NAME}:${COLLATOR_IMAGE_TAG} + script: + - /home/nonroot/zombie-net/scripts/ci/run-test-env-manager.sh + --github-remote-dir="${GH_DIR}" + --test="0001-paritydb.feature" + allow_failure: false + retry: 2 + tags: + - zombienet-polkadot-integration-test + +zombienet-tests-misc-upgrade-node: + stage: zombienet + image: "${ZOMBIENET_IMAGE}" + extends: + - .kubernetes-env + - .zombienet-refs + needs: + - job: publish-polkadot-debug-image + - job: publish-test-collators-image + - job: build-linux-stable + artifacts: true + variables: + GH_DIR: "https://github.com/paritytech/polkadot/tree/${CI_COMMIT_SHORT_SHA}/zombienet_tests/misc" + before_script: + - echo "Zombie-net Tests Config" + - echo "${ZOMBIENET_IMAGE_NAME}" + - echo "${PARACHAINS_IMAGE_NAME} ${PARACHAINS_IMAGE_TAG}" + - echo "${GH_DIR}" + - export DEBUG=zombie,zombie::network-node + - BUILD_RELEASE_VERSION="$(cat ./artifacts/BUILD_RELEASE_VERSION)" + - export ZOMBIENET_INTEGRATION_TEST_IMAGE="docker.io/parity/polkadot:${BUILD_RELEASE_VERSION}" + - export COL_IMAGE=${COLLATOR_IMAGE_NAME}:${COLLATOR_IMAGE_TAG} + - BUILD_LINUX_JOB_ID="$(cat ./artifacts/BUILD_LINUX_JOB_ID)" + - export POLKADOT_PR_BIN_URL="https://gitlab.parity.io/parity/mirrors/polkadot/-/jobs/${BUILD_LINUX_JOB_ID}/artifacts/raw/artifacts/polkadot" + script: + - /home/nonroot/zombie-net/scripts/ci/run-test-env-manager.sh + --github-remote-dir="${GH_DIR}" + --test="0002-upgrade-node.feature" + allow_failure: false + retry: 2 + tags: + - zombienet-polkadot-integration-test + +zombienet-tests-malus-dispute-valid: + stage: zombienet + image: "${ZOMBIENET_IMAGE}" + extends: + - .kubernetes-env + - .zombienet-refs + needs: + - job: publish-polkadot-debug-image + - job: publish-malus-image + - job: publish-test-collators-image + variables: + GH_DIR: "https://github.com/paritytech/polkadot/tree/${CI_COMMIT_SHORT_SHA}/node/malus/integrationtests" + before_script: + - echo "Zombie-net Tests Config" + - echo "${ZOMBIENET_IMAGE_NAME}" + - echo "${PARACHAINS_IMAGE_NAME} ${PARACHAINS_IMAGE_TAG}" + - echo "${MALUS_IMAGE_NAME} ${MALUS_IMAGE_TAG}" + - echo "${GH_DIR}" + - export DEBUG=zombie* + - export ZOMBIENET_INTEGRATION_TEST_IMAGE=${PARACHAINS_IMAGE_NAME}:${PARACHAINS_IMAGE_TAG} + - export MALUS_IMAGE=${MALUS_IMAGE_NAME}:${MALUS_IMAGE_TAG} + - export COL_IMAGE=${COLLATOR_IMAGE_NAME}:${COLLATOR_IMAGE_TAG} + script: + - /home/nonroot/zombie-net/scripts/ci/run-test-env-manager.sh + --github-remote-dir="${GH_DIR}" + --test="0001-dispute-valid-block.feature" + allow_failure: false + retry: 2 + tags: + - zombienet-polkadot-integration-test + +zombienet-tests-deregister-register-validator: + stage: zombienet + image: "${ZOMBIENET_IMAGE}" + extends: + - .kubernetes-env + - .zombienet-refs + needs: + - job: publish-polkadot-debug-image + artifacts: true + variables: + GH_DIR: "https://github.com/paritytech/polkadot/tree/${CI_COMMIT_SHORT_SHA}/zombienet_tests/smoke" + before_script: + - echo "Zombie-net Tests Config" + - echo "${ZOMBIENET_IMAGE_NAME}" + - echo "${PARACHAINS_IMAGE_NAME} ${PARACHAINS_IMAGE_TAG}" + - echo "${GH_DIR}" + - export DEBUG=zombie* + - export ZOMBIENET_INTEGRATION_TEST_IMAGE=${PARACHAINS_IMAGE_NAME}:${PARACHAINS_IMAGE_TAG} + - export MALUS_IMAGE=${MALUS_IMAGE_NAME}:${MALUS_IMAGE_TAG} + script: + - /home/nonroot/zombie-net/scripts/ci/run-test-env-manager.sh + --github-remote-dir="${GH_DIR}" + --test="0003-deregister-register-validator-smoke.feature" + allow_failure: false + retry: 2 + tags: + - zombienet-polkadot-integration-test + +zombienet-tests-beefy-and-mmr: + stage: zombienet + image: "${ZOMBIENET_IMAGE}" + extends: + - .kubernetes-env + - .zombienet-refs + needs: + - job: publish-polkadot-debug-image + variables: + GH_DIR: "https://github.com/paritytech/polkadot/tree/${CI_COMMIT_SHORT_SHA}/zombienet_tests/functional" + before_script: + - echo "Zombie-net Tests Config" + - echo "${ZOMBIENET_IMAGE_NAME}" + - echo "${PARACHAINS_IMAGE_NAME} ${PARACHAINS_IMAGE_TAG}" + - echo "${GH_DIR}" + - export DEBUG=zombie* + - export ZOMBIENET_INTEGRATION_TEST_IMAGE=${PARACHAINS_IMAGE_NAME}:${PARACHAINS_IMAGE_TAG} + script: + - /home/nonroot/zombie-net/scripts/ci/run-test-env-manager.sh + --github-remote-dir="${GH_DIR}" + --test="0003-beefy-and-mmr.feature" + allow_failure: true + retry: 2 + tags: + - zombienet-polkadot-integration-test \ No newline at end of file