# .gitlab-ci.yml # # cumulus # # pipelines can be triggered manually in the web stages: - test - build - benchmarks-build - benchmarks-run - publish - integration-test default: interruptible: true retry: max: 2 when: - runner_system_failure - unknown_failure - api_failure variables: GIT_STRATEGY: fetch GIT_DEPTH: 100 CARGO_INCREMENTAL: 0 CI_IMAGE: "paritytech/ci-linux:production" DOCKER_OS: "debian:stretch" ARCH: "x86_64" ZOMBIENET_IMAGE: "docker.io/paritytech/zombienet:v1.2.59" .collect-artifacts: &collect-artifacts artifacts: name: "${CI_JOB_NAME}_${CI_COMMIT_REF_NAME}" when: on_success expire_in: 1 days paths: - ./artifacts/ .rust-info-script: &rust-info-script - rustup show - cargo --version - rustup +nightly show - cargo +nightly --version - bash --version # collecting vars for pipeline stopper # they will be used if the job fails .pipeline-stopper-vars: &pipeline-stopper-vars - echo "FAILED_JOB_URL=${CI_JOB_URL}" > pipeline-stopper.env - echo "FAILED_JOB_NAME=${CI_JOB_NAME}" >> pipeline-stopper.env - echo "FAILED_JOB_NAME=${CI_JOB_NAME}" >> pipeline-stopper.env - echo "PR_NUM=${CI_COMMIT_REF_NAME}" >> pipeline-stopper.env .pipeline-stopper-artifacts: &pipeline-stopper-artifacts artifacts: reports: dotenv: pipeline-stopper.env .common-refs: &common-refs # these jobs run always* rules: - if: $CI_PIPELINE_SOURCE == "schedule" - if: $CI_COMMIT_REF_NAME == "master" - if: $CI_COMMIT_REF_NAME =~ /^[0-9]+$/ # PRs - if: $CI_COMMIT_REF_NAME =~ /^v[0-9]+\.[0-9]+.*$/ # i.e. v1.0, v2.1rc1 - if: $CI_COMMIT_REF_NAME =~ /^release-parachains-v[0-9].*$/ # i.e. release-parachains-v1.0, release-parachains-v2.1rc1, release-parachains-v3000 - if: $CI_COMMIT_REF_NAME =~ /^polkadot-v[0-9]+\.[0-9]+.*$/ # i.e. polkadot-v1.0.99, polkadot-v2.1rc1 .pr-refs: &pr-refs # these jobs run always* rules: - if: $CI_COMMIT_REF_NAME =~ /^[0-9]+$/ # PRs .publish-refs: &publish-refs rules: - if: $CI_COMMIT_REF_NAME == "master" - if: $CI_PIPELINE_SOURCE == "schedule" - if: $CI_COMMIT_REF_NAME =~ /^v[0-9]+\.[0-9]+.*$/ # i.e. v1.0, v2.1rc1 # run benchmarks manually only on release-parachains-v* branch .benchmarks-manual-refs: &benchmarks-manual-refs rules: - if: $CI_COMMIT_REF_NAME =~ /^release-parachains-v[0-9].*$/ # i.e. release-parachains-v1.0, release-parachains-v2.1rc1, release-parachains-v3000 when: manual # run benchmarks only on release-parachains-v* branch .benchmarks-refs: &benchmarks-refs rules: - if: $CI_COMMIT_REF_NAME =~ /^release-parachains-v[0-9].*$/ # i.e. release-parachains-v1.0, release-parachains-v2.1rc1, release-parachains-v3000 .zombienet-refs: &zombienet-refs rules: - if: $CI_PIPELINE_SOURCE == "pipeline" when: never - if: $CI_PIPELINE_SOURCE == "schedule" when: never - if: $CI_COMMIT_REF_NAME == "master" - if: $CI_COMMIT_REF_NAME =~ /^[0-9]+$/ # PRs .docker-env: &docker-env image: "${CI_IMAGE}" before_script: - *rust-info-script tags: - linux-docker .kubernetes-env: &kubernetes-env image: "${CI_IMAGE}" tags: - kubernetes-parity-build #### stage: test test-linux-stable: stage: test <<: *docker-env <<: *common-refs <<: *pipeline-stopper-artifacts before_script: - *rust-info-script - *pipeline-stopper-vars variables: # Enable debug assertions since we are running optimized builds for testing # but still want to have debug assertions. RUSTFLAGS: "-Cdebug-assertions=y -Dwarnings" script: - time cargo nextest run --all --release --locked --run-ignored all test-doc: stage: test <<: *docker-env <<: *common-refs variables: # Enable debug assertions since we are running optimized builds for testing # but still want to have debug assertions. RUSTFLAGS: "-Cdebug-assertions=y -Dwarnings" script: - time cargo test --doc check-runtime-benchmarks: stage: test <<: *docker-env <<: *common-refs script: # Check that the node will compile with `runtime-benchmarks` feature flag. - time cargo check --all --features runtime-benchmarks # Check that parachain-template will compile with `runtime-benchmarks` feature flag. - time cargo check -p parachain-template-node --features runtime-benchmarks cargo-check-try-runtime: stage: test <<: *docker-env <<: *common-refs # this is an artificial job dependency, for pipeline optimization using GitLab's DAGs needs: - job: check-runtime-benchmarks artifacts: false script: # Check that the node will compile with `try-runtime` feature flag. - time cargo check --all --features try-runtime # Check that parachain-template will compile with `try-runtime` feature flag. - time cargo check -p parachain-template-node --features try-runtime check-rustdoc: stage: test <<: *docker-env <<: *common-refs variables: SKIP_WASM_BUILD: 1 RUSTDOCFLAGS: "-Dwarnings" script: - time cargo +nightly doc --workspace --all-features --verbose --no-deps cargo-check-benches: stage: test <<: *docker-env <<: *common-refs # this is an artificial job dependency, for pipeline optimization using GitLab's DAGs needs: - job: check-rustdoc artifacts: false script: - time cargo check --all --benches #### stage: build build-linux-stable: stage: build <<: *docker-env <<: *collect-artifacts <<: *common-refs variables: # Enable debug assertions since we are running optimized builds for testing # but still want to have debug assertions. RUSTFLAGS: "-Cdebug-assertions=y -Dwarnings" # this is an artificial job dependency, for pipeline optimization using GitLab's DAGs needs: - job: check-rustdoc artifacts: false script: - echo "___Building a binary, please refrain from using it in production since it goes with the debug assertions.___" - time cargo build --release --locked --bin polkadot-parachain - echo "___Packing the artifacts___" - mkdir -p ./artifacts - mv ./target/release/polkadot-parachain ./artifacts/. - echo "___The VERSION is either a tag name or the curent branch if triggered not by a tag___" - echo ${CI_COMMIT_REF_NAME} | tee ./artifacts/VERSION build-test-parachain: stage: build <<: *docker-env <<: *collect-artifacts <<: *common-refs variables: # Enable debug assertions since we are running optimized builds for testing # but still want to have debug assertions. RUSTFLAGS: "-Cdebug-assertions=y -Dwarnings" # this is an artificial job dependency, for pipeline optimization using GitLab's DAGs needs: - job: check-rustdoc artifacts: false script: - echo "___Building a binary, please refrain from using it in production since it goes with the debug assertions.___" - time cargo +nightly build --release --locked --bin test-parachain - echo "___Packing the artifacts___" - mkdir -p ./artifacts - mv ./target/release/test-parachain ./artifacts/. - mkdir -p ./artifacts/zombienet - mv ./target/release/wbuild/cumulus-test-runtime/wasm_binary_spec_version_incremented.rs.compact.compressed.wasm ./artifacts/zombienet/. # build runtime only if files in $RUNTIME_PATH/$RUNTIME_NAME were changed .build-runtime-template: &build-runtime-template stage: build <<: *docker-env <<: *pr-refs # this is an artificial job dependency, for pipeline optimization using GitLab's DAGs needs: - job: check-rustdoc artifacts: false variables: RUNTIME_PATH: "parachains/runtimes/assets" script: - cd ${RUNTIME_PATH} - for directory in $(echo */); do echo "_____Running cargo check for ${directory} ______"; cd ${directory}; pwd; SKIP_WASM_BUILD=1 cargo check; cd ..; done # DAG: build-runtime-assets -> build-runtime-collectives -> build-runtime-contracts # DAG: build-runtime-assets ->build-runtime-starters -> build-runtime-testing build-runtime-assets: <<: *build-runtime-template variables: RUNTIME_PATH: "parachains/runtimes/assets" build-runtime-collectives: <<: *build-runtime-template variables: RUNTIME_PATH: "parachains/runtimes/collectives" # this is an artificial job dependency, for pipeline optimization using GitLab's DAGs needs: - job: build-runtime-assets artifacts: false build-runtime-contracts: <<: *build-runtime-template variables: RUNTIME_PATH: "parachains/runtimes/contracts" # this is an artificial job dependency, for pipeline optimization using GitLab's DAGs needs: - job: build-runtime-collectives artifacts: false build-runtime-starters: <<: *build-runtime-template variables: RUNTIME_PATH: "parachains/runtimes/starters" # this is an artificial job dependency, for pipeline optimization using GitLab's DAGs needs: - job: build-runtime-assets artifacts: false build-runtime-testing: <<: *build-runtime-template variables: RUNTIME_PATH: "parachains/runtimes/testing" # this is an artificial job dependency, for pipeline optimization using GitLab's DAGs needs: - job: build-runtime-starters artifacts: false #### stage: publish .build-push-image: &build-push-image image: quay.io/buildah/stable variables: DOCKERFILE: "" # docker/path-to.Dockerfile IMAGE_NAME: "" # docker.io/paritypr/image_name VERSION: "${CI_COMMIT_REF_NAME}-${CI_COMMIT_SHORT_SHA}" script: - test "$PARITYPR_USER" -a "$PARITYPR_PASS" || ( echo "no docker credentials provided"; exit 1 ) - buildah bud --format=docker --build-arg VCS_REF="${CI_COMMIT_SHA}" --build-arg BUILD_DATE="$(date -u '+%Y-%m-%dT%H:%M:%SZ')" --build-arg IMAGE_NAME="${IMAGE_NAME}" --tag "$IMAGE_NAME:$VERSION" --file ${DOCKERFILE} . - echo "$PARITYPR_PASS" | buildah login --username "$PARITYPR_USER" --password-stdin docker.io - buildah info - buildah push --format=v2s2 "$IMAGE_NAME:$VERSION" after_script: - buildah logout --all build-push-image-polkadot-parachain-debug: stage: publish <<: *kubernetes-env <<: *common-refs <<: *build-push-image needs: - job: build-linux-stable artifacts: true variables: DOCKERFILE: "docker/polkadot-parachain-debug_unsigned_injected.Dockerfile" IMAGE_NAME: "docker.io/paritypr/polkadot-parachain-debug" VERSION: "${CI_COMMIT_REF_NAME}-${CI_COMMIT_SHORT_SHA}" build-push-image-test-parachain: stage: publish <<: *kubernetes-env <<: *common-refs <<: *build-push-image needs: - job: build-test-parachain artifacts: true variables: DOCKERFILE: "docker/test-parachain_injected.Dockerfile" IMAGE_NAME: "docker.io/paritypr/test-parachain" VERSION: "${CI_COMMIT_REF_NAME}-${CI_COMMIT_SHORT_SHA}" publish-s3: stage: publish <<: *kubernetes-env image: paritytech/awscli:latest <<: *publish-refs needs: - job: build-linux-stable artifacts: true variables: GIT_STRATEGY: none BUCKET: "releases.parity.io" PREFIX: "cumulus/${ARCH}-${DOCKER_OS}" script: - echo "___Publishing a binary with debug assertions!___" - echo "___VERSION = $(cat ./artifacts/VERSION) ___" - aws s3 sync ./artifacts/ s3://${BUCKET}/${PREFIX}/$(cat ./artifacts/VERSION)/ - echo "___Updating objects in latest path___" - aws s3 sync s3://${BUCKET}/${PREFIX}/$(cat ./artifacts/VERSION)/ s3://${BUCKET}/${PREFIX}/latest/ after_script: - aws s3 ls s3://${BUCKET}/${PREFIX}/latest/ --recursive --human-readable --summarize #### stage: benchmarks # Work only on release-parachains-v* branches benchmarks-build: stage: benchmarks-build <<: *docker-env <<: *collect-artifacts <<: *benchmarks-manual-refs script: - time cargo build --profile production --locked --features runtime-benchmarks - mkdir -p artifacts - cp target/production/polkadot-parachain ./artifacts/ .git-commit-push: &git-commit-push - git status # Set git config - rm -rf .git/config - git config --global user.email "${GITHUB_EMAIL}" - git config --global user.name "${GITHUB_USER}" - git config remote.origin.url "https://${GITHUB_USER}:${GITHUB_TOKEN}@github.com/paritytech/${CI_PROJECT_NAME}.git" - git config remote.origin.fetch "+refs/heads/*:refs/remotes/origin/*" # push results to github - git checkout -b $BRANCHNAME - git add parachains/* - git commit -m "[benchmarks] pr with weights" - git push origin $BRANCHNAME benchmarks-assets: stage: benchmarks-run before_script: - *rust-info-script <<: *collect-artifacts <<: *benchmarks-refs timeout: 1d script: - ./scripts/benchmarks-ci.sh assets statemine ./artifacts - ./scripts/benchmarks-ci.sh assets statemint ./artifacts - ./scripts/benchmarks-ci.sh assets westmint ./artifacts - export BRANCHNAME="weights-statemint-${CI_COMMIT_BRANCH}" - *git-commit-push # create PR to release-parachains-v* branch - curl -u ${GITHUB_USER}:${GITHUB_TOKEN} -d '{"title":"[benchmarks] Update weights for statemine/t","body":"This PR is generated automatically by CI. (Once merged please backport to master and node release branch.)","head":"'${BRANCHNAME}'","base":"'${CI_COMMIT_BRANCH}'"}' -X POST https://api.github.com/repos/paritytech/${CI_PROJECT_NAME}/pulls after_script: - rm -rf .git/config tags: - weights benchmarks-collectives: stage: benchmarks-run before_script: - *rust-info-script <<: *collect-artifacts <<: *benchmarks-refs script: - ./scripts/benchmarks-ci.sh collectives collectives-polkadot ./artifacts - git status - export BRANCHNAME="weights-collectives-${CI_COMMIT_BRANCH}" - *git-commit-push # create PR - curl -u ${GITHUB_USER}:${GITHUB_TOKEN} -d '{"title":"[benchmarks] Update weights for collectives","body":"This PR is generated automatically by CI.","head":"'$BRANCHNAME'","base":"'${CI_COMMIT_BRANCH}'"}' -X POST https://api.github.com/repos/paritytech/${CI_PROJECT_NAME}/pulls after_script: - rm -rf .git/config tags: - weights publish-benchmarks-assets-s3: &publish-benchmarks stage: publish <<: *kubernetes-env image: paritytech/awscli:latest <<: *benchmarks-refs needs: - job: benchmarks-assets artifacts: true variables: GIT_STRATEGY: none BUCKET: "releases.parity.io" PREFIX: "cumulus/$CI_COMMIT_REF_NAME/benchmarks" script: - echo "___Removing binary from artifacts___" - rm -f ./artifacts/polkadot-parachain - echo "___Publishing benchmark results___" - aws s3 sync ./artifacts/ s3://${BUCKET}/${PREFIX}/ after_script: - aws s3 ls s3://${BUCKET}/${PREFIX}/ --recursive --human-readable --summarize publish-benchmarks-collectives-s3: <<: *publish-benchmarks needs: - job: benchmarks-collectives artifacts: true # Ref: https://github.com/paritytech/opstooling/issues/111 update-parachain-template: stage: publish extends: .kubernetes-env rules: - if: $CI_COMMIT_REF_NAME =~ /^polkadot-v[0-9]+\.[0-9]+.*$/ # i.e. polkadot-v1.0.99, polkadot-v2.1rc1 script: - git clone --depth=1 --branch="$PIPELINE_SCRIPTS_TAG" https://github.com/paritytech/pipeline-scripts - ./pipeline-scripts/update_substrate_template.sh --repo-name "substrate-parachain-template" --template-path "parachain-template" --github-api-token "$GITHUB_TOKEN" --polkadot-branch "$CI_COMMIT_REF_NAME" #### stage: integration-test zombienet-0001-sync_blocks_from_tip_without_connected_collator: stage: integration-test image: "${ZOMBIENET_IMAGE}" <<: *zombienet-refs needs: - job: build-push-image-test-parachain variables: POLKADOT_IMAGE: "docker.io/paritypr/polkadot-debug:master" GH_DIR: "https://github.com/paritytech/cumulus/tree/${CI_COMMIT_SHORT_SHA}/zombienet_tests" COL_IMAGE: "docker.io/paritypr/test-parachain:${CI_COMMIT_REF_NAME}-${CI_COMMIT_SHORT_SHA}" before_script: - echo "Zombie-net Tests Config" - echo "${ZOMBIENET_IMAGE}" - echo "${RELAY_IMAGE}" - echo "${COL_IMAGE}" - echo "${GH_DIR}" - export DEBUG=zombie - export RELAY_IMAGE=${POLKADOT_IMAGE} - export COL_IMAGE=${COL_IMAGE} script: - /home/nonroot/zombie-net/scripts/ci/run-test-env-manager.sh --github-remote-dir="${GH_DIR}" --concurrency=1 --test="0001-sync_blocks_from_tip_without_connected_collator.feature" allow_failure: true retry: 2 tags: - zombienet-polkadot-integration-test zombienet-0002-pov_recovery: stage: integration-test image: "${ZOMBIENET_IMAGE}" <<: *zombienet-refs needs: - job: build-push-image-test-parachain variables: POLKADOT_IMAGE: "docker.io/paritypr/polkadot-debug:master" GH_DIR: "https://github.com/paritytech/cumulus/tree/${CI_COMMIT_SHORT_SHA}/zombienet_tests" COL_IMAGE: "docker.io/paritypr/test-parachain:${CI_COMMIT_REF_NAME}-${CI_COMMIT_SHORT_SHA}" before_script: - echo "Zombie-net Tests Config" - echo "${ZOMBIENET_IMAGE}" - echo "${RELAY_IMAGE}" - echo "${COL_IMAGE}" - echo "${GH_DIR}" - export DEBUG=zombie - export RELAY_IMAGE=${POLKADOT_IMAGE} - export COL_IMAGE=${COL_IMAGE} script: - /home/nonroot/zombie-net/scripts/ci/run-test-env-manager.sh --github-remote-dir="${GH_DIR}" --concurrency=1 --test="0002-pov_recovery.feature" allow_failure: true retry: 2 tags: - zombienet-polkadot-integration-test zombienet-0003-full_node_catching_up: stage: integration-test image: "${ZOMBIENET_IMAGE}" <<: *zombienet-refs needs: - job: build-push-image-test-parachain variables: POLKADOT_IMAGE: "docker.io/paritypr/polkadot-debug:master" GH_DIR: "https://github.com/paritytech/cumulus/tree/${CI_COMMIT_SHORT_SHA}/zombienet_tests" COL_IMAGE: "docker.io/paritypr/test-parachain:${CI_COMMIT_REF_NAME}-${CI_COMMIT_SHORT_SHA}" before_script: - echo "Zombie-net Tests Config" - echo "${ZOMBIENET_IMAGE}" - echo "${RELAY_IMAGE}" - echo "${COL_IMAGE}" - echo "${GH_DIR}" - export DEBUG=zombie - export RELAY_IMAGE=${POLKADOT_IMAGE} - export COL_IMAGE=${COL_IMAGE} script: - /home/nonroot/zombie-net/scripts/ci/run-test-env-manager.sh --github-remote-dir="${GH_DIR}" --concurrency=1 --test="0003-full_node_catching_up.feature" allow_failure: true retry: 2 tags: - zombienet-polkadot-integration-test zombienet-0004-runtime_upgrade: stage: integration-test image: "${ZOMBIENET_IMAGE}" <<: *zombienet-refs needs: - job: build-push-image-test-parachain - job: build-test-parachain artifacts: true variables: POLKADOT_IMAGE: "docker.io/paritypr/polkadot-debug:master" GH_DIR: "https://github.com/paritytech/cumulus/tree/${CI_COMMIT_SHORT_SHA}/zombienet_tests" COL_IMAGE: "docker.io/paritypr/test-parachain:${CI_COMMIT_REF_NAME}-${CI_COMMIT_SHORT_SHA}" before_script: - ls -ltr * - cp ./artifacts/zombienet/wasm_binary_spec_version_incremented.rs.compact.compressed.wasm /tmp/ - ls /tmp - echo "Zombie-net Tests Config" - echo "${ZOMBIENET_IMAGE}" - echo "${RELAY_IMAGE}" - echo "${COL_IMAGE}" - echo "${GH_DIR}" - export DEBUG=zombie - export RELAY_IMAGE=${POLKADOT_IMAGE} - export COL_IMAGE=${COL_IMAGE} script: - /home/nonroot/zombie-net/scripts/ci/run-test-env-manager.sh --github-remote-dir="${GH_DIR}" --concurrency=1 --test="0004-runtime_upgrade.feature" allow_failure: true retry: 2 tags: - zombienet-polkadot-integration-test zombienet-0005-migrate_solo_to_para: stage: integration-test image: "${ZOMBIENET_IMAGE}" <<: *zombienet-refs needs: - job: build-push-image-test-parachain - job: build-test-parachain artifacts: true variables: POLKADOT_IMAGE: "docker.io/paritypr/polkadot-debug:master" GH_DIR: "https://github.com/paritytech/cumulus/tree/${CI_COMMIT_SHORT_SHA}/zombienet_tests" COL_IMAGE: "docker.io/paritypr/test-parachain:${CI_COMMIT_REF_NAME}-${CI_COMMIT_SHORT_SHA}" before_script: - ls -ltr * - echo "Zombie-net Tests Config" - echo "${ZOMBIENET_IMAGE}" - echo "${RELAY_IMAGE}" - echo "${COL_IMAGE}" - echo "${GH_DIR}" - export DEBUG=zombie - export RELAY_IMAGE=${POLKADOT_IMAGE} - export COL_IMAGE=${COL_IMAGE} script: - /home/nonroot/zombie-net/scripts/ci/run-test-env-manager.sh --github-remote-dir="${GH_DIR}" --concurrency=1 --test="0005-migrate_solo_to_para.feature" allow_failure: true retry: 2 tags: - zombienet-polkadot-integration-test #### stage: .post # This job cancels the whole pipeline if any of provided jobs fail. # In a DAG, every jobs chain is executed independently of others. The `fail_fast` principle suggests # to fail the pipeline as soon as possible to shorten the feedback loop. .cancel-pipeline: stage: .post needs: - job: test-linux-stable rules: - if: $CI_COMMIT_REF_NAME =~ /^[0-9]+$/ # PRs when: on_failure variables: PROJECT_ID: "${CI_PROJECT_ID}" PROJECT_NAME: "${CI_PROJECT_NAME}" PIPELINE_ID: "${CI_PIPELINE_ID}" FAILED_JOB_URL: "${FAILED_JOB_URL}" FAILED_JOB_NAME: "${FAILED_JOB_NAME}" PR_NUM: "${PR_NUM}" trigger: project: "parity/infrastructure/ci_cd/pipeline-stopper"