# .gitlab-ci.yml # # substrate # # pipelines can be triggered manually in the web # # Currently the file is divided into subfiles. Each stage has a different file which # can be found here: scripts/ci/gitlab/pipeline/.yml # # Instead of YAML anchors "extends" is used. # Useful links: # https://docs.gitlab.com/ee/ci/yaml/index.html#extends # https://docs.gitlab.com/ee/ci/yaml/yaml_optimization.html#reference-tags # # SAMPLE JOB TEMPLATE - This is not a complete example but is enough to build a # simple CI job. For full documentation, visit https://docs.gitlab.com/ee/ci/yaml/ # # my-example-job: # stage: test # One of the stages listed below this job (required) # image: paritytech/tools:latest # Any docker image (required) # allow_failure: true # Allow the pipeline to continue if this job fails (default: false) # needs: # - job: test-linux # Any jobs that are required to run before this job (optional) # variables: # MY_ENVIRONMENT_VARIABLE: "some useful value" # Environment variables passed to the job (optional) # script: # - echo "List of shell commands to run in your job" # - echo "You can also just specify a script here, like so:" # - ./scripts/ci/gitlab/my_amazing_script.sh stages: - check - test - build - publish - zombienet - deploy - notify workflow: rules: - if: $CI_COMMIT_TAG - if: $CI_COMMIT_BRANCH variables: GIT_STRATEGY: fetch GIT_DEPTH: 100 CARGO_INCREMENTAL: 0 DOCKER_OS: "debian:stretch" ARCH: "x86_64" CI_IMAGE: "paritytech/ci-linux:production" BUILDAH_IMAGE: "quay.io/buildah/stable:v1.27" RUSTY_CACHIER_SINGLE_BRANCH: master RUSTY_CACHIER_DONT_OPERATE_ON_MAIN_BRANCH: "true" RUSTY_CACHIER_COMPRESSION_METHOD: zstd NEXTEST_FAILURE_OUTPUT: immediate-final NEXTEST_SUCCESS_OUTPUT: final ZOMBIENET_IMAGE: "docker.io/paritytech/zombienet:v1.3.34" .shared-default: &shared-default retry: max: 2 when: - runner_system_failure - unknown_failure - api_failure cache: {} .default-pipeline-definitions: default: <<: *shared-default interruptible: true .crate-publishing-pipeline-definitions: default: <<: *shared-default # The crate-publishing pipeline defaults to `interruptible: false` so that we'll be able to # reach and run the publishing jobs despite the "Auto-cancel redundant pipelines" CI setting. # The setting is relevant because the crate-publishing pipeline runs on `master`, thus future # pipelines on `master` (e.g. created for new commits or other schedules) might unintendedly # cancel the publishing jobs or its dependencies before we get to actually publish the crates. interruptible: false .collect-artifacts: artifacts: name: "${CI_JOB_NAME}_${CI_COMMIT_REF_NAME}" when: on_success expire_in: 7 days paths: - artifacts/ .collect-artifacts-short: artifacts: name: "${CI_JOB_NAME}_${CI_COMMIT_REF_NAME}" when: on_success expire_in: 3 hours paths: - artifacts/ .prepare-env: before_script: # TODO: remove unset invocation when we'll be free from 'ENV RUSTC_WRAPPER=sccache' & sccache # itself in all images - unset RUSTC_WRAPPER # $WASM_BUILD_WORKSPACE_HINT enables wasm-builder to find the Cargo.lock from within generated # packages - export WASM_BUILD_WORKSPACE_HINT="$PWD" .job-switcher: before_script: - if echo "$CI_DISABLED_JOBS" | grep -xF "$CI_JOB_NAME"; then echo "The job has been cancelled in CI settings"; exit 0; fi .kubernetes-env: image: "${CI_IMAGE}" before_script: - !reference [.job-switcher, before_script] - !reference [.prepare-env, before_script] tags: - kubernetes-parity-build .rust-info-script: script: - rustup show - cargo --version - rustup +nightly show - cargo +nightly --version .pipeline-stopper-vars: script: - !reference [.job-switcher, before_script] - echo "Collecting env variables for the cancel-pipeline job" - echo "FAILED_JOB_URL=${CI_JOB_URL}" > pipeline-stopper.env - echo "FAILED_JOB_NAME=${CI_JOB_NAME}" >> pipeline-stopper.env - echo "PR_NUM=${CI_COMMIT_REF_NAME}" >> pipeline-stopper.env .pipeline-stopper-artifacts: artifacts: reports: dotenv: pipeline-stopper.env .docker-env: image: "${CI_IMAGE}" before_script: - !reference [.job-switcher, before_script] - !reference [.prepare-env, before_script] - !reference [.rust-info-script, script] - !reference [.rusty-cachier, before_script] - !reference [.pipeline-stopper-vars, script] after_script: - !reference [.rusty-cachier, after_script] tags: - linux-docker # rusty-cachier's hidden job. Parts of this job are used to instrument the pipeline's other real jobs with rusty-cachier # Description of the commands is available here - https://gitlab.parity.io/parity/infrastructure/ci_cd/rusty-cachier/client#description .rusty-cachier: before_script: - curl -s https://gitlab-ci-token:${CI_JOB_TOKEN}@gitlab.parity.io/parity/infrastructure/ci_cd/rusty-cachier/client/-/raw/release/util/install.sh | bash - rusty-cachier environment check --gracefully - $(rusty-cachier environment inject) - rusty-cachier project mtime after_script: - env RUSTY_CACHIER_SUPRESS_OUTPUT=true rusty-cachier snapshot destroy .test-refs: rules: - if: $CI_PIPELINE_SOURCE == "web" - if: $CI_PIPELINE_SOURCE == "schedule" - if: $CI_COMMIT_REF_NAME == "master" - if: $CI_COMMIT_REF_NAME =~ /^[0-9]+$/ # PRs - if: $CI_COMMIT_REF_NAME =~ /^v[0-9]+\.[0-9]+.*$/ # i.e. v1.0, v2.1rc1 # handle the specific case where benches could store incorrect bench data because of the downstream staging runs # exclude cargo-check-benches from such runs .test-refs-check-benches: rules: - if: $CI_COMMIT_REF_NAME == "master" && $CI_PIPELINE_SOURCE == "parent_pipeline" && $CI_IMAGE =~ /staging$/ when: never - if: $CI_PIPELINE_SOURCE == "web" - if: $CI_PIPELINE_SOURCE == "schedule" - if: $CI_COMMIT_REF_NAME == "master" - if: $CI_COMMIT_REF_NAME =~ /^[0-9]+$/ # PRs - if: $CI_COMMIT_REF_NAME =~ /^v[0-9]+\.[0-9]+.*$/ # i.e. v1.0, v2.1rc1 .test-refs-no-trigger: rules: - if: $CI_PIPELINE_SOURCE == "pipeline" when: never - if: $CI_PIPELINE_SOURCE == "web" - if: $CI_PIPELINE_SOURCE == "schedule" - if: $CI_COMMIT_REF_NAME == "master" - if: $CI_COMMIT_REF_NAME =~ /^[0-9]+$/ # PRs - if: $CI_COMMIT_REF_NAME =~ /^v[0-9]+\.[0-9]+.*$/ # i.e. v1.0, v2.1rc1 - if: $CI_COMMIT_REF_NAME =~ /^ci-release-.*$/ .test-refs-no-trigger-prs-only: rules: - if: $CI_PIPELINE_SOURCE == "pipeline" when: never - if: $CI_PIPELINE_SOURCE == "web" - if: $CI_PIPELINE_SOURCE == "schedule" - if: $CI_COMMIT_REF_NAME =~ /^[0-9]+$/ # PRs .publish-refs: rules: - if: $CI_PIPELINE_SOURCE == "pipeline" when: never - if: $CI_PIPELINE_SOURCE == "web" - if: $CI_PIPELINE_SOURCE == "schedule" - if: $CI_COMMIT_REF_NAME == "master" - if: $CI_COMMIT_REF_NAME =~ /^v[0-9]+\.[0-9]+.*$/ # i.e. v1.0, v2.1rc1 .build-refs: # publish-refs + PRs rules: - if: $CI_PIPELINE_SOURCE == "pipeline" when: never - if: $CI_PIPELINE_SOURCE == "web" - if: $CI_PIPELINE_SOURCE == "schedule" - if: $CI_COMMIT_REF_NAME == "master" - if: $CI_COMMIT_REF_NAME =~ /^v[0-9]+\.[0-9]+.*$/ # i.e. v1.0, v2.1rc1 - if: $CI_COMMIT_REF_NAME =~ /^[0-9]+$/ # PRs .zombienet-refs: extends: .build-refs .nightly-pipeline: rules: # this job runs only on nightly pipeline with the mentioned variable, against `master` branch - if: $CI_COMMIT_REF_NAME == "master" && $CI_PIPELINE_SOURCE == "schedule" && $PIPELINE == "nightly" .crates-publishing-variables: variables: CRATESIO_CRATES_OWNER: parity-crate-owner REPO: substrate REPO_OWNER: paritytech .crates-publishing-pipeline: extends: .crates-publishing-variables rules: - if: $CI_PIPELINE_SOURCE == "schedule" && $CI_COMMIT_REF_NAME == "master" && $PIPELINE == "automatic-crate-publishing" .crates-publishing-template: extends: - .docker-env - .crates-publishing-variables # collect artifacts even on failure so that we know how the crates were generated (they'll be # generated to the artifacts folder according to SPUB_TMP further down) artifacts: name: "${CI_JOB_NAME}_${CI_COMMIT_REF_NAME}" when: always expire_in: 7 days paths: - artifacts/ variables: SPUB_TMP: artifacts #### stage: .pre skip-if-draft: extends: .kubernetes-env variables: CI_IMAGE: "paritytech/tools:latest" stage: .pre rules: - if: $CI_COMMIT_REF_NAME =~ /^[0-9]+$/ # PRs script: - echo "Commit message is ${CI_COMMIT_MESSAGE}" - echo "Ref is ${CI_COMMIT_REF_NAME}" - echo "pipeline source is ${CI_PIPELINE_SOURCE}" - ./scripts/ci/gitlab/skip_if_draft.sh allow_failure: true check-crates-publishing-pipeline: stage: .pre extends: - .kubernetes-env - .crates-publishing-pipeline script: - git clone --depth 1 --branch "$RELENG_SCRIPTS_BRANCH" https://github.com/paritytech/releng-scripts.git - ONLY_CHECK_PIPELINE=true ./releng-scripts/publish-crates include: # check jobs - scripts/ci/gitlab/pipeline/check.yml # tests jobs - scripts/ci/gitlab/pipeline/test.yml # build jobs - scripts/ci/gitlab/pipeline/build.yml # publish jobs - scripts/ci/gitlab/pipeline/publish.yml # zombienet jobs - scripts/ci/gitlab/pipeline/zombienet.yml # The crate-publishing pipeline requires a customized `interruptible` configuration. Unfortunately # `interruptible` can't currently be dynamically set based on variables as per: # - https://gitlab.com/gitlab-org/gitlab/-/issues/38349 # - https://gitlab.com/gitlab-org/gitlab/-/issues/194023 # Thus we work around that limitation by using conditional includes. # For crate-publishing pipelines: run it with defaults + `interruptible: false`. The WHOLE # pipeline is made uninterruptible to ensure that test jobs also get a chance to run to # completion, because the publishing jobs depends on them AS INTENDED: crates should not be # published before their source code is checked. - local: scripts/ci/gitlab/crate-publishing-pipeline.yml rules: - if: $PIPELINE == "automatic-crate-publishing" # For normal pipelines: run it with defaults + `interruptible: true` - local: scripts/ci/gitlab/default-pipeline.yml rules: - if: $PIPELINE != "automatic-crate-publishing" #### stage: deploy deploy-prometheus-alerting-rules: stage: deploy needs: - job: test-prometheus-alerting-rules artifacts: false allow_failure: true trigger: project: parity/infrastructure/cloud-infra variables: SUBSTRATE_CI_COMMIT_NAME: "${CI_COMMIT_REF_NAME}" SUBSTRATE_CI_COMMIT_REF: "${CI_COMMIT_SHORT_SHA}" UPSTREAM_TRIGGER_PROJECT: "${CI_PROJECT_PATH}" rules: - if: $CI_PIPELINE_SOURCE == "pipeline" when: never - if: $CI_COMMIT_REF_NAME == "master" changes: - .gitlab-ci.yml - ./scripts/ci/monitoring/**/* #### stage: notify # This job notifies rusty-cachier about the latest commit with the cache. # This info is later used for the cache distribution and an overlay creation. # Note that we don't use any .rusty-cachier references as we assume that a pipeline has reached this stage with working rusty-cachier. rusty-cachier-notify: stage: notify extends: .kubernetes-env variables: CI_IMAGE: paritytech/rusty-cachier-env:latest GIT_STRATEGY: none dependencies: [] script: - curl -s https://gitlab-ci-token:${CI_JOB_TOKEN}@gitlab.parity.io/parity/infrastructure/ci_cd/rusty-cachier/client/-/raw/release/util/install.sh | bash - rusty-cachier cache notify #### stage: .post # This job cancels the whole pipeline if any of provided jobs fail. # In a DAG, every jobs chain is executed independently of others. The `fail_fast` principle suggests # to fail the pipeline as soon as possible to shorten the feedback loop. .cancel-pipeline-template: stage: .post rules: - if: $CI_COMMIT_REF_NAME =~ /^[0-9]+$/ # PRs when: on_failure variables: PROJECT_ID: "${CI_PROJECT_ID}" PROJECT_NAME: "${CI_PROJECT_NAME}" PIPELINE_ID: "${CI_PIPELINE_ID}" FAILED_JOB_URL: "${FAILED_JOB_URL}" FAILED_JOB_NAME: "${FAILED_JOB_NAME}" PR_NUM: "${PR_NUM}" trigger: project: "parity/infrastructure/ci_cd/pipeline-stopper" remove-cancel-pipeline-message: stage: .post rules: - if: $CI_COMMIT_REF_NAME =~ /^[0-9]+$/ # PRs variables: PROJECT_ID: "${CI_PROJECT_ID}" PROJECT_NAME: "${CI_PROJECT_NAME}" PIPELINE_ID: "${CI_PIPELINE_ID}" FAILED_JOB_URL: "https://gitlab.com" FAILED_JOB_NAME: "nope" PR_NUM: "${CI_COMMIT_REF_NAME}" trigger: project: "parity/infrastructure/ci_cd/pipeline-stopper" branch: "as-improve" # need to copy jobs this way because otherwise gitlab will wait # for all 3 jobs to finish instead of cancelling if one fails cancel-pipeline-test-linux-stable1: extends: .cancel-pipeline-template needs: - job: "test-linux-stable 1/3" cancel-pipeline-test-linux-stable2: extends: .cancel-pipeline-template needs: - job: "test-linux-stable 2/3" cancel-pipeline-test-linux-stable3: extends: .cancel-pipeline-template needs: - job: "test-linux-stable 3/3" cancel-pipeline-cargo-check-benches1: extends: .cancel-pipeline-template needs: - job: "cargo-check-benches 1/2" cancel-pipeline-cargo-check-benches2: extends: .cancel-pipeline-template needs: - job: "cargo-check-benches 2/2" cancel-pipeline-test-linux-stable-int: extends: .cancel-pipeline-template needs: - job: test-linux-stable-int cancel-pipeline-cargo-check-each-crate-1: extends: .cancel-pipeline-template needs: - job: "cargo-check-each-crate 1/2" cancel-pipeline-cargo-check-each-crate-2: extends: .cancel-pipeline-template needs: - job: "cargo-check-each-crate 2/2" cancel-pipeline-cargo-check-each-crate-macos: extends: .cancel-pipeline-template needs: - job: cargo-check-each-crate-macos cancel-pipeline-check-tracing: extends: .cancel-pipeline-template needs: - job: check-tracing