[ci] Add missing components to CI (#1201)

* [ci] Add missing components to CI

* add timestamp

* add pipeline-stopper-artifacts to jobs

* break clippy

* mv definition

* fix clippy

* comment timestamp, rm verbose

* commen cancel

* rm --verbose from build jobs

* disable rusty-cachier before_script

* enable timestamp back

* rollback .gitignore

* rollback .gitignore

* rollback .gitignore
This commit is contained in:
Alexander Samusev
2023-08-28 20:33:16 +02:00
committed by GitHub
parent c168a77e26
commit 6e394f84f8
4 changed files with 161 additions and 63 deletions
+144 -46
View File
@@ -1,32 +1,9 @@
# .gitlab-ci.yml
# polkadot-sdk | CI definitions (via GitLab CI)
#
# substrate
# FYI: Pipelines can be triggered manually through the web UI (if you have enough permissions)
#
# pipelines can be triggered manually in the web
#
# Currently the file is divided into subfiles. Each stage has a different file which
# can be found here: gitlab/pipeline/<stage_name>.yml
#
# Instead of YAML anchors "extends" is used.
# Useful links:
# https://docs.gitlab.com/ee/ci/yaml/index.html#extends
# https://docs.gitlab.com/ee/ci/yaml/yaml_optimization.html#reference-tags
#
# SAMPLE JOB TEMPLATE - This is not a complete example but is enough to build a
# simple CI job. For full documentation, visit https://docs.gitlab.com/ee/ci/yaml/
#
# my-example-job:
# stage: test # One of the stages listed below this job (required)
# image: paritytech/tools:latest # Any docker image (required)
# allow_failure: true # Allow the pipeline to continue if this job fails (default: false)
# needs:
# - job: test-linux # Any jobs that are required to run before this job (optional)
# variables:
# MY_ENVIRONMENT_VARIABLE: "some useful value" # Environment variables passed to the job (optional)
# script:
# - echo "List of shell commands to run in your job"
# - echo "You can also just specify a script here, like so:"
# - ./gitlab/my_amazing_script.sh
# Currently, entire CI instructions are split into different subfiles. Each CI stage has a corresponding
# file which can be found here: .gitlab/pipeline/<stage_name>.yml
stages:
- check
@@ -84,15 +61,18 @@ default:
.prepare-env:
before_script:
# TODO: remove unset invocation when we'll be free from 'ENV RUSTC_WRAPPER=sccache' & sccache
# itself in all images
- unset RUSTC_WRAPPER
# $WASM_BUILD_WORKSPACE_HINT enables wasm-builder to find the Cargo.lock from within generated
# packages
- export WASM_BUILD_WORKSPACE_HINT="$PWD"
# ensure that RUSTFLAGS are set correctly
- echo $RUSTFLAGS
.common-before-script:
before_script:
- !reference [.job-switcher, before_script]
- !reference [.timestamp, before_script]
- !reference [.pipeline-stopper-vars, script]
.job-switcher:
before_script:
- if echo "$CI_DISABLED_JOBS" | grep -xF "$CI_JOB_NAME"; then echo "The job has been cancelled in CI settings"; exit 0; fi
@@ -100,7 +80,7 @@ default:
.kubernetes-env:
image: "${CI_IMAGE}"
before_script:
# - !reference [.job-switcher, before_script]
- !reference [.common-before-script, before_script]
- !reference [.prepare-env, before_script]
tags:
- kubernetes-parity-build
@@ -112,9 +92,10 @@ default:
- rustup +nightly show
- cargo +nightly --version
# collecting vars for pipeline stopper
# they will be used if the job fails
.pipeline-stopper-vars:
script:
- !reference [.job-switcher, before_script]
- echo "Collecting env variables for the cancel-pipeline job"
- echo "FAILED_JOB_URL=${CI_JOB_URL}" > pipeline-stopper.env
- echo "FAILED_JOB_NAME=${CI_JOB_NAME}" >> pipeline-stopper.env
@@ -128,28 +109,26 @@ default:
.docker-env:
image: "${CI_IMAGE}"
before_script:
- !reference [.job-switcher, before_script]
- !reference [.common-before-script, before_script]
- !reference [.prepare-env, before_script]
- !reference [.rust-info-script, script]
- !reference [.rusty-cachier, before_script]
- !reference [.pipeline-stopper-vars, script]
after_script:
- !reference [.rusty-cachier, after_script]
tags:
- linux-docker-vm-c2
# rusty-cachier's hidden job. Parts of this job are used to instrument the pipeline's other real jobs with rusty-cachier
# Description of the commands is available here - https://gitlab.parity.io/parity/infrastructure/ci_cd/rusty-cachier/client#description
# rusty-cachier's commands are described here: https://gitlab.parity.io/parity/infrastructure/ci_cd/rusty-cachier/client#description
.rusty-cachier:
before_script:
# - curl -s https://gitlab-ci-token:${CI_JOB_TOKEN}@gitlab.parity.io/parity/infrastructure/ci_cd/rusty-cachier/client/-/raw/release/util/install.sh | bash
# - rusty-cachier environment check --gracefully
# - $(rusty-cachier environment inject)
# - rusty-cachier project mtime
# - mkdir -p cargo_home cargo_target_dir
# - export CARGO_HOME=$CI_PROJECT_DIR/cargo_home
# - export CARGO_TARGET_DIR=$CI_PROJECT_DIR/cargo_target_dir
# - find . \( -path ./cargo_target_dir -o -path ./cargo_home \) -prune -o -type f -exec touch -t 202005260100 {} +
# - git restore-mtime
# - rusty-cachier --version
# - rusty-cachier project touch-changed
- echo tbd
after_script:
- echo tbd
# - env RUSTY_CACHIER_SUPRESS_OUTPUT=true rusty-cachier snapshot destroy
.common-refs:
rules:
@@ -233,6 +212,125 @@ include:
# zombienet jobs
- .gitlab/pipeline/zombienet.yml
# # timestamp handler
# - project: parity/infrastructure/ci_cd/shared
# ref: v0.2
# file: /common/timestamp.yml
- project: parity/infrastructure/ci_cd/shared
ref: v0.2
file: /common/timestamp.yml
# This job cancels the whole pipeline if any of provided jobs fail.
# In a DAG, every jobs chain is executed independently of others. The `fail_fast` principle suggests
# to fail the pipeline as soon as possible to shorten the feedback loop.
.cancel-pipeline-template:
stage: .post
rules:
- if: $CI_COMMIT_REF_NAME =~ /^[0-9]+$/ # PRs
when: on_failure
variables:
PROJECT_ID: "${CI_PROJECT_ID}"
PROJECT_NAME: "${CI_PROJECT_NAME}"
PIPELINE_ID: "${CI_PIPELINE_ID}"
FAILED_JOB_URL: "${FAILED_JOB_URL}"
FAILED_JOB_NAME: "${FAILED_JOB_NAME}"
PR_NUM: "${PR_NUM}"
trigger:
project: "parity/infrastructure/ci_cd/pipeline-stopper"
branch: "as-improve"
remove-cancel-pipeline-message:
stage: .post
rules:
- if: $CI_COMMIT_REF_NAME =~ /^[0-9]+$/ # PRs
variables:
PROJECT_ID: "${CI_PROJECT_ID}"
PROJECT_NAME: "${CI_PROJECT_NAME}"
PIPELINE_ID: "${CI_PIPELINE_ID}"
FAILED_JOB_URL: "https://gitlab.com"
FAILED_JOB_NAME: "nope"
PR_NUM: "${CI_COMMIT_REF_NAME}"
trigger:
project: "parity/infrastructure/ci_cd/pipeline-stopper"
# need to copy jobs this way because otherwise gitlab will wait
# for all 3 jobs to finish instead of cancelling if one fails
cancel-pipeline-test-linux-stable1:
extends: .cancel-pipeline-template
needs:
- job: "test-linux-stable 1/3"
cancel-pipeline-test-linux-stable2:
extends: .cancel-pipeline-template
needs:
- job: "test-linux-stable 2/3"
cancel-pipeline-test-linux-stable3:
extends: .cancel-pipeline-template
needs:
- job: "test-linux-stable 3/3"
cancel-pipeline-test-linux-stable-additional-tests:
extends: .cancel-pipeline-template
needs:
- job: "test-linux-stable-additional-tests"
cancel-pipeline-test-linux-stable-slow:
extends: .cancel-pipeline-template
needs:
- job: "test-linux-stable-slow"
cancel-pipeline-cargo-check-benches1:
extends: .cancel-pipeline-template
needs:
- job: "cargo-check-benches 1/2"
cancel-pipeline-cargo-check-benches2:
extends: .cancel-pipeline-template
needs:
- job: "cargo-check-benches 2/2"
cancel-pipeline-test-linux-stable-int:
extends: .cancel-pipeline-template
needs:
- job: test-linux-stable-int
cancel-pipeline-cargo-check-each-crate-1:
extends: .cancel-pipeline-template
needs:
- job: "cargo-check-each-crate 1/6"
cancel-pipeline-cargo-check-each-crate-2:
extends: .cancel-pipeline-template
needs:
- job: "cargo-check-each-crate 2/6"
cancel-pipeline-cargo-check-each-crate-3:
extends: .cancel-pipeline-template
needs:
- job: "cargo-check-each-crate 3/6"
cancel-pipeline-cargo-check-each-crate-4:
extends: .cancel-pipeline-template
needs:
- job: "cargo-check-each-crate 4/6"
cancel-pipeline-cargo-check-each-crate-5:
extends: .cancel-pipeline-template
needs:
- job: "cargo-check-each-crate 5/6"
cancel-pipeline-cargo-check-each-crate-6:
extends: .cancel-pipeline-template
needs:
- job: "cargo-check-each-crate 6/6"
cancel-pipeline-cargo-check-each-crate-macos:
extends: .cancel-pipeline-template
needs:
- job: cargo-check-each-crate-macos
cancel-pipeline-check-tracing:
extends: .cancel-pipeline-template
needs:
- job: check-tracing
cancel-pipeline-cargo-clippy:
extends: .cancel-pipeline-template
needs:
- job: cargo-clippy