[ci] Add missing components to CI (#1201)

* [ci] Add missing components to CI

* add timestamp

* add pipeline-stopper-artifacts to jobs

* break clippy

* mv definition

* fix clippy

* comment timestamp, rm verbose

* commen cancel

* rm --verbose from build jobs

* disable rusty-cachier before_script

* enable timestamp back

* rollback .gitignore

* rollback .gitignore

* rollback .gitignore

.gitlab-ci.yml

@@ -1,32 +1,9 @@
-# .gitlab-ci.yml
+# polkadot-sdk | CI definitions (via GitLab CI)
 #
-# substrate
+# FYI: Pipelines can be triggered manually through the web UI (if you have enough permissions)
 #
-# pipelines can be triggered manually in the web
-#
-# Currently the file is divided into subfiles. Each stage has a different file which
-# can be found here: gitlab/pipeline/<stage_name>.yml
-#
-# Instead of YAML anchors "extends" is used.
-# Useful links:
-#    https://docs.gitlab.com/ee/ci/yaml/index.html#extends
-#    https://docs.gitlab.com/ee/ci/yaml/yaml_optimization.html#reference-tags
-#
-# SAMPLE JOB TEMPLATE - This is not a complete example but is enough to build a
-# simple CI job. For full documentation, visit https://docs.gitlab.com/ee/ci/yaml/
-#
-# my-example-job:
-#   stage:                           test # One of the stages listed below this job (required)
-#   image:                           paritytech/tools:latest # Any docker image (required)
-#   allow_failure:                   true # Allow the pipeline to continue if this job fails (default: false)
-#   needs:
-#     - job:                         test-linux # Any jobs that are required to run before this job (optional)
-#   variables:
-#     MY_ENVIRONMENT_VARIABLE:       "some useful value" # Environment variables passed to the job (optional)
-#   script:
-#     - echo "List of shell commands to run in your job"
-#     - echo "You can also just specify a script here, like so:"
-#     - ./gitlab/my_amazing_script.sh
+# Currently, entire CI instructions are split into different subfiles. Each CI stage has a corresponding
+# file which can be found here: .gitlab/pipeline/<stage_name>.yml
 
 stages:
   - check
@@ -84,15 +61,18 @@ default:
 
 .prepare-env:
   before_script:
-    # TODO: remove unset invocation when we'll be free from 'ENV RUSTC_WRAPPER=sccache' & sccache
-    # itself in all images
-    - unset RUSTC_WRAPPER
     # $WASM_BUILD_WORKSPACE_HINT enables wasm-builder to find the Cargo.lock from within generated
     # packages
     - export WASM_BUILD_WORKSPACE_HINT="$PWD"
     # ensure that RUSTFLAGS are set correctly
     - echo $RUSTFLAGS
 
+.common-before-script:
+  before_script:
+    - !reference [.job-switcher, before_script]
+    - !reference [.timestamp, before_script]
+    - !reference [.pipeline-stopper-vars, script]
+
 .job-switcher:
   before_script:
     - if echo "$CI_DISABLED_JOBS" | grep -xF "$CI_JOB_NAME"; then echo "The job has been cancelled in CI settings"; exit 0; fi
@@ -100,7 +80,7 @@ default:
 .kubernetes-env:
   image: "${CI_IMAGE}"
   before_script:
-    # - !reference [.job-switcher, before_script]
+    - !reference [.common-before-script, before_script]
     - !reference [.prepare-env, before_script]
   tags:
     - kubernetes-parity-build
@@ -112,9 +92,10 @@ default:
     - rustup +nightly show
     - cargo +nightly --version
 
+# collecting vars for pipeline stopper
+# they will be used if the job fails
 .pipeline-stopper-vars:
   script:
-    - !reference [.job-switcher, before_script]
     - echo "Collecting env variables for the cancel-pipeline job"
     - echo "FAILED_JOB_URL=${CI_JOB_URL}" > pipeline-stopper.env
     - echo "FAILED_JOB_NAME=${CI_JOB_NAME}" >> pipeline-stopper.env
@@ -128,28 +109,26 @@ default:
 .docker-env:
   image: "${CI_IMAGE}"
   before_script:
-    - !reference [.job-switcher, before_script]
+    - !reference [.common-before-script, before_script]
     - !reference [.prepare-env, before_script]
     - !reference [.rust-info-script, script]
     - !reference [.rusty-cachier, before_script]
-    - !reference [.pipeline-stopper-vars, script]
-  after_script:
-    - !reference [.rusty-cachier, after_script]
   tags:
     - linux-docker-vm-c2
 
 # rusty-cachier's hidden job. Parts of this job are used to instrument the pipeline's other real jobs with rusty-cachier
-# Description of the commands is available here - https://gitlab.parity.io/parity/infrastructure/ci_cd/rusty-cachier/client#description
+# rusty-cachier's commands are described here: https://gitlab.parity.io/parity/infrastructure/ci_cd/rusty-cachier/client#description
 .rusty-cachier:
   before_script:
     # - curl -s https://gitlab-ci-token:${CI_JOB_TOKEN}@gitlab.parity.io/parity/infrastructure/ci_cd/rusty-cachier/client/-/raw/release/util/install.sh | bash
-    # - rusty-cachier environment check --gracefully
-    # - $(rusty-cachier environment inject)
-    # - rusty-cachier project mtime
+    # - mkdir -p cargo_home cargo_target_dir
+    # - export CARGO_HOME=$CI_PROJECT_DIR/cargo_home
+    # - export CARGO_TARGET_DIR=$CI_PROJECT_DIR/cargo_target_dir
+    # - find . \( -path ./cargo_target_dir -o -path ./cargo_home \) -prune -o -type f -exec touch -t 202005260100 {} +
+    # - git restore-mtime
+    # - rusty-cachier --version
+    # - rusty-cachier project touch-changed
     - echo tbd
-  after_script:
-    - echo tbd
-    # - env RUSTY_CACHIER_SUPRESS_OUTPUT=true rusty-cachier snapshot destroy
 
 .common-refs:
   rules:
@@ -233,6 +212,125 @@ include:
   # zombienet jobs
   - .gitlab/pipeline/zombienet.yml
   # # timestamp handler
-  # - project: parity/infrastructure/ci_cd/shared
-  #   ref: v0.2
-  #   file: /common/timestamp.yml
+  - project: parity/infrastructure/ci_cd/shared
+    ref: v0.2
+    file: /common/timestamp.yml
+
+# This job cancels the whole pipeline if any of provided jobs fail.
+# In a DAG, every jobs chain is executed independently of others. The `fail_fast` principle suggests
+# to fail the pipeline as soon as possible to shorten the feedback loop.
+.cancel-pipeline-template:
+  stage: .post
+  rules:
+    - if: $CI_COMMIT_REF_NAME =~ /^[0-9]+$/ # PRs
+      when: on_failure
+  variables:
+    PROJECT_ID: "${CI_PROJECT_ID}"
+    PROJECT_NAME: "${CI_PROJECT_NAME}"
+    PIPELINE_ID: "${CI_PIPELINE_ID}"
+    FAILED_JOB_URL: "${FAILED_JOB_URL}"
+    FAILED_JOB_NAME: "${FAILED_JOB_NAME}"
+    PR_NUM: "${PR_NUM}"
+  trigger:
+    project: "parity/infrastructure/ci_cd/pipeline-stopper"
+    branch: "as-improve"
+
+remove-cancel-pipeline-message:
+  stage: .post
+  rules:
+    - if: $CI_COMMIT_REF_NAME =~ /^[0-9]+$/ # PRs
+  variables:
+    PROJECT_ID: "${CI_PROJECT_ID}"
+    PROJECT_NAME: "${CI_PROJECT_NAME}"
+    PIPELINE_ID: "${CI_PIPELINE_ID}"
+    FAILED_JOB_URL: "https://gitlab.com"
+    FAILED_JOB_NAME: "nope"
+    PR_NUM: "${CI_COMMIT_REF_NAME}"
+  trigger:
+    project: "parity/infrastructure/ci_cd/pipeline-stopper"
+# need to copy jobs this way because otherwise gitlab will wait
+# for all 3 jobs to finish instead of cancelling if one fails
+cancel-pipeline-test-linux-stable1:
+  extends: .cancel-pipeline-template
+  needs:
+    - job: "test-linux-stable 1/3"
+
+cancel-pipeline-test-linux-stable2:
+  extends: .cancel-pipeline-template
+  needs:
+    - job: "test-linux-stable 2/3"
+
+cancel-pipeline-test-linux-stable3:
+  extends: .cancel-pipeline-template
+  needs:
+    - job: "test-linux-stable 3/3"
+
+cancel-pipeline-test-linux-stable-additional-tests:
+  extends: .cancel-pipeline-template
+  needs:
+    - job: "test-linux-stable-additional-tests"
+
+cancel-pipeline-test-linux-stable-slow:
+  extends: .cancel-pipeline-template
+  needs:
+    - job: "test-linux-stable-slow"
+
+cancel-pipeline-cargo-check-benches1:
+  extends: .cancel-pipeline-template
+  needs:
+    - job: "cargo-check-benches 1/2"
+
+cancel-pipeline-cargo-check-benches2:
+  extends: .cancel-pipeline-template
+  needs:
+    - job: "cargo-check-benches 2/2"
+
+cancel-pipeline-test-linux-stable-int:
+  extends: .cancel-pipeline-template
+  needs:
+    - job: test-linux-stable-int
+
+cancel-pipeline-cargo-check-each-crate-1:
+  extends: .cancel-pipeline-template
+  needs:
+    - job: "cargo-check-each-crate 1/6"
+
+cancel-pipeline-cargo-check-each-crate-2:
+  extends: .cancel-pipeline-template
+  needs:
+    - job: "cargo-check-each-crate 2/6"
+
+cancel-pipeline-cargo-check-each-crate-3:
+  extends: .cancel-pipeline-template
+  needs:
+    - job: "cargo-check-each-crate 3/6"
+
+cancel-pipeline-cargo-check-each-crate-4:
+  extends: .cancel-pipeline-template
+  needs:
+    - job: "cargo-check-each-crate 4/6"
+
+cancel-pipeline-cargo-check-each-crate-5:
+  extends: .cancel-pipeline-template
+  needs:
+    - job: "cargo-check-each-crate 5/6"
+
+cancel-pipeline-cargo-check-each-crate-6:
+  extends: .cancel-pipeline-template
+  needs:
+    - job: "cargo-check-each-crate 6/6"
+
+cancel-pipeline-cargo-check-each-crate-macos:
+  extends: .cancel-pipeline-template
+  needs:
+    - job: cargo-check-each-crate-macos
+
+cancel-pipeline-check-tracing:
+  extends: .cancel-pipeline-template
+  needs:
+    - job: check-tracing
+
+cancel-pipeline-cargo-clippy:
+  extends: .cancel-pipeline-template
+  needs:
+    - job: cargo-clippy

.gitlab/pipeline/build.yml

@@ -46,8 +46,8 @@ build-test-collators:
     - .run-immediately
     - .collect-artifacts
   script:
-    - time cargo build --locked --profile testnet --verbose -p test-parachain-adder-collator
-    - time cargo build --locked --profile testnet --verbose -p test-parachain-undying-collator
+    - time cargo build --locked --profile testnet -p test-parachain-adder-collator
+    - time cargo build --locked --profile testnet -p test-parachain-undying-collator
     # pack artifacts
     - mkdir -p ./artifacts
     - mv ./target/testnet/adder-collator ./artifacts/.
@@ -66,7 +66,7 @@ build-malus:
     - .run-immediately
     - .collect-artifacts
   script:
-    - time cargo build --locked --profile testnet --verbose -p polkadot-test-malus --bin malus --bin polkadot-prepare-worker --bin polkadot-execute-worker
+    - time cargo build --locked --profile testnet -p polkadot-test-malus --bin malus --bin polkadot-prepare-worker --bin polkadot-execute-worker
     # pack artifacts
     - mkdir -p ./artifacts
     - mv ./target/testnet/malus ./artifacts/.
@@ -111,7 +111,7 @@ build-rustdoc:
   script:
     # FIXME: it fails with `RUSTDOCFLAGS="-Dwarnings"` and `--all-features`
     # FIXME: return to stable when https://github.com/rust-lang/rust/issues/96937 gets into stable
-    - time cargo doc --workspace --verbose --no-deps
+    - time cargo doc --workspace --no-deps
     - rm -f ./target/doc/.lock
     - mv ./target/doc ./crate-docs
     # FIXME: remove me after CI image gets nonroot
@@ -280,7 +280,7 @@ build-linux-substrate:
     # see https://github.com/paritytech/ci_cd/issues/682#issuecomment-1340953589
     - git checkout -B "$CI_COMMIT_REF_NAME" "$CI_COMMIT_SHA"
   script:
-    - WASM_BUILD_NO_COLOR=1 time cargo build --locked --release --verbose
+    - WASM_BUILD_NO_COLOR=1 time cargo build --locked --release
     - mv $CARGO_TARGET_DIR/release/substrate-node ./artifacts/substrate/substrate
     - echo -n "Substrate version = "
     - if [ "${CI_COMMIT_TAG}" ]; then
@@ -308,7 +308,7 @@ build-linux-substrate:
     - mkdir -p ./artifacts/subkey
   script:
     - cd ./substrate/bin/utils/subkey
-    - SKIP_WASM_BUILD=1 time cargo build --locked --release --verbose
+    - SKIP_WASM_BUILD=1 time cargo build --locked --release
     # - cd -
     # - mv $CARGO_TARGET_DIR/release/subkey ./artifacts/subkey/.
     # - echo -n "Subkey version = "
@@ -331,7 +331,7 @@ build-subkey-linux:
 #     - mkdir -p ./artifacts/subkey
 #   script:
 #     - cd ./bin/utils/subkey
-#     - SKIP_WASM_BUILD=1 time cargo build --locked --release --verbose
+#     - SKIP_WASM_BUILD=1 time cargo build --locked --release
 #     - cd -
 #     - mv ./target/release/subkey ./artifacts/subkey/.
 #     - echo -n "Subkey version = "

.gitlab/pipeline/check.yml

@@ -3,10 +3,9 @@ cargo-clippy:
   extends:
     - .docker-env
     - .common-refs
+    - .pipeline-stopper-artifacts
   script:
     - SKIP_WASM_BUILD=1 env -u RUSTFLAGS cargo clippy --all-targets --locked --workspace
-  # fixme!
-  allow_failure: true
 
 check-try-runtime:
   stage: check
@@ -43,7 +42,6 @@ cargo-deny-licenses:
   script:
     - $CARGO_DENY_CMD --hide-inclusion-graph
   after_script:
-    # - !reference [.rusty-cachier, after_script]
     - echo "___The complete log is in the artifacts___"
     - $CARGO_DENY_CMD 2> deny.log
     - if [ $CI_JOB_STATUS != 'success' ]; then

.gitlab/pipeline/test.yml

@@ -11,6 +11,7 @@ test-linux-stable:
     - .docker-env
     - .common-refs
     - .run-immediately
+    - .pipeline-stopper-artifacts
   variables:
     RUST_TOOLCHAIN: stable
     # Enable debug assertions since we are running optimized builds for testing
@@ -32,7 +33,6 @@ test-linux-stable:
         --workspace \
         --locked \
         --release \
-        --verbose \
         --no-fail-fast \
         --features runtime-benchmarks,try-runtime,experimental \
         --partition count:${CI_NODE_INDEX}/${CI_NODE_TOTAL}
@@ -66,7 +66,6 @@ test-linux-oldkernel-stable:
 #         --workspace \
 #         --locked \
 #         --release \
-#         --verbose \
 #         --no-fail-fast \
 #         --features runtime-benchmarks,try-runtime \
 #         --partition count:${CI_NODE_INDEX}/${CI_NODE_TOTAL}
@@ -79,6 +78,7 @@ test-linux-stable-additional-tests:
     - .docker-env
     - .common-refs
     - .run-immediately
+    - .pipeline-stopper-artifacts
   variables:
     RUST_TOOLCHAIN: stable
     # Enable debug assertions since we are running optimized builds for testing
@@ -91,7 +91,6 @@ test-linux-stable-additional-tests:
         --workspace \
         --locked \
         --release \
-        --verbose \
         --features runtime-benchmarks,try-runtime
   allow_failure: true
 
@@ -104,6 +103,7 @@ test-linux-stable-slow:
     - .docker-env
     - .common-refs
     - .run-immediately
+    - .pipeline-stopper-artifacts
   variables:
     RUST_TOOLCHAIN: stable
     # Enable debug assertions since we are running optimized builds for testing
@@ -116,7 +116,6 @@ test-linux-stable-slow:
         --workspace \
         --locked \
         --release \
-        --verbose \
         --features runtime-benchmarks,try-runtime
   allow_failure: true
 
@@ -166,7 +165,7 @@ test-rustdoc:
     SKIP_WASM_BUILD: 1
     RUSTDOCFLAGS: "-Dwarnings"
   script:
-    - time cargo doc --workspace --all-features --verbose --no-deps
+    - time cargo doc --workspace --all-features --no-deps
   allow_failure: true
 
 cargo-check-all-benches:
@@ -216,6 +215,7 @@ cargo-check-benches:
     - .common-refs
     - .run-immediately
     - .collect-artifacts
+    - .pipeline-stopper-artifacts
   before_script:
     # TODO: DON'T FORGET TO CHANGE FOR PROD VALUES!!!
     # merges in the master branch on PRs. skip if base is not master
@@ -348,6 +348,7 @@ test-linux-stable-int:
     - .docker-env
     - .common-refs
     - .run-immediately
+    - .pipeline-stopper-artifacts
   variables:
     # Enable debug assertions since we are running optimized builds for testing
     # but still want to have debug assertions.
@@ -370,6 +371,7 @@ check-tracing:
     - .docker-env
     - .common-refs
     - .run-immediately
+    - .pipeline-stopper-artifacts
   script:
     # with-tracing must be explicitly activated, we run a test to ensure this works as expected in both cases
     - time cargo test --locked --manifest-path ./substrate/primitives/tracing/Cargo.toml --no-default-features
@@ -390,9 +392,9 @@ test-full-crypto-feature:
     RUST_BACKTRACE: 1
   script:
     - cd substrate/primitives/core/
-    - time cargo build --locked --verbose --no-default-features --features full_crypto
+    - time cargo build --locked --no-default-features --features full_crypto
     - cd ../application-crypto
-    - time cargo build --locked --verbose --no-default-features --features full_crypto
+    - time cargo build --locked --no-default-features --features full_crypto
 
 cargo-check-each-crate:
   stage: test