[ci] Divide .gitlab-ci.yml into several files (#11333)

* rebase

* update Cargo.lock

* revert Cargo.lock

* fix Cargo.lock from 11384

substrate/scripts/ci/gitlab/pipeline/test.yml

@@ -0,0 +1,271 @@
+# This file is part of .gitlab-ci.yml
+# Here are all jobs that are executed during "test" stage
+
+cargo-deny:
+  stage:                           test
+  extends:
+    - .docker-env
+    - .nightly-pipeline
+  script:
+    - cargo deny check --hide-inclusion-graph -c ./scripts/ci/deny.toml
+  after_script:
+    - echo "___The complete log is in the artifacts___"
+    - cargo deny check -c ./scripts/ci/deny.toml 2> deny.log
+  artifacts:
+    name:                          $CI_COMMIT_SHORT_SHA
+    expire_in:                     3 days
+    when:                          always
+    paths:
+      - deny.log
+  # FIXME: Temporarily allow to fail.
+  allow_failure:                   true
+
+cargo-fmt:
+  stage:                           test
+  extends:
+    - .docker-env
+    - .test-refs
+  script:
+    - cargo +nightly fmt --all -- --check
+
+cargo-clippy:
+  stage:                           test
+  extends:
+    - .docker-env
+    - .test-refs
+  script:
+    - SKIP_WASM_BUILD=1 env -u RUSTFLAGS cargo +nightly clippy --all-targets
+
+cargo-check-nixos:
+  stage:                           test
+  extends:
+    - .docker-env
+    - .test-refs
+  before_script:                   []
+  # Don't use CI_IMAGE here because it breaks nightly checks of paritytech/ci-linux image
+  image:                           nixos/nix
+  variables:
+    SNAP:                          "DUMMY"
+    WS_API:                        "DUMMY"
+  script:
+    - nix-channel --update
+    - nix-shell shell.nix
+    - nix-shell --run "cargo check --workspace --all-targets --all-features"
+
+cargo-check-benches:
+  stage:                           test
+  extends:
+    - .docker-env
+    - .test-refs
+    - .collect-artifacts
+  before_script:
+    # merges in the master branch on PRs
+    - if [ $CI_COMMIT_REF_NAME != "master" ]; then
+        git fetch origin +master:master;
+        git fetch origin +$CI_COMMIT_REF_NAME:$CI_COMMIT_REF_NAME;
+        git checkout master;
+        git config user.email "ci@gitlab.parity.io";
+        git merge $CI_COMMIT_REF_NAME --verbose --no-edit;
+      fi
+    - !reference [.rust-info-script, script]
+  script:
+    - mkdir -p ./artifacts/benches/$CI_COMMIT_REF_NAME-$CI_COMMIT_SHORT_SHA
+    - SKIP_WASM_BUILD=1 time cargo +nightly check --benches --all
+    - 'cargo run --release -p node-bench -- ::node::import::native::sr25519::transfer_keep_alive::paritydb::small --json
+      | tee ./artifacts/benches/$CI_COMMIT_REF_NAME-$CI_COMMIT_SHORT_SHA/::node::import::native::sr25519::transfer_keep_alive::paritydb::small.json'
+    - 'cargo run --release -p node-bench -- ::trie::read::small --json
+      | tee ./artifacts/benches/$CI_COMMIT_REF_NAME-$CI_COMMIT_SHORT_SHA/::trie::read::small.json'
+    - sccache -s
+  tags:
+    - linux-docker-benches
+
+node-bench-regression-guard:
+  # it's not belong to `build` semantically, but dag jobs can't depend on each other
+  # within the single stage - https://gitlab.com/gitlab-org/gitlab/-/issues/30632
+  # more: https://github.com/paritytech/substrate/pull/8519#discussion_r608012402
+  stage:                           build
+  extends:
+    - .docker-env
+    - .test-refs-no-trigger-prs-only
+  needs:
+    # this is a DAG
+    - job:                         cargo-check-benches
+      artifacts:                   true
+    # this does not like a DAG, just polls the artifact
+    - project:                     $CI_PROJECT_PATH
+      job:                         cargo-check-benches
+      ref:                         master
+      artifacts:                   true
+  variables:
+    CI_IMAGE:                      "paritytech/node-bench-regression-guard:latest"
+  before_script: [""]
+  script:
+    - echo "------- IMPORTANT -------"
+    - echo "node-bench-regression-guard depends on the results of a cargo-check-benches job"
+    - echo "In case of this job failure, check your pipeline's cargo-check-benches"
+    - 'node-bench-regression-guard --reference artifacts/benches/master-*
+       --compare-with artifacts/benches/$CI_COMMIT_REF_NAME-$CI_COMMIT_SHORT_SHA'
+
+cargo-check-subkey:
+  stage:                           test
+  extends:
+    - .docker-env
+    - .test-refs
+  script:
+    - cd ./bin/utils/subkey
+    - SKIP_WASM_BUILD=1 time cargo check --release
+
+cargo-check-try-runtime:
+  stage:                           test
+  extends:
+    - .docker-env
+    - .test-refs
+  script:
+    - time cargo check --features try-runtime
+
+cargo-check-wasmer-sandbox:
+  stage:                           test
+  extends:
+    - .docker-env
+    - .test-refs
+  script:
+    - time cargo check --features wasmer-sandbox
+
+test-deterministic-wasm:
+  stage:                           test
+  extends:
+    - .docker-env
+    - .test-refs
+  variables:
+    WASM_BUILD_NO_COLOR:           1
+  script:
+    # build runtime
+    - cargo build --verbose --release -p node-runtime
+    # make checksum
+    - sha256sum ./target/release/wbuild/node-runtime/target/wasm32-unknown-unknown/release/node_runtime.wasm > checksum.sha256
+    # clean up – FIXME: can we reuse some of the artifacts?
+    - cargo clean
+    # build again
+    - cargo build --verbose --release -p node-runtime
+    # confirm checksum
+    - sha256sum -c ./checksum.sha256
+
+test-linux-stable:
+  stage:                           test
+  extends:
+    - .docker-env
+    - .test-refs
+  variables:
+    # Enable debug assertions since we are running optimized builds for testing
+    # but still want to have debug assertions.
+    RUSTFLAGS:                     "-Cdebug-assertions=y -Dwarnings"
+    RUST_BACKTRACE:                1
+    WASM_BUILD_NO_COLOR:           1
+    # Ensure we run the UI tests.
+    RUN_UI_TESTS:                  1
+  script:
+    # this job runs all tests in former runtime-benchmarks, frame-staking and wasmtime tests
+    - time cargo test --workspace --locked --release --verbose --features runtime-benchmarks --manifest-path ./bin/node/cli/Cargo.toml
+    - time cargo test -p frame-support-test --features=conditional-storage,no-metadata-docs --manifest-path ./frame/support/test/Cargo.toml --test pallet # does not reuse cache 1 min 44 sec
+    - SUBSTRATE_TEST_TIMEOUT=1 time cargo test -p substrate-test-utils --release --verbose --locked -- --ignored timeout
+
+test-frame-examples-compile-to-wasm:
+  # into one job
+  stage:                           test
+  extends:
+    - .docker-env
+    - .test-refs
+  variables:
+    # Enable debug assertions since we are running optimized builds for testing
+    # but still want to have debug assertions.
+    RUSTFLAGS:                     "-Cdebug-assertions=y"
+    RUST_BACKTRACE: 1
+  script:
+    - cd ./frame/examples/offchain-worker/
+    - cargo +nightly build --target=wasm32-unknown-unknown --no-default-features
+    - cd ../basic
+    - cargo +nightly build --target=wasm32-unknown-unknown --no-default-features
+
+test-linux-stable-int:
+  stage:                           test
+  extends:
+    - .docker-env
+    - .test-refs
+  variables:
+    # Enable debug assertions since we are running optimized builds for testing
+    # but still want to have debug assertions.
+    RUSTFLAGS:                     "-Cdebug-assertions=y -Dwarnings"
+    RUST_BACKTRACE:                1
+    WASM_BUILD_NO_COLOR:           1
+    # Ensure we run the UI tests.
+    RUN_UI_TESTS:                  1
+  script:
+    - echo "___Logs will be partly shown at the end in case of failure.___"
+    - echo "___Full log will be saved to the job artifacts only in case of failure.___"
+    - WASM_BUILD_NO_COLOR=1
+      RUST_LOG=sync=trace,consensus=trace,client=trace,state-db=trace,db=trace,forks=trace,state_db=trace,storage_cache=trace
+        time cargo test -p node-cli --release --verbose --locked -- --ignored
+        &> ${CI_COMMIT_SHORT_SHA}_int_failure.log
+  after_script:
+    - awk '/FAILED|^error\[/,0' ${CI_COMMIT_SHORT_SHA}_int_failure.log
+  artifacts:
+    name:                          $CI_COMMIT_SHORT_SHA
+    when:                          on_failure
+    expire_in:                     3 days
+    paths:
+      - ${CI_COMMIT_SHORT_SHA}_int_failure.log
+
+check-tracing:
+  stage:                           test
+  extends:
+    - .docker-env
+    - .test-refs
+  script:
+    # with-tracing must be explicitly activated, we run a test to ensure this works as expected in both cases
+    - time cargo +nightly test --manifest-path ./primitives/tracing/Cargo.toml --no-default-features
+    - time cargo +nightly test --manifest-path ./primitives/tracing/Cargo.toml --no-default-features --features=with-tracing
+
+test-full-crypto-feature:
+  stage:                           test
+  extends:
+    - .docker-env
+    - .test-refs
+  variables:
+    # Enable debug assertions since we are running optimized builds for testing
+    # but still want to have debug assertions.
+    RUSTFLAGS:                     "-Cdebug-assertions=y"
+    RUST_BACKTRACE: 1
+  script:
+    - cd primitives/core/
+    - time cargo +nightly build --verbose --no-default-features --features full_crypto
+    - cd ../application-crypto
+    - time cargo +nightly build --verbose --no-default-features --features full_crypto
+
+test-wasmer-sandbox:
+  stage:                           test
+  extends:
+    - .docker-env
+    - .test-refs-wasmer-sandbox
+  script:
+    - time cargo test --release --features runtime-benchmarks,wasmer-sandbox,disable-ui-tests
+
+cargo-check-macos:
+  stage:                           test
+  extends:                         .test-refs-no-trigger
+  before_script:
+    - !reference [.rust-info-script, script]
+  script:
+    - SKIP_WASM_BUILD=1 time cargo check --release
+  tags:
+    - osx
+
+check-rustdoc:
+  stage:                           test
+  extends:
+    - .docker-env
+    - .test-refs
+  variables:
+    SKIP_WASM_BUILD:               1
+    RUSTDOCFLAGS:                  "-Dwarnings"
+  script:
+    - time cargo +nightly doc --workspace --all-features --verbose --no-deps