[ci] Split gitlab-ci config to separate files (#6031)

* [WIP][ci] Split gitlab-ci config to separate files

* add stage

* move short benches

* rename stage short_benchmark

* include short-benchmarks

* add check

* add test stage

* quickfix check-no-default-features

* add build stage

* add weights and publish

* rename stage

* rename stage

* fix publish miner image

* fix publish miner image

* add dag

* fix image

* start build-short-benchmark with the pipeline

* some more DAGs

* empty line to short-benchmarks

* add change from master

Co-authored-by: parity-processbot <>

polkadot/scripts/ci/gitlab/pipeline/build.yml

@@ -0,0 +1,193 @@
+# This file is part of .gitlab-ci.yml
+# Here are all jobs that are executed during "build" stage
+
+build-linux-stable:
+  stage:                           build
+  # this is an artificial job dependency, for pipeline optimization using GitLab's DAGs
+  # the job can be found in check.yml
+  needs:
+    - job:                         job-starter
+      artifacts:                   false
+  extends:
+    - .docker-env
+    - .common-refs
+    - .compiler-info
+    - .collect-artifacts
+  variables:
+    RUST_TOOLCHAIN: stable
+    # Enable debug assertions since we are running optimized builds for testing
+    # but still want to have debug assertions.
+    RUSTFLAGS: "-Cdebug-assertions=y -Dwarnings"
+    # Ensure we run the UI tests.
+    RUN_UI_TESTS: 1
+  script:
+    - time cargo build --profile testnet --features pyroscope --verbose --bin polkadot
+    - sccache -s
+    # pack artifacts
+    - mkdir -p ./artifacts
+    - VERSION="${CI_COMMIT_REF_NAME}" # will be tag or branch name
+    - mv ./target/testnet/polkadot ./artifacts/.
+    - pushd artifacts
+    - sha256sum polkadot | tee polkadot.sha256
+    - shasum -c polkadot.sha256
+    - popd
+    - EXTRATAG="${CI_COMMIT_REF_NAME}-${CI_COMMIT_SHORT_SHA}"
+    - echo "Polkadot version = ${VERSION} (EXTRATAG = ${EXTRATAG})"
+    - echo -n ${VERSION} > ./artifacts/VERSION
+    - echo -n ${EXTRATAG} > ./artifacts/EXTRATAG
+    - echo -n ${CI_JOB_ID} > ./artifacts/BUILD_LINUX_JOB_ID
+    - RELEASE_VERSION=$(./artifacts/polkadot -V | awk '{print $2}'| awk -F "-" '{print $1}')
+    - echo -n "v${RELEASE_VERSION}" > ./artifacts/BUILD_RELEASE_VERSION
+    - cp -r scripts/* ./artifacts
+
+check-transaction-versions:
+  stage:                           build
+  extends:
+    - .docker-env
+    - .test-refs
+  image:                           node:15
+  needs:
+    - job:                         build-linux-stable
+      artifacts:                   true
+  before_script:
+    - apt-get -y update; apt-get -y install jq lsof
+    - npm install --ignore-scripts -g @polkadot/metadata-cmp
+    # Set git config
+    - git config remote.origin.url "https://github.com/paritytech/polkadot.git"
+    - git fetch origin release
+  script:
+    - ./scripts/ci/gitlab/check_extrinsics_ordering.sh
+
+build-test-collators:
+  stage:                           build
+  # this is an artificial job dependency, for pipeline optimization using GitLab's DAGs
+  # the job can be found in check.yml
+  needs:
+    - job:                         job-starter
+      artifacts:                   false
+  extends:
+    - .docker-env
+    - .common-refs
+    - .compiler-info
+    - .collect-artifacts
+  script:
+    - time cargo build --profile testnet --verbose -p test-parachain-adder-collator
+    - time cargo build --profile testnet --verbose -p test-parachain-undying-collator
+    - sccache -s
+    # pack artifacts
+    - mkdir -p ./artifacts
+    - mv ./target/testnet/adder-collator ./artifacts/.
+    - mv ./target/testnet/undying-collator ./artifacts/.
+    - echo -n "${CI_COMMIT_REF_NAME}" > ./artifacts/VERSION
+    - echo -n "${CI_COMMIT_REF_NAME}-${CI_COMMIT_SHORT_SHA}" > ./artifacts/EXTRATAG
+    - echo "adder-collator version = $(cat ./artifacts/VERSION) (EXTRATAG = $(cat ./artifacts/EXTRATAG))"
+    - echo "undying-collator version = $(cat ./artifacts/VERSION) (EXTRATAG = $(cat ./artifacts/EXTRATAG))"
+    - cp -r ./scripts/* ./artifacts
+
+build-malus:
+  stage:                           build
+  # this is an artificial job dependency, for pipeline optimization using GitLab's DAGs
+  # the job can be found in check.yml
+  needs:
+    - job:                         job-starter
+      artifacts:                   false
+  extends:
+    - .docker-env
+    - .common-refs
+    - .compiler-info
+    - .collect-artifacts
+  script:
+    - time cargo build --profile testnet --verbose -p polkadot-test-malus
+    - sccache -s
+    # pack artifacts
+    - mkdir -p ./artifacts
+    - mv ./target/testnet/malus ./artifacts/.
+    - echo -n "${CI_COMMIT_REF_NAME}" > ./artifacts/VERSION
+    - echo -n "${CI_COMMIT_REF_NAME}-${CI_COMMIT_SHORT_SHA}" > ./artifacts/EXTRATAG
+    - echo "polkadot-test-malus = $(cat ./artifacts/VERSION) (EXTRATAG = $(cat ./artifacts/EXTRATAG))"
+    - cp -r ./scripts/* ./artifacts
+
+build-staking-miner:
+  stage:                           build
+  # this is an artificial job dependency, for pipeline optimization using GitLab's DAGs
+  # the job can be found in check.yml
+  needs:
+    - job:                         job-starter
+      artifacts:                   false
+  extends:
+    - .docker-env
+    - .common-refs
+    - .compiler-info
+    - .collect-artifacts
+  script:
+    - time cargo build --locked --release --package staking-miner
+    # pack artifacts
+    - mkdir -p ./artifacts
+    - mv ./target/release/staking-miner ./artifacts/.
+    - echo -n "${CI_COMMIT_REF_NAME}" > ./artifacts/VERSION
+    - echo -n "${CI_COMMIT_REF_NAME}-${CI_COMMIT_SHORT_SHA}" > ./artifacts/EXTRATAG
+    - echo "staking-miner = $(cat ./artifacts/VERSION) (EXTRATAG = $(cat ./artifacts/EXTRATAG))"
+    - cp -r ./scripts/* ./artifacts
+
+build-rustdoc:
+  stage:                           build
+  # this is an artificial job dependency, for pipeline optimization using GitLab's DAGs
+  # the job can be found in test.yml
+  needs:
+    - job:                         test-deterministic-wasm
+      artifacts:                   false
+  extends:
+    - .docker-env
+    - .test-refs
+  variables:
+    SKIP_WASM_BUILD:               1
+  artifacts:
+    name:                          "${CI_JOB_NAME}_${CI_COMMIT_REF_NAME}-doc"
+    when:                          on_success
+    expire_in:                     1 days
+    paths:
+    - ./crate-docs/
+  script:
+    # FIXME: it fails with `RUSTDOCFLAGS="-Dwarnings"` and `--all-features`
+    # FIXME: return to stable when https://github.com/rust-lang/rust/issues/96937 gets into stable
+    - time cargo +nightly doc --workspace --verbose --no-deps
+    - rm -f ./target/doc/.lock
+    - mv ./target/doc ./crate-docs
+    # FIXME: remove me after CI image gets nonroot
+    - chown -R nonroot:nonroot ./crate-docs
+    - echo "<meta http-equiv=refresh content=0;url=polkadot_service/index.html>" > ./crate-docs/index.html
+
+build-implementers-guide:
+  stage:                           build
+  # this is an artificial job dependency, for pipeline optimization using GitLab's DAGs
+  # the job can be found in test.yml
+  needs:
+    - job:                         test-deterministic-wasm
+      artifacts:                   false
+  extends:
+    - .docker-env
+    - .test-refs
+    - .collect-artifacts-short
+  script:
+    - cargo install mdbook mdbook-mermaid mdbook-linkcheck
+    - mdbook build ./roadmap/implementers-guide
+    - mkdir -p artifacts
+    - mv roadmap/implementers-guide/book artifacts/
+    # FIXME: remove me after CI image gets nonroot
+    - chown -R nonroot:nonroot artifacts/
+
+build-short-benchmark:
+  stage:                           build
+  # this is an artificial job dependency, for pipeline optimization using GitLab's DAGs
+  # the job can be found in check.yml
+  needs:
+    - job:                         job-starter
+      artifacts:                   false
+  extends:
+    - .docker-env
+    - .test-refs
+    - .collect-artifacts
+  script:
+    - cargo +nightly build --profile release --locked --features=runtime-benchmarks
+    - mkdir artifacts
+    - cp ./target/release/polkadot ./artifacts/

polkadot/scripts/ci/gitlab/pipeline/check.yml

@@ -0,0 +1,97 @@
+# This file is part of .gitlab-ci.yml
+# Here are all jobs that are executed during "check" stage
+
+check-runtime:
+  stage:                           check
+  image:                           paritytech/tools:latest
+  extends:
+    - .kubernetes-env
+  rules:
+    - if: $CI_COMMIT_REF_NAME =~ /^release-v[0-9]+\.[0-9]+.*$/      # i.e. release-v0.9.27
+  variables:
+    GITLAB_API:                    "https://gitlab.parity.io/api/v4"
+    GITHUB_API_PROJECT:            "parity%2Finfrastructure%2Fgithub-api"
+  script:
+    - ./scripts/ci/gitlab/check_runtime.sh
+  allow_failure:                   true
+
+cargo-fmt:
+  stage:                           check
+  extends:
+    - .docker-env
+    - .test-refs
+  script:
+    - cargo +nightly --version
+    - cargo +nightly fmt --all -- --check
+  allow_failure:                   true
+
+spellcheck:
+  stage:                           check
+  extends:
+    - .docker-env
+    - .test-refs
+  script:
+    - cargo spellcheck --version
+    # compare with the commit parent to the PR, given it's from a default branch
+    - git fetch origin +${CI_DEFAULT_BRANCH}:${CI_DEFAULT_BRANCH}
+    - echo "___Spellcheck is going to check your diff___"
+    - cargo spellcheck list-files -vvv $(git diff --diff-filter=AM --name-only $(git merge-base ${CI_COMMIT_SHA} ${CI_DEFAULT_BRANCH} -- :^bridges))
+    - time cargo spellcheck check -vvv --cfg=scripts/ci/gitlab/spellcheck.toml --checkers hunspell --code 1
+        $(git diff --diff-filter=AM --name-only $(git merge-base ${CI_COMMIT_SHA} ${CI_DEFAULT_BRANCH} -- :^bridges))
+  allow_failure:                   true
+
+check-try-runtime:
+  stage:                           check
+  extends:
+    - .docker-env
+    - .test-refs
+    - .compiler-info
+  script:
+    # Check that everything compiles with `try-runtime` feature flag.
+    - cargo check --features try-runtime --all
+
+# More info can be found here: https://github.com/paritytech/polkadot/pull/5865
+# Works only in PRs
+check-runtime-migration:
+  stage:                           check
+  extends:
+    - .docker-env
+    - .test-pr-refs
+    - .compiler-info
+  script:
+    - |
+      export has_runtimemigration_label=$(curl -sS -H "Accept: application/vnd.github+json" \
+        -H "Authorization: token $GITHUB_PR_TOKEN" \
+        https://api.github.com/repos/paritytech/polkadot/issues/$CI_COMMIT_REF_NAME/labels | grep "E1" | wc -l)
+    - |
+      if [[ $has_runtimemigration_label != 0 ]]; then
+        echo "Found label runtimemigration. Running tests"
+        export RUST_LOG=remote-ext=debug,runtime=debug
+        time cargo test --release -p westend-runtime -p polkadot-runtime -p kusama-runtime --features try-runtime
+      else
+        echo "runtimemigration label not found. Skipping"
+      fi
+  allow_failure:                   true
+
+# is broken, need to fix
+check-no-default-features:
+  stage:                           check
+  extends:
+    - .docker-env
+    - .test-refs
+    - .compiler-info
+  script:
+    # Check that polkadot-cli will compile no default features.
+    - pushd ./node/service && cargo check --no-default-features && popd
+    - pushd ./cli && cargo check --no-default-features --features "service" && popd
+    - exit 0
+
+# this is artificial job to run some build and tests using DAG
+job-starter:
+  stage:                           check
+  image:                           paritytech/tools:latest
+  extends:
+    - .kubernetes-env
+    - .common-refs
+  script:
+    - echo ok

polkadot/scripts/ci/gitlab/pipeline/publish.yml

@@ -0,0 +1,209 @@
+# This file is part of .gitlab-ci.yml
+# Here are all jobs that are executed during "publish" stage
+
+# This image is used in testnets
+# Release image is handled by the Github Action here:
+# .github/workflows/publish-docker-release.yml
+publish-polkadot-debug-image:
+  stage:                           publish
+  extends:
+    - .build-push-image
+  rules:
+    # Don't run when triggered from another pipeline
+    - if: $CI_PIPELINE_SOURCE == "pipeline"
+      when: never
+    - if: $CI_PIPELINE_SOURCE == "web"
+    - if: $CI_PIPELINE_SOURCE == "schedule"
+    - if: $CI_COMMIT_REF_NAME == "master"
+    - if: $CI_COMMIT_REF_NAME =~ /^[0-9]+$/                         # PRs
+    - if: $CI_COMMIT_REF_NAME =~ /^v[0-9]+\.[0-9]+.*$/              # i.e. v1.0, v2.1rc1
+  variables:
+    GIT_STRATEGY:                  none
+    DOCKER_USER:                   ${PARITYPR_USER}
+    DOCKER_PASS:                   ${PARITYPR_PASS}
+    # scripts/ci/dockerfiles/polkadot_injected_debug.Dockerfile
+    DOCKERFILE:                    ci/dockerfiles/polkadot_injected_debug.Dockerfile
+    IMAGE_NAME:                    docker.io/paritypr/polkadot-debug
+  needs:
+    - job:                         build-linux-stable
+      artifacts:                   true
+  after_script:
+    # pass artifacts to the zombienet-tests job
+    # https://docs.gitlab.com/ee/ci/multi_project_pipelines.html#with-variable-inheritance
+    - echo "PARACHAINS_IMAGE_NAME=${IMAGE_NAME}" > ./artifacts/parachains.env
+    - echo "PARACHAINS_IMAGE_TAG=$(cat ./artifacts/EXTRATAG)" >> ./artifacts/parachains.env
+  artifacts:
+    reports:
+      # this artifact is used in zombienet-tests job
+      dotenv: ./artifacts/parachains.env
+    expire_in:                     1 days
+
+publish-test-collators-image:
+  # service image for Simnet
+  stage:                           publish
+  extends:
+    - .build-push-image
+    - .zombienet-refs
+  variables:
+    GIT_STRATEGY:                  none
+    DOCKER_USER:                   ${PARITYPR_USER}
+    DOCKER_PASS:                   ${PARITYPR_PASS}
+    # scripts/ci/dockerfiles/collator_injected.Dockerfile
+    DOCKERFILE:                    ci/dockerfiles/collator_injected.Dockerfile
+    IMAGE_NAME:                    docker.io/paritypr/colander
+  needs:
+    - job:                         build-test-collators
+      artifacts:                   true
+  after_script:
+    - buildah logout --all
+    # pass artifacts to the zombienet-tests job
+    - echo "COLLATOR_IMAGE_NAME=${IMAGE_NAME}" > ./artifacts/collator.env
+    - echo "COLLATOR_IMAGE_TAG=$(cat ./artifacts/EXTRATAG)" >> ./artifacts/collator.env
+  artifacts:
+    reports:
+      # this artifact is used in zombienet-tests job
+      dotenv: ./artifacts/collator.env
+
+publish-malus-image:
+  # service image for Simnet
+  stage:                           publish
+  extends:
+    - .build-push-image
+    - .zombienet-refs
+  variables:
+    GIT_STRATEGY:                  none
+    DOCKER_USER:                   ${PARITYPR_USER}
+    DOCKER_PASS:                   ${PARITYPR_PASS}
+    # scripts/ci/dockerfiles/malus_injected.Dockerfile
+    DOCKERFILE:                    ci/dockerfiles/malus_injected.Dockerfile
+    IMAGE_NAME:                    docker.io/paritypr/malus
+  needs:
+    - job:                         build-malus
+      artifacts:                   true
+  after_script:
+    - buildah logout "$IMAGE_NAME"
+    # pass artifacts to the zombienet-tests job
+    - echo "MALUS_IMAGE_NAME=${IMAGE_NAME}" > ./artifacts/malus.env
+    - echo "MALUS_IMAGE_TAG=$(cat ./artifacts/EXTRATAG)" >> ./artifacts/malus.env
+  artifacts:
+    reports:
+      # this artifact is used in zombienet-tests job
+      dotenv: ./artifacts/malus.env
+
+publish-staking-miner-image:
+  stage:                           publish
+  extends:
+    - .build-push-image
+    - .publish-refs
+  variables:
+    # scripts/ci/dockerfiles/staking-miner/staking-miner_injected.Dockerfile
+    DOCKERFILE:                    ci/dockerfiles/staking-miner/staking-miner_injected.Dockerfile
+    IMAGE_NAME:                    docker.io/paritytech/staking-miner
+    GIT_STRATEGY:                  none
+    DOCKER_USER:                   ${Docker_Hub_User_Parity}
+    DOCKER_PASS:                   ${Docker_Hub_Pass_Parity}
+  needs:
+    - job:                         build-staking-miner
+      artifacts:                   true
+
+
+publish-s3-release:
+  stage:                           publish
+  extends:
+    - .kubernetes-env
+  needs:
+    - job:                         build-linux-stable
+      artifacts:                   true
+  image:                           paritytech/awscli:latest
+  variables:
+    GIT_STRATEGY:                  none
+    PREFIX:                        "builds/polkadot/${ARCH}-${DOCKER_OS}"
+  rules:
+    - if: $CI_PIPELINE_SOURCE == "pipeline"
+      when: never
+    # publishing binaries nightly
+    - if: $CI_PIPELINE_SOURCE == "schedule"
+  before_script:
+    - !reference [.build-push-image, before_script]
+  script:
+    - echo "uploading objects to https://releases.parity.io/${PREFIX}/${VERSION}"
+    - aws s3 sync --acl public-read ./artifacts/ s3://${AWS_BUCKET}/${PREFIX}/${VERSION}/
+    - echo "update objects at https://releases.parity.io/${PREFIX}/${EXTRATAG}"
+    - find ./artifacts -type f | while read file; do
+        name="${file#./artifacts/}";
+        aws s3api copy-object
+          --copy-source ${AWS_BUCKET}/${PREFIX}/${VERSION}/${name}
+          --bucket ${AWS_BUCKET} --key ${PREFIX}/${EXTRATAG}/${name};
+      done
+    - |
+      cat <<-EOM
+      |
+      |  polkadot binary paths:
+      |
+      |  - https://releases.parity.io/${PREFIX}/${EXTRATAG}/polkadot
+      |  - https://releases.parity.io/${PREFIX}/${VERSION}/polkadot
+      |
+      EOM
+  after_script:
+    - aws s3 ls s3://${AWS_BUCKET}/${PREFIX}/${EXTRATAG}/
+        --recursive --human-readable --summarize
+
+publish-rustdoc:
+  stage:                           publish
+  extends:
+    - .kubernetes-env
+  image:                           paritytech/tools:latest
+  variables:
+    GIT_DEPTH:                     100
+  rules:
+    - if: $CI_PIPELINE_SOURCE == "pipeline"
+      when: never
+    - if: $CI_PIPELINE_SOURCE == "web" && $CI_COMMIT_REF_NAME == "master"
+    - if: $CI_COMMIT_REF_NAME == "master"
+  # `needs:` can be removed after CI image gets nonroot. In this case `needs:` stops other
+  # artifacts from being dowloaded by this job.
+  needs:
+    - job:                         build-rustdoc
+      artifacts:                   true
+    - job:                         build-implementers-guide
+      artifacts:                   true
+  script:
+    # Save README and docs
+    - cp -r ./crate-docs/ /tmp/doc/
+    - cp -r ./artifacts/book/ /tmp/
+    # setup ssh
+    - eval $(ssh-agent)
+    - ssh-add - <<< ${GITHUB_SSH_PRIV_KEY}
+    - mkdir ~/.ssh && touch ~/.ssh/known_hosts
+    - ssh-keyscan -t rsa github.com >> ~/.ssh/known_hosts
+    # Set git config
+    - git config user.email "devops-team@parity.io"
+    - git config user.name "${GITHUB_USER}"
+    - git config remote.origin.url "git@github.com:/paritytech/${CI_PROJECT_NAME}.git"
+    - git config remote.origin.fetch "+refs/heads/*:refs/remotes/origin/*"
+    - git fetch origin gh-pages
+    - git checkout gh-pages
+    # Remove everything and restore generated docs and README
+    - cp index.html /tmp
+    - cp README.md /tmp
+    - rm -rf ./*
+    # dir for rustdoc
+    - mkdir -p doc
+    # dir for implementors guide
+    - mkdir -p book
+    - mv /tmp/doc/* doc/
+    - mv /tmp/book/html/* book/
+    - mv /tmp/index.html .
+    - mv /tmp/README.md .
+    # Upload files
+    - git add --all --force
+    # `git commit` has an exit code of > 0 if there is nothing to commit.
+    # This causes GitLab to exit immediately and marks this job failed.
+    # We don't want to mark the entire job failed if there's nothing to
+    # publish though, hence the `|| true`.
+    - git commit -m "Updated docs for ${CI_COMMIT_REF_NAME}" ||
+        echo "___Nothing to commit___"
+    - git push origin gh-pages --force
+    - echo "___Rustdoc was successfully published to https://paritytech.github.io/polkadot/___"
+  after_script:
+    - rm -rf .git/ ./*

polkadot/scripts/ci/gitlab/pipeline/short-benchmarks.yml

@@ -0,0 +1,27 @@
+# This file is part of .gitlab-ci.yml
+# Here are all jobs that are executed during "short-benchmarks" stage
+
+# Run all pallet benchmarks only once to check if there are any errors
+short-benchmark-polkadot:          &short-bench
+  stage:                           short-benchmarks
+  extends:
+    - .test-pr-refs
+    - .docker-env
+  # this is an artificial job dependency, for pipeline optimization using GitLab's DAGs
+  needs:
+    - job:                         build-short-benchmark
+      artifacts:                   true
+  variables:
+    RUNTIME:                       polkadot
+  script:
+    - ./artifacts/polkadot benchmark pallet --execution wasm --wasm-execution compiled --chain $RUNTIME-dev --pallet "*" --extrinsic "*" --steps 2 --repeat 1
+
+short-benchmark-kusama:
+  <<:                              *short-bench
+  variables:
+    RUNTIME:                       kusama
+
+short-benchmark-westend:
+  <<:                              *short-bench
+  variables:
+    RUNTIME:                       westend

polkadot/scripts/ci/gitlab/pipeline/test.yml

@@ -0,0 +1,74 @@
+# This file is part of .gitlab-ci.yml
+# Here are all jobs that are executed during "test" stage
+
+test-linux-stable:
+  stage:                           test
+  # this is an artificial job dependency, for pipeline optimization using GitLab's DAGs
+  # the job can be found in check.yml
+  needs:
+    - job:                         job-starter
+      artifacts:                   false
+  extends:
+    - .docker-env
+    - .common-refs
+    - .pipeline-stopper-artifacts
+  before_script:
+    - rustup show
+    - cargo --version
+    - !reference [.pipeline-stopper-vars, before_script]
+  variables:
+    RUST_TOOLCHAIN: stable
+    # Enable debug assertions since we are running optimized builds for testing
+    # but still want to have debug assertions.
+    RUSTFLAGS: "-Cdebug-assertions=y -Dwarnings"
+  script:
+    - time cargo test --workspace --profile testnet --verbose --locked --features=runtime-benchmarks,runtime-metrics
+
+.check-dependent-project:          &check-dependent-project
+  stage:                           test
+  extends:
+    - .docker-env
+    - .test-pr-refs
+  script:
+    - git clone
+        --depth=1
+        "--branch=$PIPELINE_SCRIPTS_TAG"
+        https://github.com/paritytech/pipeline-scripts
+    - ./pipeline-scripts/check_dependent_project.sh
+        --org paritytech
+        --dependent-repo "$DEPENDENT_REPO"
+        --github-api-token "$GITHUB_PR_TOKEN"
+        --extra-dependencies "$EXTRA_DEPENDENCIES"
+        --companion-overrides "$COMPANION_OVERRIDES"
+
+check-dependent-cumulus:
+  <<: *check-dependent-project
+  variables:
+    DEPENDENT_REPO: cumulus
+    EXTRA_DEPENDENCIES: substrate
+    COMPANION_OVERRIDES: |
+      polkadot: release-v*
+      cumulus: polkadot-v*
+
+test-node-metrics:
+  stage:                           test
+  extends:
+    - .docker-env
+    - .test-refs
+    - .compiler-info
+  variables:
+    RUST_TOOLCHAIN: stable
+    # Enable debug assertions since we are running optimized builds for testing
+    # but still want to have debug assertions.
+    RUSTFLAGS: "-Cdebug-assertions=y -Dwarnings"
+  script:
+    - time cargo test  --profile testnet --verbose --locked --features=runtime-metrics -p polkadot-node-metrics
+
+test-deterministic-wasm:
+  stage:                           test
+  extends:
+    - .docker-env
+    - .test-refs
+    - .compiler-info
+  script:
+    - ./scripts/ci/gitlab/test_deterministic_wasm.sh

polkadot/scripts/ci/gitlab/pipeline/weights.yml

@@ -0,0 +1,33 @@
+# This file is part of .gitlab-ci.yml
+# Here are all jobs that are executed during "weights" stage
+
+update_polkadot_weights:           &update-weights
+  stage:                           weights
+  timeout:                         1d
+  when:                            manual
+  variables:
+    RUNTIME:                       polkadot
+  artifacts:
+    paths:
+      - ${RUNTIME}_weights_${CI_COMMIT_SHORT_SHA}.patch
+  script:
+    - ./scripts/ci/run_benches_for_runtime.sh $RUNTIME
+    - git diff -P > ${RUNTIME}_weights_${CI_COMMIT_SHORT_SHA}.patch
+  # uses the "shell" executors
+  tags:
+    - weights
+
+update_kusama_weights:
+  <<:                              *update-weights
+  variables:
+    RUNTIME:                       kusama
+
+update_westend_weights:
+  <<:                              *update-weights
+  variables:
+    RUNTIME:                       westend
+
+update_rococo_weights:
+  <<:                              *update-weights
+  variables:
+    RUNTIME:                       rococo

polkadot/scripts/ci/gitlab/pipeline/zombienet.yml

@@ -0,0 +1,302 @@
+# This file is part of .gitlab-ci.yml
+# Here are all jobs that are executed during "zombienet" stage
+
+zombienet-tests-parachains-smoke-test:
+  stage:                           zombienet
+  image:                           "${ZOMBIENET_IMAGE}"
+  extends:
+    - .kubernetes-env
+    - .zombienet-refs
+  needs:
+    - job:                         publish-polkadot-debug-image
+    - job:                         publish-malus-image
+    - job:                         publish-test-collators-image
+  variables:
+    GH_DIR:                        "https://github.com/paritytech/polkadot/tree/${CI_COMMIT_SHORT_SHA}/zombienet_tests/smoke"
+  before_script:
+    - echo "Zombie-net Tests Config"
+    - echo "${ZOMBIENET_IMAGE}"
+    - echo "${PARACHAINS_IMAGE_NAME} ${PARACHAINS_IMAGE_TAG}"
+    - echo "${MALUS_IMAGE_NAME} ${MALUS_IMAGE_TAG}"
+    - echo "${GH_DIR}"
+    - export DEBUG=zombie,zombie::network-node
+    - export ZOMBIENET_INTEGRATION_TEST_IMAGE=${PARACHAINS_IMAGE_NAME}:${PARACHAINS_IMAGE_TAG}
+    - export MALUS_IMAGE=${MALUS_IMAGE_NAME}:${MALUS_IMAGE_TAG}
+    - export COL_IMAGE="docker.io/paritypr/colander:4519" # The collator image is fixed
+  script:
+    - /home/nonroot/zombie-net/scripts/ci/run-test-env-manager.sh
+        --github-remote-dir="${GH_DIR}"
+        --test="0001-parachains-smoke-test.feature"
+  allow_failure:                   false
+  retry: 2
+  tags:
+    - zombienet-polkadot-integration-test
+
+zombienet-tests-parachains-pvf:
+  stage:                           zombienet
+  image:                           "${ZOMBIENET_IMAGE}"
+  extends:
+    - .kubernetes-env
+    - .zombienet-refs
+  needs:
+    - job:                         publish-polkadot-debug-image
+    - job:                         publish-test-collators-image
+  variables:
+    GH_DIR:                        "https://github.com/paritytech/polkadot/tree/${CI_COMMIT_SHORT_SHA}/zombienet_tests/functional"
+  before_script:
+    - echo "Zombie-net Tests Config"
+    - echo "${ZOMBIENET_IMAGE}"
+    - echo "${PARACHAINS_IMAGE_NAME} ${PARACHAINS_IMAGE_TAG}"
+    - echo "COL_IMAGE=${COLLATOR_IMAGE_NAME}:${COLLATOR_IMAGE_TAG}"
+    - echo "${GH_DIR}"
+    - export DEBUG=zombie,zombie::network-node
+    - export ZOMBIENET_INTEGRATION_TEST_IMAGE=${PARACHAINS_IMAGE_NAME}:${PARACHAINS_IMAGE_TAG}
+    - export MALUS_IMAGE=${MALUS_IMAGE_NAME}:${MALUS_IMAGE_TAG}
+    - export COL_IMAGE=${COLLATOR_IMAGE_NAME}:${COLLATOR_IMAGE_TAG}
+  script:
+    - /home/nonroot/zombie-net/scripts/ci/run-test-env-manager.sh
+        --github-remote-dir="${GH_DIR}"
+        --test="0001-parachains-pvf.feature"
+  allow_failure:                   false
+  retry: 2
+  tags:
+    - zombienet-polkadot-integration-test
+
+zombienet-tests-parachains-disputes:
+  stage:                           zombienet
+  image:                           "${ZOMBIENET_IMAGE}"
+  extends:
+    - .kubernetes-env
+    - .zombienet-refs
+  needs:
+    - job:                         publish-polkadot-debug-image
+    - job:                         publish-test-collators-image
+    - job:                         publish-malus-image
+  variables:
+    GH_DIR:                        "https://github.com/paritytech/polkadot/tree/${CI_COMMIT_SHORT_SHA}/zombienet_tests/functional"
+  before_script:
+    - echo "Zombie-net Tests Config"
+    - echo "${ZOMBIENET_IMAGE_NAME}"
+    - echo "${PARACHAINS_IMAGE_NAME} ${PARACHAINS_IMAGE_TAG}"
+    - echo "${MALUS_IMAGE_NAME} ${MALUS_IMAGE_TAG}"
+    - echo "${GH_DIR}"
+    - export DEBUG=zombie,zombie::network-node
+    - export ZOMBIENET_INTEGRATION_TEST_IMAGE=${PARACHAINS_IMAGE_NAME}:${PARACHAINS_IMAGE_TAG}
+    - export MALUS_IMAGE=${MALUS_IMAGE_NAME}:${MALUS_IMAGE_TAG}
+    - export COL_IMAGE=${COLLATOR_IMAGE_NAME}:${COLLATOR_IMAGE_TAG}
+  script:
+    - /home/nonroot/zombie-net/scripts/ci/run-test-env-manager.sh
+        --github-remote-dir="${GH_DIR}"
+        --test="0002-parachains-disputes.feature"
+  allow_failure:                   false
+  retry: 2
+  tags:
+    - zombienet-polkadot-integration-test
+
+zombienet-tests-parachains-disputes-garbage-candidate:
+  stage:                           zombienet
+  image:                           "${ZOMBIENET_IMAGE}"
+  extends:
+    - .kubernetes-env
+    - .zombienet-refs
+  needs:
+    - job:                         publish-polkadot-debug-image
+    - job:                         publish-test-collators-image
+    - job:                         publish-malus-image
+  variables:
+    GH_DIR:                        "https://github.com/paritytech/polkadot/tree/${CI_COMMIT_SHORT_SHA}/zombienet_tests/functional"
+  before_script:
+    - echo "Zombie-net Tests Config"
+    - echo "${ZOMBIENET_IMAGE_NAME}"
+    - echo "${PARACHAINS_IMAGE_NAME} ${PARACHAINS_IMAGE_TAG}"
+    - echo "${MALUS_IMAGE_NAME} ${MALUS_IMAGE_TAG}"
+    - echo "${GH_DIR}"
+    - export DEBUG=zombie,zombie::network-node
+    - export ZOMBIENET_INTEGRATION_TEST_IMAGE=${PARACHAINS_IMAGE_NAME}:${PARACHAINS_IMAGE_TAG}
+    - export MALUS_IMAGE=${MALUS_IMAGE_NAME}:${MALUS_IMAGE_TAG}
+    - export COL_IMAGE=${COLLATOR_IMAGE_NAME}:${COLLATOR_IMAGE_TAG}
+  script:
+    - /home/nonroot/zombie-net/scripts/ci/run-test-env-manager.sh
+        --github-remote-dir="${GH_DIR}"
+        --test="0003-parachains-garbage-candidate.feature"
+  allow_failure:                   false
+  retry: 2
+  tags:
+    - zombienet-polkadot-integration-test
+
+zombienet-test-parachains-upgrade-smoke-test:
+  stage:                           zombienet
+  image:                           "${ZOMBIENET_IMAGE}"
+  extends:
+    - .kubernetes-env
+    - .zombienet-refs
+  needs:
+    - job:                         publish-polkadot-debug-image
+    - job:                         publish-malus-image
+    - job:                         publish-test-collators-image
+  variables:
+    GH_DIR:                        'https://github.com/paritytech/polkadot/tree/${CI_COMMIT_SHORT_SHA}/zombienet_tests/smoke'
+  before_script:
+    - echo "ZombieNet Tests Config"
+    - echo "${PARACHAINS_IMAGE_NAME}:${PARACHAINS_IMAGE_TAG}"
+    - echo "docker.io/parity/polkadot-collator:latest"
+    - echo "${ZOMBIENET_IMAGE}"
+    - echo "${GH_DIR}"
+    - export DEBUG=zombie,zombie::network-node
+    - export ZOMBIENET_INTEGRATION_TEST_IMAGE=${PARACHAINS_IMAGE_NAME}:${PARACHAINS_IMAGE_TAG}
+    - export COL_IMAGE="docker.io/parity/polkadot-collator:latest" # Use cumulus lastest image
+  script:
+    - /home/nonroot/zombie-net/scripts/ci/run-test-env-manager.sh
+        --github-remote-dir="${GH_DIR}"
+        --test="0002-parachains-upgrade-smoke-test.feature"
+  allow_failure:                   true
+  retry: 2
+  tags:
+    - zombienet-polkadot-integration-test
+
+zombienet-tests-misc-paritydb:
+  stage:                           zombienet
+  image:                           "${ZOMBIENET_IMAGE}"
+  extends:
+    - .kubernetes-env
+    - .zombienet-refs
+  needs:
+    - job:                         publish-polkadot-debug-image
+    - job:                         publish-test-collators-image
+      artifacts:                   true
+  variables:
+    GH_DIR:                        "https://github.com/paritytech/polkadot/tree/${CI_COMMIT_SHORT_SHA}/zombienet_tests/misc"
+  before_script:
+    - echo "Zombie-net Tests Config"
+    - echo "${ZOMBIENET_IMAGE_NAME}"
+    - echo "${PARACHAINS_IMAGE_NAME} ${PARACHAINS_IMAGE_TAG}"
+    - echo "${GH_DIR}"
+    - export DEBUG=zombie,zombie::network-node
+    - export ZOMBIENET_INTEGRATION_TEST_IMAGE=${PARACHAINS_IMAGE_NAME}:${PARACHAINS_IMAGE_TAG}
+    - export COL_IMAGE=${COLLATOR_IMAGE_NAME}:${COLLATOR_IMAGE_TAG}
+  script:
+    - /home/nonroot/zombie-net/scripts/ci/run-test-env-manager.sh
+        --github-remote-dir="${GH_DIR}"
+        --test="0001-paritydb.feature"
+  allow_failure:                   false
+  retry: 2
+  tags:
+    - zombienet-polkadot-integration-test
+
+zombienet-tests-misc-upgrade-node:
+  stage:                           zombienet
+  image:                           "${ZOMBIENET_IMAGE}"
+  extends:
+    - .kubernetes-env
+    - .zombienet-refs
+  needs:
+    - job:                         publish-polkadot-debug-image
+    - job:                         publish-test-collators-image
+    - job:                         build-linux-stable
+      artifacts:                   true
+  variables:
+    GH_DIR:                        "https://github.com/paritytech/polkadot/tree/${CI_COMMIT_SHORT_SHA}/zombienet_tests/misc"
+  before_script:
+    - echo "Zombie-net Tests Config"
+    - echo "${ZOMBIENET_IMAGE_NAME}"
+    - echo "${PARACHAINS_IMAGE_NAME} ${PARACHAINS_IMAGE_TAG}"
+    - echo "${GH_DIR}"
+    - export DEBUG=zombie,zombie::network-node
+    - BUILD_RELEASE_VERSION="$(cat ./artifacts/BUILD_RELEASE_VERSION)"
+    - export ZOMBIENET_INTEGRATION_TEST_IMAGE="docker.io/parity/polkadot:${BUILD_RELEASE_VERSION}"
+    - export COL_IMAGE=${COLLATOR_IMAGE_NAME}:${COLLATOR_IMAGE_TAG}
+    - BUILD_LINUX_JOB_ID="$(cat ./artifacts/BUILD_LINUX_JOB_ID)"
+    - export POLKADOT_PR_BIN_URL="https://gitlab.parity.io/parity/mirrors/polkadot/-/jobs/${BUILD_LINUX_JOB_ID}/artifacts/raw/artifacts/polkadot"
+  script:
+    - /home/nonroot/zombie-net/scripts/ci/run-test-env-manager.sh
+        --github-remote-dir="${GH_DIR}"
+        --test="0002-upgrade-node.feature"
+  allow_failure:                   false
+  retry: 2
+  tags:
+    - zombienet-polkadot-integration-test
+
+zombienet-tests-malus-dispute-valid:
+  stage:                           zombienet
+  image:                           "${ZOMBIENET_IMAGE}"
+  extends:
+    - .kubernetes-env
+    - .zombienet-refs
+  needs:
+    - job:                         publish-polkadot-debug-image
+    - job:                         publish-malus-image
+    - job:                         publish-test-collators-image
+  variables:
+    GH_DIR:                        "https://github.com/paritytech/polkadot/tree/${CI_COMMIT_SHORT_SHA}/node/malus/integrationtests"
+  before_script:
+    - echo "Zombie-net Tests Config"
+    - echo "${ZOMBIENET_IMAGE_NAME}"
+    - echo "${PARACHAINS_IMAGE_NAME} ${PARACHAINS_IMAGE_TAG}"
+    - echo "${MALUS_IMAGE_NAME} ${MALUS_IMAGE_TAG}"
+    - echo "${GH_DIR}"
+    - export DEBUG=zombie*
+    - export ZOMBIENET_INTEGRATION_TEST_IMAGE=${PARACHAINS_IMAGE_NAME}:${PARACHAINS_IMAGE_TAG}
+    - export MALUS_IMAGE=${MALUS_IMAGE_NAME}:${MALUS_IMAGE_TAG}
+    - export COL_IMAGE=${COLLATOR_IMAGE_NAME}:${COLLATOR_IMAGE_TAG}
+  script:
+    - /home/nonroot/zombie-net/scripts/ci/run-test-env-manager.sh
+        --github-remote-dir="${GH_DIR}"
+        --test="0001-dispute-valid-block.feature"
+  allow_failure:                   false
+  retry: 2
+  tags:
+    - zombienet-polkadot-integration-test
+
+zombienet-tests-deregister-register-validator:
+  stage:                           zombienet
+  image:                           "${ZOMBIENET_IMAGE}"
+  extends:
+    - .kubernetes-env
+    - .zombienet-refs
+  needs:
+    - job:                         publish-polkadot-debug-image
+      artifacts:                   true
+  variables:
+    GH_DIR:                        "https://github.com/paritytech/polkadot/tree/${CI_COMMIT_SHORT_SHA}/zombienet_tests/smoke"
+  before_script:
+    - echo "Zombie-net Tests Config"
+    - echo "${ZOMBIENET_IMAGE_NAME}"
+    - echo "${PARACHAINS_IMAGE_NAME} ${PARACHAINS_IMAGE_TAG}"
+    - echo "${GH_DIR}"
+    - export DEBUG=zombie*
+    - export ZOMBIENET_INTEGRATION_TEST_IMAGE=${PARACHAINS_IMAGE_NAME}:${PARACHAINS_IMAGE_TAG}
+    - export MALUS_IMAGE=${MALUS_IMAGE_NAME}:${MALUS_IMAGE_TAG}
+  script:
+    - /home/nonroot/zombie-net/scripts/ci/run-test-env-manager.sh
+        --github-remote-dir="${GH_DIR}"
+        --test="0003-deregister-register-validator-smoke.feature"
+  allow_failure:                   false
+  retry: 2
+  tags:
+    - zombienet-polkadot-integration-test
+
+zombienet-tests-beefy-and-mmr:
+  stage:                           zombienet
+  image:                           "${ZOMBIENET_IMAGE}"
+  extends:
+    - .kubernetes-env
+    - .zombienet-refs
+  needs:
+    - job:                         publish-polkadot-debug-image
+  variables:
+    GH_DIR:                        "https://github.com/paritytech/polkadot/tree/${CI_COMMIT_SHORT_SHA}/zombienet_tests/functional"
+  before_script:
+    - echo "Zombie-net Tests Config"
+    - echo "${ZOMBIENET_IMAGE_NAME}"
+    - echo "${PARACHAINS_IMAGE_NAME} ${PARACHAINS_IMAGE_TAG}"
+    - echo "${GH_DIR}"
+    - export DEBUG=zombie*
+    - export ZOMBIENET_INTEGRATION_TEST_IMAGE=${PARACHAINS_IMAGE_NAME}:${PARACHAINS_IMAGE_TAG}
+  script:
+    - /home/nonroot/zombie-net/scripts/ci/run-test-env-manager.sh
+      --github-remote-dir="${GH_DIR}"
+      --test="0003-beefy-and-mmr.feature"
+  allow_failure:                   true
+  retry: 2
+  tags:
+    - zombienet-polkadot-integration-test