[ci] Split .gitlab-ci.yml (#1701)

* [WIP][ci] Split .gitlab-ci.yml

* comment unused includes

* fix typo

* fix needs

* add all stages

* exclude zombienet from check

scripts/ci/gitlab/pipeline/benchmarks.yml

@@ -0,0 +1,61 @@
+# This file is part of .gitlab-ci.yml
+# Here are all jobs that are executed during "benchmarks" stage
+# Work only on release-parachains-v* branches
+
+benchmarks-build:
+  stage:                           benchmarks-build
+  extends:
+    - .docker-env
+    - .collect-artifacts
+    - .benchmarks-manual-refs
+  script:
+    - time cargo build --profile production --locked --features runtime-benchmarks
+    - mkdir -p artifacts
+    - cp target/production/polkadot-parachain ./artifacts/
+
+benchmarks-assets:
+  stage:                           benchmarks-run
+  extends:
+    - .collect-artifacts
+    - .benchmarks-refs
+  before_script:
+    - !reference [.docker-env, before_script]
+  timeout:                         1d
+  script:
+    - ./scripts/benchmarks-ci.sh assets statemine ./artifacts
+    - ./scripts/benchmarks-ci.sh assets statemint ./artifacts
+    - ./scripts/benchmarks-ci.sh assets westmint ./artifacts
+    - export CURRENT_TIME=$(date '+%s')
+    - export BRANCHNAME="weights-statemint-${CI_COMMIT_BRANCH}-${CURRENT_TIME}"
+    - !reference [.git-commit-push, script]
+    # create PR to release-parachains-v* branch
+    - curl -u ${GITHUB_USER}:${GITHUB_TOKEN}
+      -d '{"title":"[benchmarks] Update weights for statemine/t","body":"This PR is generated automatically by CI. (Once merged please backport to master and node release branch.)","head":"'${BRANCHNAME}'","base":"'${CI_COMMIT_BRANCH}'"}'
+      -X POST https://api.github.com/repos/paritytech/${CI_PROJECT_NAME}/pulls
+  after_script:
+    - rm -rf .git/config
+  tags:
+    - weights
+
+benchmarks-collectives:
+  stage:                           benchmarks-run
+  extends:
+    - .collect-artifacts
+    - .benchmarks-refs
+  before_script:
+    - !reference [.docker-env, before_script]
+  script:
+    - ./scripts/benchmarks-ci.sh collectives collectives-polkadot ./artifacts
+    - git status
+    - export CURRENT_TIME=$(date '+%s')
+    - export BRANCHNAME="weights-collectives-${CI_COMMIT_BRANCH}-${CURRENT_TIME}"
+    - !reference [.git-commit-push, script]
+    # create PR
+    - curl -u ${GITHUB_USER}:${GITHUB_TOKEN}
+      -d '{"title":"[benchmarks] Update weights for collectives","body":"This PR is generated automatically by CI.","head":"'$BRANCHNAME'","base":"'${CI_COMMIT_BRANCH}'"}'
+      -X POST https://api.github.com/repos/paritytech/${CI_PROJECT_NAME}/pulls
+  after_script:
+    - rm -rf .git/config
+  tags:
+    - weights
+

scripts/ci/gitlab/pipeline/build.yml

@@ -0,0 +1,113 @@
+# This file is part of .gitlab-ci.yml
+# Here are all jobs that are executed during "build" stage
+
+build-linux-stable:
+  stage:                           build
+  extends:
+    - .docker-env
+    - .common-refs
+    - .collect-artifacts
+  variables:
+    # Enable debug assertions since we are running optimized builds for testing
+    # but still want to have debug assertions.
+    RUSTFLAGS: "-Cdebug-assertions=y -Dwarnings"
+  # this is an artificial job dependency, for pipeline optimization using GitLab's DAGs
+  needs:
+    - job:                         check-rustdoc
+      artifacts:                   false
+  script:
+    - echo "___Building a binary, please refrain from using it in production since it goes with the debug assertions.___"
+    - time cargo build --release --locked --bin polkadot-parachain
+    - echo "___Packing the artifacts___"
+    - mkdir -p ./artifacts
+    - mv ./target/release/polkadot-parachain ./artifacts/.
+    - echo "___The VERSION is either a tag name or the curent branch if triggered not by a tag___"
+    - echo ${CI_COMMIT_REF_NAME} | tee ./artifacts/VERSION
+
+build-test-parachain:
+  stage:                           build
+  extends:
+    - .docker-env
+    - .common-refs
+    - .collect-artifacts
+  variables:
+    # Enable debug assertions since we are running optimized builds for testing
+    # but still want to have debug assertions.
+    RUSTFLAGS:                     "-Cdebug-assertions=y -Dwarnings"
+  # this is an artificial job dependency, for pipeline optimization using GitLab's DAGs
+  needs:
+    - job:                         check-rustdoc
+      artifacts:                   false
+  script:
+    - echo "___Building a binary, please refrain from using it in production since it goes with the debug assertions.___"
+    - time cargo +nightly build --release --locked --bin test-parachain
+    - echo "___Packing the artifacts___"
+    - mkdir -p ./artifacts
+    - mv ./target/release/test-parachain ./artifacts/.
+    - mkdir -p ./artifacts/zombienet
+    - mv ./target/release/wbuild/cumulus-test-runtime/wasm_binary_spec_version_incremented.rs.compact.compressed.wasm ./artifacts/zombienet/.
+
+# build runtime only if files in $RUNTIME_PATH/$RUNTIME_NAME were changed
+.build-runtime-template:           &build-runtime-template
+  stage:                           build
+  extends:
+    - .docker-env
+    - .pr-refs
+  # this is an artificial job dependency, for pipeline optimization using GitLab's DAGs
+  needs:
+    - job:                         check-rustdoc
+      artifacts:                   false
+  variables:
+    RUNTIME_PATH:                  "parachains/runtimes/assets"
+  script:
+    - cd ${RUNTIME_PATH}
+    - for directory in $(echo */); do
+        echo "_____Running cargo check for ${directory} ______";
+        cd ${directory};
+        pwd;
+        SKIP_WASM_BUILD=1 cargo check;
+        cd ..;
+      done
+
+# DAG: build-runtime-assets -> build-runtime-collectives -> build-runtime-contracts
+# DAG: build-runtime-assets ->build-runtime-starters -> build-runtime-testing
+build-runtime-assets:
+  <<: *build-runtime-template
+  variables:
+    RUNTIME_PATH:                  "parachains/runtimes/assets"
+
+build-runtime-collectives:
+  <<: *build-runtime-template
+  variables:
+    RUNTIME_PATH:                  "parachains/runtimes/collectives"
+  # this is an artificial job dependency, for pipeline optimization using GitLab's DAGs
+  needs:
+    - job:                         build-runtime-assets
+      artifacts:                   false
+
+build-runtime-contracts:
+  <<: *build-runtime-template
+  variables:
+    RUNTIME_PATH:                  "parachains/runtimes/contracts"
+  # this is an artificial job dependency, for pipeline optimization using GitLab's DAGs
+  needs:
+    - job:                         build-runtime-collectives
+      artifacts:                   false
+
+build-runtime-starters:
+  <<: *build-runtime-template
+  variables:
+    RUNTIME_PATH:                  "parachains/runtimes/starters"
+  # this is an artificial job dependency, for pipeline optimization using GitLab's DAGs
+  needs:
+    - job:                         build-runtime-assets
+      artifacts:                   false
+
+build-runtime-testing:
+  <<: *build-runtime-template
+  variables:
+    RUNTIME_PATH:                  "parachains/runtimes/testing"
+  # this is an artificial job dependency, for pipeline optimization using GitLab's DAGs
+  needs:
+    - job:                         build-runtime-starters
+      artifacts:                   false

scripts/ci/gitlab/pipeline/integration_tests.yml

@@ -0,0 +1,2 @@
+# This file is part of .gitlab-ci.yml
+# Here are all jobs that are executed during "integration_stage" stage

scripts/ci/gitlab/pipeline/publish.yml

@@ -0,0 +1,117 @@
+# This file is part of .gitlab-ci.yml
+# Here are all jobs that are executed during "publish" stage
+
+.build-push-image:
+  image:                           quay.io/buildah/stable
+  variables:
+    DOCKERFILE:                    "" # docker/path-to.Dockerfile
+    IMAGE_NAME:                    "" # docker.io/paritypr/image_name
+    VERSION:                       "${CI_COMMIT_REF_NAME}-${CI_COMMIT_SHORT_SHA}"
+  script:
+    - test "$PARITYPR_USER" -a "$PARITYPR_PASS" ||
+        ( echo "no docker credentials provided"; exit 1 )
+    - buildah bud
+        --format=docker
+        --build-arg VCS_REF="${CI_COMMIT_SHA}"
+        --build-arg BUILD_DATE="$(date -u '+%Y-%m-%dT%H:%M:%SZ')"
+        --build-arg IMAGE_NAME="${IMAGE_NAME}"
+        --tag "$IMAGE_NAME:$VERSION"
+        --file ${DOCKERFILE} .
+    - echo "$PARITYPR_PASS" |
+        buildah login --username "$PARITYPR_USER" --password-stdin docker.io
+    - buildah info
+    - buildah push --format=v2s2 "$IMAGE_NAME:$VERSION"
+  after_script:
+    - buildah logout --all
+
+build-push-image-polkadot-parachain-debug:
+  stage:                           publish
+  extends:
+    - .kubernetes-env
+    - .common-refs
+    - .build-push-image
+  needs:
+    - job:                         build-linux-stable
+      artifacts:                   true
+  variables:
+    DOCKERFILE:                    "docker/polkadot-parachain-debug_unsigned_injected.Dockerfile"
+    IMAGE_NAME:                    "docker.io/paritypr/polkadot-parachain-debug"
+    VERSION:                       "${CI_COMMIT_REF_NAME}-${CI_COMMIT_SHORT_SHA}"
+
+build-push-image-test-parachain:
+  stage:                           publish
+  extends:
+    - .kubernetes-env
+    - .common-refs
+    - .build-push-image
+  needs:
+    - job:                         build-test-parachain
+      artifacts:                   true
+  variables:
+    DOCKERFILE:                    "docker/test-parachain_injected.Dockerfile"
+    IMAGE_NAME:                    "docker.io/paritypr/test-parachain"
+    VERSION:                       "${CI_COMMIT_REF_NAME}-${CI_COMMIT_SHORT_SHA}"
+
+publish-s3:
+  stage:                           publish
+  extends:
+    - .kubernetes-env
+    - .publish-refs
+  image:                           paritytech/awscli:latest
+  needs:
+    - job:                         build-linux-stable
+      artifacts:                   true
+  variables:
+    GIT_STRATEGY:                  none
+    BUCKET:                        "releases.parity.io"
+    PREFIX:                        "cumulus/${ARCH}-${DOCKER_OS}"
+  script:
+    - echo "___Publishing a binary with debug assertions!___"
+    - echo "___VERSION = $(cat ./artifacts/VERSION) ___"
+    - aws s3 sync ./artifacts/ s3://${BUCKET}/${PREFIX}/$(cat ./artifacts/VERSION)/
+    - echo "___Updating objects in latest path___"
+    - aws s3 sync s3://${BUCKET}/${PREFIX}/$(cat ./artifacts/VERSION)/ s3://${BUCKET}/${PREFIX}/latest/
+  after_script:
+    - aws s3 ls s3://${BUCKET}/${PREFIX}/latest/
+        --recursive --human-readable --summarize
+
+publish-benchmarks-assets-s3:      &publish-benchmarks
+  stage:                           publish
+  extends:
+    - .kubernetes-env
+    - .benchmarks-refs
+  image:                           paritytech/awscli:latest
+  needs:
+    - job:                         benchmarks-assets
+      artifacts:                   true
+  variables:
+    GIT_STRATEGY:                  none
+    BUCKET:                        "releases.parity.io"
+    PREFIX:                        "cumulus/$CI_COMMIT_REF_NAME/benchmarks"
+  script:
+    - echo "___Removing binary from artifacts___"
+    - rm -f ./artifacts/polkadot-parachain
+    - echo "___Publishing benchmark results___"
+    - aws s3 sync ./artifacts/ s3://${BUCKET}/${PREFIX}/
+  after_script:
+    - aws s3 ls s3://${BUCKET}/${PREFIX}/ --recursive --human-readable --summarize
+
+publish-benchmarks-collectives-s3:
+  <<:                              *publish-benchmarks
+  needs:
+    - job:                         benchmarks-collectives
+      artifacts:                   true
+
+# Ref: https://github.com/paritytech/opstooling/issues/111
+update-parachain-template:
+  stage:                           publish
+  extends:                         .kubernetes-env
+  rules:
+    - if: $CI_COMMIT_REF_NAME =~ /^polkadot-v[0-9]+\.[0-9]+.*$/ # i.e. polkadot-v1.0.99, polkadot-v2.1rc1
+  script:
+    - git clone --depth=1 --branch="$PIPELINE_SCRIPTS_TAG" https://github.com/paritytech/pipeline-scripts
+    - ./pipeline-scripts/update_substrate_template.sh
+      --repo-name "substrate-parachain-template"
+      --template-path "parachain-template"
+      --github-api-token "$GITHUB_TOKEN"
+      --polkadot-branch "$CI_COMMIT_REF_NAME"

scripts/ci/gitlab/pipeline/test.yml

@@ -0,0 +1,79 @@
+# This file is part of .gitlab-ci.yml
+# Here are all jobs that are executed during "test" stage
+
+test-linux-stable:
+  stage:                           test
+  extends:
+    - .docker-env
+    - .common-refs
+    - .pipeline-stopper-artifacts
+  before_script:
+    - !reference [.docker-env, before_script]
+    - !reference [.pipeline-stopper-vars, before_script]
+  variables:
+    # Enable debug assertions since we are running optimized builds for testing
+    # but still want to have debug assertions.
+    RUSTFLAGS: "-Cdebug-assertions=y -Dwarnings"
+  script:
+    - time cargo nextest run --all --release --locked --run-ignored all
+
+test-doc:
+  stage:                           test
+  extends:
+    - .docker-env
+    - .common-refs
+  variables:
+    # Enable debug assertions since we are running optimized builds for testing
+    # but still want to have debug assertions.
+    RUSTFLAGS: "-Cdebug-assertions=y -Dwarnings"
+  script:
+    - time cargo test --doc
+
+check-runtime-benchmarks:
+  stage:                           test
+  extends:
+    - .docker-env
+    - .common-refs
+  script:
+    # Check that the node will compile with `runtime-benchmarks` feature flag.
+    - time cargo check --all --features runtime-benchmarks
+    # Check that parachain-template will compile with `runtime-benchmarks` feature flag.
+    - time cargo check -p parachain-template-node --features runtime-benchmarks
+
+cargo-check-try-runtime:
+  stage:                           test
+  extends:
+    - .docker-env
+    - .common-refs
+  # this is an artificial job dependency, for pipeline optimization using GitLab's DAGs
+  needs:
+    - job:                         check-runtime-benchmarks
+      artifacts:                   false
+  script:
+    # Check that the node will compile with `try-runtime` feature flag.
+    - time cargo check --all --features try-runtime
+    # Check that parachain-template will compile with `try-runtime` feature flag.
+    - time cargo check -p parachain-template-node --features try-runtime
+
+check-rustdoc:
+  stage:                           test
+  extends:
+    - .docker-env
+    - .common-refs
+  variables:
+    SKIP_WASM_BUILD:               1
+    RUSTDOCFLAGS:                  "-Dwarnings"
+  script:
+    - time cargo +nightly doc --workspace --all-features --verbose --no-deps
+
+cargo-check-benches:
+  stage:                           test
+  extends:
+    - .docker-env
+    - .common-refs
+  # this is an artificial job dependency, for pipeline optimization using GitLab's DAGs
+  needs:
+    - job:                         check-rustdoc
+      artifacts:                   false
+  script:
+    - time cargo check --all --benches

scripts/ci/gitlab/pipeline/zombienet.yml

@@ -0,0 +1,101 @@
+# This file is part of .gitlab-ci.yml
+# Here are all jobs that are executed during "zombienet" stage
+
+.zombienet-before-script:
+  before_script:
+    - echo "Zombie-net Tests Config"
+    - echo "${ZOMBIENET_IMAGE}"
+    - echo "${RELAY_IMAGE}"
+    - echo "${COL_IMAGE}"
+    - echo "${GH_DIR}"
+    - export DEBUG=zombie
+    - export RELAY_IMAGE=${POLKADOT_IMAGE}
+    - export COL_IMAGE=${COL_IMAGE}
+
+# common settings for all zombienet jobs
+.zombienet-common:
+  stage:                           zombienet
+  image:                           "${ZOMBIENET_IMAGE}"
+  needs:
+    - job:                         build-push-image-test-parachain
+      artifacts:                   true
+  variables:
+    POLKADOT_IMAGE:                "docker.io/paritypr/polkadot-debug:master"
+    GH_DIR:                        "https://github.com/paritytech/cumulus/tree/${CI_COMMIT_SHORT_SHA}/zombienet_tests"
+    COL_IMAGE:                     "docker.io/paritypr/test-parachain:${CI_COMMIT_REF_NAME}-${CI_COMMIT_SHORT_SHA}"
+  allow_failure:                   true
+  retry: 2
+  tags:
+    - zombienet-polkadot-integration-test
+
+zombienet-0001-sync_blocks_from_tip_without_connected_collator:
+  extends:
+    - .zombienet-common
+    - .zombienet-refs
+    - .zombienet-before-script
+  script:
+    - /home/nonroot/zombie-net/scripts/ci/run-test-env-manager.sh
+        --github-remote-dir="${GH_DIR}"
+        --concurrency=1
+        --test="0001-sync_blocks_from_tip_without_connected_collator.feature"
+
+zombienet-0002-pov_recovery:
+  extends:
+    - .zombienet-common
+    - .zombienet-refs
+    - .zombienet-before-script
+  script:
+    - /home/nonroot/zombie-net/scripts/ci/run-test-env-manager.sh
+        --github-remote-dir="${GH_DIR}"
+        --concurrency=1
+        --test="0002-pov_recovery.feature"
+
+zombienet-0003-full_node_catching_up:
+  extends:
+    - .zombienet-common
+    - .zombienet-refs
+    - .zombienet-before-script
+  script:
+    - /home/nonroot/zombie-net/scripts/ci/run-test-env-manager.sh
+        --github-remote-dir="${GH_DIR}"
+        --concurrency=1
+        --test="0003-full_node_catching_up.feature"
+
+zombienet-0004-runtime_upgrade:
+  extends:
+    - .zombienet-common
+    - .zombienet-refs
+    - .zombienet-before-script
+  needs:
+    - !reference [.zombienet-common, needs]
+    - job:                         build-test-parachain
+      artifacts:                   true
+  before_script:
+    - ls -ltr *
+    - cp ./artifacts/zombienet/wasm_binary_spec_version_incremented.rs.compact.compressed.wasm /tmp/
+    - ls /tmp
+    - !reference [.zombienet-before-script, before_script]
+  script:
+    - /home/nonroot/zombie-net/scripts/ci/run-test-env-manager.sh
+        --github-remote-dir="${GH_DIR}"
+        --concurrency=1
+        --test="0004-runtime_upgrade.feature"
+
+zombienet-0005-migrate_solo_to_para:
+  extends:
+    - .zombienet-common
+    - .zombienet-refs
+    - .zombienet-before-script
+  needs:
+    - !reference [.zombienet-common, needs]
+    - job:                         build-test-parachain
+      artifacts:                   true
+  before_script:
+    - ls -ltr *
+    - !reference [.zombienet-before-script, before_script]
+  script:
+    - /home/nonroot/zombie-net/scripts/ci/run-test-env-manager.sh
+        --github-remote-dir="${GH_DIR}"
+        --concurrency=1
+        --test="0005-migrate_solo_to_para.feature"
+