CI: optimizations (#2885)

* CI: do not publish unnecessary things to s3 and to release registry

* CI: remove the build-linux-release job

* CI: publish less

* CI: workspace = all

* CI: more logs for the pipeline trigger

* CI: chore

* CI: nicer

* CI: new bucket for bins

* CI: no need pushing every master image to product registry

* CI: default variables in image jobs

* CI: optimize the bin build

polkadot/.gitlab-ci.yml

@@ -109,23 +109,39 @@ test-deterministic-wasm:
   script:
     - ./scripts/gitlab/test_deterministic_wasm.sh
 
-test-linux-stable:
+test-build-linux-stable:
   stage:                           test
-  <<:                              *rules-test
   <<:                              *docker-env
   <<:                              *compiler-info
+  <<:                              *collect-artifacts
   variables:
     RUST_TOOLCHAIN: stable
     # Enable debug assertions since we are running optimized builds for testing
     # but still want to have debug assertions.
     RUSTFLAGS: "-Cdebug-assertions=y -Dwarnings"
-    TARGET: native
-  artifacts:
-    paths:
-      - ./target/release/polkadot
+  rules:
+    - if: $CI_PIPELINE_SOURCE == "schedule"
+    - if: $CI_COMMIT_REF_NAME == "master"
+    - if: $CI_COMMIT_REF_NAME =~ /^v[0-9]+\.[0-9]+.*$/              # i.e. v1.0, v2.1rc1
+    - if: $CI_COMMIT_REF_NAME =~ /^[0-9]+$/                         # PRs
+    - if: $CI_COMMIT_REF_NAME == "rococo-v1"
   script:
     - ./scripts/gitlab/test_linux_stable.sh
+    # we're using the bin built here, instead of having a parallel `build-linux-release`
+    - time cargo build --release --verbose --bin polkadot
     - sccache -s
+    # pack-artifacts
+    - mkdir -p ./artifacts
+    - VERSION="${CI_COMMIT_REF_NAME}" # will be tag or branch name
+    - mv ./target/release/polkadot ./artifacts/.
+    - sha256sum ./artifacts/polkadot | tee ./artifacts/polkadot.sha256
+    - EXTRATAG="$(./artifacts/polkadot --version |
+        sed -n -r 's/^polkadot ([0-9.]+.*-[0-9a-f]{7,13})-.*$/\1/p')"
+    - EXTRATAG="${CI_COMMIT_REF_NAME}-${EXTRATAG}-$(cut -c 1-8 ./artifacts/polkadot.sha256)"
+    - echo "Polkadot version = ${VERSION} (EXTRATAG = ${EXTRATAG})"
+    - echo -n ${VERSION} > ./artifacts/VERSION
+    - echo -n ${EXTRATAG} > ./artifacts/EXTRATAG
+    - cp -r scripts/docker/* ./artifacts
 
 check-web-wasm:
   stage:                           test
@@ -148,43 +164,6 @@ check-runtime-benchmarks:
     - ./scripts/gitlab/check_runtime_benchmarks.sh
     - sccache -s
 
-.pack-artifacts:                   &pack-artifacts
-  - mkdir -p ./artifacts
-  - VERSION="${CI_COMMIT_REF_NAME}" # will be tag or branch name
-  - mv ./target/release/polkadot ./artifacts/.
-  - sha256sum ./artifacts/polkadot | tee ./artifacts/polkadot.sha256
-  - if [ "${CI_COMMIT_TAG}" ]; then
-      EXTRATAG="latest";
-    else
-      EXTRATAG="$(./artifacts/polkadot --version |
-        sed -n -r 's/^polkadot ([0-9.]+.*-[0-9a-f]{7,13})-.*$/\1/p')";
-      EXTRATAG="${CI_COMMIT_REF_NAME}-${EXTRATAG}-$(cut -c 1-8 ./artifacts/polkadot.sha256)";
-    fi
-  - echo "Polkadot version = ${VERSION} (EXTRATAG = ${EXTRATAG})"
-  - echo -n ${VERSION} > ./artifacts/VERSION
-  - echo -n ${EXTRATAG} > ./artifacts/EXTRATAG
-  - cp -r scripts/docker/* ./artifacts
-
-build-linux-release:
-  stage:                           test
-  <<:                              *collect-artifacts
-  <<:                              *docker-env
-  <<:                              *compiler-info
-  rules:
-    - if: $CI_PIPELINE_SOURCE == "schedule"
-    - if: $CI_COMMIT_REF_NAME == "master"
-    - if: $CI_COMMIT_REF_NAME =~ /^v[0-9]+\.[0-9]+.*$/              # i.e. v1.0, v2.1rc1
-    - if: $CI_COMMIT_REF_NAME =~ /^[0-9]+$/                         # PRs
-      variables:
-        RUSTFLAGS:                     "-Cdebug-assertions=y"
-    - if: $CI_COMMIT_REF_NAME == "rococo-v1"
-      variables:
-        RUSTFLAGS:                     "-Cdebug-assertions=y"
-  script:
-    - time cargo build --release --verbose
-    - sccache -s
-    - *pack-artifacts
-
 build-adder-collator:
   stage:                           test
   <<:                              *collect-artifacts
@@ -213,7 +192,7 @@ check-transaction-versions:
   <<:                              *rules-test
   <<:                              *docker-env
   needs:
-    - job:                         test-linux-stable
+    - job:                         test-build-linux-stable
       artifacts:                   true
   before_script:
     - apt-get -y update; apt-get -y install jq lsof
@@ -235,19 +214,16 @@ generate-impl-guide:
 .build-push-image:                 &build-push-image
   <<:                              *kubernetes-env
   image:                           quay.io/buildah/stable
-  variables:
+  variables:                       &image-variables
     GIT_STRATEGY:                  none
-    # scripts/docker/Dockerfile
-    DOCKERFILE:                    Dockerfile
-    IMAGE_NAME:                    docker.io/parity/polkadot
-    DOCKER_USER:                   ${Docker_Hub_User_Parity}
-    DOCKER_PASS:                   ${Docker_Hub_Pass_Parity}
+    DOCKER_USER:                   ${PARITYPR_USER}
+    DOCKER_PASS:                   ${PARITYPR_PASS}
   before_script:                   &check-versions
     - test -s ./artifacts/VERSION || exit 1
     - test -s ./artifacts/EXTRATAG || exit 1
     - VERSION="$(cat ./artifacts/VERSION)"
     - EXTRATAG="$(cat ./artifacts/EXTRATAG)"
-    - echo "Polkadot version = ${VERSION} (EXTRATAG ${EXTRATAG})"
+    - echo "Polkadot version = ${VERSION} (EXTRATAG = ${EXTRATAG})"
   script:
     - test "$DOCKER_USER" -a "$DOCKER_PASS" ||
         ( echo "no docker credentials provided"; exit 1 )
@@ -280,6 +256,11 @@ generate-impl-guide:
 publish-polkadot-image:
   stage:                           build
   <<:                              *build-push-image
+  variables:
+    <<:                            *image-variables
+    # scripts/docker/Dockerfile
+    DOCKERFILE:                    Dockerfile
+    IMAGE_NAME:                    docker.io/paritypr/synth-wave
   rules:
   # Don't run on releases - this is handled by the Github Action here:
   # .github/workflows/publish-docker-release.yml
@@ -288,15 +269,14 @@ publish-polkadot-image:
     - if: $CI_PIPELINE_SOURCE == "schedule"
     - if: $CI_COMMIT_REF_NAME == "master"
     - if: $CI_COMMIT_REF_NAME =~ /^[0-9]+$/                         # PRs
-      variables:
-        IMAGE_NAME:                    docker.io/paritypr/synth-wave
-        DOCKER_USER:                   ${PARITYPR_USER}
-        DOCKER_PASS:                   ${PARITYPR_PASS}
     - if: $CI_COMMIT_REF_NAME == "rococo-v1"
       variables:
+        <<:                        *image-variables
         IMAGE_NAME:                docker.io/parity/rococo
+        DOCKER_USER:               ${Docker_Hub_User_Parity}
+        DOCKER_PASS:               ${Docker_Hub_Pass_Parity}
   needs:
-    - job:                         build-linux-release
+    - job:                         test-build-linux-stable
       artifacts:                   true
 
 publish-adder-collator-image:
@@ -304,15 +284,13 @@ publish-adder-collator-image:
   stage:                           build
   <<:                              *build-push-image
   variables:
+    <<:                            *image-variables
     # scripts/docker/collator.Dockerfile
     DOCKERFILE:                    collator.Dockerfile
     IMAGE_NAME:                    docker.io/paritypr/colander
-    DOCKER_USER:                   ${PARITYPR_USER}
-    DOCKER_PASS:                   ${PARITYPR_PASS}
   rules:
     - if: $CI_PIPELINE_SOURCE == "schedule"
     - if: $CI_COMMIT_REF_NAME == "master"
-    # FIXME: remove me after merging
     - if: $CI_COMMIT_REF_NAME =~ /^[0-9]+$/                         # PRs
   needs:
     - job:                         build-adder-collator
@@ -332,51 +310,41 @@ publish-adder-collator-image:
 publish-s3-release:                &publish-s3
   stage:                           publish
   needs:
-    - job:                         build-linux-release
+    - job:                         test-build-linux-stable
       artifacts:                   true
   <<:                              *kubernetes-env
   image:                           paritytech/awscli:latest
   variables:
     GIT_STRATEGY:                  none
-    BUCKET:                        "releases.parity.io"
-    PREFIX:                        "polkadot/${ARCH}-${DOCKER_OS}"
+    PREFIX:                        "builds/polkadot/${ARCH}-${DOCKER_OS}"
   rules:
+    # publishing binaries nightly
     - if: $CI_PIPELINE_SOURCE == "schedule"
-    - if: $CI_COMMIT_REF_NAME == "master"
-    - if: $CI_COMMIT_REF_NAME =~ /^v[0-9]+\.[0-9]+.*$/              # i.e. v1.0, v2.1rc1
   before_script:
     - *check-versions
   script:
-    - echo "uploading objects to https://${BUCKET}/${PREFIX}/${VERSION}"
-    - aws s3 sync ./artifacts/ s3://${BUCKET}/${PREFIX}/${VERSION}/
-    - echo "update objects at https://${BUCKET}/${PREFIX}/${EXTRATAG}"
+    - echo "uploading objects to https://releases.parity.io/${PREFIX}/${VERSION}"
+    - aws s3 sync --acl public-read ./artifacts/ s3://${AWS_BUCKET}/${PREFIX}/${VERSION}/
+    - echo "update objects at https://releases.parity.io/${PREFIX}/${EXTRATAG}"
     - find ./artifacts -type f | while read file; do
         name="${file#./artifacts/}";
         aws s3api copy-object
-        --copy-source ${BUCKET}/${PREFIX}/${VERSION}/${name}
-        --bucket ${BUCKET} --key ${PREFIX}/${EXTRATAG}/${name};
+          --copy-source ${AWS_BUCKET}/${PREFIX}/${VERSION}/${name}
+          --bucket ${AWS_BUCKET} --key ${PREFIX}/${EXTRATAG}/${name};
       done
     - |
       cat <<-EOM
       |
       |  polkadot binary paths:
       |
-      |  - https://${BUCKET}/${PREFIX}/${EXTRATAG}/polkadot
-      |  - https://${BUCKET}/${PREFIX}/${VERSION}/polkadot
+      |  - https://releases.parity.io/${PREFIX}/${EXTRATAG}/polkadot
+      |  - https://releases.parity.io/${PREFIX}/${VERSION}/polkadot
       |
       EOM
   after_script:
-    - aws s3 ls s3://${BUCKET}/${PREFIX}/${EXTRATAG}/
+    - aws s3 ls s3://${AWS_BUCKET}/${PREFIX}/${EXTRATAG}/
         --recursive --human-readable --summarize
 
-publish-s3-adder-collator:
-  <<:                              *publish-s3
-  variables:
-    PREFIX:                        "rococo/${ARCH}-${DOCKER_OS}"
-  needs:
-    - job:                         build-adder-collator
-      artifacts:                   true
-
 #### stage:                        deploy
 
 deploy-polkasync-kusama:

polkadot/scripts/gitlab/test_linux_stable.sh

@@ -4,4 +4,4 @@ set -e
 #shellcheck source=../common/lib.sh
 source "$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )/../common/lib.sh"
 
-time cargo test --all --release --verbose --locked --features=runtime-benchmarks
+time cargo test --workspace --release --verbose --locked --features=runtime-benchmarks

polkadot/scripts/gitlab/trigger_pipeline.sh

@@ -1,7 +1,10 @@
 #!/bin/bash
 
 set -eu
+
 # API trigger another project's pipeline
+echo "Triggering Simnet pipeline."
+
 curl --silent \
     -X POST \
     -F "token=${CI_JOB_TOKEN}" \
@@ -12,13 +15,14 @@ curl --silent \
     -F "variables[IMAGE_TAG]=${IMAGE_TAG}" \
     -F "variables[COLLATOR_IMAGE_TAG]=${COLLATOR_IMAGE_TAG}" \
     "https://${CI_SERVER_HOST}/api/v4/projects/${DWNSTRM_ID}/trigger/pipeline" | \
-        tee pipeline
+        tee pipeline;
 
 PIPELINE_ID=$(cat pipeline | jq ".id")
-echo "\nWaiting on ${PIPELINE_ID} status..."
+PIPELINE_URL=$(cat pipeline | jq ".web_url")
+echo
+echo "Simnet pipeline ${PIPELINE_URL} was successfully triggered."
+echo "Now we're polling it to obtain the distinguished status."
 
-# This part polls for the triggered pipeline status, the native
-# `trigger` job does not return this status via API.
 # This is a workaround for a Gitlab bug, waits here until
 # https://gitlab.com/gitlab-org/gitlab/-/issues/326137 gets fixed.
 # The timeout is 360 curls with 8 sec interval, roughly an hour.
@@ -30,17 +34,19 @@ function get_status() {
             jq --raw-output ".status";
 }
 
+echo "Waiting on ${PIPELINE_ID} status..."
+
 for i in $(seq 1 360); do
     STATUS=$(get_status);
     echo "Triggered pipeline status is ${STATUS}";
     if [[ ${STATUS} =~ ^(pending|running|created)$ ]]; then
-        echo "Busy...";
+        echo;
     elif [[ ${STATUS} =~ ^(failed|canceled|skipped|manual)$ ]]; then
-        exit 1;
+        echo "Something's broken in: ${PIPELINE_URL}"; exit 1;
     elif [[ ${STATUS} =~ ^(success)$ ]]; then
-        exit 0;
+        echo "Look how green it is: ${PIPELINE_URL}"; exit 0;
     else
-        exit 1;
+        echo "Something else has happened in ${PIPELINE_URL}"; exit 1;
     fi
 sleep 8;
 done