CI updates (#8633)

* CI: opt bench jobs; add bench to triggered job

* CI: no need in manual builds; build=publish

* CI: more logs to the trigger job

* CI: DAGs and louts

substrate/.gitlab-ci.yml

@@ -105,18 +105,6 @@ default:
     - if: $CI_COMMIT_REF_NAME =~ /^[0-9]+$/                         # PRs
 
 .build-refs:                       &build-refs
-  rules:
-    # .publish-refs with manual on PRs
-    - if: $CI_PIPELINE_SOURCE == "pipeline"
-      when: never
-    - if: $CI_PIPELINE_SOURCE == "web"
-    - if: $CI_COMMIT_REF_NAME == "master"
-    - if: $CI_COMMIT_REF_NAME =~ /^v[0-9]+\.[0-9]+.*$/              # i.e. v1.0, v2.1rc1
-    - if: $CI_COMMIT_REF_NAME =~ /^[0-9]+$/                         # PRs
-      when: manual
-      allow_failure: true
-
-.publish-refs:                     &publish-refs
   rules:
     - if: $CI_PIPELINE_SOURCE == "pipeline"
       when: never
@@ -131,11 +119,13 @@ default:
     - if: $CI_COMMIT_REF_NAME == "master" && $CI_PIPELINE_SOURCE == "schedule" && $PIPELINE == "nightly"
 
 .merge-ref-into-master-script:     &merge-ref-into-master-script
-  - git fetch origin +master:master
+  - if [ $CI_COMMIT_REF_NAME != "master" ]; then
-  - git fetch origin +$CI_COMMIT_REF_NAME:$CI_COMMIT_REF_NAME
+      git fetch origin +master:master;
-  - git checkout master
+      git fetch origin +$CI_COMMIT_REF_NAME:$CI_COMMIT_REF_NAME;
-  - git config user.email "ci@gitlab.parity.io"
+      git checkout master;
-  - git merge $CI_COMMIT_REF_NAME --verbose --no-edit
+      git config user.email "ci@gitlab.parity.io";
+      git merge $CI_COMMIT_REF_NAME --verbose --no-edit;
+    fi
 
 .cargo-check-benches-script:       &cargo-check-benches-script
   - mkdir -p artifacts/benches/$CI_COMMIT_REF_NAME-$CI_COMMIT_SHORT_SHA
@@ -244,17 +234,10 @@ cargo-deny:
 cargo-check-benches:
   stage:                           test
   <<:                              *docker-env
-  <<:                              *test-refs-no-trigger
+  <<:                              *test-refs
-  <<:                              *collect-artifacts
-  script:
-    - *cargo-check-benches-script
-
-cargo-check-benches-merged:
-  stage:                           test
-  <<:                              *docker-env
-  <<:                              *test-refs-no-trigger-prs-only
   <<:                              *collect-artifacts
   before_script:
+    # merges in the master branch on PRs
     - *merge-ref-into-master-script
     - *rust-info-script
   script:
@@ -268,8 +251,10 @@ node-bench-regression-guard:
   <<:                              *docker-env
   <<:                              *test-refs-no-trigger-prs-only
   needs:
-    - job:                         cargo-check-benches-merged
+    # this is a DAG
+    - job:                         cargo-check-benches
       artifacts:                   true
+    # this does not like a DAG, just polls the artifact
     - project:                     $CI_PROJECT_PATH
       job:                         cargo-check-benches
       ref:                         master
@@ -550,7 +535,7 @@ build-rust-doc:
 #### stage:                        publish
 
 .build-push-docker-image:          &build-push-docker-image
-  <<:                              *publish-refs
+  <<:                              *build-refs
   <<:                              *kubernetes-build
   image:                           quay.io/buildah/stable
   variables:                       &docker-build-vars
@@ -612,7 +597,7 @@ publish-docker-subkey:
 
 publish-s3-release:
   stage:                           publish
-  <<:                              *publish-refs
+  <<:                              *build-refs
   <<:                              *kubernetes-build
   needs:
     - job:                         build-linux-substrate
@@ -641,7 +626,7 @@ publish-s3-doc:
       artifacts:                   true
     - job:                         build-linux-substrate
       artifacts:                   false
-  <<:                              *publish-refs
+  <<:                              *build-refs
   <<:                              *kubernetes-build
   variables:
     GIT_STRATEGY:                  none
@@ -713,6 +698,8 @@ trigger-simnet:
   stage:                           deploy
   image:                           paritytech/tools:latest
   rules:
+    - if: $CI_PIPELINE_SOURCE == "pipeline"
+      when: never
     - if: $CI_PIPELINE_SOURCE == "web" && $CI_COMMIT_REF_NAME == "master"
     - if: $CI_COMMIT_REF_NAME == "master"
   needs:

substrate/.maintain/gitlab/trigger_pipeline.sh

@@ -2,6 +2,8 @@
 
 set -eu
 # API trigger another project's pipeline
+echo "Triggering Simnet pipeline."
+
 curl --silent \
     -X POST \
     -F "token=${CI_JOB_TOKEN}" \
@@ -14,10 +16,11 @@ curl --silent \
         tee pipeline
 
 PIPELINE_ID=$(cat pipeline | jq ".id")
-echo "\nWaiting on ${PIPELINE_ID} status..."
+PIPELINE_URL=$(cat pipeline | jq ".web_url")
+echo
+echo "Simnet pipeline ${PIPELINE_URL} was successfully triggered."
+echo "Now we're polling it to obtain the distinguished status."
 
-# This part polls for the triggered pipeline status, the native
-# `trigger` job does not return this status via API.
 # This is a workaround for a Gitlab bug, waits here until
 # https://gitlab.com/gitlab-org/gitlab/-/issues/326137 gets fixed.
 # The timeout is 360 curls with 8 sec interval, roughly an hour.
@@ -29,17 +32,19 @@ function get_status() {
             jq --raw-output ".status";
 }
 
+echo "Waiting on ${PIPELINE_ID} status..."
+
 for i in $(seq 1 360); do
     STATUS=$(get_status);
     echo "Triggered pipeline status is ${STATUS}";
     if [[ ${STATUS} =~ ^(pending|running|created)$ ]]; then
-        echo "Busy...";
+        echo "${STATUS}"...";
     elif [[ ${STATUS} =~ ^(failed|canceled|skipped|manual)$ ]]; then
-        exit 1;
+        echo "Oh noes! Something's broken in: ${PIPELINE_URL}"; exit 1;
     elif [[ ${STATUS} =~ ^(success)$ ]]; then
-        exit 0;
+        echo "Look how green it is: ${PIPELINE_URL}"; exit 0;
     else
-        exit 1;
+        echo "Something else has happened in ${PIPELINE_URL}"; exit 1;
     fi
 sleep 8;
 done