From 4dcbac4b58ff3499f0498735e7d4ae45c279b85e Mon Sep 17 00:00:00 2001 From: Denis Pisarev Date: Fri, 16 Apr 2021 21:35:08 +0200 Subject: [PATCH] CI updates (#8633) * CI: opt bench jobs; add bench to triggered job * CI: no need in manual builds; build=publish * CI: more logs to the trigger job * CI: DAGs and louts --- substrate/.gitlab-ci.yml | 47 +++++++------------ .../.maintain/gitlab/trigger_pipeline.sh | 19 +++++--- 2 files changed, 29 insertions(+), 37 deletions(-) diff --git a/substrate/.gitlab-ci.yml b/substrate/.gitlab-ci.yml index c0d7838014..9de2f79b03 100644 --- a/substrate/.gitlab-ci.yml +++ b/substrate/.gitlab-ci.yml @@ -105,18 +105,6 @@ default: - if: $CI_COMMIT_REF_NAME =~ /^[0-9]+$/ # PRs .build-refs: &build-refs - rules: - # .publish-refs with manual on PRs - - if: $CI_PIPELINE_SOURCE == "pipeline" - when: never - - if: $CI_PIPELINE_SOURCE == "web" - - if: $CI_COMMIT_REF_NAME == "master" - - if: $CI_COMMIT_REF_NAME =~ /^v[0-9]+\.[0-9]+.*$/ # i.e. v1.0, v2.1rc1 - - if: $CI_COMMIT_REF_NAME =~ /^[0-9]+$/ # PRs - when: manual - allow_failure: true - -.publish-refs: &publish-refs rules: - if: $CI_PIPELINE_SOURCE == "pipeline" when: never @@ -131,11 +119,13 @@ default: - if: $CI_COMMIT_REF_NAME == "master" && $CI_PIPELINE_SOURCE == "schedule" && $PIPELINE == "nightly" .merge-ref-into-master-script: &merge-ref-into-master-script - - git fetch origin +master:master - - git fetch origin +$CI_COMMIT_REF_NAME:$CI_COMMIT_REF_NAME - - git checkout master - - git config user.email "ci@gitlab.parity.io" - - git merge $CI_COMMIT_REF_NAME --verbose --no-edit + - if [ $CI_COMMIT_REF_NAME != "master" ]; then + git fetch origin +master:master; + git fetch origin +$CI_COMMIT_REF_NAME:$CI_COMMIT_REF_NAME; + git checkout master; + git config user.email "ci@gitlab.parity.io"; + git merge $CI_COMMIT_REF_NAME --verbose --no-edit; + fi .cargo-check-benches-script: &cargo-check-benches-script - mkdir -p artifacts/benches/$CI_COMMIT_REF_NAME-$CI_COMMIT_SHORT_SHA @@ -244,17 +234,10 @@ cargo-deny: cargo-check-benches: stage: test <<: *docker-env - <<: *test-refs-no-trigger - <<: *collect-artifacts - script: - - *cargo-check-benches-script - -cargo-check-benches-merged: - stage: test - <<: *docker-env - <<: *test-refs-no-trigger-prs-only + <<: *test-refs <<: *collect-artifacts before_script: + # merges in the master branch on PRs - *merge-ref-into-master-script - *rust-info-script script: @@ -268,8 +251,10 @@ node-bench-regression-guard: <<: *docker-env <<: *test-refs-no-trigger-prs-only needs: - - job: cargo-check-benches-merged + # this is a DAG + - job: cargo-check-benches artifacts: true + # this does not like a DAG, just polls the artifact - project: $CI_PROJECT_PATH job: cargo-check-benches ref: master @@ -550,7 +535,7 @@ build-rust-doc: #### stage: publish .build-push-docker-image: &build-push-docker-image - <<: *publish-refs + <<: *build-refs <<: *kubernetes-build image: quay.io/buildah/stable variables: &docker-build-vars @@ -612,7 +597,7 @@ publish-docker-subkey: publish-s3-release: stage: publish - <<: *publish-refs + <<: *build-refs <<: *kubernetes-build needs: - job: build-linux-substrate @@ -641,7 +626,7 @@ publish-s3-doc: artifacts: true - job: build-linux-substrate artifacts: false - <<: *publish-refs + <<: *build-refs <<: *kubernetes-build variables: GIT_STRATEGY: none @@ -713,6 +698,8 @@ trigger-simnet: stage: deploy image: paritytech/tools:latest rules: + - if: $CI_PIPELINE_SOURCE == "pipeline" + when: never - if: $CI_PIPELINE_SOURCE == "web" && $CI_COMMIT_REF_NAME == "master" - if: $CI_COMMIT_REF_NAME == "master" needs: diff --git a/substrate/.maintain/gitlab/trigger_pipeline.sh b/substrate/.maintain/gitlab/trigger_pipeline.sh index a2678bfa48..dd9da8102d 100755 --- a/substrate/.maintain/gitlab/trigger_pipeline.sh +++ b/substrate/.maintain/gitlab/trigger_pipeline.sh @@ -2,6 +2,8 @@ set -eu # API trigger another project's pipeline +echo "Triggering Simnet pipeline." + curl --silent \ -X POST \ -F "token=${CI_JOB_TOKEN}" \ @@ -14,10 +16,11 @@ curl --silent \ tee pipeline PIPELINE_ID=$(cat pipeline | jq ".id") -echo "\nWaiting on ${PIPELINE_ID} status..." +PIPELINE_URL=$(cat pipeline | jq ".web_url") +echo +echo "Simnet pipeline ${PIPELINE_URL} was successfully triggered." +echo "Now we're polling it to obtain the distinguished status." -# This part polls for the triggered pipeline status, the native -# `trigger` job does not return this status via API. # This is a workaround for a Gitlab bug, waits here until # https://gitlab.com/gitlab-org/gitlab/-/issues/326137 gets fixed. # The timeout is 360 curls with 8 sec interval, roughly an hour. @@ -29,17 +32,19 @@ function get_status() { jq --raw-output ".status"; } +echo "Waiting on ${PIPELINE_ID} status..." + for i in $(seq 1 360); do STATUS=$(get_status); echo "Triggered pipeline status is ${STATUS}"; if [[ ${STATUS} =~ ^(pending|running|created)$ ]]; then - echo "Busy..."; + echo "${STATUS}"..."; elif [[ ${STATUS} =~ ^(failed|canceled|skipped|manual)$ ]]; then - exit 1; + echo "Oh noes! Something's broken in: ${PIPELINE_URL}"; exit 1; elif [[ ${STATUS} =~ ^(success)$ ]]; then - exit 0; + echo "Look how green it is: ${PIPELINE_URL}"; exit 0; else - exit 1; + echo "Something else has happened in ${PIPELINE_URL}"; exit 1; fi sleep 8; done