From 82e52b8be8c7b1d82f626da81999ee6015156f79 Mon Sep 17 00:00:00 2001 From: Denis Pisarev Date: Fri, 19 Feb 2021 15:48:08 +0100 Subject: [PATCH] CI: test and update ci image, codeowners (#8142) * CI: prep for the new ff deployment * CI: variable for CI image * git: add CI team and remove Max from CODEOWNERS * CI: diener should be updated in CI image, not here. * CI: diener should be updated in CI image, not here. * CI: run cargo deny on changes to manifests and lock; run build jobs on schedules [skip ci] * CI: remove flaming-fir deployment, it will be handled from s3 updates [skip ci] * CI: trigger simnet with a certain substrate version * CI: remove cargo-audit in favor of cargo-deny; prepare for being triggered * CI: prepare to be triggered * CI: chore --- substrate/.gitlab-ci.yml | 141 +++++++----------- substrate/.maintain/flamingfir-deploy.sh | 35 ----- .../gitlab/check_polkadot_companion_build.sh | 2 - substrate/docs/CODEOWNERS | 12 +- 4 files changed, 62 insertions(+), 128 deletions(-) delete mode 100755 substrate/.maintain/flamingfir-deploy.sh diff --git a/substrate/.gitlab-ci.yml b/substrate/.gitlab-ci.yml index d87f6e3e98..9cd755bc79 100644 --- a/substrate/.gitlab-ci.yml +++ b/substrate/.gitlab-ci.yml @@ -39,6 +39,7 @@ variables: &default-vars CARGO_INCREMENTAL: 0 DOCKER_OS: "debian:stretch" ARCH: "x86_64" + CI_IMAGE: "paritytech/ci-linux:production" # FIXME set to release CARGO_UNLEASH_INSTALL_PARAMS: "--version 1.0.0-alpha.11" CARGO_UNLEASH_PKG_DEF: "--skip node node-* pallet-template pallet-example pallet-example-* subkey chain-spec-builder" @@ -60,7 +61,7 @@ default: interruptible: true .docker-env: &docker-env - image: paritytech/ci-linux:production + image: "${CI_IMAGE}" before_script: - rustup show - cargo --version @@ -84,12 +85,39 @@ default: - if: $CI_COMMIT_REF_NAME =~ /^[0-9]+$/ # PRs - if: $CI_COMMIT_REF_NAME =~ /^v[0-9]+\.[0-9]+.*$/ # i.e. v1.0, v2.1rc1 +.test-refs-no-trigger: &test-refs-no-trigger + rules: + - if: $CI_PIPELINE_SOURCE == "pipeline" + when: never + - if: $CI_PIPELINE_SOURCE == "web" + - if: $CI_PIPELINE_SOURCE == "schedule" + - if: $CI_COMMIT_REF_NAME == "master" + - if: $CI_COMMIT_REF_NAME == "tags" + - if: $CI_COMMIT_REF_NAME =~ /^[0-9]+$/ # PRs + - if: $CI_COMMIT_REF_NAME =~ /^v[0-9]+\.[0-9]+.*$/ # i.e. v1.0, v2.1rc1 + .build-refs: &build-refs rules: + # .publish-refs with manual on PRs + - if: $CI_PIPELINE_SOURCE == "pipeline" + when: never - if: $CI_PIPELINE_SOURCE == "web" - if: $CI_COMMIT_REF_NAME == "master" - if: $CI_COMMIT_REF_NAME == "tags" - if: $CI_COMMIT_REF_NAME =~ /^v[0-9]+\.[0-9]+.*$/ # i.e. v1.0, v2.1rc1 + - if: $CI_COMMIT_REF_NAME =~ /^[0-9]+$/ # PRs + when: manual + allow_failure: true + +.publish-refs: &publish-refs + rules: + - if: $CI_PIPELINE_SOURCE == "pipeline" + when: never + - if: $CI_PIPELINE_SOURCE == "web" + - if: $CI_PIPELINE_SOURCE == "schedule" + - if: $CI_COMMIT_REF_NAME == "master" + - if: $CI_COMMIT_REF_NAME == "tags" + - if: $CI_COMMIT_REF_NAME =~ /^v[0-9]+\.[0-9]+.*$/ # i.e. v1.0, v2.1rc1 #### stage: .pre @@ -145,6 +173,8 @@ test-dependency-rules: stage: check image: paritytech/tools:latest <<: *kubernetes-build + rules: + - if: $CI_COMMIT_REF_NAME =~ /^[0-9]+$/ # PRs script: - .maintain/ensure-deps.sh @@ -153,36 +183,28 @@ test-prometheus-alerting-rules: image: paritytech/tools:latest <<: *kubernetes-build rules: + - if: $CI_PIPELINE_SOURCE == "pipeline" + when: never - if: $CI_COMMIT_BRANCH changes: - .gitlab-ci.yml - .maintain/monitoring/**/* script: - promtool check rules .maintain/monitoring/alerting-rules/alerting-rules.yaml - - cat .maintain/monitoring/alerting-rules/alerting-rules.yaml | promtool test rules .maintain/monitoring/alerting-rules/alerting-rule-tests.yaml + - cat .maintain/monitoring/alerting-rules/alerting-rules.yaml | + promtool test rules .maintain/monitoring/alerting-rules/alerting-rule-tests.yaml #### stage: test -cargo-audit: - stage: test - <<: *docker-env - rules: - - if: $CI_COMMIT_REF_NAME =~ /^[0-9]+$/ # PRs - when: never - - if: $CI_PIPELINE_SOURCE == "web" - - if: $CI_PIPELINE_SOURCE == "schedule" - - if: $CI_COMMIT_REF_NAME == "master" - - if: $CI_COMMIT_REF_NAME =~ /^v[0-9]+\.[0-9]+.*$/ # i.e. v1.0, v2.1rc1 - script: - - cargo audit - allow_failure: true - cargo-deny: stage: test <<: *docker-env rules: - - if: $CI_COMMIT_MESSAGE =~ /skip-checks/ + - if: $CI_PIPELINE_SOURCE == "pipeline" when: never + - changes: + - "Cargo.lock" + - "**/Cargo.toml" - if: $CI_PIPELINE_SOURCE == "web" - if: $CI_PIPELINE_SOURCE == "schedule" - if: $CI_COMMIT_REF_NAME == "master" @@ -205,7 +227,7 @@ cargo-deny: cargo-check-benches: stage: test <<: *docker-env - <<: *test-refs + <<: *test-refs-no-trigger script: - SKIP_WASM_BUILD=1 time cargo +nightly check --benches --all - cargo run --release -p node-bench -- ::node::import::native::sr25519::transfer_keep_alive::paritydb::small @@ -262,9 +284,8 @@ unleash-check: stage: test <<: *docker-env rules: - - if: $CI_COMMIT_MESSAGE =~ /skip-checks/ + - if: $CI_PIPELINE_SOURCE == "pipeline" when: never - # .test-refs - if: $CI_COMMIT_REF_NAME == "master" - if: $CI_COMMIT_REF_NAME == "tags" - if: $CI_COMMIT_REF_NAME =~ /^v[0-9]+\.[0-9]+.*$/ # i.e. v1.0, v2.1rc1 @@ -281,7 +302,7 @@ test-frame-examples-compile-to-wasm: <<: *default-vars # Enable debug assertions since we are running optimized builds for testing # but still want to have debug assertions. - RUSTFLAGS: -Cdebug-assertions=y + RUSTFLAGS: "-Cdebug-assertions=y" RUST_BACKTRACE: 1 script: - cd frame/example-offchain-worker/ @@ -333,7 +354,7 @@ test-full-crypto-feature: <<: *default-vars # Enable debug assertions since we are running optimized builds for testing # but still want to have debug assertions. - RUSTFLAGS: -Cdebug-assertions=y + RUSTFLAGS: "-Cdebug-assertions=y" RUST_BACKTRACE: 1 script: - cd primitives/core/ @@ -346,7 +367,7 @@ cargo-check-macos: stage: test # shell runner on mac ignores the image set in *docker-env <<: *docker-env - <<: *test-refs + <<: *test-refs-no-trigger script: - SKIP_WASM_BUILD=1 time cargo check --release - sccache -s @@ -367,7 +388,7 @@ check-polkadot-companion-status: check-polkadot-companion-build: stage: build <<: *docker-env - <<: *test-refs + <<: *test-refs-no-trigger needs: - job: test-linux-stable-int artifacts: false @@ -396,15 +417,7 @@ build-linux-substrate: &build-binary stage: build <<: *collect-artifacts <<: *docker-env - rules: - # .build-refs with manual on PRs - - if: $CI_PIPELINE_SOURCE == "web" - - if: $CI_COMMIT_REF_NAME == "master" - - if: $CI_COMMIT_REF_NAME == "tags" - - if: $CI_COMMIT_REF_NAME =~ /^v[0-9]+\.[0-9]+.*$/ # i.e. v1.0, v2.1rc1 - - if: $CI_COMMIT_REF_NAME =~ /^[0-9]+$/ # PRs - when: manual - allow_failure: true + <<: *build-refs needs: - job: test-linux-stable artifacts: false @@ -431,15 +444,7 @@ build-linux-subkey: &build-subkey stage: build <<: *collect-artifacts <<: *docker-env - rules: - # .build-refs with manual on PRs - - if: $CI_PIPELINE_SOURCE == "web" - - if: $CI_COMMIT_REF_NAME == "master" - - if: $CI_COMMIT_REF_NAME == "tags" - - if: $CI_COMMIT_REF_NAME =~ /^v[0-9]+\.[0-9]+.*$/ # i.e. v1.0, v2.1rc1 - - if: $CI_COMMIT_REF_NAME =~ /^[0-9]+$/ # PRs - when: manual - allow_failure: true + <<: *build-refs needs: - job: cargo-check-subkey artifacts: false @@ -466,7 +471,7 @@ build-macos-subkey: #### stage: publish .build-push-docker-image: &build-push-docker-image - <<: *build-refs + <<: *publish-refs <<: *kubernetes-build image: quay.io/buildah/stable variables: &docker-build-vars @@ -499,8 +504,6 @@ build-macos-subkey: publish-docker-substrate: stage: publish <<: *build-push-docker-image - # collect VERSION artifact here to pass it on to kubernetes - <<: *collect-artifacts needs: - job: build-linux-substrate artifacts: true @@ -508,8 +511,12 @@ publish-docker-substrate: <<: *docker-build-vars PRODUCT: substrate after_script: - # only VERSION information is needed for the deployment - - find ./artifacts/ -depth -not -name VERSION -type f -delete + - echo "VERSION=${VERSION}" >> build.env + artifacts: + reports: + # this artifact is used in trigger-simnet job + # https://docs.gitlab.com/ee/ci/multi_project_pipelines.html#with-variable-inheritance + dotenv: artifacts/substrate/build.env publish-docker-subkey: stage: publish @@ -523,7 +530,7 @@ publish-docker-subkey: publish-s3-release: stage: publish - <<: *build-refs + <<: *publish-refs <<: *kubernetes-build needs: - job: build-linux-substrate @@ -587,6 +594,8 @@ deploy-prometheus-alerting-rules: - kubectl -n ${NAMESPACE} patch prometheusrule ${PROMETHEUSRULE} --type=merge --patch "$(sed 's/^/ /;1s/^/spec:\n/' ${RULES})" rules: + - if: $CI_PIPELINE_SOURCE == "pipeline" + when: never - if: $CI_COMMIT_REF_NAME == "master" changes: - .gitlab-ci.yml @@ -599,43 +608,7 @@ trigger-simnet: - if: $CI_COMMIT_REF_NAME == "master" && $CI_PIPELINE_SOURCE == "schedule" && $PIPELINE == "nightly" needs: - job: publish-docker-substrate - artifacts: false trigger: project: parity/simnet branch: master strategy: depend - -.validator-deploy: &validator-deploy - stage: deploy - rules: - # this job runs only on nightly pipeline with the mentioned variable, against `master` branch - - if: $CI_COMMIT_REF_NAME == "master" && $CI_PIPELINE_SOURCE == "schedule" && $PIPELINE == "nightly" - needs: - # script will fail if there is no artifacts/substrate/VERSION - - job: publish-docker-substrate - artifacts: true - image: parity/azure-ansible:v2 - allow_failure: true - interruptible: true - tags: - - linux-docker - -validator 1 4: - <<: *validator-deploy - script: - - ./.maintain/flamingfir-deploy.sh flamingfir-validator1 - -validator 2 4: - <<: *validator-deploy - script: - - ./.maintain/flamingfir-deploy.sh flamingfir-validator2 - -validator 3 4: - <<: *validator-deploy - script: - - ./.maintain/flamingfir-deploy.sh flamingfir-validator3 - -validator 4 4: - <<: *validator-deploy - script: - - ./.maintain/flamingfir-deploy.sh flamingfir-validator4 diff --git a/substrate/.maintain/flamingfir-deploy.sh b/substrate/.maintain/flamingfir-deploy.sh deleted file mode 100755 index 8f0fb3a2bc..0000000000 --- a/substrate/.maintain/flamingfir-deploy.sh +++ /dev/null @@ -1,35 +0,0 @@ -#!/bin/bash - -RETRY_COUNT=10 -RETRY_ATTEMPT=0 -SLEEP_TIME=15 -TARGET_HOST="$1" -COMMIT=$(cat artifacts/substrate/VERSION) -DOWNLOAD_URL="https://releases.parity.io/substrate/x86_64-debian:stretch/${COMMIT}/substrate/substrate" -POST_DATA='{"extra_vars":{"artifact_path":"'${DOWNLOAD_URL}'","target_host":"'${TARGET_HOST}'"}}' - -JOB_ID=$(wget -O - --header "Authorization: Bearer ${AWX_TOKEN}" --header "Content-type: application/json" --post-data "${POST_DATA}" https://ansible-awx.parity.io/api/v2/job_templates/32/launch/ | jq .job) - -echo "Launched job: $JOB_ID" - - -while [ ${RETRY_ATTEMPT} -le ${RETRY_COUNT} ] ; do - export RETRY_RESULT=$(wget -O - --header "Authorization: Bearer ${AWX_TOKEN}" https://ansible-awx.parity.io/api/v2/jobs/${JOB_ID}/ | jq .status) - RETRY_ATTEMPT=$(( $RETRY_ATTEMPT +1 )) - sleep $SLEEP_TIME - if [ $(echo $RETRY_RESULT | egrep -e successful -e failed) ] ; then - break - fi -done - -AWX_OUTPUT=$(wget -O - --header "Authorization: Bearer ${AWX_TOKEN}" https://ansible-awx.parity.io/api/v2/jobs/${JOB_ID}/stdout?format=txt_download) - -echo "AWX job log:" -echo "${AWX_OUTPUT}" - - -JOB_STATUS=$(wget -O - --header "Authorization: Bearer ${AWX_TOKEN}" https://ansible-awx.parity.io/api/v2/jobs/${JOB_ID}/ | jq .status ) - -echo "===================================" -echo -e "Ansible AWX Remote Job: ${JOB_ID} \x1B[31mStatus: ${JOB_STATUS}\x1B[0m" -echo "===================================" diff --git a/substrate/.maintain/gitlab/check_polkadot_companion_build.sh b/substrate/.maintain/gitlab/check_polkadot_companion_build.sh index e5b308d038..bf8fbf5aaf 100755 --- a/substrate/.maintain/gitlab/check_polkadot_companion_build.sh +++ b/substrate/.maintain/gitlab/check_polkadot_companion_build.sh @@ -51,8 +51,6 @@ git merge origin/master # ancestor for successfully performing merges below. git clone --depth 20 https://github.com/paritytech/polkadot.git -cargo install -f diener - cd polkadot # either it's a pull request then check for a companion otherwise use diff --git a/substrate/docs/CODEOWNERS b/substrate/docs/CODEOWNERS index a3837e1677..865c8d56df 100644 --- a/substrate/docs/CODEOWNERS +++ b/substrate/docs/CODEOWNERS @@ -18,6 +18,11 @@ # are more recognizable on GitHub, you can use them for mentioning unlike an email. # - The latest matching rule, if multiple, takes precedence. +# CI +/.maintain/ @paritytech/ci +/.github/ @paritytech/ci +/.gitlab-ci.yml @paritytech/ci + # Block production /client/basic-authorship/ @NikVolf @@ -56,10 +61,3 @@ # Transaction weight stuff /frame/support/src/weights.rs @shawntabrizi - -# Authority discovery -/client/authority-discovery/ @mxinden -/frame/authority-discovery/ @mxinden - -# Prometheus endpoint -/utils/prometheus/ @mxinden