mirror of
https://github.com/pezkuwichain/pezkuwi-subxt.git
synced 2026-04-30 21:17:56 +00:00
Vladimir Istyufeev
ad308386d8
* Introduce `linux-docker-benches` * Add additional info to `node-bench-regression-guard` job's run
966 lines
38 KiB
YAML
966 lines
38 KiB
YAML
# .gitlab-ci.yml
|
||
#
|
||
# substrate
|
||
#
|
||
# pipelines can be triggered manually in the web
|
||
|
||
# SAMPLE JOB TEMPLATE - This is not a complete example but is enough to build a
|
||
# simple CI job. For full documentation, visit https://docs.gitlab.com/ee/ci/yaml/
|
||
#
|
||
# my-example-job:
|
||
# stage: test # One of the stages listed below this job (required)
|
||
# image: paritytech/tools:latest # Any docker image (required)
|
||
# allow_failure: true # Allow the pipeline to continue if this job fails (default: false)
|
||
# needs:
|
||
# - job: test-linux # Any jobs that are required to run before this job (optional)
|
||
# variables:
|
||
# MY_ENVIRONMENT_VARIABLE: "some useful value" # Environment variables passed to the job (optional)
|
||
# script:
|
||
# - echo "List of shell commands to run in your job"
|
||
# - echo "You can also just specify a script here, like so:"
|
||
# - ./.maintain/gitlab/my_amazing_script.sh
|
||
|
||
stages:
|
||
- check
|
||
- test
|
||
- build
|
||
- publish
|
||
- deploy
|
||
|
||
workflow:
|
||
rules:
|
||
- if: $CI_COMMIT_TAG
|
||
- if: $CI_COMMIT_BRANCH
|
||
|
||
variables: &default-vars
|
||
GIT_STRATEGY: fetch
|
||
GIT_DEPTH: 100
|
||
CARGO_INCREMENTAL: 0
|
||
DOCKER_OS: "debian:stretch"
|
||
ARCH: "x86_64"
|
||
CI_IMAGE: "paritytech/ci-linux:production"
|
||
# FIXME set to release
|
||
CARGO_UNLEASH_INSTALL_PARAMS: "--version 1.0.0-alpha.12"
|
||
CARGO_UNLEASH_PKG_DEF: "--skip node node-* pallet-template pallet-example pallet-example-* subkey chain-spec-builder"
|
||
VAULT_SERVER_URL: "https://vault.parity-mgmt-vault.parity.io"
|
||
VAULT_AUTH_PATH: "gitlab-parity-io-jwt"
|
||
VAULT_AUTH_ROLE: "cicd_gitlab_parity_${CI_PROJECT_NAME}"
|
||
SIMNET_FEATURES_PATH: "simnet_tests/tests"
|
||
|
||
default:
|
||
cache: {}
|
||
|
||
.collect-artifacts: &collect-artifacts
|
||
artifacts:
|
||
name: "${CI_JOB_NAME}_${CI_COMMIT_REF_NAME}"
|
||
when: on_success
|
||
expire_in: 7 days
|
||
paths:
|
||
- artifacts/
|
||
|
||
.collect-artifacts-short: &collect-artifacts-short
|
||
artifacts:
|
||
name: "${CI_JOB_NAME}_${CI_COMMIT_REF_NAME}"
|
||
when: on_success
|
||
expire_in: 3 hours
|
||
paths:
|
||
- artifacts/
|
||
|
||
.kubernetes-env: &kubernetes-env
|
||
retry:
|
||
max: 2
|
||
when:
|
||
- runner_system_failure
|
||
- unknown_failure
|
||
- api_failure
|
||
interruptible: true
|
||
tags:
|
||
- kubernetes-parity-build
|
||
|
||
.rust-info-script: &rust-info-script
|
||
- rustup show
|
||
- cargo --version
|
||
- rustup +nightly show
|
||
- cargo +nightly --version
|
||
- sccache -s
|
||
|
||
.docker-env: &docker-env
|
||
image: "${CI_IMAGE}"
|
||
before_script:
|
||
- *rust-info-script
|
||
retry:
|
||
max: 2
|
||
when:
|
||
- runner_system_failure
|
||
- unknown_failure
|
||
- api_failure
|
||
interruptible: true
|
||
tags:
|
||
- linux-docker
|
||
|
||
.test-refs: &test-refs
|
||
rules:
|
||
- if: $CI_PIPELINE_SOURCE == "web"
|
||
- if: $CI_PIPELINE_SOURCE == "schedule"
|
||
- if: $CI_COMMIT_REF_NAME == "master"
|
||
- if: $CI_COMMIT_REF_NAME =~ /^[0-9]+$/ # PRs
|
||
- if: $CI_COMMIT_REF_NAME =~ /^v[0-9]+\.[0-9]+.*$/ # i.e. v1.0, v2.1rc1
|
||
|
||
.test-refs-no-trigger: &test-refs-no-trigger
|
||
rules:
|
||
- if: $CI_PIPELINE_SOURCE == "pipeline"
|
||
when: never
|
||
- if: $CI_PIPELINE_SOURCE == "web"
|
||
- if: $CI_PIPELINE_SOURCE == "schedule"
|
||
- if: $CI_COMMIT_REF_NAME == "master"
|
||
- if: $CI_COMMIT_REF_NAME =~ /^[0-9]+$/ # PRs
|
||
- if: $CI_COMMIT_REF_NAME =~ /^v[0-9]+\.[0-9]+.*$/ # i.e. v1.0, v2.1rc1
|
||
- if: $CI_COMMIT_REF_NAME =~ /^ci-release-.*$/
|
||
|
||
.test-refs-no-trigger-prs-only: &test-refs-no-trigger-prs-only
|
||
rules:
|
||
- if: $CI_PIPELINE_SOURCE == "pipeline"
|
||
when: never
|
||
- if: $CI_PIPELINE_SOURCE == "web"
|
||
- if: $CI_PIPELINE_SOURCE == "schedule"
|
||
- if: $CI_COMMIT_REF_NAME =~ /^[0-9]+$/ # PRs
|
||
|
||
.test-refs-wasmer-sandbox: &test-refs-wasmer-sandbox
|
||
rules:
|
||
- if: $CI_PIPELINE_SOURCE == "web"
|
||
- if: $CI_PIPELINE_SOURCE == "schedule"
|
||
- if: $CI_COMMIT_REF_NAME == "master"
|
||
changes:
|
||
- client/executor/**/*
|
||
- frame/contracts/**/*
|
||
- primitives/sandbox/**/*
|
||
- if: $CI_COMMIT_REF_NAME =~ /^[0-9]+$/ # PRs
|
||
changes:
|
||
- client/executor/**/*
|
||
- frame/contracts/**/*
|
||
- primitives/sandbox/**/*
|
||
- if: $CI_COMMIT_REF_NAME =~ /^v[0-9]+\.[0-9]+.*$/ # i.e. v1.0, v2.1rc1
|
||
changes:
|
||
- client/executor/**/*
|
||
- frame/contracts/**/*
|
||
- primitives/sandbox/**/*
|
||
|
||
.build-refs: &build-refs
|
||
rules:
|
||
- if: $CI_PIPELINE_SOURCE == "pipeline"
|
||
when: never
|
||
- if: $CI_PIPELINE_SOURCE == "web"
|
||
- if: $CI_PIPELINE_SOURCE == "schedule"
|
||
- if: $CI_COMMIT_REF_NAME == "master"
|
||
- if: $CI_COMMIT_REF_NAME =~ /^v[0-9]+\.[0-9]+.*$/ # i.e. v1.0, v2.1rc1
|
||
|
||
.nightly-pipeline: &nightly-pipeline
|
||
rules:
|
||
# this job runs only on nightly pipeline with the mentioned variable, against `master` branch
|
||
- if: $CI_COMMIT_REF_NAME == "master" && $CI_PIPELINE_SOURCE == "schedule" && $PIPELINE == "nightly"
|
||
|
||
.merge-ref-into-master-script: &merge-ref-into-master-script
|
||
- if [ $CI_COMMIT_REF_NAME != "master" ]; then
|
||
git fetch origin +master:master;
|
||
git fetch origin +$CI_COMMIT_REF_NAME:$CI_COMMIT_REF_NAME;
|
||
git checkout master;
|
||
git config user.email "ci@gitlab.parity.io";
|
||
git merge $CI_COMMIT_REF_NAME --verbose --no-edit;
|
||
fi
|
||
|
||
.cargo-check-benches-script: &cargo-check-benches-script
|
||
- mkdir -p artifacts/benches/$CI_COMMIT_REF_NAME-$CI_COMMIT_SHORT_SHA
|
||
- SKIP_WASM_BUILD=1 time cargo +nightly check --benches --all
|
||
- 'cargo run --release -p node-bench -- ::node::import::native::sr25519::transfer_keep_alive::paritydb::small --json
|
||
| tee artifacts/benches/$CI_COMMIT_REF_NAME-$CI_COMMIT_SHORT_SHA/::node::import::native::sr25519::transfer_keep_alive::paritydb::small.json'
|
||
- 'cargo run --release -p node-bench -- ::trie::read::small --json
|
||
| tee artifacts/benches/$CI_COMMIT_REF_NAME-$CI_COMMIT_SHORT_SHA/::trie::read::small.json'
|
||
- sccache -s
|
||
|
||
.build-linux-substrate-script: &build-linux-substrate-script
|
||
- WASM_BUILD_NO_COLOR=1 time cargo build --release --verbose
|
||
- mv ./target/release/substrate ./artifacts/substrate/.
|
||
- echo -n "Substrate version = "
|
||
- if [ "${CI_COMMIT_TAG}" ]; then
|
||
echo "${CI_COMMIT_TAG}" | tee ./artifacts/substrate/VERSION;
|
||
else
|
||
./artifacts/substrate/substrate --version |
|
||
sed -n -E 's/^substrate ([0-9.]+.*-[0-9a-f]{7,13})-.*$/\1/p' |
|
||
tee ./artifacts/substrate/VERSION;
|
||
fi
|
||
- sha256sum ./artifacts/substrate/substrate | tee ./artifacts/substrate/substrate.sha256
|
||
- cp -r .maintain/docker/substrate.Dockerfile ./artifacts/substrate/
|
||
- sccache -s
|
||
|
||
#### Vault secrets
|
||
.vault-secrets: &vault-secrets
|
||
secrets:
|
||
DOCKER_HUB_USER:
|
||
vault: cicd/gitlab/parity/DOCKER_HUB_USER@kv
|
||
file: false
|
||
DOCKER_HUB_PASS:
|
||
vault: cicd/gitlab/parity/DOCKER_HUB_PASS@kv
|
||
file: false
|
||
GITHUB_PR_TOKEN:
|
||
vault: cicd/gitlab/parity/GITHUB_PR_TOKEN@kv
|
||
file: false
|
||
AWS_ACCESS_KEY_ID:
|
||
vault: cicd/gitlab/$CI_PROJECT_PATH/AWS_ACCESS_KEY_ID@kv
|
||
file: false
|
||
AWS_SECRET_ACCESS_KEY:
|
||
vault: cicd/gitlab/$CI_PROJECT_PATH/AWS_SECRET_ACCESS_KEY@kv
|
||
file: false
|
||
AWX_TOKEN:
|
||
vault: cicd/gitlab/$CI_PROJECT_PATH/AWX_TOKEN@kv
|
||
file: false
|
||
CRATES_TOKEN:
|
||
vault: cicd/gitlab/$CI_PROJECT_PATH/CRATES_TOKEN@kv
|
||
file: false
|
||
DOCKER_CHAOS_TOKEN:
|
||
vault: cicd/gitlab/$CI_PROJECT_PATH/DOCKER_CHAOS_TOKEN@kv
|
||
file: false
|
||
DOCKER_CHAOS_USER:
|
||
vault: cicd/gitlab/$CI_PROJECT_PATH/DOCKER_CHAOS_USER@kv
|
||
file: false
|
||
GITHUB_EMAIL:
|
||
vault: cicd/gitlab/$CI_PROJECT_PATH/GITHUB_EMAIL@kv
|
||
file: false
|
||
GITHUB_RELEASE_TOKEN:
|
||
vault: cicd/gitlab/$CI_PROJECT_PATH/GITHUB_RELEASE_TOKEN@kv
|
||
file: false
|
||
GITHUB_TOKEN:
|
||
vault: cicd/gitlab/$CI_PROJECT_PATH/GITHUB_TOKEN@kv
|
||
file: false
|
||
GITHUB_USER:
|
||
vault: cicd/gitlab/$CI_PROJECT_PATH/GITHUB_USER@kv
|
||
file: false
|
||
MATRIX_ACCESS_TOKEN:
|
||
vault: cicd/gitlab/$CI_PROJECT_PATH/MATRIX_ACCESS_TOKEN@kv
|
||
file: false
|
||
MATRIX_ROOM_ID:
|
||
vault: cicd/gitlab/$CI_PROJECT_PATH/MATRIX_ROOM_ID@kv
|
||
file: false
|
||
PIPELINE_TOKEN:
|
||
vault: cicd/gitlab/$CI_PROJECT_PATH/PIPELINE_TOKEN@kv
|
||
file: false
|
||
VALIDATOR_KEYS:
|
||
vault: cicd/gitlab/$CI_PROJECT_PATH/VALIDATOR_KEYS@kv
|
||
file: false
|
||
VALIDATOR_KEYS_CHAOS:
|
||
vault: cicd/gitlab/$CI_PROJECT_PATH/VALIDATOR_KEYS_CHAOS@kv
|
||
file: false
|
||
|
||
#### stage: .pre
|
||
|
||
skip-if-draft:
|
||
image: paritytech/tools:latest
|
||
<<: *kubernetes-env
|
||
<<: *vault-secrets
|
||
stage: .pre
|
||
rules:
|
||
- if: $CI_COMMIT_REF_NAME =~ /^[0-9]+$/ # PRs
|
||
script:
|
||
- echo "Commit message is ${CI_COMMIT_MESSAGE}"
|
||
- echo "Ref is ${CI_COMMIT_REF_NAME}"
|
||
- echo "pipeline source is ${CI_PIPELINE_SOURCE}"
|
||
- ./.maintain/gitlab/skip_if_draft.sh
|
||
|
||
#### stage: check
|
||
|
||
check-runtime:
|
||
stage: check
|
||
image: paritytech/tools:latest
|
||
<<: *kubernetes-env
|
||
<<: *vault-secrets
|
||
rules:
|
||
- if: $CI_COMMIT_REF_NAME =~ /^[0-9]+$/ # PRs
|
||
variables:
|
||
<<: *default-vars
|
||
GITLAB_API: "https://gitlab.parity.io/api/v4"
|
||
GITHUB_API_PROJECT: "parity%2Finfrastructure%2Fgithub-api"
|
||
script:
|
||
- ./.maintain/gitlab/check_runtime.sh
|
||
allow_failure: true
|
||
|
||
check-signed-tag:
|
||
stage: check
|
||
image: paritytech/tools:latest
|
||
<<: *kubernetes-env
|
||
<<: *vault-secrets
|
||
rules:
|
||
- if: $CI_COMMIT_REF_NAME =~ /^ci-release-.*$/
|
||
- if: $CI_COMMIT_REF_NAME =~ /^v[0-9]+\.[0-9]+.*$/ # i.e. v1.0, v2.1rc1
|
||
script:
|
||
- ./.maintain/gitlab/check_signed.sh
|
||
|
||
test-dependency-rules:
|
||
stage: check
|
||
image: paritytech/tools:latest
|
||
<<: *kubernetes-env
|
||
rules:
|
||
- if: $CI_COMMIT_REF_NAME =~ /^[0-9]+$/ # PRs
|
||
script:
|
||
- .maintain/ensure-deps.sh
|
||
|
||
test-prometheus-alerting-rules:
|
||
stage: check
|
||
image: paritytech/tools:latest
|
||
<<: *kubernetes-env
|
||
rules:
|
||
- if: $CI_PIPELINE_SOURCE == "pipeline"
|
||
when: never
|
||
- if: $CI_COMMIT_BRANCH
|
||
changes:
|
||
- .gitlab-ci.yml
|
||
- .maintain/monitoring/**/*
|
||
script:
|
||
- promtool check rules .maintain/monitoring/alerting-rules/alerting-rules.yaml
|
||
- cat .maintain/monitoring/alerting-rules/alerting-rules.yaml |
|
||
promtool test rules .maintain/monitoring/alerting-rules/alerting-rule-tests.yaml
|
||
|
||
#### stage: test
|
||
|
||
cargo-deny:
|
||
stage: test
|
||
<<: *docker-env
|
||
<<: *nightly-pipeline
|
||
script:
|
||
- cargo deny check --hide-inclusion-graph -c .maintain/deny.toml
|
||
after_script:
|
||
- echo "___The complete log is in the artifacts___"
|
||
- cargo deny check -c .maintain/deny.toml 2> deny.log
|
||
artifacts:
|
||
name: $CI_COMMIT_SHORT_SHA
|
||
expire_in: 3 days
|
||
when: always
|
||
paths:
|
||
- deny.log
|
||
# FIXME: Temporarily allow to fail.
|
||
allow_failure: true
|
||
|
||
cargo-fmt:
|
||
stage: test
|
||
<<: *docker-env
|
||
<<: *test-refs
|
||
script:
|
||
- cargo +nightly fmt --all -- --check
|
||
|
||
cargo-clippy:
|
||
stage: test
|
||
<<: *docker-env
|
||
<<: *test-refs
|
||
script:
|
||
- SKIP_WASM_BUILD=1 env -u RUSTFLAGS cargo +nightly clippy
|
||
|
||
cargo-check-benches:
|
||
stage: test
|
||
<<: *docker-env
|
||
<<: *test-refs
|
||
<<: *collect-artifacts
|
||
before_script:
|
||
# merges in the master branch on PRs
|
||
- *merge-ref-into-master-script
|
||
- *rust-info-script
|
||
script:
|
||
- *cargo-check-benches-script
|
||
tags:
|
||
- linux-docker-benches
|
||
|
||
node-bench-regression-guard:
|
||
# it's not belong to `build` semantically, but dag jobs can't depend on each other
|
||
# within the single stage - https://gitlab.com/gitlab-org/gitlab/-/issues/30632
|
||
# more: https://github.com/paritytech/substrate/pull/8519#discussion_r608012402
|
||
stage: build
|
||
<<: *docker-env
|
||
<<: *test-refs-no-trigger-prs-only
|
||
needs:
|
||
# this is a DAG
|
||
- job: cargo-check-benches
|
||
artifacts: true
|
||
# this does not like a DAG, just polls the artifact
|
||
- project: $CI_PROJECT_PATH
|
||
job: cargo-check-benches
|
||
ref: master
|
||
artifacts: true
|
||
variables:
|
||
CI_IMAGE: "paritytech/node-bench-regression-guard:latest"
|
||
before_script: [""]
|
||
script:
|
||
- echo "------- IMPORTANT -------"
|
||
- echo "node-bench-regression-guard depends on the results of a cargo-check-benches job"
|
||
- echo "In case of this job failure, check your pipeline's cargo-check-benches"
|
||
- 'node-bench-regression-guard --reference artifacts/benches/master-*
|
||
--compare-with artifacts/benches/$CI_COMMIT_REF_NAME-$CI_COMMIT_SHORT_SHA'
|
||
|
||
cargo-check-subkey:
|
||
stage: test
|
||
<<: *docker-env
|
||
<<: *test-refs
|
||
script:
|
||
- cd ./bin/utils/subkey
|
||
- SKIP_WASM_BUILD=1 time cargo check --release
|
||
- sccache -s
|
||
|
||
cargo-check-try-runtime:
|
||
stage: test
|
||
<<: *docker-env
|
||
<<: *test-refs
|
||
script:
|
||
- time cargo check --features try-runtime
|
||
- sccache -s
|
||
|
||
cargo-check-wasmer-sandbox:
|
||
stage: test
|
||
<<: *docker-env
|
||
<<: *test-refs
|
||
script:
|
||
- time cargo check --features wasmer-sandbox
|
||
- sccache -s
|
||
|
||
test-deterministic-wasm:
|
||
stage: test
|
||
<<: *docker-env
|
||
<<: *test-refs
|
||
variables:
|
||
<<: *default-vars
|
||
WASM_BUILD_NO_COLOR: 1
|
||
script:
|
||
# build runtime
|
||
- cargo build --verbose --release -p node-runtime
|
||
# make checksum
|
||
- sha256sum target/release/wbuild/node-runtime/target/wasm32-unknown-unknown/release/node_runtime.wasm > checksum.sha256
|
||
# clean up – FIXME: can we reuse some of the artifacts?
|
||
- cargo clean
|
||
# build again
|
||
- cargo build --verbose --release -p node-runtime
|
||
# confirm checksum
|
||
- sha256sum -c checksum.sha256
|
||
- sccache -s
|
||
|
||
test-linux-stable: &test-linux
|
||
stage: test
|
||
<<: *docker-env
|
||
<<: *test-refs
|
||
variables:
|
||
<<: *default-vars
|
||
# Enable debug assertions since we are running optimized builds for testing
|
||
# but still want to have debug assertions.
|
||
RUSTFLAGS: "-Cdebug-assertions=y -Dwarnings"
|
||
RUST_BACKTRACE: 1
|
||
WASM_BUILD_NO_COLOR: 1
|
||
script:
|
||
# this job runs all tests in former runtime-benchmarks, frame-staking and wasmtime tests
|
||
- time cargo test --workspace --locked --release --verbose --features runtime-benchmarks --manifest-path bin/node/cli/Cargo.toml
|
||
- time cargo test -p frame-support-test --features=conditional-storage --manifest-path frame/support/test/Cargo.toml --test pallet # does not reuse cache 1 min 44 sec
|
||
- SUBSTRATE_TEST_TIMEOUT=1 time cargo test -p substrate-test-utils --release --verbose --locked -- --ignored timeout
|
||
- sccache -s
|
||
|
||
#unleash-check:
|
||
#stage: test
|
||
#<<: *docker-env
|
||
#<<: *test-refs-no-trigger
|
||
#script:
|
||
#- cargo install cargo-unleash ${CARGO_UNLEASH_INSTALL_PARAMS}
|
||
#- cargo unleash de-dev-deps
|
||
# Reuse build artifacts when running checks (cuts down check time by 3x)
|
||
# TODO: Implement this optimization in cargo-unleash rather than here
|
||
#- mkdir -p target/unleash
|
||
#- export CARGO_TARGET_DIR=target/unleash
|
||
#- cargo unleash check ${CARGO_UNLEASH_PKG_DEF}
|
||
# FIXME: this job must not fail, or unleash-to-crates-io will publish broken stuff
|
||
#allow_failure: true
|
||
|
||
test-frame-examples-compile-to-wasm:
|
||
# into one job
|
||
stage: test
|
||
<<: *docker-env
|
||
<<: *test-refs
|
||
variables:
|
||
<<: *default-vars
|
||
# Enable debug assertions since we are running optimized builds for testing
|
||
# but still want to have debug assertions.
|
||
RUSTFLAGS: "-Cdebug-assertions=y"
|
||
RUST_BACKTRACE: 1
|
||
script:
|
||
- cd frame/example-offchain-worker/
|
||
- cargo +nightly build --target=wasm32-unknown-unknown --no-default-features
|
||
- cd ../example
|
||
- cargo +nightly build --target=wasm32-unknown-unknown --no-default-features
|
||
- sccache -s
|
||
|
||
test-linux-stable-int:
|
||
<<: *test-linux
|
||
stage: test
|
||
script:
|
||
- echo "___Logs will be partly shown at the end in case of failure.___"
|
||
- echo "___Full log will be saved to the job artifacts only in case of failure.___"
|
||
- WASM_BUILD_NO_COLOR=1
|
||
RUST_LOG=sync=trace,consensus=trace,client=trace,state-db=trace,db=trace,forks=trace,state_db=trace,storage_cache=trace
|
||
time cargo test -p node-cli --release --verbose --locked -- --ignored
|
||
&> ${CI_COMMIT_SHORT_SHA}_int_failure.log
|
||
- sccache -s
|
||
after_script:
|
||
- awk '/FAILED|^error\[/,0' ${CI_COMMIT_SHORT_SHA}_int_failure.log
|
||
artifacts:
|
||
name: $CI_COMMIT_SHORT_SHA
|
||
when: on_failure
|
||
expire_in: 3 days
|
||
paths:
|
||
- ${CI_COMMIT_SHORT_SHA}_int_failure.log
|
||
|
||
check-tracing:
|
||
stage: test
|
||
<<: *docker-env
|
||
<<: *test-refs
|
||
script:
|
||
# with-tracing must be explicitly activated, we run a test to ensure this works as expected in both cases
|
||
- time cargo +nightly test --manifest-path primitives/tracing/Cargo.toml --no-default-features
|
||
- time cargo +nightly test --manifest-path primitives/tracing/Cargo.toml --no-default-features --features=with-tracing
|
||
- sccache -s
|
||
|
||
test-full-crypto-feature:
|
||
stage: test
|
||
<<: *docker-env
|
||
<<: *test-refs
|
||
variables:
|
||
<<: *default-vars
|
||
# Enable debug assertions since we are running optimized builds for testing
|
||
# but still want to have debug assertions.
|
||
RUSTFLAGS: "-Cdebug-assertions=y"
|
||
RUST_BACKTRACE: 1
|
||
script:
|
||
- cd primitives/core/
|
||
- time cargo +nightly build --verbose --no-default-features --features full_crypto
|
||
- cd ../application-crypto
|
||
- time cargo +nightly build --verbose --no-default-features --features full_crypto
|
||
- sccache -s
|
||
|
||
test-wasmer-sandbox:
|
||
stage: test
|
||
<<: *docker-env
|
||
<<: *test-refs-wasmer-sandbox
|
||
variables:
|
||
<<: *default-vars
|
||
script:
|
||
- time cargo test --release --features runtime-benchmarks,wasmer-sandbox,disable-ui-tests
|
||
- sccache -s
|
||
|
||
cargo-check-macos:
|
||
stage: test
|
||
# shell runner on mac ignores the image set in *docker-env
|
||
<<: *docker-env
|
||
<<: *test-refs-no-trigger
|
||
script:
|
||
- SKIP_WASM_BUILD=1 time cargo check --release
|
||
- sccache -s
|
||
tags:
|
||
- osx
|
||
|
||
#### stage: build
|
||
|
||
.check-dependent-project: &check-dependent-project
|
||
stage: build
|
||
<<: *docker-env
|
||
<<: *test-refs-no-trigger
|
||
<<: *vault-secrets
|
||
script:
|
||
- git clone
|
||
--depth=1
|
||
"--branch=$PIPELINE_SCRIPTS_TAG"
|
||
https://github.com/paritytech/pipeline-scripts
|
||
- ./pipeline-scripts/check_dependent_project.sh
|
||
paritytech
|
||
substrate
|
||
--substrate
|
||
"$DEPENDENT_REPO"
|
||
"$GITHUB_PR_TOKEN"
|
||
|
||
# Individual jobs are set up for each dependent project so that they can be ran in parallel.
|
||
# Arguably we could generate a job for each companion in the PR's description using Gitlab's
|
||
# parent-child pipelines but that's more complicated.
|
||
|
||
check-dependent-polkadot:
|
||
<<: *check-dependent-project
|
||
variables:
|
||
DEPENDENT_REPO: polkadot
|
||
|
||
check-dependent-cumulus:
|
||
<<: *check-dependent-project
|
||
variables:
|
||
DEPENDENT_REPO: cumulus
|
||
|
||
|
||
build-linux-substrate:
|
||
stage: build
|
||
<<: *collect-artifacts
|
||
<<: *docker-env
|
||
<<: *build-refs
|
||
needs:
|
||
- job: test-linux-stable
|
||
artifacts: false
|
||
before_script:
|
||
- mkdir -p ./artifacts/substrate/
|
||
script:
|
||
- *build-linux-substrate-script
|
||
- printf '\n# building node-template\n\n'
|
||
- ./.maintain/node-template-release.sh ./artifacts/substrate/substrate-node-template.tar.gz
|
||
|
||
|
||
#Build binary for simnet quick tests.
|
||
build-linux-substrate-simnet:
|
||
stage: build
|
||
<<: *collect-artifacts-short
|
||
<<: *docker-env
|
||
<<: *test-refs-no-trigger-prs-only
|
||
before_script:
|
||
- mkdir -p ./artifacts/substrate/
|
||
script:
|
||
- *build-linux-substrate-script
|
||
|
||
build-linux-subkey: &build-subkey
|
||
stage: build
|
||
<<: *collect-artifacts
|
||
<<: *docker-env
|
||
<<: *build-refs
|
||
needs:
|
||
- job: cargo-check-subkey
|
||
artifacts: false
|
||
before_script:
|
||
- mkdir -p ./artifacts/subkey
|
||
script:
|
||
- cd ./bin/utils/subkey
|
||
- SKIP_WASM_BUILD=1 time cargo build --release --verbose
|
||
- cd -
|
||
- mv ./target/release/subkey ./artifacts/subkey/.
|
||
- echo -n "Subkey version = "
|
||
- ./artifacts/subkey/subkey --version |
|
||
sed -n -E 's/^subkey ([0-9.]+.*)/\1/p' |
|
||
tee ./artifacts/subkey/VERSION;
|
||
- sha256sum ./artifacts/subkey/subkey | tee ./artifacts/subkey/subkey.sha256
|
||
- cp -r .maintain/docker/subkey.Dockerfile ./artifacts/subkey/
|
||
- sccache -s
|
||
|
||
build-macos-subkey:
|
||
<<: *build-subkey
|
||
tags:
|
||
- osx
|
||
|
||
build-rustdoc:
|
||
stage: build
|
||
<<: *docker-env
|
||
<<: *test-refs
|
||
variables:
|
||
<<: *default-vars
|
||
SKIP_WASM_BUILD: 1
|
||
DOC_INDEX_PAGE: "sc_service/index.html" # default redirected page
|
||
artifacts:
|
||
name: "${CI_JOB_NAME}_${CI_COMMIT_REF_NAME}-doc"
|
||
when: on_success
|
||
expire_in: 7 days
|
||
paths:
|
||
- ./crate-docs/
|
||
script:
|
||
# FIXME: it fails with `RUSTDOCFLAGS="-Dwarnings"`
|
||
- RUSTDOCFLAGS="--html-in-header $(pwd)/.maintain/rustdoc-header.html"
|
||
time cargo +nightly doc --no-deps --workspace --all-features --verbose
|
||
- rm -f ./target/doc/.lock
|
||
- mv ./target/doc ./crate-docs
|
||
# FIXME: remove me after CI image gets nonroot
|
||
- chown -R nonroot:nonroot ./crate-docs
|
||
- echo "<meta http-equiv=refresh content=0;url=${DOC_INDEX_PAGE}>" > ./crate-docs/index.html
|
||
- sccache -s
|
||
|
||
#### stage: publish
|
||
|
||
.build-push-docker-image: &build-push-docker-image
|
||
<<: *build-refs
|
||
<<: *kubernetes-env
|
||
<<: *vault-secrets
|
||
image: quay.io/buildah/stable
|
||
variables: &docker-build-vars
|
||
<<: *default-vars
|
||
GIT_STRATEGY: none
|
||
DOCKERFILE: $PRODUCT.Dockerfile
|
||
IMAGE_NAME: docker.io/parity/$PRODUCT
|
||
before_script:
|
||
- cd ./artifacts/$PRODUCT/
|
||
- VERSION="$(cat ./VERSION)"
|
||
- echo "${PRODUCT} version = ${VERSION}"
|
||
- test -z "${VERSION}" && exit 1
|
||
script:
|
||
- test "$DOCKER_HUB_USER" -a "$DOCKER_HUB_PASS" ||
|
||
( echo "no docker credentials provided"; exit 1 )
|
||
- buildah bud
|
||
--format=docker
|
||
--build-arg VCS_REF="${CI_COMMIT_SHA}"
|
||
--build-arg BUILD_DATE="$(date -u '+%Y-%m-%dT%H:%M:%SZ')"
|
||
--tag "$IMAGE_NAME:$VERSION"
|
||
--tag "$IMAGE_NAME:latest"
|
||
--file "$DOCKERFILE" .
|
||
- echo "$DOCKER_HUB_PASS" |
|
||
buildah login --username "$DOCKER_HUB_USER" --password-stdin docker.io
|
||
- buildah info
|
||
- buildah push --format=v2s2 "$IMAGE_NAME:$VERSION"
|
||
- buildah push --format=v2s2 "$IMAGE_NAME:latest"
|
||
after_script:
|
||
- buildah logout --all
|
||
# pass artifacts to the trigger-simnet job
|
||
- echo "SUBSTRATE_IMAGE_NAME=${IMAGE_NAME}" | tee -a ./artifacts/$PRODUCT/build.env
|
||
- IMAGE_TAG="$(cat ./artifacts/$PRODUCT/VERSION)"
|
||
- echo "SUBSTRATE_IMAGE_TAG=${IMAGE_TAG}" | tee -a ./artifacts/$PRODUCT/build.env
|
||
- cat ./artifacts/$PRODUCT/build.env
|
||
|
||
publish-docker-substrate:
|
||
stage: publish
|
||
<<: *build-push-docker-image
|
||
<<: *build-refs
|
||
needs:
|
||
- job: build-linux-substrate
|
||
artifacts: true
|
||
variables:
|
||
<<: *docker-build-vars
|
||
PRODUCT: substrate
|
||
artifacts:
|
||
reports:
|
||
# this artifact is used in trigger-simnet job
|
||
# https://docs.gitlab.com/ee/ci/multi_project_pipelines.html#with-variable-inheritance
|
||
dotenv: ./artifacts/substrate/build.env
|
||
|
||
#publish artifact for quick simnet tests
|
||
publish-docker-substrate-simnet:
|
||
stage: publish
|
||
<<: *build-push-docker-image
|
||
<<: *test-refs-no-trigger-prs-only
|
||
needs:
|
||
- job: build-linux-substrate-simnet
|
||
artifacts: true
|
||
variables:
|
||
<<: *default-vars
|
||
GIT_STRATEGY: none
|
||
DOCKERFILE: $PRODUCT.Dockerfile
|
||
PRODUCT: substrate
|
||
IMAGE_NAME: docker.io/paritypr/$PRODUCT
|
||
artifacts:
|
||
reports:
|
||
# this artifact is used in simnet-tests-quick job
|
||
dotenv: ./artifacts/substrate/build.env
|
||
|
||
publish-docker-subkey:
|
||
stage: publish
|
||
<<: *build-push-docker-image
|
||
needs:
|
||
- job: build-linux-subkey
|
||
artifacts: true
|
||
variables:
|
||
<<: *docker-build-vars
|
||
PRODUCT: subkey
|
||
|
||
publish-s3-release:
|
||
stage: publish
|
||
<<: *build-refs
|
||
<<: *kubernetes-env
|
||
<<: *vault-secrets
|
||
needs:
|
||
- job: build-linux-substrate
|
||
artifacts: true
|
||
- job: build-linux-subkey
|
||
artifacts: true
|
||
image: paritytech/awscli:latest
|
||
variables:
|
||
GIT_STRATEGY: none
|
||
BUCKET: "releases.parity.io"
|
||
PREFIX: "substrate/${ARCH}-${DOCKER_OS}"
|
||
script:
|
||
- aws s3 sync ./artifacts/ s3://${BUCKET}/${PREFIX}/$(cat ./artifacts/substrate/VERSION)/
|
||
- echo "update objects in latest path"
|
||
- aws s3 sync s3://${BUCKET}/${PREFIX}/$(cat ./artifacts/substrate/VERSION)/ s3://${BUCKET}/${PREFIX}/latest/
|
||
after_script:
|
||
- aws s3 ls s3://${BUCKET}/${PREFIX}/latest/
|
||
--recursive --human-readable --summarize
|
||
|
||
publish-rustdoc:
|
||
stage: publish
|
||
<<: *kubernetes-env
|
||
<<: *vault-secrets
|
||
image: node:16
|
||
variables:
|
||
GIT_DEPTH: 100
|
||
# --- Following variables are for rustdocs deployment ---
|
||
# Space separated values of branches and tags to generate rustdocs
|
||
RUSTDOCS_DEPLOY_REFS: "master monthly-2021-09+1 monthly-2021-08 v3.0.0"
|
||
# Location of the docs index template
|
||
INDEX_TPL: ".maintain/docs-index-tpl.ejs"
|
||
# Where the `/latest` symbolic link links to. One of the $RUSTDOCS_DEPLOY_REFS value.
|
||
LATEST: "monthly-2021-09+1"
|
||
rules:
|
||
- if: $CI_PIPELINE_SOURCE == "pipeline"
|
||
when: never
|
||
- if: $CI_PIPELINE_SOURCE == "web" && $CI_COMMIT_REF_NAME == "master"
|
||
- if: $CI_COMMIT_REF_NAME == "master"
|
||
- if: $CI_COMMIT_REF_NAME =~ /^monthly-20[0-9]{2}-[0-9]{2}.*$/ # to support: monthly-2021-09+1
|
||
- if: $CI_COMMIT_REF_NAME =~ /^v[0-9]+\.[0-9]+.*$/ # i.e. v1.0, v2.1rc1
|
||
# `needs:` can be removed after CI image gets nonroot. In this case `needs:` stops other
|
||
# artifacts from being dowloaded by this job.
|
||
needs:
|
||
- job: build-rustdoc
|
||
artifacts: true
|
||
script:
|
||
# If $CI_COMMIT_REF_NAME doesn't match one of $RUSTDOCS_DEPLOY_REFS space-separated values, we
|
||
# exit immediately.
|
||
# Putting spaces at the front and back to ensure we are not matching just any substring, but the
|
||
# whole space-separated value.
|
||
- '[[ " ${RUSTDOCS_DEPLOY_REFS} " =~ " ${CI_COMMIT_REF_NAME} " ]] || exit 0'
|
||
- rm -rf /tmp/*
|
||
# Set git config
|
||
- rm -rf .git/config
|
||
- git config user.email "devops-team@parity.io"
|
||
- git config user.name "${GITHUB_USER}"
|
||
- git config remote.origin.url "https://${GITHUB_TOKEN}@github.com/paritytech/${CI_PROJECT_NAME}.git"
|
||
- git config remote.origin.fetch "+refs/heads/*:refs/remotes/origin/*"
|
||
- git fetch origin gh-pages
|
||
# Install `ejs` and generate index.html based on RUSTDOCS_DEPLOY_REFS
|
||
- yarn global add ejs
|
||
- 'ejs ${INDEX_TPL} -i "{\"deploy_refs\":\"${RUSTDOCS_DEPLOY_REFS}\",\"repo_name\":\"${CI_PROJECT_NAME}\",\"latest\":\"${LATEST}\"}" > /tmp/index.html'
|
||
# Save README and docs
|
||
- cp -r ./crate-docs/ /tmp/doc/
|
||
- cp README.md /tmp/doc/
|
||
- git checkout gh-pages
|
||
# Remove directories no longer necessary, as specified in $RUSTDOCS_DEPLOY_REFS.
|
||
# Also ensure $RUSTDOCS_DEPLOY_REFS is not just empty spaces.
|
||
# Even though this block spans multiple lines, they are concatenated to run as a single line
|
||
# command, so note for the semi-colons in the inner-most code block.
|
||
- if [[ ! -z ${RUSTDOCS_DEPLOY_REFS// } ]]; then
|
||
for FILE in *; do
|
||
if [[ ! " $RUSTDOCS_DEPLOY_REFS " =~ " $FILE " ]]; then
|
||
echo "Removing ${FILE}...";
|
||
rm -rf $FILE;
|
||
fi
|
||
done
|
||
fi
|
||
# Move the index page & built back
|
||
- mv -f /tmp/index.html .
|
||
# Ensure the destination dir doesn't exist.
|
||
- rm -rf ${CI_COMMIT_REF_NAME}
|
||
- mv -f /tmp/doc ${CI_COMMIT_REF_NAME}
|
||
# Add the symlink
|
||
- '[[ -e "$LATEST" ]] && ln -sf "${LATEST}" latest'
|
||
# Upload files
|
||
- git add --all --force
|
||
# `git commit` has an exit code of > 0 if there is nothing to commit.
|
||
# This causes GitLab to exit immediately and marks this job failed.
|
||
# We don't want to mark the entire job failed if there's nothing to
|
||
# publish though, hence the `|| true`.
|
||
- git commit -m "___Updated docs for ${CI_COMMIT_REF_NAME}___" ||
|
||
echo "___Nothing to commit___"
|
||
- git push origin gh-pages --force
|
||
after_script:
|
||
- rm -rf .git/ ./*
|
||
|
||
publish-draft-release:
|
||
stage: publish
|
||
<<: *vault-secrets
|
||
image: paritytech/tools:latest
|
||
rules:
|
||
- if: $CI_COMMIT_REF_NAME =~ /^ci-release-.*$/
|
||
- if: $CI_COMMIT_REF_NAME =~ /^v[0-9]+\.[0-9]+.*$/ # i.e. v1.0, v2.1rc1
|
||
script:
|
||
- ./.maintain/gitlab/publish_draft_release.sh
|
||
allow_failure: true
|
||
|
||
unleash-to-crates-io:
|
||
stage: publish
|
||
<<: *docker-env
|
||
<<: *vault-secrets
|
||
rules:
|
||
- if: $CI_COMMIT_REF_NAME =~ /^ci-release-.*$/
|
||
# FIXME: wait until https://github.com/paritytech/cargo-unleash/issues/50 is fixed, also
|
||
# remove allow_failure: true on the check job
|
||
# - if: $CI_COMMIT_REF_NAME =~ /^v[0-9]+\.[0-9]+.*$/ # i.e. v1.0, v2.1rc1
|
||
script:
|
||
- cargo install cargo-unleash ${CARGO_UNLEASH_INSTALL_PARAMS}
|
||
- cargo unleash em-dragons --no-check --owner github:paritytech:core-devs ${CARGO_UNLEASH_PKG_DEF}
|
||
allow_failure: true
|
||
|
||
#### stage: deploy
|
||
|
||
deploy-prometheus-alerting-rules:
|
||
stage: deploy
|
||
needs:
|
||
- job: test-prometheus-alerting-rules
|
||
artifacts: false
|
||
allow_failure: true
|
||
trigger:
|
||
project: parity/infrastructure/cloud-infra
|
||
variables:
|
||
SUBSTRATE_CI_COMMIT_NAME: "${CI_COMMIT_REF_NAME}"
|
||
SUBSTRATE_CI_COMMIT_REF: "${CI_COMMIT_SHORT_SHA}"
|
||
UPSTREAM_TRIGGER_PROJECT: "${CI_PROJECT_PATH}"
|
||
rules:
|
||
- if: $CI_PIPELINE_SOURCE == "pipeline"
|
||
when: never
|
||
- if: $CI_COMMIT_REF_NAME == "master"
|
||
changes:
|
||
- .gitlab-ci.yml
|
||
- .maintain/monitoring/**/*
|
||
|
||
# Runs "quick" and "long" tests on nightly schedule and on commit / merge to master
|
||
# A "quick" test is a smoke test where basic check-expect tests run by
|
||
# checking values from metrics exposed by the app.
|
||
# A "long" test is the load testing where we send 50K transactions into the
|
||
# network and check if all completed successfully
|
||
simnet-tests:
|
||
stage: deploy
|
||
image: docker.io/paritytech/simnet:${SIMNET_REF}
|
||
<<: *kubernetes-env
|
||
<<: *vault-secrets
|
||
rules:
|
||
- if: $CI_PIPELINE_SOURCE == "pipeline"
|
||
when: never
|
||
- if: $CI_PIPELINE_SOURCE == "web" && $CI_COMMIT_REF_NAME == "master"
|
||
- if: $CI_COMMIT_REF_NAME == "master"
|
||
needs:
|
||
- job: publish-docker-substrate
|
||
# variables:
|
||
# `build.env` brings here `${SUBSTRATE_IMAGE_NAME}` and `${SUBSTRATE_IMAGE_TAG}`
|
||
# (`$VERSION` here, # i.e. `2643-0.8.29-5f689e0a-6b24dc54`).
|
||
# ${SIMNET_REF} is a gitlab variable
|
||
before_script:
|
||
- echo "Simnet Tests Config
|
||
docker.io/paritytech/simnet:${SIMNET_REF}
|
||
${SUBSTRATE_IMAGE_NAME} ${SUBSTRATE_IMAGE_TAG}"
|
||
script:
|
||
- /home/nonroot/simnet/gurke/scripts/run-test-environment-manager.sh
|
||
--github-remote-dir="https://github.com/paritytech/substrate/tree/master/simnet_tests"
|
||
--config="simnet_tests/configs/default_local_testnet.toml"
|
||
--image="${SUBSTRATE_IMAGE_NAME}:${SUBSTRATE_IMAGE_TAG}"
|
||
retry: 2
|
||
tags:
|
||
- substrate-simnet
|
||
|
||
#run quick simnet-tests for each PR
|
||
simnet-tests-quick:
|
||
stage: deploy
|
||
image: docker.io/paritytech/simnet:${SIMNET_REF}
|
||
<<: *kubernetes-env
|
||
<<: *vault-secrets
|
||
<<: *test-refs-no-trigger-prs-only
|
||
variables:
|
||
SIMNET_FEATURES: "${SIMNET_FEATURES_PATH}/quick"
|
||
needs:
|
||
- job: publish-docker-substrate-simnet
|
||
before_script:
|
||
- echo "Simnet Tests Config
|
||
docker.io/paritytech/simnet:${SIMNET_REF}
|
||
${SUBSTRATE_IMAGE_NAME} ${SUBSTRATE_IMAGE_TAG}"
|
||
script:
|
||
- echo "Image=${SUBSTRATE_IMAGE_NAME}:${SUBSTRATE_IMAGE_TAG}"
|
||
- echo "Features=${SIMNET_FEATURES}"
|
||
- /home/nonroot/simnet/gurke/scripts/run-test-environment-manager.sh
|
||
--github-remote-dir="https://github.com/paritytech/substrate/tree/master/simnet_tests"
|
||
--config="simnet_tests/configs/default_local_testnet.toml"
|
||
--image="${SUBSTRATE_IMAGE_NAME}:${SUBSTRATE_IMAGE_TAG}"
|
||
--features="${SIMNET_FEATURES}"
|
||
tags:
|
||
- substrate-simnet
|
||
|