pezkuwichain/pezkuwi-subxt
1
0
Fork 0
mirror of https://github.com/pezkuwichain/pezkuwi-subxt.git synced 2026-06-12 03:01:07 +00:00
Code Issues Packages Projects Releases Wiki Activity

[ci] add more jobs for pipeline cancel, cleanup (#1314)

Browse Source
This commit is contained in:
Alexander Samusev
2023-08-31 11:36:38 +02:00
committed by GitHub
parent 7cef7cdfb0
commit aabed6757e
56 changed files with 55 additions and 3705 deletions
Download Patch File Download Diff File Expand all files Collapse all files
substrate/scripts/ci/common/lib.sh
-117
View File
@@ -1,117 +0,0 @@
#!/bin/sh
api_base="https://api.github.com/repos"
# Function to take 2 git tags/commits and get any lines from commit messages
# that contain something that looks like a PR reference: e.g., (#1234)
sanitised_git_logs(){
git --no-pager log --pretty=format:"%s" "$1...$2" |
# Only find messages referencing a PR
grep -E '\(#[0-9]+\)' |
# Strip any asterisks
sed 's/^* //g' |
# And add them all back
sed 's/^/* /g'
}
# Returns the last published release on github
# Note: we can't just use /latest because that ignores prereleases
# repo: 'organization/repo'
# Usage: last_github_release "$repo"
last_github_release(){
i=0
# Iterate over releases until we find the last release that's not just a draft
while [ $i -lt 29 ]; do
out=$(curl -H "Authorization: token $GITHUB_RELEASE_TOKEN" -s "$api_base/$1/releases" | jq ".[$i]")
echo "$out"
# Ugh when echoing to jq, we need to translate newlines into spaces :/
if [ "$(echo "$out" | tr '\r\n' ' ' | jq '.draft')" = "false" ]; then
echo "$out" | tr '\r\n' ' ' | jq '.tag_name'
return
else
i=$((i + 1))
fi
done
}
# Checks whether a tag on github has been verified
# repo: 'organization/repo'
# tagver: 'v1.2.3'
# Usage: check_tag $repo $tagver
check_tag () {
repo=$1
tagver=$2
tag_out=$(curl -H "Authorization: token $GITHUB_RELEASE_TOKEN" -s "$api_base/$repo/git/refs/tags/$tagver")
tag_sha=$(echo "$tag_out" | jq -r .object.sha)
object_url=$(echo "$tag_out" | jq -r .object.url)
if [ "$tag_sha" = "null" ]; then
return 2
fi
verified_str=$(curl -H "Authorization: token $GITHUB_RELEASE_TOKEN" -s "$object_url" | jq -r .verification.verified)
if [ "$verified_str" = "true" ]; then
# Verified, everything is good
return 0
else
# Not verified. Bad juju.
return 1
fi
}
# Checks whether a given PR has a given label.
# repo: 'organization/repo'
# pr_id: 12345
# label: B1-silent
# Usage: has_label $repo $pr_id $label
has_label(){
repo="$1"
pr_id="$2"
label="$3"
# These will exist if the function is called in Gitlab.
# If the function's called in Github, we should have GITHUB_ACCESS_TOKEN set
# already.
if [ -n "$GITHUB_RELEASE_TOKEN" ]; then
GITHUB_TOKEN="$GITHUB_RELEASE_TOKEN"
elif [ -n "$GITHUB_PR_TOKEN" ]; then
GITHUB_TOKEN="$GITHUB_PR_TOKEN"
fi
out=$(curl -H "Authorization: token $GITHUB_TOKEN" -s "$api_base/$repo/pulls/$pr_id")
[ -n "$(echo "$out" | tr -d '\r\n' | jq ".labels | .[] | select(.name==\"$label\")")" ]
}
# Formats a message into a JSON string for posting to Matrix
# message: 'any plaintext message'
# formatted_message: '<strong>optional message formatted in <em>html</em></strong>'
# Usage: structure_message $content $formatted_content (optional)
structure_message() {
if [ -z "$2" ]; then
body=$(jq -Rs --arg body "$1" '{"msgtype": "m.text", $body}' < /dev/null)
else
body=$(jq -Rs --arg body "$1" --arg formatted_body "$2" '{"msgtype": "m.text", $body, "format": "org.matrix.custom.html", $formatted_body}' < /dev/null)
fi
echo "$body"
}
# Post a message to a matrix room
# body: '{body: "JSON string produced by structure_message"}'
# room_id: !fsfSRjgjBWEWffws:matrix.parity.io
# access_token: see https://matrix.org/docs/guides/client-server-api/
# Usage: send_message $body (json formatted) $room_id $access_token
send_message() {
curl -XPOST -d "$1" "https://m.parity.io/_matrix/client/r0/rooms/$2/send/m.room.message?access_token=$3"
}
# Check for runtime changes between two commits. This is defined as any changes
# to bin/node/src/runtime, frame/ and primitives/sr_* trees.
has_runtime_changes() {
from=$1
to=$2
if git diff --name-only "${from}...${to}" \
| grep -q -e '^frame/' -e '^primitives/'
then
return 0
else
return 1
fi
}
substrate/scripts/ci/github/check_labels.sh
-68
View File
@@ -1,68 +0,0 @@
#!/usr/bin/env bash
set -e
#shellcheck source=../common/lib.sh
source "$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )/../common/lib.sh"
repo="$GITHUB_REPOSITORY"
pr="$GITHUB_PR"
ensure_labels() {
for label in "$@"; do
if has_label "$repo" "$pr" "$label"; then
return 0
fi
done
return 1
}
# Must have one of the following labels
releasenotes_labels=(
'B0-silent'
'B3-apinoteworthy'
'B5-clientnoteworthy'
'B7-runtimenoteworthy'
)
criticality_labels=(
'C1-low 📌'
'C3-medium 📣'
'C7-high ❗️'
'C9-critical ‼️'
)
audit_labels=(
'D1-audited 👍'
'D2-notlive 💤'
'D3-trivial 🧸'
'D5-nicetohaveaudit ⚠️'
'D9-needsaudit 👮'
)
echo "[+] Checking release notes (B) labels"
if ensure_labels "${releasenotes_labels[@]}"; then
echo "[+] Release notes label detected. All is well."
else
echo "[!] Release notes label not detected. Please add one of: ${releasenotes_labels[*]}"
exit 1
fi
echo "[+] Checking release criticality (C) labels"
if ensure_labels "${criticality_labels[@]}"; then
echo "[+] Release criticality label detected. All is well."
else
echo "[!] Release criticality label not detected. Please add one of: ${criticality_labels[*]}"
exit 1
fi
if has_runtime_changes origin/master "${HEAD_SHA}"; then
echo "[+] Runtime changes detected. Checking audit (D) labels"
if ensure_labels "${audit_labels[@]}"; then
echo "[+] Release audit label detected. All is well."
else
echo "[!] Release audit label not detected. Please add one of: ${audit_labels[*]}"
exit 1
fi
fi
exit 0
substrate/scripts/ci/github/generate_changelog.sh
-85
View File
@@ -1,85 +0,0 @@
#!/usr/bin/env bash
# shellcheck source=../common/lib.sh
source "$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )/../common/lib.sh"
version="$2"
last_version="$1"
all_changes="$(sanitised_git_logs "$last_version" "$version")"
runtime_changes=""
api_changes=""
client_changes=""
changes=""
migrations=""
while IFS= read -r line; do
pr_id=$(echo "$line" | sed -E 's/.*#([0-9]+)\)$/\1/')
# Skip if the PR has the silent label - this allows us to skip a few requests
if has_label 'paritytech/substrate' "$pr_id" 'B0-silent'; then
continue
fi
if has_label 'paritytech/substrate' "$pr_id" 'B3-apinoteworthy' ; then
api_changes="$api_changes
$line"
fi
if has_label 'paritytech/substrate' "$pr_id" 'B5-clientnoteworthy'; then
client_changes="$client_changes
$line"
fi
if has_label 'paritytech/substrate' "$pr_id" 'B7-runtimenoteworthy'; then
runtime_changes="$runtime_changes
$line"
fi
if has_label 'paritytech/substrate' "$pr_id" 'E1-runtime-migration'; then
migrations="$migrations
$line"
fi
done <<< "$all_changes"
# Make the substrate section if there are any substrate changes
if [ -n "$runtime_changes" ] ||
[ -n "$api_changes" ] ||
[ -n "$client_changes" ] ||
[ -n "$migrations" ]; then
changes=$(cat << EOF
Substrate changes
-----------------
EOF
)
if [ -n "$runtime_changes" ]; then
changes="$changes
Runtime
-------
$runtime_changes"
fi
if [ -n "$client_changes" ]; then
changes="$changes
Client
------
$client_changes"
fi
if [ -n "$api_changes" ]; then
changes="$changes
API
---
$api_changes"
fi
release_text="$release_text
$changes"
fi
if [ -n "$migrations" ]; then
changes="$changes
Runtime Migrations
------------------
$migrations"
fi
echo "$changes"
substrate/scripts/ci/gitlab/check-each-crate.py
-57
View File
@@ -1,57 +0,0 @@
#!/usr/bin/env python3
# A script that checks each workspace crate individually.
# It's relevant to check workspace crates individually because otherwise their compilation problems
# due to feature misconfigurations won't be caught, as exemplified by
# https://github.com/paritytech/substrate/issues/12705
#
# `check-each-crate.py target_group groups_total`
#
# - `target_group`: Integer starting from 1, the group this script should execute.
# - `groups_total`: Integer starting from 1, total number of groups.
import subprocess, sys
# Get all crates
output = subprocess.check_output(["cargo", "tree", "--locked", "--workspace", "--depth", "0", "--prefix", "none"])
# Convert the output into a proper list
crates = []
for line in output.splitlines():
if line != b"":
crates.append(line.decode('utf8').split(" ")[0])
# Make the list unique and sorted
crates = list(set(crates))
crates.sort()
target_group = int(sys.argv[1]) - 1
groups_total = int(sys.argv[2])
if len(crates) == 0:
print("No crates detected!", file=sys.stderr)
sys.exit(1)
print(f"Total crates: {len(crates)}", file=sys.stderr)
crates_per_group = len(crates) // groups_total
# If this is the last runner, we need to take care of crates
# after the group that we lost because of the integer division.
if target_group + 1 == groups_total:
overflow_crates = len(crates) % groups_total
else:
overflow_crates = 0
print(f"Crates per group: {crates_per_group}", file=sys.stderr)
# Check each crate
for i in range(0, crates_per_group + overflow_crates):
crate = crates_per_group * target_group + i
print(f"Checking {crates[crate]}", file=sys.stderr)
res = subprocess.run(["cargo", "check", "--locked", "-p", crates[crate]])
if res.returncode != 0:
sys.exit(1)
substrate/scripts/ci/gitlab/check_runtime.sh
-121
View File
@@ -1,121 +0,0 @@
#!/bin/sh
#
#
# check for any changes in the node/src/runtime, frame/ and primitives/sr_* trees. if
# there are any changes found, it should mark the PR breaksconsensus and
# "auto-fail" the PR if there isn't a change in the runtime/src/lib.rs file
# that alters the version.
set -e # fail on any error
#shellcheck source=../common/lib.sh
. "$(dirname "${0}")/../common/lib.sh"
VERSIONS_FILE="bin/node/runtime/src/lib.rs"
boldprint () { printf "|\n| \033[1m%s\033[0m\n|\n" "${@}"; }
boldcat () { printf "|\n"; while read -r l; do printf "| \033[1m%s\033[0m\n" "${l}"; done; printf "|\n" ; }
github_label () {
echo
echo "# run github-api job for labeling it ${1}"
curl -sS -X POST \
-F "token=${CI_JOB_TOKEN}" \
-F "ref=master" \
-F "variables[LABEL]=${1}" \
-F "variables[PRNO]=${CI_COMMIT_REF_NAME}" \
"${GITLAB_API}/projects/${GITHUB_API_PROJECT}/trigger/pipeline"
}
boldprint "latest 10 commits of ${CI_COMMIT_REF_NAME}"
git log --graph --oneline --decorate=short -n 10
boldprint "make sure the master branch and release tag are available in shallow clones"
git fetch --depth="${GIT_DEPTH:-100}" origin master
git fetch --depth="${GIT_DEPTH:-100}" origin release
git tag -f release FETCH_HEAD
git log -n1 release
boldprint "check if the wasm sources changed"
if ! has_runtime_changes origin/master "${CI_COMMIT_SHA}"
then
boldcat <<-EOT
no changes to the runtime source code detected
EOT
exit 0
fi
# check for spec_version updates: if the spec versions changed, then there is
# consensus-critical logic that has changed. the runtime wasm blobs must be
# rebuilt.
add_spec_version="$(git diff tags/release ${CI_COMMIT_SHA} -- "${VERSIONS_FILE}" \
| sed -n -r "s/^\+[[:space:]]+spec_version: +([0-9]+),$/\1/p")"
sub_spec_version="$(git diff tags/release ${CI_COMMIT_SHA} -- "${VERSIONS_FILE}" \
| sed -n -r "s/^\-[[:space:]]+spec_version: +([0-9]+),$/\1/p")"
if [ "${add_spec_version}" != "${sub_spec_version}" ]
then
boldcat <<-EOT
changes to the runtime sources and changes in the spec version.
spec_version: ${sub_spec_version} -> ${add_spec_version}
EOT
exit 0
else
# check for impl_version updates: if only the impl versions changed, we assume
# there is no consensus-critical logic that has changed.
add_impl_version="$(git diff tags/release ${CI_COMMIT_SHA} -- "${VERSIONS_FILE}" \
| sed -n -r 's/^\+[[:space:]]+impl_version: +([0-9]+),$/\1/p')"
sub_impl_version="$(git diff tags/release ${CI_COMMIT_SHA} -- "${VERSIONS_FILE}" \
| sed -n -r 's/^\-[[:space:]]+impl_version: +([0-9]+),$/\1/p')"
# see if the impl version changed
if [ "${add_impl_version}" != "${sub_impl_version}" ]
then
boldcat <<-EOT
changes to the runtime sources and changes in the impl version.
impl_version: ${sub_impl_version} -> ${add_impl_version}
EOT
exit 0
fi
boldcat <<-EOT
wasm source files changed but not the spec/impl version. If changes made do not alter logic,
just bump 'impl_version'. If they do change logic, bump 'spec_version'.
source file directories:
- bin/node/src/runtime
- frame
- primitives/sr-*
versions file: ${VERSIONS_FILE}
EOT
fi
# dropped through. there's something wrong; exit 1.
exit 1
# vim: noexpandtab
substrate/scripts/ci/gitlab/check_signed.sh
-16
View File
@@ -1,16 +0,0 @@
#!/usr/bin/env bash
# shellcheck source=../common/lib.sh
source "$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )/../common/lib.sh"
version="$CI_COMMIT_TAG"
echo '[+] Checking tag has been signed'
check_tag "paritytech/substrate" "$version"
case $? in
0) echo '[+] Tag found and has been signed'; exit 0
;;
1) echo '[!] Tag found but has not been signed. Aborting release.'; exit 1
;;
2) echo '[!] Tag not found. Aborting release.'; exit 1
esac
substrate/scripts/ci/gitlab/ensure-deps.sh
-80
View File
@@ -1,80 +0,0 @@
#!/usr/bin/env bash
# The script is meant to check if the rules regarding packages
# dependencies are satisfied.
# The general format is:
# [top-lvl-dir] MESSAGE/[other-top-dir]
# For instance no crate within `./client` directory
# is allowed to import any crate with a directory path containing `frame`.
# Such rule is just: `client crates must not depend on anything in /frame`.
# The script should be run from the main repo directory!
set -u
# HARD FAILING
MUST_NOT=(
"client crates must not depend on anything in /frame"
"client crates must not depend on anything in /node"
"frame crates must not depend on anything in /node"
"frame crates must not depend on anything in /client"
"primitives crates must not depend on anything in /frame"
)
# ONLY DISPLAYED, script still succeeds
PLEASE_DONT=(
"primitives crates should not depend on anything in /client"
)
VIOLATIONS=()
PACKAGES=()
function check_rule() {
rule=$1
from=$(echo $rule | cut -f1 -d\ )
to=$(echo $rule | cut -f2 -d\/)
cd $from
echo "Checking rule '$rule'"
packages=$(find -name Cargo.toml | xargs grep -wn "path.*\.\.\/$to")
has_references=$(echo -n $packages | wc -c)
if [ "$has_references" != "0" ]; then
VIOLATIONS+=("$rule")
# Find packages that violate:
PACKAGES+=("$packages")
fi
cd - > /dev/null
}
for rule in "${MUST_NOT[@]}"
do
check_rule "$rule";
done
# Only the MUST NOT will be counted towards failure
HARD_VIOLATIONS=${#VIOLATIONS[@]}
for rule in "${PLEASE_DONT[@]}"
do
check_rule "$rule";
done
# Display violations and fail
I=0
for v in "${VIOLATIONS[@]}"
do
cat << EOF
===========================================
======= Violation of rule: $v
===========================================
${PACKAGES[$I]}
EOF
I=$I+1
done
exit $HARD_VIOLATIONS
substrate/scripts/ci/gitlab/pipeline/build.yml
-215
View File
@@ -1,215 +0,0 @@
# This file is part of .gitlab-ci.yml
# Here are all jobs that are executed during "build" stage
# PIPELINE_SCRIPTS_TAG can be found in the project variables
.check-dependent-project:
stage: build
# DAG: this is artificial dependency
needs:
- job: cargo-clippy
artifacts: false
extends:
- .docker-env
- .test-refs-no-trigger-prs-only
variables:
RUSTFLAGS: "-D warnings"
script:
- cargo install --locked --git https://github.com/paritytech/try-runtime-cli --rev a93c9b5abe5d31a4cf1936204f7e5c489184b521
- git clone
--depth=1
--branch="$PIPELINE_SCRIPTS_TAG"
https://github.com/paritytech/pipeline-scripts
- ./pipeline-scripts/check_dependent_project.sh
--org paritytech
--dependent-repo "$DEPENDENT_REPO"
--github-api-token "$GITHUB_PR_TOKEN"
--extra-dependencies "$EXTRA_DEPENDENCIES"
--companion-overrides "$COMPANION_OVERRIDES"
.check-runtime-migration:
extends:
- .check-dependent-project
- .test-refs-no-trigger-prs-only
variables:
DEPENDENT_REPO: polkadot
COMPANION_OVERRIDES: |
substrate: polkadot-v*
polkadot: release-v*
COMPANION_CHECK_COMMAND: >
time cargo build --release -p "$NETWORK"-runtime --features try-runtime &&
time try-runtime \
--runtime ./target/release/wbuild/"$NETWORK"-runtime/target/wasm32-unknown-unknown/release/"$NETWORK"_runtime.wasm \
on-runtime-upgrade --checks=pre-and-post live --uri wss://${NETWORK}-try-runtime-node.parity-chains.parity.io:443
# Individual jobs are set up for each dependent project so that they can be ran in parallel.
# Arguably we could generate a job for each companion in the PR's description using Gitlab's
# parent-child pipelines but that's more complicated.
check-runtime-migration-polkadot:
extends:
- .check-runtime-migration
variables:
NETWORK: polkadot
check-runtime-migration-kusama:
extends: .check-runtime-migration
variables:
NETWORK: kusama
check-runtime-migration-rococo:
extends: .check-runtime-migration
variables:
NETWORK: rococo
allow_failure: true
check-runtime-migration-westend:
extends: .check-runtime-migration
variables:
NETWORK: westend
check-dependent-polkadot:
extends: .check-dependent-project
variables:
DEPENDENT_REPO: polkadot
COMPANION_OVERRIDES: |
substrate: polkadot-v*
polkadot: release-v*
# enable the same feature flags as polkadot's test-linux-stable
COMPANION_CHECK_COMMAND: >
cargo check --all-targets --workspace
--features=runtime-benchmarks,runtime-metrics,try-runtime
rules:
- if: $CI_COMMIT_REF_NAME =~ /^[0-9]+$/ #PRs
check-dependent-cumulus:
extends: .check-dependent-project
variables:
DEPENDENT_REPO: cumulus
EXTRA_DEPENDENCIES: polkadot
COMPANION_OVERRIDES: |
substrate: polkadot-v*
polkadot: release-v*
rules:
- if: $CI_COMMIT_REF_NAME =~ /^[0-9]+$/ #PRs
build-linux-substrate:
stage: build
extends:
- .collect-artifacts
- .docker-env
- .build-refs
variables:
# this variable gets overriden by "rusty-cachier environment inject", use the value as default
CARGO_TARGET_DIR: "$CI_PROJECT_DIR/target"
needs:
- job: test-linux-stable
artifacts: false
before_script:
- !reference [.timestamp, before_script]
- !reference [.job-switcher, before_script]
- mkdir -p ./artifacts/substrate/
- !reference [.rusty-cachier, before_script]
# tldr: we need to checkout the branch HEAD explicitly because of our dynamic versioning approach while building the substrate binary
# see https://github.com/paritytech/ci_cd/issues/682#issuecomment-1340953589
- git checkout -B "$CI_COMMIT_REF_NAME" "$CI_COMMIT_SHA"
script:
- rusty-cachier snapshot create
- WASM_BUILD_NO_COLOR=1 time cargo build --locked --release -p node-cli --verbose
- mv $CARGO_TARGET_DIR/release/substrate-node ./artifacts/substrate/substrate
- echo -n "Substrate version = "
- if [ "${CI_COMMIT_TAG}" ]; then
echo "${CI_COMMIT_TAG}" | tee ./artifacts/substrate/VERSION;
else
./artifacts/substrate/substrate --version |
cut -d ' ' -f 2 | tee ./artifacts/substrate/VERSION;
fi
- sha256sum ./artifacts/substrate/substrate | tee ./artifacts/substrate/substrate.sha256
- cp -r ./scripts/ci/docker/substrate.Dockerfile ./artifacts/substrate/
- printf '\n# building node-template\n\n'
- ./scripts/ci/node-template-release.sh ./artifacts/substrate/substrate-node-template.tar.gz
- rusty-cachier cache upload
.build-subkey:
stage: build
extends:
- .collect-artifacts
- .docker-env
- .publish-refs
variables:
# this variable gets overriden by "rusty-cachier environment inject", use the value as default
CARGO_TARGET_DIR: "$CI_PROJECT_DIR/target"
before_script:
- !reference [.timestamp, before_script]
- !reference [.job-switcher, before_script]
- mkdir -p ./artifacts/subkey
- !reference [.rusty-cachier, before_script]
script:
- rusty-cachier snapshot create
- cd ./bin/utils/subkey
- SKIP_WASM_BUILD=1 time cargo build --locked --release --verbose
- cd -
- mv $CARGO_TARGET_DIR/release/subkey ./artifacts/subkey/.
- echo -n "Subkey version = "
- ./artifacts/subkey/subkey --version |
sed -n -E 's/^subkey ([0-9.]+.*)/\1/p' |
tee ./artifacts/subkey/VERSION;
- sha256sum ./artifacts/subkey/subkey | tee ./artifacts/subkey/subkey.sha256
- cp -r ./scripts/ci/docker/subkey.Dockerfile ./artifacts/subkey/
- rusty-cachier cache upload
build-subkey-linux:
extends: .build-subkey
build-subkey-macos:
extends: .build-subkey
# duplicating before_script & script sections from .build-subkey hidden job
# to overwrite rusty-cachier integration as it doesn't work on macos
before_script:
# skip timestamp script, the osx bash doesn't support printf %()T
- !reference [.job-switcher, before_script]
- mkdir -p ./artifacts/subkey
script:
- cd ./bin/utils/subkey
- SKIP_WASM_BUILD=1 time cargo build --locked --release --verbose
- cd -
- mv ./target/release/subkey ./artifacts/subkey/.
- echo -n "Subkey version = "
- ./artifacts/subkey/subkey --version |
sed -n -E 's/^subkey ([0-9.]+.*)/\1/p' |
tee ./artifacts/subkey/VERSION;
- sha256sum ./artifacts/subkey/subkey | tee ./artifacts/subkey/subkey.sha256
- cp -r ./scripts/ci/docker/subkey.Dockerfile ./artifacts/subkey/
after_script: [""]
tags:
- osx
build-rustdoc:
stage: build
extends:
- .docker-env
- .test-refs
variables:
SKIP_WASM_BUILD: 1
DOC_INDEX_PAGE: "substrate/index.html" # default redirected page
# this variable gets overriden by "rusty-cachier environment inject", use the value as default
CARGO_TARGET_DIR: "$CI_PROJECT_DIR/target"
artifacts:
name: "${CI_JOB_NAME}_${CI_COMMIT_REF_NAME}-doc"
when: on_success
expire_in: 7 days
paths:
- ./crate-docs/
# DAG: this is artificial dependency
needs:
- job: cargo-clippy
artifacts: false
script:
- rusty-cachier snapshot create
- time cargo doc --locked --workspace --all-features --verbose --no-deps
- rm -f $CARGO_TARGET_DIR/doc/.lock
- mv $CARGO_TARGET_DIR/doc ./crate-docs
# FIXME: remove me after CI image gets nonroot
- chown -R nonroot:nonroot ./crate-docs
- echo "<meta http-equiv=refresh content=0;url=${DOC_INDEX_PAGE}>" > ./crate-docs/index.html
- rusty-cachier cache upload
substrate/scripts/ci/gitlab/pipeline/check.yml
-78
View File
@@ -1,78 +0,0 @@
# This file is part of .gitlab-ci.yml
# Here are all jobs that are executed during "check" stage
check-runtime:
stage: check
extends:
- .kubernetes-env
- .test-refs-no-trigger-prs-only
variables:
CI_IMAGE: "paritytech/tools:latest"
GITLAB_API: "https://gitlab.parity.io/api/v4"
GITHUB_API_PROJECT: "parity%2Finfrastructure%2Fgithub-api"
script:
- ./scripts/ci/gitlab/check_runtime.sh
allow_failure: true
check-signed-tag:
stage: check
extends: .kubernetes-env
variables:
CI_IMAGE: "paritytech/tools:latest"
rules:
- if: $CI_COMMIT_REF_NAME =~ /^ci-release-.*$/
- if: $CI_COMMIT_REF_NAME =~ /^v[0-9]+\.[0-9]+.*$/ # i.e. v1.0, v2.1rc1
script:
- ./scripts/ci/gitlab/check_signed.sh
test-dependency-rules:
stage: check
extends:
- .kubernetes-env
- .test-refs-no-trigger-prs-only
variables:
CI_IMAGE: "paritytech/tools:latest"
script:
- ./scripts/ci/gitlab/ensure-deps.sh
test-rust-features:
stage: check
extends:
- .kubernetes-env
- .test-refs-no-trigger-prs-only
script:
- git clone
--depth=1
--branch="$PIPELINE_SCRIPTS_TAG"
https://github.com/paritytech/pipeline-scripts
- bash ./pipeline-scripts/rust-features.sh .
test-rust-feature-propagation:
stage: check
extends:
- .kubernetes-env
- .test-refs-no-trigger-prs-only
script:
- cargo install --locked --version 0.7.4 -q -f zepter && zepter --version
- echo "👉 Hello developer! If you see this CI check failing then it means that one of the crates is missing a feature for one of its dependencies. The output below tells you which feature needs to be added for which dependency to which crate. You can do this by modifying the Cargo.toml file. For more context see the MR where this check was introduced https://github.com/paritytech/substrate/pull/14660"
- zepter lint propagate-feature --feature try-runtime --left-side-feature-missing=ignore --workspace --feature-enables-dep="try-runtime:frame-try-runtime" --locked
- zepter lint propagate-feature --feature runtime-benchmarks --left-side-feature-missing=ignore --workspace --feature-enables-dep="runtime-benchmarks:frame-benchmarking" --locked
- zepter lint propagate-feature --feature std --left-side-feature-missing=ignore --workspace --locked
allow_failure: true # Experimental
test-prometheus-alerting-rules:
stage: check
extends: .kubernetes-env
variables:
CI_IMAGE: "paritytech/tools:latest"
rules:
- if: $CI_PIPELINE_SOURCE == "pipeline"
when: never
- if: $CI_COMMIT_BRANCH
changes:
- .gitlab-ci.yml
- ./scripts/ci/monitoring/**/*
script:
- promtool check rules ./scripts/ci/monitoring/alerting-rules/alerting-rules.yaml
- cat ./scripts/ci/monitoring/alerting-rules/alerting-rules.yaml |
promtool test rules ./scripts/ci/monitoring/alerting-rules/alerting-rule-tests.yaml
substrate/scripts/ci/gitlab/pipeline/publish.yml
-270
View File
@@ -1,270 +0,0 @@
# This file is part of .gitlab-ci.yml
# Here are all jobs that are executed during "publish" stage
.build-push-docker-image-common:
extends:
- .kubernetes-env
stage: publish
variables:
CI_IMAGE: $BUILDAH_IMAGE
GIT_STRATEGY: none
DOCKERFILE: $PRODUCT.Dockerfile
IMAGE_NAME: docker.io/$IMAGE_PATH
before_script:
- !reference [.kubernetes-env, before_script]
- cd ./artifacts/$PRODUCT/
- VERSION="$(cat ./VERSION)"
- echo "${PRODUCT} version = ${VERSION}"
- test -z "${VERSION}" && exit 1
script:
- test "$DOCKER_USER" -a "$DOCKER_PASS" ||
( echo "no docker credentials provided"; exit 1 )
- $BUILDAH_COMMAND build
--format=docker
--build-arg VCS_REF="${CI_COMMIT_SHA}"
--build-arg BUILD_DATE="$(date -u '+%Y-%m-%dT%H:%M:%SZ')"
--build-arg IMAGE_NAME="${IMAGE_PATH}"
--tag "$IMAGE_NAME:$VERSION"
--tag "$IMAGE_NAME:latest"
--file "$DOCKERFILE" .
- echo "$DOCKER_PASS" |
buildah login --username "$DOCKER_USER" --password-stdin docker.io
- $BUILDAH_COMMAND info
- $BUILDAH_COMMAND push --format=v2s2 "$IMAGE_NAME:$VERSION"
- $BUILDAH_COMMAND push --format=v2s2 "$IMAGE_NAME:latest"
after_script:
- buildah logout --all
- echo "SUBSTRATE_IMAGE_NAME=${IMAGE_NAME}" | tee -a ./artifacts/$PRODUCT/build.env
- IMAGE_TAG="$(cat ./artifacts/$PRODUCT/VERSION)"
- echo "SUBSTRATE_IMAGE_TAG=${IMAGE_TAG}" | tee -a ./artifacts/$PRODUCT/build.env
- cat ./artifacts/$PRODUCT/build.env
.build-push-docker-image:
extends:
- .publish-refs
- .build-push-docker-image-common
variables:
IMAGE_PATH: parity/$PRODUCT
DOCKER_USER: $Docker_Hub_User_Parity
DOCKER_PASS: $Docker_Hub_Pass_Parity
.push-docker-image-description:
stage: publish
extends:
- .kubernetes-env
variables:
CI_IMAGE: paritytech/dockerhub-description
DOCKERHUB_REPOSITORY: parity/$PRODUCT
DOCKER_USERNAME: $Docker_Hub_User_Parity
DOCKER_PASSWORD: $Docker_Hub_Pass_Parity
README_FILEPATH: $CI_PROJECT_DIR/scripts/ci/docker/$PRODUCT.Dockerfile.README.md
rules:
- if: $CI_COMMIT_REF_NAME == $CI_DEFAULT_BRANCH && $CI_PIPELINE_SOURCE == "push"
changes:
- scripts/ci/docker/$PRODUCT.Dockerfile.README.md
before_script:
- echo
script:
- cd / && sh entrypoint.sh
# publish image to docker.io/paritypr, (e.g. for later use in zombienet testing)
.build-push-image-temporary:
extends:
- .build-refs
- .build-push-docker-image-common
variables:
IMAGE_PATH: paritypr/$PRODUCT
DOCKER_USER: $PARITYPR_USER
DOCKER_PASS: $PARITYPR_PASS
publish-docker-substrate:
extends: .build-push-docker-image
needs:
- job: build-linux-substrate
artifacts: true
variables:
PRODUCT: substrate
publish-docker-description-substrate:
extends: .push-docker-image-description
variables:
PRODUCT: substrate
SHORT_DESCRIPTION: "Substrate Docker Image."
publish-docker-substrate-temporary:
extends: .build-push-image-temporary
needs:
- job: build-linux-substrate
artifacts: true
variables:
PRODUCT: substrate
artifacts:
reports:
# this artifact is used in zombienet-tests job
# https://docs.gitlab.com/ee/ci/multi_project_pipelines.html#with-variable-inheritance
dotenv: ./artifacts/$PRODUCT/build.env
expire_in: 24h
publish-docker-subkey:
extends: .build-push-docker-image
needs:
- job: build-subkey-linux
artifacts: true
variables:
PRODUCT: subkey
publish-docker-description-subkey:
extends: .push-docker-image-description
variables:
PRODUCT: subkey
SHORT_DESCRIPTION: "The subkey program is a key management utility for Substrate-based blockchains."
publish-s3-release:
stage: publish
extends:
- .publish-refs
- .kubernetes-env
needs:
- job: build-linux-substrate
artifacts: true
- job: build-subkey-linux
artifacts: true
image: paritytech/awscli:latest
variables:
GIT_STRATEGY: none
BUCKET: "releases.parity.io"
PREFIX: "substrate/${ARCH}-${DOCKER_OS}"
script:
- aws s3 sync ./artifacts/ s3://${BUCKET}/${PREFIX}/$(cat ./artifacts/substrate/VERSION)/
- echo "update objects in latest path"
- aws s3 sync s3://${BUCKET}/${PREFIX}/$(cat ./artifacts/substrate/VERSION)/ s3://${BUCKET}/${PREFIX}/latest/
after_script:
- aws s3 ls s3://${BUCKET}/${PREFIX}/latest/
--recursive --human-readable --summarize
publish-rustdoc:
stage: publish
extends: .kubernetes-env
variables:
CI_IMAGE: node:16
GIT_DEPTH: 100
RUSTDOCS_DEPLOY_REFS: "master"
rules:
- if: $CI_PIPELINE_SOURCE == "pipeline"
when: never
- if: $CI_PIPELINE_SOURCE == "web" && $CI_COMMIT_REF_NAME == "master"
- if: $CI_COMMIT_REF_NAME == "master"
- if: $CI_COMMIT_REF_NAME =~ /^monthly-20[0-9]{2}-[0-9]{2}.*$/ # to support: monthly-2021-09+1
- if: $CI_COMMIT_REF_NAME =~ /^v[0-9]+\.[0-9]+.*$/ # i.e. v1.0, v2.1rc1
# `needs:` can be removed after CI image gets nonroot. In this case `needs:` stops other
# artifacts from being dowloaded by this job.
needs:
- job: build-rustdoc
artifacts: true
script:
# If $CI_COMMIT_REF_NAME doesn't match one of $RUSTDOCS_DEPLOY_REFS space-separated values, we
# exit immediately.
# Putting spaces at the front and back to ensure we are not matching just any substring, but the
# whole space-separated value.
- '[[ " ${RUSTDOCS_DEPLOY_REFS} " =~ " ${CI_COMMIT_REF_NAME} " ]] || exit 0'
# setup ssh
- eval $(ssh-agent)
- ssh-add - <<< ${GITHUB_SSH_PRIV_KEY}
- mkdir ~/.ssh && touch ~/.ssh/known_hosts
- ssh-keyscan -t rsa github.com >> ~/.ssh/known_hosts
# Set git config
- git config user.email "devops-team@parity.io"
- git config user.name "${GITHUB_USER}"
- git config remote.origin.url "git@github.com:/paritytech/${CI_PROJECT_NAME}.git"
- git config remote.origin.fetch "+refs/heads/*:refs/remotes/origin/*"
- git fetch origin gh-pages
# Save README and docs
- cp -r ./crate-docs/ /tmp/doc/
- cp README.md /tmp/doc/
# we don't need to commit changes because we copy docs to /tmp
- git checkout gh-pages --force
# Install `index-tpl-crud` and generate index.html based on RUSTDOCS_DEPLOY_REFS
- which index-tpl-crud &> /dev/null || yarn global add @substrate/index-tpl-crud
- index-tpl-crud upsert ./index.html ${CI_COMMIT_REF_NAME}
# Ensure the destination dir doesn't exist.
- rm -rf ${CI_COMMIT_REF_NAME}
- mv -f /tmp/doc ${CI_COMMIT_REF_NAME}
# Upload files
- git add --all
# `git commit` has an exit code of > 0 if there is nothing to commit.
# This causes GitLab to exit immediately and marks this job failed.
# We don't want to mark the entire job failed if there's nothing to
# publish though, hence the `|| true`.
- git commit -m "___Updated docs for ${CI_COMMIT_REF_NAME}___" ||
echo "___Nothing to commit___"
- git push origin gh-pages --force
after_script:
- rm -rf .git/ ./*
publish-draft-release:
stage: publish
image: paritytech/tools:latest
rules:
- if: $CI_COMMIT_REF_NAME =~ /^ci-release-.*$/
- if: $CI_COMMIT_REF_NAME =~ /^v[0-9]+\.[0-9]+.*$/ # i.e. v1.0, v2.1rc1
script:
- ./scripts/ci/gitlab/publish_draft_release.sh
allow_failure: true
.publish-crates-template:
stage: publish
extends:
- .crates-publishing-template
- .crates-publishing-pipeline
# We don't want multiple jobs racing to publish crates as it's redundant and they might overwrite
# the releases of one another. Use resource_group to ensure that at most one instance of this job
# is running at any given time.
resource_group: crates-publishing
# crates.io currently rate limits crate publishing at 1 per minute:
# https://github.com/paritytech/release-engineering/issues/123#issuecomment-1335509748
# Taking into account the 202 (as of Dec 07, 2022) publishable Substrate crates, in the worst
# case, due to the rate limits alone, we'd have to wait through at least 202 minutes of delay.
# Taking into account also the verification steps and extra synchronization delays after
# publishing the crate, the job needs to have a much higher timeout than average.
timeout: 9h
# A custom publishing environment is used for us to be able to set up protected secrets
# specifically for it
environment: publish-crates
script:
- rusty-cachier snapshot create
- git clone
--depth 1
--branch "$RELENG_SCRIPTS_BRANCH"
https://github.com/paritytech/releng-scripts.git
- CRATESIO_TARGET_INSTANCE=default ./releng-scripts/publish-crates
- rusty-cachier cache upload
publish-crates:
extends: .publish-crates-template
# publish-crates should only be run if publish-crates-locally passes
needs:
- job: check-crate-publishing
artifacts: false
publish-crates-manual:
extends: .publish-crates-template
when: manual
interruptible: false
check-crate-publishing:
stage: publish
extends:
- .crates-publishing-template
- .crates-publishing-pipeline
# When lots of crates are taken into account (for example on master where all crates are tested)
# the job might take a long time, as evidenced by:
# https://gitlab.parity.io/parity/mirrors/substrate/-/jobs/2269364
timeout: 4h
script:
- rusty-cachier snapshot create
- git clone
--depth 1
--branch "$RELENG_SCRIPTS_BRANCH"
https://github.com/paritytech/releng-scripts.git
- CRATESIO_TARGET_INSTANCE=local ./releng-scripts/publish-crates
- rusty-cachier cache upload
substrate/scripts/ci/gitlab/pipeline/test.yml
-494
View File
@@ -1,494 +0,0 @@
# This file is part of .gitlab-ci.yml
# Here are all jobs that are executed during "test" stage
# It's more like a check and it belongs to the previous stage, but we want to run this job with real tests in parallel
find-fail-ci-phrase:
stage: test
variables:
CI_IMAGE: "paritytech/tools:latest"
ASSERT_REGEX: "FAIL-CI"
GIT_DEPTH: 1
extends:
- .kubernetes-env
script:
- set +e
- rg --line-number --hidden --type rust --glob '!{.git,target}' "$ASSERT_REGEX" .; exit_status=$?
- if [ $exit_status -eq 0 ]; then
echo "$ASSERT_REGEX was found, exiting with 1";
exit 1;
else
echo "No $ASSERT_REGEX was found, exiting with 0";
exit 0;
fi
cargo-deny-licenses:
stage: test
extends:
- .docker-env
- .test-refs
variables:
CARGO_DENY_CMD: "cargo deny --all-features check licenses -c ./scripts/ci/deny.toml"
script:
- rusty-cachier snapshot create
- $CARGO_DENY_CMD --hide-inclusion-graph
- rusty-cachier cache upload
after_script:
- !reference [.rusty-cachier, after_script]
- echo "___The complete log is in the artifacts___"
- $CARGO_DENY_CMD 2> deny.log
- if [ $CI_JOB_STATUS != 'success' ]; then
echo 'Please check license of your crate or add an exception to scripts/ci/deny.toml';
fi
artifacts:
name: $CI_COMMIT_SHORT_SHA
expire_in: 3 days
when: always
paths:
- deny.log
cargo-fmt:
stage: test
variables:
RUSTY_CACHIER_TOOLCHAIN: nightly
extends:
- .docker-env
- .test-refs
script:
- rusty-cachier snapshot create
- cargo +nightly fmt --all -- --check
- rusty-cachier cache upload
cargo-fmt-manifest:
stage: test
extends:
- .docker-env
- .test-refs
script:
- cargo install zepter --locked --version 0.11.1 -q -f --no-default-features && zepter --version
- echo "👉 Hello developer! If you see this CI check failing then it means that one of the your changes in a Cargo.toml file introduced ill-formatted or unsorted features. Please take a look at 'docs/STYLE_GUIDE.md#manifest-formatting' to find out more."
- zepter format features --check
allow_failure: true # Experimental
cargo-clippy:
stage: test
# this is an artificial job dependency, for pipeline optimization using GitLab's DAGs
needs:
- job: cargo-fmt
artifacts: false
extends:
- .docker-env
- .test-refs
script:
- echo $RUSTFLAGS
- cargo version && cargo clippy --version
- rusty-cachier snapshot create
- SKIP_WASM_BUILD=1 env -u RUSTFLAGS cargo clippy --locked --all-targets --workspace
- rusty-cachier cache upload
cargo-check-benches:
stage: test
variables:
CI_JOB_NAME: "cargo-check-benches"
extends:
- .docker-env
- .test-refs-check-benches
- .collect-artifacts
- .pipeline-stopper-artifacts
before_script:
- !reference [.timestamp, before_script]
# perform rusty-cachier operations before any further modifications to the git repo to make cargo feel cheated not so much
- !reference [.rust-info-script, script]
- !reference [.job-switcher, before_script]
- !reference [.rusty-cachier, before_script]
- !reference [.pipeline-stopper-vars, script]
# merges in the master branch on PRs. skip if base is not master
- 'if [ $CI_COMMIT_REF_NAME != "master" ]; then
BASE=$(curl -s -H "Authorization: Bearer ${GITHUB_PR_TOKEN}" https://api.github.com/repos/paritytech/substrate/pulls/${CI_COMMIT_REF_NAME} | jq -r .base.ref);
printf "Merging base branch %s\n" "${BASE:=master}";
if [ $BASE != "master" ]; then
echo "$BASE is not master, skipping merge";
else
git config user.email "ci@gitlab.parity.io";
git fetch origin "refs/heads/${BASE}";
git merge --verbose --no-edit FETCH_HEAD;
fi
fi'
parallel: 2
script:
- rusty-cachier snapshot create
- mkdir -p ./artifacts/benches/$CI_COMMIT_REF_NAME-$CI_COMMIT_SHORT_SHA
# this job is executed in parallel on two runners
- echo "___Running benchmarks___";
- case ${CI_NODE_INDEX} in
1)
SKIP_WASM_BUILD=1 time cargo check --locked --benches --all;
cargo run --locked --release -p node-bench -- ::trie::read::small --json
| tee ./artifacts/benches/$CI_COMMIT_REF_NAME-$CI_COMMIT_SHORT_SHA/::trie::read::small.json;
echo "___Uploading cache for rusty-cachier___";
rusty-cachier cache upload
;;
2)
cargo run --locked --release -p node-bench -- ::node::import::sr25519::transfer_keep_alive::paritydb::small --json
| tee ./artifacts/benches/$CI_COMMIT_REF_NAME-$CI_COMMIT_SHORT_SHA/::node::import::sr25519::transfer_keep_alive::paritydb::small.json
;;
esac
node-bench-regression-guard:
# it's not belong to `build` semantically, but dag jobs can't depend on each other
# within the single stage - https://gitlab.com/gitlab-org/gitlab/-/issues/30632
# more: https://github.com/paritytech/substrate/pull/8519#discussion_r608012402
stage: build
extends:
- .docker-env
- .test-refs-no-trigger-prs-only
needs:
# this is a DAG
- job: cargo-check-benches
artifacts: true
# polls artifact from master to compare with current result
# need to specify both parallel jobs from master because of the bug
# https://gitlab.com/gitlab-org/gitlab/-/issues/39063
- project: $CI_PROJECT_PATH
job: "cargo-check-benches 1/2"
ref: master
artifacts: true
- project: $CI_PROJECT_PATH
job: "cargo-check-benches 2/2"
ref: master
artifacts: true
variables:
CI_IMAGE: "paritytech/node-bench-regression-guard:latest"
before_script:
- !reference [.timestamp, before_script]
script:
- echo "------- IMPORTANT -------"
- echo "node-bench-regression-guard depends on the results of a cargo-check-benches job"
- echo "In case of this job failure, check your pipeline's cargo-check-benches"
- "node-bench-regression-guard --reference artifacts/benches/master-*
--compare-with artifacts/benches/$CI_COMMIT_REF_NAME-$CI_COMMIT_SHORT_SHA"
after_script: [""]
cargo-check-try-runtime-and-experimental:
stage: test
extends:
- .docker-env
- .test-refs
script:
- rusty-cachier snapshot create
- time cargo check --workspace --locked --features try-runtime,experimental
- rusty-cachier cache upload
test-deterministic-wasm:
stage: test
# this is an artificial job dependency, for pipeline optimization using GitLab's DAGs
needs:
- job: cargo-check-try-runtime-and-experimental
artifacts: false
extends:
- .docker-env
- .test-refs
variables:
WASM_BUILD_NO_COLOR: 1
# this variable gets overriden by "rusty-cachier environment inject", use the value as default
CARGO_TARGET_DIR: "$CI_PROJECT_DIR/target"
script:
- rusty-cachier snapshot create
# build runtime
- cargo build --locked --verbose --release -p kitchensink-runtime
# make checksum
- sha256sum $CARGO_TARGET_DIR/release/wbuild/kitchensink-runtime/target/wasm32-unknown-unknown/release/kitchensink_runtime.wasm > checksum.sha256
# clean up
- rm -rf $CARGO_TARGET_DIR/release/wbuild
# build again
- cargo build --locked --verbose --release -p kitchensink-runtime
# confirm checksum
- sha256sum -c ./checksum.sha256
# clean up again, don't put release binaries into the cache
- rm -rf $CARGO_TARGET_DIR/release/wbuild
- rusty-cachier cache upload
test-linux-stable:
stage: test
extends:
- .docker-env
- .test-refs
- .pipeline-stopper-artifacts
variables:
# Enable debug assertions since we are running optimized builds for testing
# but still want to have debug assertions.
RUSTFLAGS: "-C debug-assertions -D warnings"
RUST_BACKTRACE: 1
WASM_BUILD_NO_COLOR: 1
WASM_BUILD_RUSTFLAGS: "-C debug-assertions -D warnings"
# Ensure we run the UI tests.
RUN_UI_TESTS: 1
# needed for rusty-cachier to keep cache in test-linux-stable folder and not in test-linux-stable-1/3
CI_JOB_NAME: "test-linux-stable"
parallel: 3
script:
- rusty-cachier snapshot create
# this job runs all tests in former runtime-benchmarks, frame-staking and wasmtime tests
# tests are partitioned by nextest and executed in parallel on $CI_NODE_TOTAL runners
- echo "Node index - ${CI_NODE_INDEX}. Total amount - ${CI_NODE_TOTAL}"
- time cargo nextest run --workspace
--locked
--release
--verbose
--features runtime-benchmarks,try-runtime,experimental
--manifest-path ./bin/node/cli/Cargo.toml
--partition count:${CI_NODE_INDEX}/${CI_NODE_TOTAL}
# run runtime-api tests with `enable-staging-api` feature
- time cargo nextest run -p sp-api-test --features enable-staging-api
# we need to update cache only from one job
- if [ ${CI_NODE_INDEX} == 1 ]; then rusty-cachier cache upload; fi
# Upload tests results to Elasticsearch
- echo "Upload test results to Elasticsearch"
- cat target/nextest/default/junit.xml | xq . > target/nextest/default/junit.json
- |
curl -v -XPOST --http1.1 \
-u ${ELASTIC_USERNAME}:${ELASTIC_PASSWORD} \
https://elasticsearch.parity-build.parity.io/unit-tests/_doc/${CI_JOB_ID} \
-H 'Content-Type: application/json' \
-d @target/nextest/default/junit.json || echo "failed to upload junit report"
artifacts:
when: always
paths:
- target/nextest/default/junit.xml
reports:
junit: target/nextest/default/junit.xml
test-frame-support:
stage: test
extends:
- .docker-env
- .test-refs
variables:
# Enable debug assertions since we are running optimized builds for testing
# but still want to have debug assertions.
RUSTFLAGS: "-C debug-assertions -D warnings"
RUST_BACKTRACE: 1
WASM_BUILD_NO_COLOR: 1
WASM_BUILD_RUSTFLAGS: "-C debug-assertions -D warnings"
# Ensure we run the UI tests.
RUN_UI_TESTS: 1
script:
- rusty-cachier snapshot create
- cat /cargo_target_dir/debug/.fingerprint/memory_units-759eddf317490d2b/lib-memory_units.json || true
- time cargo test --verbose --locked -p frame-support-test --features=frame-feature-testing,no-metadata-docs,try-runtime,experimental --manifest-path ./frame/support/test/Cargo.toml
- time cargo test --verbose --locked -p frame-support-test --features=frame-feature-testing,frame-feature-testing-2,no-metadata-docs,try-runtime,experimental --manifest-path ./frame/support/test/Cargo.toml
- SUBSTRATE_TEST_TIMEOUT=1 time cargo test -p substrate-test-utils --release --verbose --locked -- --ignored timeout
- cat /cargo_target_dir/debug/.fingerprint/memory_units-759eddf317490d2b/lib-memory_units.json || true
- rusty-cachier cache upload
# This job runs tests that don't work with cargo-nextest in test-linux-stable
test-linux-stable-extra:
stage: test
extends:
- .docker-env
- .test-refs
variables:
# Enable debug assertions since we are running optimized builds for testing
# but still want to have debug assertions.
RUSTFLAGS: "-C debug-assertions -D warnings"
RUST_BACKTRACE: 1
WASM_BUILD_NO_COLOR: 1
WASM_BUILD_RUSTFLAGS: "-C debug-assertions -D warnings"
# Ensure we run the UI tests.
RUN_UI_TESTS: 1
script:
- rusty-cachier snapshot create
# Run node-cli tests
# TODO: add to test-linux-stable-nextest after fix https://github.com/paritytech/substrate/issues/11321
- time cargo test node-cli --workspace --locked --release --verbose --features runtime-benchmarks --manifest-path ./bin/node/cli/Cargo.toml
# Run doctests
# TODO: add to test-linux-stable-nextest after fix https://github.com/nextest-rs/nextest/issues/16
- time cargo test --doc --workspace --locked --release --verbose --features runtime-benchmarks --manifest-path ./bin/node/cli/Cargo.toml
- rusty-cachier cache upload
# This job runs all benchmarks defined in the `/bin/node/runtime` once to check that there are no errors.
quick-benchmarks:
stage: test
extends:
- .docker-env
- .test-refs
variables:
# Enable debug assertions since we are running optimized builds for testing
# but still want to have debug assertions.
RUSTFLAGS: "-C debug-assertions -D warnings"
RUST_BACKTRACE: "full"
WASM_BUILD_NO_COLOR: 1
WASM_BUILD_RUSTFLAGS: "-C debug-assertions -D warnings"
script:
- rusty-cachier snapshot create
- time cargo run --locked --release -p node-cli --features runtime-benchmarks -- benchmark pallet --wasm-execution compiled --chain dev --pallet "*" --extrinsic "*" --steps 2 --repeat 1
- rusty-cachier cache upload
test-frame-examples-compile-to-wasm:
# into one job
stage: test
extends:
- .docker-env
- .test-refs
variables:
# Enable debug assertions since we are running optimized builds for testing
# but still want to have debug assertions.
RUSTFLAGS: "-C debug-assertions"
RUST_BACKTRACE: 1
script:
- rusty-cachier snapshot create
- cd ./frame/examples/offchain-worker/
- cargo build --locked --target=wasm32-unknown-unknown --no-default-features
- cd ../basic
- cargo build --locked --target=wasm32-unknown-unknown --no-default-features
- rusty-cachier cache upload
test-linux-stable-int:
stage: test
extends:
- .docker-env
- .test-refs
- .pipeline-stopper-artifacts
variables:
# Enable debug assertions since we are running optimized builds for testing
# but still want to have debug assertions.
RUSTFLAGS: "-C debug-assertions -D warnings"
RUST_BACKTRACE: 1
WASM_BUILD_NO_COLOR: 1
WASM_BUILD_RUSTFLAGS: "-C debug-assertions -D warnings"
# Ensure we run the UI tests.
RUN_UI_TESTS: 1
script:
- rusty-cachier snapshot create
- WASM_BUILD_NO_COLOR=1
RUST_LOG=sync=trace,consensus=trace,client=trace,state-db=trace,db=trace,forks=trace,state_db=trace,storage_cache=trace
time cargo test -p node-cli --release --verbose --locked -- --ignored
- rusty-cachier cache upload
# more information about this job can be found here:
# https://github.com/paritytech/substrate/pull/6916
check-tracing:
stage: test
# this is an artificial job dependency, for pipeline optimization using GitLab's DAGs
needs:
- job: test-linux-stable-int
artifacts: false
extends:
- .docker-env
- .test-refs
- .pipeline-stopper-artifacts
script:
- rusty-cachier snapshot create
# with-tracing must be explicitly activated, we run a test to ensure this works as expected in both cases
- time cargo test --locked --manifest-path ./primitives/tracing/Cargo.toml --no-default-features
- time cargo test --locked --manifest-path ./primitives/tracing/Cargo.toml --no-default-features --features=with-tracing
- rusty-cachier cache upload
# more information about this job can be found here:
# https://github.com/paritytech/substrate/pull/3778
test-full-crypto-feature:
stage: test
# this is an artificial job dependency, for pipeline optimization using GitLab's DAGs
needs:
- job: check-tracing
artifacts: false
extends:
- .docker-env
- .test-refs
variables:
# Enable debug assertions since we are running optimized builds for testing
# but still want to have debug assertions.
RUSTFLAGS: "-C debug-assertions"
RUST_BACKTRACE: 1
script:
- rusty-cachier snapshot create
- cd primitives/core/
- time cargo build --locked --verbose --no-default-features --features full_crypto
- cd ../application-crypto
- time cargo build --locked --verbose --no-default-features --features full_crypto
- rusty-cachier cache upload
check-rustdoc:
stage: test
extends:
- .docker-env
- .test-refs
variables:
SKIP_WASM_BUILD: 1
RUSTDOCFLAGS: "-Dwarnings"
script:
- rusty-cachier snapshot create
- time cargo doc --locked --workspace --all-features --verbose --no-deps
- rusty-cachier cache upload
cargo-check-each-crate:
stage: test
extends:
- .docker-env
- .test-refs
- .collect-artifacts
- .pipeline-stopper-artifacts
variables:
# $CI_JOB_NAME is set manually so that rusty-cachier can share the cache for all
# "cargo-check-each-crate I/N" jobs
CI_JOB_NAME: cargo-check-each-crate
script:
- rusty-cachier snapshot create
- PYTHONUNBUFFERED=x time ./scripts/ci/gitlab/check-each-crate.py "$CI_NODE_INDEX" "$CI_NODE_TOTAL"
# need to update cache only from one job
- if [ "$CI_NODE_INDEX" == 1 ]; then rusty-cachier cache upload; fi
parallel: 2
cargo-check-each-crate-macos:
stage: test
extends:
- .test-refs
- .collect-artifacts
- .pipeline-stopper-artifacts
before_script:
# skip timestamp script, the osx bash doesn't support printf %()T
- !reference [.job-switcher, before_script]
- !reference [.rust-info-script, script]
- !reference [.pipeline-stopper-vars, script]
variables:
SKIP_WASM_BUILD: 1
script:
# TODO: enable rusty-cachier once it supports Mac
# TODO: use parallel jobs, as per cargo-check-each-crate, once more Mac runners are available
# - time ./scripts/ci/gitlab/check-each-crate.py 1 1
- time cargo check --workspace --locked
tags:
- osx
cargo-hfuzz:
stage: test
extends:
- .docker-env
- .test-refs
- .pipeline-stopper-artifacts
variables:
# max 10s per iteration, 60s per file
HFUZZ_RUN_ARGS: >
--exit_upon_crash
--exit_code_upon_crash 1
--timeout 10
--run_time 60
# use git version of honggfuzz-rs until v0.5.56 is out, we need a few recent changes:
# https://github.com/rust-fuzz/honggfuzz-rs/pull/75 to avoid breakage on debian
# https://github.com/rust-fuzz/honggfuzz-rs/pull/81 fix to the above pr
# https://github.com/rust-fuzz/honggfuzz-rs/pull/82 fix for handling rusty-cachier's absolute CARGO_TARGET_DIR
HFUZZ_BUILD_ARGS: >
--config=patch.crates-io.honggfuzz.git="https://github.com/altaua/honggfuzz-rs"
--config=patch.crates-io.honggfuzz.rev="205f7c8c059a0d98fe1cb912cdac84f324cb6981"
artifacts:
name: "hfuzz-$CI_COMMIT_SHORT_SHA"
expire_in: 7 days
when: on_failure
paths:
- primitives/arithmetic/fuzzer/hfuzz_workspace/
script:
- cd ./primitives/arithmetic/fuzzer
- rusty-cachier snapshot create
- cargo hfuzz build
- rusty-cachier cache upload
- for target in $(cargo read-manifest | jq -r '.targets | .[] | .name'); do
cargo hfuzz run "$target" || { printf "fuzzing failure for %s\n" "$target"; exit 1; }; done
substrate/scripts/ci/gitlab/pipeline/zombienet.yml
-67
View File
@@ -1,67 +0,0 @@
# This file is part of .gitlab-ci.yml
# Here are all jobs that are executed during "zombienet" stage
# common settings for all zombienet jobs
.zombienet-common:
before_script:
- echo "Zombie-net Tests Config"
- echo "${ZOMBIENET_IMAGE}"
- echo "${SUBSTRATE_IMAGE_NAME} ${SUBSTRATE_IMAGE_TAG}"
- echo "${GH_DIR}"
- export DEBUG=zombie,zombie::network-node
- export ZOMBIENET_INTEGRATION_TEST_IMAGE=${SUBSTRATE_IMAGE_NAME}:${SUBSTRATE_IMAGE_TAG}
- echo "${ZOMBIENET_INTEGRATION_TEST_IMAGE}"
stage: zombienet
image: "${ZOMBIENET_IMAGE}"
needs:
- job: publish-docker-substrate-temporary
extends:
- .kubernetes-env
- .zombienet-refs
variables:
GH_DIR: "https://github.com/paritytech/substrate/tree/${CI_COMMIT_SHA}/zombienet"
FF_DISABLE_UMASK_FOR_DOCKER_EXECUTOR: 1
artifacts:
name: "${CI_JOB_NAME}_${CI_COMMIT_REF_NAME}"
when: always
expire_in: 2 days
paths:
- ./zombienet-logs
after_script:
- mkdir -p ./zombienet-logs
- cp /tmp/zombie*/logs/* ./zombienet-logs/
retry: 2
tags:
- zombienet-polkadot-integration-test
zombienet-0000-block-building:
extends:
- .zombienet-common
script:
- /home/nonroot/zombie-net/scripts/ci/run-test-env-manager.sh
--github-remote-dir="${GH_DIR}/0000-block-building"
--test="block-building.zndsl"
zombienet-0001-basic-warp-sync:
extends:
- .zombienet-common
script:
- /home/nonroot/zombie-net/scripts/ci/run-test-env-manager.sh
--github-remote-dir="${GH_DIR}/0001-basic-warp-sync"
--test="test-warp-sync.zndsl"
zombienet-0002-validators-warp-sync:
extends:
- .zombienet-common
script:
- /home/nonroot/zombie-net/scripts/ci/run-test-env-manager.sh
--github-remote-dir="${GH_DIR}/0002-validators-warp-sync"
--test="test-validators-warp-sync.zndsl"
zombienet-0003-block-building-warp-sync:
extends:
- .zombienet-common
script:
- /home/nonroot/zombie-net/scripts/ci/run-test-env-manager.sh
--github-remote-dir="${GH_DIR}/0003-block-building-warp-sync"
--test="test-block-building-warp-sync.zndsl"
substrate/scripts/ci/gitlab/prettier.sh
-6
View File
@@ -1,6 +0,0 @@
#!/bin/sh
# meant to be installed via
# git config filter.ci-prettier.clean "scripts/ci/gitlab/prettier.sh"
prettier --parser yaml
substrate/scripts/ci/gitlab/publish_draft_release.sh
-54
View File
@@ -1,54 +0,0 @@
#!/usr/bin/env bash
# shellcheck source=../common/lib.sh
source "$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )/../common/lib.sh"
version="$CI_COMMIT_TAG"
# Note that this is not the last *tagged* version, but the last *published* version
last_version=$(last_github_release 'paritytech/substrate')
release_text="$(./generate_release_text.sh "$last_version" "$version")"
echo "[+] Pushing release to github"
# Create release on github
release_name="Substrate $version"
data=$(jq -Rs --arg version "$version" \
--arg release_name "$release_name" \
--arg release_text "$release_text" \
'{
"tag_name": $version,
"target_commitish": "master",
"name": $release_name,
"body": $release_text,
"draft": true,
"prerelease": false
}' < /dev/null)
out=$(curl -s -X POST --data "$data" -H "Authorization: token $GITHUB_RELEASE_TOKEN" "$api_base/paritytech/substrate/releases")
html_url=$(echo "$out" | jq -r .html_url)
if [ "$html_url" == "null" ]
then
echo "[!] Something went wrong posting:"
echo "$out"
else
echo "[+] Release draft created: $html_url"
fi
echo '[+] Sending draft release URL to Matrix'
msg_body=$(cat <<EOF
**Release pipeline for Substrate $version complete.**
Draft release created: $html_url
EOF
)
formatted_msg_body=$(cat <<EOF
<strong>Release pipeline for Substrate $version complete.</strong><br />
Draft release created: $html_url
EOF
)
send_message "$(structure_message "$msg_body" "$formatted_msg_body")" "!aJymqQYtCjjqImFLSb:parity.io" "$RELEASENOTES_MATRIX_V2_ACCESS_TOKEN"
echo "[+] Done! Maybe the release worked..."