mirror of
https://github.com/pezkuwichain/pezkuwi-subxt.git
synced 2026-06-12 05:21:08 +00:00
Move scripts used in CI to the new location (#11008)
Move scripts used in CI to the new location - **./scripts/ci/** * Move github scripts * Move more files * Move ci scripts and fix dependencies * Update docs/node-template-release.md Co-authored-by: João Paulo Silva de Souza <77391175+joao-paulo-parity@users.noreply.github.com> * Remove Cargo.lock * Apply suggestions from code review Co-authored-by: Denis Pisarev <denis.pisarev@parity.io> * Make more paths uniform Co-authored-by: João Paulo Silva de Souza <77391175+joao-paulo-parity@users.noreply.github.com> Co-authored-by: Denis Pisarev <denis.pisarev@parity.io>
This commit is contained in:
Sergejs Kostjucenko
committed by
GitHub
GitHub
parent
42484508d7
commit
46891e849f
Executable
+117
@@ -0,0 +1,117 @@
|
||||
#!/bin/sh
|
||||
|
||||
api_base="https://api.github.com/repos"
|
||||
|
||||
# Function to take 2 git tags/commits and get any lines from commit messages
|
||||
# that contain something that looks like a PR reference: e.g., (#1234)
|
||||
sanitised_git_logs(){
|
||||
git --no-pager log --pretty=format:"%s" "$1...$2" |
|
||||
# Only find messages referencing a PR
|
||||
grep -E '\(#[0-9]+\)' |
|
||||
# Strip any asterisks
|
||||
sed 's/^* //g' |
|
||||
# And add them all back
|
||||
sed 's/^/* /g'
|
||||
}
|
||||
|
||||
# Returns the last published release on github
|
||||
# Note: we can't just use /latest because that ignores prereleases
|
||||
# repo: 'organization/repo'
|
||||
# Usage: last_github_release "$repo"
|
||||
last_github_release(){
|
||||
i=0
|
||||
# Iterate over releases until we find the last release that's not just a draft
|
||||
while [ $i -lt 29 ]; do
|
||||
out=$(curl -H "Authorization: token $GITHUB_RELEASE_TOKEN" -s "$api_base/$1/releases" | jq ".[$i]")
|
||||
echo "$out"
|
||||
# Ugh when echoing to jq, we need to translate newlines into spaces :/
|
||||
if [ "$(echo "$out" | tr '\r\n' ' ' | jq '.draft')" = "false" ]; then
|
||||
echo "$out" | tr '\r\n' ' ' | jq '.tag_name'
|
||||
return
|
||||
else
|
||||
i=$((i + 1))
|
||||
fi
|
||||
done
|
||||
}
|
||||
|
||||
# Checks whether a tag on github has been verified
|
||||
# repo: 'organization/repo'
|
||||
# tagver: 'v1.2.3'
|
||||
# Usage: check_tag $repo $tagver
|
||||
check_tag () {
|
||||
repo=$1
|
||||
tagver=$2
|
||||
tag_out=$(curl -H "Authorization: token $GITHUB_RELEASE_TOKEN" -s "$api_base/$repo/git/refs/tags/$tagver")
|
||||
tag_sha=$(echo "$tag_out" | jq -r .object.sha)
|
||||
object_url=$(echo "$tag_out" | jq -r .object.url)
|
||||
if [ "$tag_sha" = "null" ]; then
|
||||
return 2
|
||||
fi
|
||||
verified_str=$(curl -H "Authorization: token $GITHUB_RELEASE_TOKEN" -s "$object_url" | jq -r .verification.verified)
|
||||
if [ "$verified_str" = "true" ]; then
|
||||
# Verified, everything is good
|
||||
return 0
|
||||
else
|
||||
# Not verified. Bad juju.
|
||||
return 1
|
||||
fi
|
||||
}
|
||||
|
||||
# Checks whether a given PR has a given label.
|
||||
# repo: 'organization/repo'
|
||||
# pr_id: 12345
|
||||
# label: B1-silent
|
||||
# Usage: has_label $repo $pr_id $label
|
||||
has_label(){
|
||||
repo="$1"
|
||||
pr_id="$2"
|
||||
label="$3"
|
||||
|
||||
# These will exist if the function is called in Gitlab.
|
||||
# If the function's called in Github, we should have GITHUB_ACCESS_TOKEN set
|
||||
# already.
|
||||
if [ -n "$GITHUB_RELEASE_TOKEN" ]; then
|
||||
GITHUB_TOKEN="$GITHUB_RELEASE_TOKEN"
|
||||
elif [ -n "$GITHUB_PR_TOKEN" ]; then
|
||||
GITHUB_TOKEN="$GITHUB_PR_TOKEN"
|
||||
fi
|
||||
|
||||
out=$(curl -H "Authorization: token $GITHUB_TOKEN" -s "$api_base/$repo/pulls/$pr_id")
|
||||
[ -n "$(echo "$out" | tr -d '\r\n' | jq ".labels | .[] | select(.name==\"$label\")")" ]
|
||||
}
|
||||
|
||||
# Formats a message into a JSON string for posting to Matrix
|
||||
# message: 'any plaintext message'
|
||||
# formatted_message: '<strong>optional message formatted in <em>html</em></strong>'
|
||||
# Usage: structure_message $content $formatted_content (optional)
|
||||
structure_message() {
|
||||
if [ -z "$2" ]; then
|
||||
body=$(jq -Rs --arg body "$1" '{"msgtype": "m.text", $body}' < /dev/null)
|
||||
else
|
||||
body=$(jq -Rs --arg body "$1" --arg formatted_body "$2" '{"msgtype": "m.text", $body, "format": "org.matrix.custom.html", $formatted_body}' < /dev/null)
|
||||
fi
|
||||
echo "$body"
|
||||
}
|
||||
|
||||
# Post a message to a matrix room
|
||||
# body: '{body: "JSON string produced by structure_message"}'
|
||||
# room_id: !fsfSRjgjBWEWffws:matrix.parity.io
|
||||
# access_token: see https://matrix.org/docs/guides/client-server-api/
|
||||
# Usage: send_message $body (json formatted) $room_id $access_token
|
||||
send_message() {
|
||||
curl -XPOST -d "$1" "https://matrix.parity.io/_matrix/client/r0/rooms/$2/send/m.room.message?access_token=$3"
|
||||
}
|
||||
|
||||
# Check for runtime changes between two commits. This is defined as any changes
|
||||
# to bin/node/src/runtime, frame/ and primitives/sr_* trees.
|
||||
has_runtime_changes() {
|
||||
from=$1
|
||||
to=$2
|
||||
if git diff --name-only "${from}...${to}" \
|
||||
| grep -q -e '^frame/' -e '^primitives/'
|
||||
then
|
||||
return 0
|
||||
else
|
||||
return 1
|
||||
fi
|
||||
}
|
||||
@@ -0,0 +1,193 @@
|
||||
# This template contains all of the possible sections and their default values
|
||||
|
||||
# Note that all fields that take a lint level have these possible values:
|
||||
# * deny - An error will be produced and the check will fail
|
||||
# * warn - A warning will be produced, but the check will not fail
|
||||
# * allow - No warning or error will be produced, though in some cases a note
|
||||
# will be
|
||||
|
||||
# The values provided in this template are the default values that will be used
|
||||
# when any section or field is not specified in your own configuration
|
||||
|
||||
# If 1 or more target triples (and optionally, target_features) are specified,
|
||||
# only the specified targets will be checked when running `cargo deny check`.
|
||||
# This means, if a particular package is only ever used as a target specific
|
||||
# dependency, such as, for example, the `nix` crate only being used via the
|
||||
# `target_family = "unix"` configuration, that only having windows targets in
|
||||
# this list would mean the nix crate, as well as any of its exclusive
|
||||
# dependencies not shared by any other crates, would be ignored, as the target
|
||||
# list here is effectively saying which targets you are building for.
|
||||
targets = [
|
||||
# The triple can be any string, but only the target triples built in to
|
||||
# rustc (as of 1.40) can be checked against actual config expressions
|
||||
#{ triple = "x86_64-unknown-linux-musl" },
|
||||
# You can also specify which target_features you promise are enabled for a
|
||||
# particular target. target_features are currently not validated against
|
||||
# the actual valid features supported by the target architecture.
|
||||
#{ triple = "wasm32-unknown-unknown", features = ["atomics"] },
|
||||
]
|
||||
|
||||
# This section is considered when running `cargo deny check advisories`
|
||||
# More documentation for the advisories section can be found here:
|
||||
# https://embarkstudios.github.io/cargo-deny/checks/advisories/cfg.html
|
||||
[advisories]
|
||||
# The path where the advisory database is cloned/fetched into
|
||||
db-path = "~/.cargo/advisory-db"
|
||||
# The url of the advisory database to use
|
||||
db-url = "https://github.com/rustsec/advisory-db"
|
||||
# The lint level for security vulnerabilities
|
||||
vulnerability = "deny"
|
||||
# The lint level for unmaintained crates
|
||||
unmaintained = "warn"
|
||||
# The lint level for crates that have been yanked from their source registry
|
||||
yanked = "warn"
|
||||
# The lint level for crates with security notices. Note that as of
|
||||
# 2019-12-17 there are no security notice advisories in
|
||||
# https://github.com/rustsec/advisory-db
|
||||
notice = "warn"
|
||||
# A list of advisory IDs to ignore. Note that ignored advisories will still
|
||||
# output a note when they are encountered.
|
||||
ignore = [
|
||||
#"RUSTSEC-0000-0000",
|
||||
]
|
||||
# Threshold for security vulnerabilities, any vulnerability with a CVSS score
|
||||
# lower than the range specified will be ignored. Note that ignored advisories
|
||||
# will still output a note when they are encountered.
|
||||
# * None - CVSS Score 0.0
|
||||
# * Low - CVSS Score 0.1 - 3.9
|
||||
# * Medium - CVSS Score 4.0 - 6.9
|
||||
# * High - CVSS Score 7.0 - 8.9
|
||||
# * Critical - CVSS Score 9.0 - 10.0
|
||||
#severity-threshold =
|
||||
|
||||
# This section is considered when running `cargo deny check licenses`
|
||||
# More documentation for the licenses section can be found here:
|
||||
# https://embarkstudios.github.io/cargo-deny/checks/licenses/cfg.html
|
||||
[licenses]
|
||||
# The lint level for crates which do not have a detectable license
|
||||
unlicensed = "deny"
|
||||
# List of explictly allowed licenses
|
||||
# See https://spdx.org/licenses/ for list of possible licenses
|
||||
# [possible values: any SPDX 3.7 short identifier (+ optional exception)].
|
||||
allow = [
|
||||
#"MIT",
|
||||
#"Apache-2.0",
|
||||
#"Apache-2.0 WITH LLVM-exception",
|
||||
]
|
||||
# List of explictly disallowed licenses
|
||||
# See https://spdx.org/licenses/ for list of possible licenses
|
||||
# [possible values: any SPDX 3.7 short identifier (+ optional exception)].
|
||||
deny = [
|
||||
#"Nokia",
|
||||
]
|
||||
# Lint level for licenses considered copyleft
|
||||
copyleft = "allow"
|
||||
# Blanket approval or denial for OSI-approved or FSF Free/Libre licenses
|
||||
# * both - The license will be approved if it is both OSI-approved *AND* FSF
|
||||
# * either - The license will be approved if it is either OSI-approved *OR* FSF
|
||||
# * osi-only - The license will be approved if is OSI-approved *AND NOT* FSF
|
||||
# * fsf-only - The license will be approved if is FSF *AND NOT* OSI-approved
|
||||
# * neither - This predicate is ignored and the default lint level is used
|
||||
allow-osi-fsf-free = "either"
|
||||
# Lint level used when no other predicates are matched
|
||||
# 1. License isn't in the allow or deny lists
|
||||
# 2. License isn't copyleft
|
||||
# 3. License isn't OSI/FSF, or allow-osi-fsf-free = "neither"
|
||||
default = "deny"
|
||||
# The confidence threshold for detecting a license from license text.
|
||||
# The higher the value, the more closely the license text must be to the
|
||||
# canonical license text of a valid SPDX license file.
|
||||
# [possible values: any between 0.0 and 1.0].
|
||||
confidence-threshold = 0.9
|
||||
# Allow 1 or more licenses on a per-crate basis, so that particular licenses
|
||||
# aren't accepted for every possible crate as with the normal allow list
|
||||
exceptions = [
|
||||
# Each entry is the crate and version constraint, and its specific allow
|
||||
# list
|
||||
#{ allow = ["Zlib"], name = "adler32", version = "*" },
|
||||
]
|
||||
|
||||
# Some crates don't have (easily) machine readable licensing information,
|
||||
# adding a clarification entry for it allows you to manually specify the
|
||||
# licensing information
|
||||
[[licenses.clarify]]
|
||||
# The name of the crate the clarification applies to
|
||||
name = "ring"
|
||||
# THe optional version constraint for the crate
|
||||
#version = "*"
|
||||
# The SPDX expression for the license requirements of the crate
|
||||
expression = "OpenSSL"
|
||||
# One or more files in the crate's source used as the "source of truth" for
|
||||
# the license expression. If the contents match, the clarification will be used
|
||||
# when running the license check, otherwise the clarification will be ignored
|
||||
# and the crate will be checked normally, which may produce warnings or errors
|
||||
# depending on the rest of your configuration
|
||||
license-files = [
|
||||
# Each entry is a crate relative path, and the (opaque) hash of its contents
|
||||
{ path = "LICENSE", hash = 0xbd0eed23 }
|
||||
]
|
||||
[[licenses.clarify]]
|
||||
name = "webpki"
|
||||
expression = "ISC"
|
||||
license-files = [{ path = "LICENSE", hash = 0x001c7e6c }]
|
||||
|
||||
[licenses.private]
|
||||
# If true, ignores workspace crates that aren't published, or are only
|
||||
# published to private registries
|
||||
ignore = false
|
||||
# One or more private registries that you might publish crates to, if a crate
|
||||
# is only published to private registries, and ignore is true, the crate will
|
||||
# not have its license(s) checked
|
||||
registries = [
|
||||
#"https://sekretz.com/registry
|
||||
]
|
||||
|
||||
# This section is considered when running `cargo deny check bans`.
|
||||
# More documentation about the 'bans' section can be found here:
|
||||
# https://embarkstudios.github.io/cargo-deny/checks/bans/cfg.html
|
||||
[bans]
|
||||
# Lint level for when multiple versions of the same crate are detected
|
||||
multiple-versions = "warn"
|
||||
# The graph highlighting used when creating dotgraphs for crates
|
||||
# with multiple versions
|
||||
# * lowest-version - The path to the lowest versioned duplicate is highlighted
|
||||
# * simplest-path - The path to the version with the fewest edges is highlighted
|
||||
# * all - Both lowest-version and simplest-path are used
|
||||
highlight = "lowest-version"
|
||||
# List of crates that are allowed. Use with care!
|
||||
allow = [
|
||||
#{ name = "ansi_term", version = "=0.11.0" },
|
||||
]
|
||||
# List of crates to deny
|
||||
deny = [
|
||||
{ name = "parity-util-mem", version = "<0.6" }
|
||||
# Each entry the name of a crate and a version range. If version is
|
||||
# not specified, all versions will be matched.
|
||||
]
|
||||
# Certain crates/versions that will be skipped when doing duplicate detection.
|
||||
skip = [
|
||||
#{ name = "ansi_term", version = "=0.11.0" },
|
||||
]
|
||||
# Similarly to `skip` allows you to skip certain crates during duplicate
|
||||
# detection. Unlike skip, it also includes the entire tree of transitive
|
||||
# dependencies starting at the specified crate, up to a certain depth, which is
|
||||
# by default infinite
|
||||
skip-tree = [
|
||||
#{ name = "ansi_term", version = "=0.11.0", depth = 20 },
|
||||
]
|
||||
|
||||
# This section is considered when running `cargo deny check sources`.
|
||||
# More documentation about the 'sources' section can be found here:
|
||||
# https://embarkstudios.github.io/cargo-deny/checks/sources/cfg.html
|
||||
[sources]
|
||||
# Lint level for what to happen when a crate from a crate registry that is not
|
||||
# in the allow list is encountered
|
||||
unknown-registry = "deny"
|
||||
# Lint level for what to happen when a crate from a git repository that is not
|
||||
# in the allow list is encountered
|
||||
unknown-git = "warn"
|
||||
# List of URLs for allowed crate registries. Defaults to the crates.io index
|
||||
# if not specified. If it is specified but empty, no registries are allowed.
|
||||
allow-registry = ["https://github.com/rust-lang/crates.io-index"]
|
||||
# List of URLs for allowed Git repositories
|
||||
allow-git = []
|
||||
@@ -0,0 +1,30 @@
|
||||
FROM docker.io/library/ubuntu:20.04
|
||||
|
||||
# metadata
|
||||
ARG VCS_REF
|
||||
ARG BUILD_DATE
|
||||
|
||||
LABEL io.parity.image.authors="devops-team@parity.io" \
|
||||
io.parity.image.vendor="Parity Technologies" \
|
||||
io.parity.image.title="parity/subkey" \
|
||||
io.parity.image.description="Subkey: key generating utility for Substrate." \
|
||||
io.parity.image.source="https://github.com/paritytech/substrate/blob/${VCS_REF}/scripts/ci/docker/subkey.Dockerfile" \
|
||||
io.parity.image.revision="${VCS_REF}" \
|
||||
io.parity.image.created="${BUILD_DATE}" \
|
||||
io.parity.image.documentation="https://github.com/paritytech/substrate/tree/${VCS_REF}/subkey"
|
||||
|
||||
# show backtraces
|
||||
ENV RUST_BACKTRACE 1
|
||||
|
||||
# add user
|
||||
RUN useradd -m -u 1000 -U -s /bin/sh -d /subkey subkey
|
||||
|
||||
# add subkey binary to docker image
|
||||
COPY ./subkey /usr/local/bin
|
||||
|
||||
USER subkey
|
||||
|
||||
# check if executable works in this container
|
||||
RUN /usr/local/bin/subkey --version
|
||||
|
||||
ENTRYPOINT ["/usr/local/bin/subkey"]
|
||||
@@ -0,0 +1,44 @@
|
||||
FROM docker.io/library/ubuntu:20.04
|
||||
|
||||
# metadata
|
||||
ARG VCS_REF
|
||||
ARG BUILD_DATE
|
||||
|
||||
LABEL io.parity.image.authors="devops-team@parity.io" \
|
||||
io.parity.image.vendor="Parity Technologies" \
|
||||
io.parity.image.title="parity/substrate" \
|
||||
io.parity.image.description="Substrate: The platform for blockchain innovators." \
|
||||
io.parity.image.source="https://github.com/paritytech/substrate/blob/${VCS_REF}/scripts/ci/docker/Dockerfile" \
|
||||
io.parity.image.revision="${VCS_REF}" \
|
||||
io.parity.image.created="${BUILD_DATE}" \
|
||||
io.parity.image.documentation="https://wiki.parity.io/Parity-Substrate"
|
||||
|
||||
# show backtraces
|
||||
ENV RUST_BACKTRACE 1
|
||||
|
||||
# install tools and dependencies
|
||||
RUN apt-get update && \
|
||||
DEBIAN_FRONTEND=noninteractive apt-get upgrade -y && \
|
||||
DEBIAN_FRONTEND=noninteractive apt-get install -y \
|
||||
libssl1.1 \
|
||||
ca-certificates \
|
||||
curl && \
|
||||
# apt cleanup
|
||||
apt-get autoremove -y && \
|
||||
apt-get clean && \
|
||||
find /var/lib/apt/lists/ -type f -not -name lock -delete; \
|
||||
# add user
|
||||
useradd -m -u 1000 -U -s /bin/sh -d /substrate substrate
|
||||
|
||||
# add substrate binary to docker image
|
||||
COPY ./substrate /usr/local/bin
|
||||
|
||||
USER substrate
|
||||
|
||||
# check if executable works in this container
|
||||
RUN /usr/local/bin/substrate --version
|
||||
|
||||
EXPOSE 30333 9933 9944
|
||||
VOLUME ["/substrate"]
|
||||
|
||||
ENTRYPOINT ["/usr/local/bin/substrate"]
|
||||
Executable
+68
@@ -0,0 +1,68 @@
|
||||
#!/usr/bin/env bash
|
||||
set -e
|
||||
|
||||
#shellcheck source=../common/lib.sh
|
||||
source "$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )/../common/lib.sh"
|
||||
|
||||
repo="$GITHUB_REPOSITORY"
|
||||
pr="$GITHUB_PR"
|
||||
|
||||
ensure_labels() {
|
||||
for label in "$@"; do
|
||||
if has_label "$repo" "$pr" "$label"; then
|
||||
return 0
|
||||
fi
|
||||
done
|
||||
return 1
|
||||
}
|
||||
|
||||
# Must have one of the following labels
|
||||
releasenotes_labels=(
|
||||
'B0-silent'
|
||||
'B3-apinoteworthy'
|
||||
'B5-clientnoteworthy'
|
||||
'B7-runtimenoteworthy'
|
||||
)
|
||||
|
||||
criticality_labels=(
|
||||
'C1-low 📌'
|
||||
'C3-medium 📣'
|
||||
'C7-high ❗️'
|
||||
'C9-critical ‼️'
|
||||
)
|
||||
|
||||
audit_labels=(
|
||||
'D1-audited 👍'
|
||||
'D2-notlive 💤'
|
||||
'D3-trivial 🧸'
|
||||
'D5-nicetohaveaudit ⚠️'
|
||||
'D9-needsaudit 👮'
|
||||
)
|
||||
|
||||
echo "[+] Checking release notes (B) labels"
|
||||
if ensure_labels "${releasenotes_labels[@]}"; then
|
||||
echo "[+] Release notes label detected. All is well."
|
||||
else
|
||||
echo "[!] Release notes label not detected. Please add one of: ${releasenotes_labels[*]}"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
echo "[+] Checking release criticality (C) labels"
|
||||
if ensure_labels "${criticality_labels[@]}"; then
|
||||
echo "[+] Release criticality label detected. All is well."
|
||||
else
|
||||
echo "[!] Release criticality label not detected. Please add one of: ${criticality_labels[*]}"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
if has_runtime_changes origin/master "${HEAD_SHA}"; then
|
||||
echo "[+] Runtime changes detected. Checking audit (D) labels"
|
||||
if ensure_labels "${audit_labels[@]}"; then
|
||||
echo "[+] Release audit label detected. All is well."
|
||||
else
|
||||
echo "[!] Release audit label not detected. Please add one of: ${audit_labels[*]}"
|
||||
exit 1
|
||||
fi
|
||||
fi
|
||||
|
||||
exit 0
|
||||
+85
@@ -0,0 +1,85 @@
|
||||
#!/usr/bin/env bash
|
||||
|
||||
# shellcheck source=../common/lib.sh
|
||||
source "$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )/../common/lib.sh"
|
||||
|
||||
version="$2"
|
||||
last_version="$1"
|
||||
|
||||
all_changes="$(sanitised_git_logs "$last_version" "$version")"
|
||||
runtime_changes=""
|
||||
api_changes=""
|
||||
client_changes=""
|
||||
changes=""
|
||||
migrations=""
|
||||
|
||||
while IFS= read -r line; do
|
||||
pr_id=$(echo "$line" | sed -E 's/.*#([0-9]+)\)$/\1/')
|
||||
|
||||
# Skip if the PR has the silent label - this allows us to skip a few requests
|
||||
if has_label 'paritytech/substrate' "$pr_id" 'B0-silent'; then
|
||||
continue
|
||||
fi
|
||||
if has_label 'paritytech/substrate' "$pr_id" 'B3-apinoteworthy' ; then
|
||||
api_changes="$api_changes
|
||||
$line"
|
||||
fi
|
||||
if has_label 'paritytech/substrate' "$pr_id" 'B5-clientnoteworthy'; then
|
||||
client_changes="$client_changes
|
||||
$line"
|
||||
fi
|
||||
if has_label 'paritytech/substrate' "$pr_id" 'B7-runtimenoteworthy'; then
|
||||
runtime_changes="$runtime_changes
|
||||
$line"
|
||||
fi
|
||||
if has_label 'paritytech/substrate' "$pr_id" 'E1-runtime-migration'; then
|
||||
migrations="$migrations
|
||||
$line"
|
||||
fi
|
||||
done <<< "$all_changes"
|
||||
|
||||
# Make the substrate section if there are any substrate changes
|
||||
if [ -n "$runtime_changes" ] ||
|
||||
[ -n "$api_changes" ] ||
|
||||
[ -n "$client_changes" ] ||
|
||||
[ -n "$migrations" ]; then
|
||||
changes=$(cat << EOF
|
||||
Substrate changes
|
||||
-----------------
|
||||
|
||||
EOF
|
||||
)
|
||||
if [ -n "$runtime_changes" ]; then
|
||||
changes="$changes
|
||||
|
||||
Runtime
|
||||
-------
|
||||
$runtime_changes"
|
||||
fi
|
||||
if [ -n "$client_changes" ]; then
|
||||
changes="$changes
|
||||
|
||||
Client
|
||||
------
|
||||
$client_changes"
|
||||
fi
|
||||
if [ -n "$api_changes" ]; then
|
||||
changes="$changes
|
||||
|
||||
API
|
||||
---
|
||||
$api_changes"
|
||||
fi
|
||||
release_text="$release_text
|
||||
|
||||
$changes"
|
||||
fi
|
||||
if [ -n "$migrations" ]; then
|
||||
changes="$changes
|
||||
|
||||
Runtime Migrations
|
||||
------------------
|
||||
$migrations"
|
||||
fi
|
||||
|
||||
echo "$changes"
|
||||
Executable
+121
@@ -0,0 +1,121 @@
|
||||
#!/bin/sh
|
||||
#
|
||||
#
|
||||
# check for any changes in the node/src/runtime, frame/ and primitives/sr_* trees. if
|
||||
# there are any changes found, it should mark the PR breaksconsensus and
|
||||
# "auto-fail" the PR if there isn't a change in the runtime/src/lib.rs file
|
||||
# that alters the version.
|
||||
|
||||
set -e # fail on any error
|
||||
|
||||
#shellcheck source=../common/lib.sh
|
||||
. "$(dirname "${0}")/../common/lib.sh"
|
||||
|
||||
VERSIONS_FILE="bin/node/runtime/src/lib.rs"
|
||||
|
||||
boldprint () { printf "|\n| \033[1m%s\033[0m\n|\n" "${@}"; }
|
||||
boldcat () { printf "|\n"; while read -r l; do printf "| \033[1m%s\033[0m\n" "${l}"; done; printf "|\n" ; }
|
||||
|
||||
github_label () {
|
||||
echo
|
||||
echo "# run github-api job for labeling it ${1}"
|
||||
curl -sS -X POST \
|
||||
-F "token=${CI_JOB_TOKEN}" \
|
||||
-F "ref=master" \
|
||||
-F "variables[LABEL]=${1}" \
|
||||
-F "variables[PRNO]=${CI_COMMIT_REF_NAME}" \
|
||||
"${GITLAB_API}/projects/${GITHUB_API_PROJECT}/trigger/pipeline"
|
||||
}
|
||||
|
||||
|
||||
boldprint "latest 10 commits of ${CI_COMMIT_REF_NAME}"
|
||||
git log --graph --oneline --decorate=short -n 10
|
||||
|
||||
boldprint "make sure the master branch and release tag are available in shallow clones"
|
||||
git fetch --depth="${GIT_DEPTH:-100}" origin master
|
||||
git fetch --depth="${GIT_DEPTH:-100}" origin release
|
||||
git tag -f release FETCH_HEAD
|
||||
git log -n1 release
|
||||
|
||||
|
||||
boldprint "check if the wasm sources changed"
|
||||
if ! has_runtime_changes origin/master "${CI_COMMIT_SHA}"
|
||||
then
|
||||
boldcat <<-EOT
|
||||
|
||||
no changes to the runtime source code detected
|
||||
|
||||
EOT
|
||||
|
||||
exit 0
|
||||
fi
|
||||
|
||||
|
||||
|
||||
# check for spec_version updates: if the spec versions changed, then there is
|
||||
# consensus-critical logic that has changed. the runtime wasm blobs must be
|
||||
# rebuilt.
|
||||
|
||||
add_spec_version="$(git diff tags/release ${CI_COMMIT_SHA} -- "${VERSIONS_FILE}" \
|
||||
| sed -n -r "s/^\+[[:space:]]+spec_version: +([0-9]+),$/\1/p")"
|
||||
sub_spec_version="$(git diff tags/release ${CI_COMMIT_SHA} -- "${VERSIONS_FILE}" \
|
||||
| sed -n -r "s/^\-[[:space:]]+spec_version: +([0-9]+),$/\1/p")"
|
||||
|
||||
|
||||
|
||||
if [ "${add_spec_version}" != "${sub_spec_version}" ]
|
||||
then
|
||||
|
||||
boldcat <<-EOT
|
||||
|
||||
changes to the runtime sources and changes in the spec version.
|
||||
|
||||
spec_version: ${sub_spec_version} -> ${add_spec_version}
|
||||
|
||||
EOT
|
||||
exit 0
|
||||
|
||||
else
|
||||
# check for impl_version updates: if only the impl versions changed, we assume
|
||||
# there is no consensus-critical logic that has changed.
|
||||
|
||||
add_impl_version="$(git diff tags/release ${CI_COMMIT_SHA} -- "${VERSIONS_FILE}" \
|
||||
| sed -n -r 's/^\+[[:space:]]+impl_version: +([0-9]+),$/\1/p')"
|
||||
sub_impl_version="$(git diff tags/release ${CI_COMMIT_SHA} -- "${VERSIONS_FILE}" \
|
||||
| sed -n -r 's/^\-[[:space:]]+impl_version: +([0-9]+),$/\1/p')"
|
||||
|
||||
|
||||
# see if the impl version changed
|
||||
if [ "${add_impl_version}" != "${sub_impl_version}" ]
|
||||
then
|
||||
boldcat <<-EOT
|
||||
|
||||
changes to the runtime sources and changes in the impl version.
|
||||
|
||||
impl_version: ${sub_impl_version} -> ${add_impl_version}
|
||||
|
||||
EOT
|
||||
exit 0
|
||||
fi
|
||||
|
||||
|
||||
boldcat <<-EOT
|
||||
|
||||
wasm source files changed but not the spec/impl version. If changes made do not alter logic,
|
||||
just bump 'impl_version'. If they do change logic, bump 'spec_version'.
|
||||
|
||||
source file directories:
|
||||
- bin/node/src/runtime
|
||||
- frame
|
||||
- primitives/sr-*
|
||||
|
||||
versions file: ${VERSIONS_FILE}
|
||||
|
||||
EOT
|
||||
fi
|
||||
|
||||
# dropped through. there's something wrong; exit 1.
|
||||
|
||||
exit 1
|
||||
|
||||
# vim: noexpandtab
|
||||
Executable
+16
@@ -0,0 +1,16 @@
|
||||
#!/usr/bin/env bash
|
||||
|
||||
# shellcheck source=../common/lib.sh
|
||||
source "$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )/../common/lib.sh"
|
||||
|
||||
version="$CI_COMMIT_TAG"
|
||||
|
||||
echo '[+] Checking tag has been signed'
|
||||
check_tag "paritytech/substrate" "$version"
|
||||
case $? in
|
||||
0) echo '[+] Tag found and has been signed'; exit 0
|
||||
;;
|
||||
1) echo '[!] Tag found but has not been signed. Aborting release.'; exit 1
|
||||
;;
|
||||
2) echo '[!] Tag not found. Aborting release.'; exit 1
|
||||
esac
|
||||
Executable
+80
@@ -0,0 +1,80 @@
|
||||
#!/usr/bin/env bash
|
||||
|
||||
# The script is meant to check if the rules regarding packages
|
||||
# dependencies are satisfied.
|
||||
# The general format is:
|
||||
# [top-lvl-dir] MESSAGE/[other-top-dir]
|
||||
|
||||
# For instance no crate within `./client` directory
|
||||
# is allowed to import any crate with a directory path containing `frame`.
|
||||
# Such rule is just: `client crates must not depend on anything in /frame`.
|
||||
|
||||
# The script should be run from the main repo directory!
|
||||
|
||||
set -u
|
||||
|
||||
# HARD FAILING
|
||||
MUST_NOT=(
|
||||
"client crates must not depend on anything in /frame"
|
||||
"client crates must not depend on anything in /node"
|
||||
"frame crates must not depend on anything in /node"
|
||||
"frame crates must not depend on anything in /client"
|
||||
"primitives crates must not depend on anything in /frame"
|
||||
)
|
||||
|
||||
# ONLY DISPLAYED, script still succeeds
|
||||
PLEASE_DONT=(
|
||||
"primitives crates should not depend on anything in /client"
|
||||
)
|
||||
|
||||
VIOLATIONS=()
|
||||
PACKAGES=()
|
||||
|
||||
function check_rule() {
|
||||
rule=$1
|
||||
from=$(echo $rule | cut -f1 -d\ )
|
||||
to=$(echo $rule | cut -f2 -d\/)
|
||||
|
||||
cd $from
|
||||
echo "Checking rule '$rule'"
|
||||
packages=$(find -name Cargo.toml | xargs grep -wn "path.*\.\.\/$to")
|
||||
has_references=$(echo -n $packages | wc -c)
|
||||
if [ "$has_references" != "0" ]; then
|
||||
VIOLATIONS+=("$rule")
|
||||
# Find packages that violate:
|
||||
PACKAGES+=("$packages")
|
||||
fi
|
||||
cd - > /dev/null
|
||||
}
|
||||
|
||||
for rule in "${MUST_NOT[@]}"
|
||||
do
|
||||
check_rule "$rule";
|
||||
done
|
||||
|
||||
# Only the MUST NOT will be counted towards failure
|
||||
HARD_VIOLATIONS=${#VIOLATIONS[@]}
|
||||
|
||||
|
||||
for rule in "${PLEASE_DONT[@]}"
|
||||
do
|
||||
check_rule "$rule";
|
||||
done
|
||||
|
||||
# Display violations and fail
|
||||
I=0
|
||||
for v in "${VIOLATIONS[@]}"
|
||||
do
|
||||
cat << EOF
|
||||
|
||||
===========================================
|
||||
======= Violation of rule: $v
|
||||
===========================================
|
||||
${PACKAGES[$I]}
|
||||
|
||||
|
||||
EOF
|
||||
I=$I+1
|
||||
done
|
||||
|
||||
exit $HARD_VIOLATIONS
|
||||
+54
@@ -0,0 +1,54 @@
|
||||
#!/usr/bin/env bash
|
||||
|
||||
# shellcheck source=../common/lib.sh
|
||||
source "$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )/../common/lib.sh"
|
||||
|
||||
version="$CI_COMMIT_TAG"
|
||||
|
||||
# Note that this is not the last *tagged* version, but the last *published* version
|
||||
last_version=$(last_github_release 'paritytech/substrate')
|
||||
|
||||
release_text="$(./generate_release_text.sh "$last_version" "$version")"
|
||||
|
||||
echo "[+] Pushing release to github"
|
||||
# Create release on github
|
||||
release_name="Substrate $version"
|
||||
data=$(jq -Rs --arg version "$version" \
|
||||
--arg release_name "$release_name" \
|
||||
--arg release_text "$release_text" \
|
||||
'{
|
||||
"tag_name": $version,
|
||||
"target_commitish": "master",
|
||||
"name": $release_name,
|
||||
"body": $release_text,
|
||||
"draft": true,
|
||||
"prerelease": false
|
||||
}' < /dev/null)
|
||||
|
||||
out=$(curl -s -X POST --data "$data" -H "Authorization: token $GITHUB_RELEASE_TOKEN" "$api_base/paritytech/substrate/releases")
|
||||
|
||||
html_url=$(echo "$out" | jq -r .html_url)
|
||||
|
||||
if [ "$html_url" == "null" ]
|
||||
then
|
||||
echo "[!] Something went wrong posting:"
|
||||
echo "$out"
|
||||
else
|
||||
echo "[+] Release draft created: $html_url"
|
||||
fi
|
||||
|
||||
echo '[+] Sending draft release URL to Matrix'
|
||||
|
||||
msg_body=$(cat <<EOF
|
||||
**Release pipeline for Substrate $version complete.**
|
||||
Draft release created: $html_url
|
||||
EOF
|
||||
)
|
||||
formatted_msg_body=$(cat <<EOF
|
||||
<strong>Release pipeline for Substrate $version complete.</strong><br />
|
||||
Draft release created: $html_url
|
||||
EOF
|
||||
)
|
||||
send_message "$(structure_message "$msg_body" "$formatted_msg_body")" "$MATRIX_ROOM_ID" "$MATRIX_ACCESS_TOKEN"
|
||||
|
||||
echo "[+] Done! Maybe the release worked..."
|
||||
Executable
+14
@@ -0,0 +1,14 @@
|
||||
#!/bin/sh
|
||||
url="https://api.github.com/repos/paritytech/substrate/pulls/${CI_COMMIT_REF_NAME}"
|
||||
echo "[+] API URL: $url"
|
||||
|
||||
draft_state=$(curl -H "Authorization: token ${GITHUB_PR_TOKEN}" "$url" | jq -r .draft)
|
||||
echo "[+] Draft state: $draft_state"
|
||||
|
||||
if [ "$draft_state" = 'true' ]; then
|
||||
echo "[!] PR is currently a draft, stopping pipeline"
|
||||
exit 1
|
||||
else
|
||||
echo "[+] PR is not a draft. Proceeding with CI pipeline"
|
||||
exit 0
|
||||
fi
|
||||
@@ -0,0 +1,239 @@
|
||||
rule_files:
|
||||
- /dev/stdin
|
||||
|
||||
evaluation_interval: 1m
|
||||
|
||||
tests:
|
||||
- interval: 1m
|
||||
input_series:
|
||||
- series: 'substrate_sub_libp2p_peers_count{
|
||||
job="substrate",
|
||||
pod="substrate-abcdef01234-abcdef",
|
||||
instance="substrate-abcdef01234-abcdef",
|
||||
}'
|
||||
values: '3 2+0x4 1+0x9' # 3 2 2 2 2 2 1 1 1 1 1 1 1 1 1 1
|
||||
|
||||
- series: 'substrate_sub_txpool_validations_scheduled{
|
||||
job="substrate",
|
||||
pod="substrate-abcdef01234-abcdef",
|
||||
instance="substrate-abcdef01234-abcdef",
|
||||
}'
|
||||
values: '11+1x10 22+2x30 10043x5'
|
||||
|
||||
- series: 'substrate_sub_txpool_validations_finished{
|
||||
job="substrate",
|
||||
pod="substrate-abcdef01234-abcdef",
|
||||
instance="substrate-abcdef01234-abcdef",
|
||||
}'
|
||||
values: '0+1x42 42x5'
|
||||
|
||||
- series: 'substrate_block_height{
|
||||
status="best", job="substrate",
|
||||
pod="substrate-abcdef01234-abcdef",
|
||||
instance="substrate-abcdef01234-abcdef",
|
||||
}'
|
||||
values: '1+1x3 4+0x13' # 1 2 3 4 4 4 4 4 4 4 4 4 ...
|
||||
|
||||
- series: 'substrate_block_height{
|
||||
status="finalized",
|
||||
job="substrate",
|
||||
pod="substrate-abcdef01234-abcdef",
|
||||
instance="substrate-abcdef01234-abcdef",
|
||||
}'
|
||||
values: '1+1x3 4+0x13' # 1 2 3 4 4 4 4 4 4 4 4 4 ...
|
||||
|
||||
alert_rule_test:
|
||||
|
||||
######################################################################
|
||||
# Block production
|
||||
######################################################################
|
||||
|
||||
- eval_time: 6m
|
||||
alertname: BlockProductionSlow
|
||||
exp_alerts:
|
||||
- eval_time: 7m
|
||||
alertname: BlockProductionSlow
|
||||
exp_alerts:
|
||||
- exp_labels:
|
||||
severity: warning
|
||||
pod: substrate-abcdef01234-abcdef
|
||||
instance: substrate-abcdef01234-abcdef
|
||||
job: substrate
|
||||
status: best
|
||||
exp_annotations:
|
||||
message: "Best block on instance
|
||||
substrate-abcdef01234-abcdef increases by less than 1 per
|
||||
minute for more than 3 minutes."
|
||||
|
||||
- eval_time: 14m
|
||||
alertname: BlockProductionSlow
|
||||
exp_alerts:
|
||||
- exp_labels:
|
||||
severity: warning
|
||||
pod: substrate-abcdef01234-abcdef
|
||||
instance: substrate-abcdef01234-abcdef
|
||||
job: substrate
|
||||
status: best
|
||||
exp_annotations:
|
||||
message: "Best block on instance
|
||||
substrate-abcdef01234-abcdef increases by less than 1 per
|
||||
minute for more than 3 minutes."
|
||||
- exp_labels:
|
||||
severity: critical
|
||||
pod: substrate-abcdef01234-abcdef
|
||||
instance: substrate-abcdef01234-abcdef
|
||||
job: substrate
|
||||
status: best
|
||||
exp_annotations:
|
||||
message: "Best block on instance
|
||||
substrate-abcdef01234-abcdef increases by less than 1 per
|
||||
minute for more than 10 minutes."
|
||||
|
||||
######################################################################
|
||||
# Block finalization
|
||||
######################################################################
|
||||
|
||||
- eval_time: 6m
|
||||
alertname: BlockFinalizationSlow
|
||||
exp_alerts:
|
||||
- eval_time: 7m
|
||||
alertname: BlockFinalizationSlow
|
||||
exp_alerts:
|
||||
- exp_labels:
|
||||
severity: warning
|
||||
pod: substrate-abcdef01234-abcdef
|
||||
instance: substrate-abcdef01234-abcdef
|
||||
job: substrate
|
||||
status: finalized
|
||||
exp_annotations:
|
||||
message: "Finalized block on instance
|
||||
substrate-abcdef01234-abcdef increases by less than 1 per
|
||||
minute for more than 3 minutes."
|
||||
|
||||
- eval_time: 14m
|
||||
alertname: BlockFinalizationSlow
|
||||
exp_alerts:
|
||||
- exp_labels:
|
||||
severity: warning
|
||||
pod: substrate-abcdef01234-abcdef
|
||||
instance: substrate-abcdef01234-abcdef
|
||||
job: substrate
|
||||
status: finalized
|
||||
exp_annotations:
|
||||
message: "Finalized block on instance
|
||||
substrate-abcdef01234-abcdef increases by less than 1 per
|
||||
minute for more than 3 minutes."
|
||||
- exp_labels:
|
||||
severity: critical
|
||||
pod: substrate-abcdef01234-abcdef
|
||||
instance: substrate-abcdef01234-abcdef
|
||||
job: substrate
|
||||
status: finalized
|
||||
exp_annotations:
|
||||
message: "Finalized block on instance
|
||||
substrate-abcdef01234-abcdef increases by less than 1 per
|
||||
minute for more than 10 minutes."
|
||||
|
||||
######################################################################
|
||||
# Transaction queue
|
||||
######################################################################
|
||||
|
||||
- eval_time: 11m
|
||||
alertname: TransactionQueueSizeIncreasing
|
||||
# Number of validations scheduled and finished both grow at a rate
|
||||
# of 1 in the first 10 minutes, thereby the queue is not increasing
|
||||
# in size, thus don't expect an alert.
|
||||
exp_alerts:
|
||||
- eval_time: 22m
|
||||
alertname: TransactionQueueSizeIncreasing
|
||||
# Number of validations scheduled is growing twice as fast as the
|
||||
# number of validations finished after minute 10. Thus expect
|
||||
# warning alert after 20 minutes.
|
||||
exp_alerts:
|
||||
- exp_labels:
|
||||
severity: warning
|
||||
pod: substrate-abcdef01234-abcdef
|
||||
instance: substrate-abcdef01234-abcdef
|
||||
job: substrate
|
||||
exp_annotations:
|
||||
message: "The transaction pool size on node
|
||||
substrate-abcdef01234-abcdef has been monotonically
|
||||
increasing for more than 10 minutes."
|
||||
- eval_time: 43m
|
||||
alertname: TransactionQueueSizeIncreasing
|
||||
# Number of validations scheduled is growing twice as fast as the
|
||||
# number of validations finished after minute 10. Thus expect
|
||||
# both warning and critical alert after 40 minutes.
|
||||
exp_alerts:
|
||||
- exp_labels:
|
||||
severity: warning
|
||||
pod: substrate-abcdef01234-abcdef
|
||||
instance: substrate-abcdef01234-abcdef
|
||||
job: substrate
|
||||
exp_annotations:
|
||||
message: "The transaction pool size on node
|
||||
substrate-abcdef01234-abcdef has been monotonically
|
||||
increasing for more than 10 minutes."
|
||||
- exp_labels:
|
||||
severity: warning
|
||||
pod: substrate-abcdef01234-abcdef
|
||||
instance: substrate-abcdef01234-abcdef
|
||||
job: substrate
|
||||
exp_annotations:
|
||||
message: "The transaction pool size on node
|
||||
substrate-abcdef01234-abcdef has been monotonically
|
||||
increasing for more than 30 minutes."
|
||||
- eval_time: 49m
|
||||
alertname: TransactionQueueSizeHigh
|
||||
# After minute 43 the number of validations scheduled jumps up
|
||||
# drastically while the number of validations finished stays the
|
||||
# same. Thus expect an alert.
|
||||
exp_alerts:
|
||||
- exp_labels:
|
||||
severity: warning
|
||||
pod: substrate-abcdef01234-abcdef
|
||||
instance: substrate-abcdef01234-abcdef
|
||||
job: substrate
|
||||
exp_annotations:
|
||||
message: "The transaction pool size on node
|
||||
substrate-abcdef01234-abcdef has been above 10_000 for more
|
||||
than 5 minutes."
|
||||
|
||||
######################################################################
|
||||
# Networking
|
||||
######################################################################
|
||||
|
||||
- eval_time: 3m # Values: 3 2 2
|
||||
alertname: NumberOfPeersLow
|
||||
exp_alerts:
|
||||
- eval_time: 4m # Values: 2 2 2
|
||||
alertname: NumberOfPeersLow
|
||||
exp_alerts:
|
||||
- exp_labels:
|
||||
severity: warning
|
||||
pod: substrate-abcdef01234-abcdef
|
||||
instance: substrate-abcdef01234-abcdef
|
||||
job: substrate
|
||||
exp_annotations:
|
||||
message: "The node substrate-abcdef01234-abcdef has less
|
||||
than 3 peers for more than 3 minutes"
|
||||
|
||||
- eval_time: 16m # Values: 3 2 2 2 2 2 1 1 1 1 1 1 1 1 1 1 1
|
||||
alertname: NumberOfPeersLow
|
||||
exp_alerts:
|
||||
- exp_labels:
|
||||
severity: warning
|
||||
pod: substrate-abcdef01234-abcdef
|
||||
instance: substrate-abcdef01234-abcdef
|
||||
job: substrate
|
||||
exp_annotations:
|
||||
message: "The node substrate-abcdef01234-abcdef has less
|
||||
than 3 peers for more than 3 minutes"
|
||||
- exp_labels:
|
||||
severity: critical
|
||||
pod: substrate-abcdef01234-abcdef
|
||||
instance: substrate-abcdef01234-abcdef
|
||||
job: substrate
|
||||
exp_annotations:
|
||||
message: "The node substrate-abcdef01234-abcdef has less
|
||||
than 3 peers for more than 15 minutes"
|
||||
@@ -0,0 +1,171 @@
|
||||
groups:
|
||||
- name: substrate.rules
|
||||
rules:
|
||||
|
||||
##############################################################################
|
||||
# Block production
|
||||
##############################################################################
|
||||
|
||||
- alert: BlockProductionSlow
|
||||
annotations:
|
||||
message: 'Best block on instance {{ $labels.instance }} increases by
|
||||
less than 1 per minute for more than 3 minutes.'
|
||||
expr: increase(substrate_block_height{status="best"}[1m]) < 1
|
||||
for: 3m
|
||||
labels:
|
||||
severity: warning
|
||||
- alert: BlockProductionSlow
|
||||
annotations:
|
||||
message: 'Best block on instance {{ $labels.instance }} increases by
|
||||
less than 1 per minute for more than 10 minutes.'
|
||||
expr: increase(substrate_block_height{status="best"}[1m]) < 1
|
||||
for: 10m
|
||||
labels:
|
||||
severity: critical
|
||||
|
||||
##############################################################################
|
||||
# Block finalization
|
||||
##############################################################################
|
||||
|
||||
- alert: BlockFinalizationSlow
|
||||
expr: increase(substrate_block_height{status="finalized"}[1m]) < 1
|
||||
for: 3m
|
||||
labels:
|
||||
severity: warning
|
||||
annotations:
|
||||
message: 'Finalized block on instance {{ $labels.instance }} increases by
|
||||
less than 1 per minute for more than 3 minutes.'
|
||||
- alert: BlockFinalizationSlow
|
||||
expr: increase(substrate_block_height{status="finalized"}[1m]) < 1
|
||||
for: 10m
|
||||
labels:
|
||||
severity: critical
|
||||
annotations:
|
||||
message: 'Finalized block on instance {{ $labels.instance }} increases by
|
||||
less than 1 per minute for more than 10 minutes.'
|
||||
- alert: BlockFinalizationLaggingBehind
|
||||
# Under the assumption of an average block production of 6 seconds,
|
||||
# "best" and "finalized" being more than 10 blocks apart would imply
|
||||
# more than a 1 minute delay between block production and finalization.
|
||||
expr: '(substrate_block_height{status="best"} - ignoring(status)
|
||||
substrate_block_height{status="finalized"}) > 10'
|
||||
for: 8m
|
||||
labels:
|
||||
severity: critical
|
||||
annotations:
|
||||
message: "Block finalization on instance {{ $labels.instance }} is behind
|
||||
block production by {{ $value }} for more than 8 minutes."
|
||||
|
||||
##############################################################################
|
||||
# Transaction queue
|
||||
##############################################################################
|
||||
|
||||
- alert: TransactionQueueSizeIncreasing
|
||||
expr: 'increase(substrate_sub_txpool_validations_scheduled[5m]) -
|
||||
increase(substrate_sub_txpool_validations_finished[5m]) > 0'
|
||||
for: 10m
|
||||
labels:
|
||||
severity: warning
|
||||
annotations:
|
||||
message: 'The transaction pool size on node {{ $labels.instance }} has
|
||||
been monotonically increasing for more than 10 minutes.'
|
||||
- alert: TransactionQueueSizeIncreasing
|
||||
expr: 'increase(substrate_sub_txpool_validations_scheduled[5m]) -
|
||||
increase(substrate_sub_txpool_validations_finished[5m]) > 0'
|
||||
for: 30m
|
||||
labels:
|
||||
severity: warning
|
||||
annotations:
|
||||
message: 'The transaction pool size on node {{ $labels.instance }} has
|
||||
been monotonically increasing for more than 30 minutes.'
|
||||
- alert: TransactionQueueSizeHigh
|
||||
expr: 'substrate_sub_txpool_validations_scheduled -
|
||||
substrate_sub_txpool_validations_finished > 10000'
|
||||
for: 5m
|
||||
labels:
|
||||
severity: warning
|
||||
annotations:
|
||||
message: 'The transaction pool size on node {{ $labels.instance }} has
|
||||
been above 10_000 for more than 5 minutes.'
|
||||
|
||||
##############################################################################
|
||||
# Networking
|
||||
##############################################################################
|
||||
|
||||
- alert: NumberOfPeersLow
|
||||
expr: substrate_sub_libp2p_peers_count < 3
|
||||
for: 3m
|
||||
labels:
|
||||
severity: warning
|
||||
annotations:
|
||||
message: 'The node {{ $labels.instance }} has less than 3 peers for more
|
||||
than 3 minutes'
|
||||
- alert: NumberOfPeersLow
|
||||
expr: substrate_sub_libp2p_peers_count < 3
|
||||
for: 15m
|
||||
labels:
|
||||
severity: critical
|
||||
annotations:
|
||||
message: 'The node {{ $labels.instance }} has less than 3 peers for more
|
||||
than 15 minutes'
|
||||
- alert: NoIncomingConnection
|
||||
expr: increase(substrate_sub_libp2p_incoming_connections_total[20m]) == 0
|
||||
labels:
|
||||
severity: warning
|
||||
annotations:
|
||||
message: 'The node {{ $labels.instance }} has not received any new incoming
|
||||
TCP connection in the past 20 minutes. Is it connected to the Internet?'
|
||||
|
||||
##############################################################################
|
||||
# System
|
||||
##############################################################################
|
||||
|
||||
- alert: NumberOfFileDescriptorsHigh
|
||||
expr: 'node_filefd_allocated{chain!=""} > 10000'
|
||||
for: 3m
|
||||
labels:
|
||||
severity: warning
|
||||
annotations:
|
||||
message: 'The node {{ $labels.instance }} has more than 10_000 file
|
||||
descriptors allocated for more than 3 minutes'
|
||||
|
||||
##############################################################################
|
||||
# Others
|
||||
##############################################################################
|
||||
|
||||
- alert: AuthorityDiscoveryDiscoveryFailureHigh
|
||||
expr: 'substrate_authority_discovery_handle_value_found_event_failure /
|
||||
ignoring(name)
|
||||
substrate_authority_discovery_dht_event_received{name="value_found"} > 0.5'
|
||||
for: 2h
|
||||
labels:
|
||||
severity: warning
|
||||
annotations:
|
||||
message: 'Authority discovery on node {{ $labels.instance }} fails to
|
||||
process more than 50 % of the values found on the DHT for more than 2
|
||||
hours.'
|
||||
|
||||
- alert: UnboundedChannelPersistentlyLarge
|
||||
expr: '(
|
||||
(substrate_unbounded_channel_len{action = "send"} -
|
||||
ignoring(action) substrate_unbounded_channel_len{action = "received"})
|
||||
or on(instance) substrate_unbounded_channel_len{action = "send"}
|
||||
) >= 200'
|
||||
for: 5m
|
||||
labels:
|
||||
severity: warning
|
||||
annotations:
|
||||
message: 'Channel {{ $labels.entity }} on node {{ $labels.instance }} contains
|
||||
more than 200 items for more than 5 minutes. Node might be frozen.'
|
||||
|
||||
- alert: UnboundedChannelVeryLarge
|
||||
expr: '(
|
||||
(substrate_unbounded_channel_len{action = "send"} -
|
||||
ignoring(action) substrate_unbounded_channel_len{action = "received"})
|
||||
or on(instance) substrate_unbounded_channel_len{action = "send"}
|
||||
) > 15000'
|
||||
labels:
|
||||
severity: warning
|
||||
annotations:
|
||||
message: 'Channel {{ $labels.entity }} on node {{ $labels.instance }} contains more than
|
||||
15000 items.'
|
||||
@@ -0,0 +1,7 @@
|
||||
## Substrate Dashboard
|
||||
|
||||
Shared templated Grafana dashboards.
|
||||
|
||||
To import the dashboards follow the [Grafana
|
||||
documentation](https://grafana.com/docs/grafana/latest/reference/export_import/).
|
||||
You can see an example setup [here](./substrate-networking.json).
|
||||
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
+16
@@ -0,0 +1,16 @@
|
||||
#!/usr/bin/env bash
|
||||
|
||||
set -e
|
||||
|
||||
export TERM=xterm
|
||||
PROJECT_ROOT=`git rev-parse --show-toplevel`
|
||||
|
||||
if [ "$#" -ne 1 ]; then
|
||||
echo "node-template-release.sh path_to_target_archive"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
PATH_TO_ARCHIVE=$1
|
||||
cd $PROJECT_ROOT/scripts/ci/node-template-release
|
||||
|
||||
cargo run $PROJECT_ROOT/bin/node-template $PROJECT_ROOT/$PATH_TO_ARCHIVE
|
||||
@@ -0,0 +1,21 @@
|
||||
[package]
|
||||
name = "node-template-release"
|
||||
version = "3.0.0"
|
||||
authors = ["Parity Technologies <admin@parity.io>"]
|
||||
edition = "2021"
|
||||
license = "GPL-3.0"
|
||||
|
||||
[dependencies]
|
||||
toml = "0.4"
|
||||
tar = "0.4"
|
||||
glob = "0.2"
|
||||
clap = { version = "3.0", features = ["derive"] }
|
||||
tempfile = "3"
|
||||
fs_extra = "1"
|
||||
git2 = "0.8"
|
||||
flate2 = "1.0"
|
||||
|
||||
[workspace]
|
||||
|
||||
[package.metadata.docs.rs]
|
||||
targets = ["x86_64-unknown-linux-gnu"]
|
||||
@@ -0,0 +1,279 @@
|
||||
use clap::Parser;
|
||||
|
||||
use std::{
|
||||
collections::HashMap,
|
||||
fs::{self, File, OpenOptions},
|
||||
io::{Read, Write},
|
||||
path::{Path, PathBuf},
|
||||
process::Command,
|
||||
};
|
||||
|
||||
use glob;
|
||||
|
||||
use fs_extra::dir::{self, CopyOptions};
|
||||
|
||||
use tempfile;
|
||||
|
||||
use git2;
|
||||
|
||||
use toml;
|
||||
|
||||
use tar;
|
||||
|
||||
use flate2::{write::GzEncoder, Compression};
|
||||
|
||||
const SUBSTRATE_GIT_URL: &str = "https://github.com/paritytech/substrate.git";
|
||||
|
||||
type CargoToml = HashMap<String, toml::Value>;
|
||||
|
||||
#[derive(Parser)]
|
||||
struct Options {
|
||||
/// The path to the `node-template` source.
|
||||
#[clap(parse(from_os_str))]
|
||||
node_template: PathBuf,
|
||||
/// The path where to output the generated `tar.gz` file.
|
||||
#[clap(parse(from_os_str))]
|
||||
output: PathBuf,
|
||||
}
|
||||
|
||||
/// Find all `Cargo.toml` files in the given path.
|
||||
fn find_cargo_tomls(path: PathBuf) -> Vec<PathBuf> {
|
||||
let path = format!("{}/**/*.toml", path.display());
|
||||
|
||||
let glob = glob::glob(&path).expect("Generates globbing pattern");
|
||||
|
||||
let mut result = Vec::new();
|
||||
glob.into_iter().for_each(|file| match file {
|
||||
Ok(file) => result.push(file),
|
||||
Err(e) => println!("{:?}", e),
|
||||
});
|
||||
|
||||
if result.is_empty() {
|
||||
panic!("Did not found any `Cargo.toml` files.");
|
||||
}
|
||||
|
||||
result
|
||||
}
|
||||
|
||||
/// Copy the `node-template` to the given path.
|
||||
fn copy_node_template(node_template: &Path, dest_path: &Path) {
|
||||
let options = CopyOptions::new();
|
||||
dir::copy(node_template, dest_path, &options).expect("Copies node-template to tmp dir");
|
||||
}
|
||||
|
||||
/// Gets the latest commit id of the repository given by `path`.
|
||||
fn get_git_commit_id(path: &Path) -> String {
|
||||
let repo = git2::Repository::discover(path)
|
||||
.expect(&format!("Node template ({}) should be in a git repository.", path.display()));
|
||||
|
||||
let commit_id = repo
|
||||
.head()
|
||||
.expect("Repository should have a head")
|
||||
.peel_to_commit()
|
||||
.expect("Head references a commit")
|
||||
.id();
|
||||
|
||||
format!("{}", commit_id)
|
||||
}
|
||||
|
||||
/// Parse the given `Cargo.toml` into a `HashMap`
|
||||
fn parse_cargo_toml(file: &Path) -> CargoToml {
|
||||
let mut content = String::new();
|
||||
File::open(file)
|
||||
.expect("Cargo.toml exists")
|
||||
.read_to_string(&mut content)
|
||||
.expect("Reads file");
|
||||
toml::from_str(&content).expect("Cargo.toml is a valid toml file")
|
||||
}
|
||||
|
||||
/// Replaces all substrate path dependencies with a git dependency.
|
||||
fn replace_path_dependencies_with_git(
|
||||
cargo_toml_path: &Path,
|
||||
commit_id: &str,
|
||||
cargo_toml: &mut CargoToml,
|
||||
) {
|
||||
let mut cargo_toml_path = cargo_toml_path.to_path_buf();
|
||||
// remove `Cargo.toml`
|
||||
cargo_toml_path.pop();
|
||||
|
||||
for &table in &["dependencies", "build-dependencies", "dev-dependencies"] {
|
||||
let mut dependencies: toml::value::Table =
|
||||
match cargo_toml.remove(table).and_then(|v| v.try_into().ok()) {
|
||||
Some(deps) => deps,
|
||||
None => continue,
|
||||
};
|
||||
|
||||
let deps_rewritten = dependencies
|
||||
.iter()
|
||||
.filter_map(|(k, v)| {
|
||||
v.clone().try_into::<toml::value::Table>().ok().map(move |v| (k, v))
|
||||
})
|
||||
.filter(|t| {
|
||||
t.1.contains_key("path") && {
|
||||
// if the path does not exists, we need to add this as git dependency
|
||||
t.1.get("path")
|
||||
.unwrap()
|
||||
.as_str()
|
||||
.map(|path| !cargo_toml_path.join(path).exists())
|
||||
.unwrap_or(false)
|
||||
}
|
||||
})
|
||||
.map(|(k, mut v)| {
|
||||
// remove `path` and add `git` and `rev`
|
||||
v.remove("path");
|
||||
v.insert("git".into(), SUBSTRATE_GIT_URL.into());
|
||||
v.insert("rev".into(), commit_id.into());
|
||||
|
||||
(k.clone(), v.into())
|
||||
})
|
||||
.collect::<HashMap<_, _>>();
|
||||
|
||||
dependencies.extend(deps_rewritten.into_iter());
|
||||
|
||||
cargo_toml.insert(table.into(), dependencies.into());
|
||||
}
|
||||
}
|
||||
|
||||
/// Update the top level (workspace) `Cargo.toml` file.
|
||||
///
|
||||
/// - Adds `profile.release` = `panic = unwind`
|
||||
/// - Adds `workspace` definition
|
||||
fn update_top_level_cargo_toml(
|
||||
cargo_toml: &mut CargoToml,
|
||||
workspace_members: Vec<&PathBuf>,
|
||||
node_template_path: &Path,
|
||||
) {
|
||||
let mut panic_unwind = toml::value::Table::new();
|
||||
panic_unwind.insert("panic".into(), "unwind".into());
|
||||
|
||||
let mut profile = toml::value::Table::new();
|
||||
profile.insert("release".into(), panic_unwind.into());
|
||||
|
||||
cargo_toml.insert("profile".into(), profile.into());
|
||||
|
||||
let members = workspace_members
|
||||
.iter()
|
||||
.map(|p| {
|
||||
p.strip_prefix(node_template_path)
|
||||
.expect("Workspace member is a child of the node template path!")
|
||||
.parent()
|
||||
// We get the `Cargo.toml` paths as workspace members, but for the `members` field
|
||||
// we just need the path.
|
||||
.expect("The given path ends with `Cargo.toml` as file name!")
|
||||
.display()
|
||||
.to_string()
|
||||
})
|
||||
.collect::<Vec<_>>();
|
||||
|
||||
let mut members_section = toml::value::Table::new();
|
||||
members_section.insert("members".into(), members.into());
|
||||
|
||||
cargo_toml.insert("workspace".into(), members_section.into());
|
||||
}
|
||||
|
||||
fn write_cargo_toml(path: &Path, cargo_toml: CargoToml) {
|
||||
let content = toml::to_string_pretty(&cargo_toml).expect("Creates `Cargo.toml`");
|
||||
let mut file = File::create(path).expect(&format!("Creates `{}`.", path.display()));
|
||||
write!(file, "{}", content).expect("Writes `Cargo.toml`");
|
||||
}
|
||||
|
||||
/// Build and test the generated node-template
|
||||
fn build_and_test(path: &Path, cargo_tomls: &[PathBuf]) {
|
||||
// Build node
|
||||
assert!(Command::new("cargo")
|
||||
.args(&["build", "--all"])
|
||||
.current_dir(path)
|
||||
.status()
|
||||
.expect("Compiles node")
|
||||
.success());
|
||||
|
||||
// Test node
|
||||
assert!(Command::new("cargo")
|
||||
.args(&["test", "--all"])
|
||||
.current_dir(path)
|
||||
.status()
|
||||
.expect("Tests node")
|
||||
.success());
|
||||
|
||||
// Remove all `target` directories
|
||||
for toml in cargo_tomls {
|
||||
let mut target_path = toml.clone();
|
||||
target_path.pop();
|
||||
target_path = target_path.join("target");
|
||||
|
||||
if target_path.exists() {
|
||||
fs::remove_dir_all(&target_path)
|
||||
.expect(&format!("Removes `{}`", target_path.display()));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn main() {
|
||||
let options = Options::parse();
|
||||
|
||||
let build_dir = tempfile::tempdir().expect("Creates temp build dir");
|
||||
|
||||
let node_template_folder = options
|
||||
.node_template
|
||||
.canonicalize()
|
||||
.expect("Node template path exists")
|
||||
.file_name()
|
||||
.expect("Node template folder is last element of path")
|
||||
.to_owned();
|
||||
|
||||
// The path to the node-template in the build dir.
|
||||
let node_template_path = build_dir.path().join(node_template_folder);
|
||||
|
||||
copy_node_template(&options.node_template, build_dir.path());
|
||||
let mut cargo_tomls = find_cargo_tomls(build_dir.path().to_owned());
|
||||
|
||||
let commit_id = get_git_commit_id(&options.node_template);
|
||||
let top_level_cargo_toml_path = node_template_path.join("Cargo.toml");
|
||||
|
||||
// Check if top level Cargo.toml exists. If not, create one in the destination
|
||||
if !cargo_tomls.contains(&top_level_cargo_toml_path) {
|
||||
// create the top_level_cargo_toml
|
||||
OpenOptions::new()
|
||||
.create(true)
|
||||
.write(true)
|
||||
.open(top_level_cargo_toml_path.clone())
|
||||
.expect("Create root level `Cargo.toml` failed.");
|
||||
|
||||
// push into our data structure
|
||||
cargo_tomls.push(PathBuf::from(top_level_cargo_toml_path.clone()));
|
||||
}
|
||||
|
||||
cargo_tomls.iter().for_each(|t| {
|
||||
let mut cargo_toml = parse_cargo_toml(&t);
|
||||
replace_path_dependencies_with_git(&t, &commit_id, &mut cargo_toml);
|
||||
|
||||
// Check if this is the top level `Cargo.toml`, as this requires some special treatments.
|
||||
if top_level_cargo_toml_path == *t {
|
||||
// All workspace member `Cargo.toml` file paths.
|
||||
let workspace_members =
|
||||
cargo_tomls.iter().filter(|p| **p != top_level_cargo_toml_path).collect();
|
||||
|
||||
update_top_level_cargo_toml(&mut cargo_toml, workspace_members, &node_template_path);
|
||||
}
|
||||
|
||||
write_cargo_toml(&t, cargo_toml);
|
||||
});
|
||||
|
||||
// adding root rustfmt to node template build path
|
||||
let node_template_rustfmt_toml_path = node_template_path.join("rustfmt.toml");
|
||||
let root_rustfmt_toml = &options.node_template.join("../../rustfmt.toml");
|
||||
if root_rustfmt_toml.exists() {
|
||||
fs::copy(&root_rustfmt_toml, &node_template_rustfmt_toml_path)
|
||||
.expect("Copying rustfmt.toml.");
|
||||
}
|
||||
|
||||
build_and_test(&node_template_path, &cargo_tomls);
|
||||
|
||||
let output = GzEncoder::new(
|
||||
File::create(&options.output).expect("Creates output file"),
|
||||
Compression::default(),
|
||||
);
|
||||
let mut tar = tar::Builder::new(output);
|
||||
tar.append_dir_all("substrate-node-template", node_template_path)
|
||||
.expect("Writes substrate-node-template archive");
|
||||
}
|
||||
Reference in New Issue
Block a user