Files
pezkuwi-sdk/.github/scripts/common/lib.sh
T
pezkuwichain ee389beb8c feat: Add rebrand CI/CD workflows to main branch
- Add 72 rebrand workflow files (polkadot→pezkuwi, substrate→bizinikiwi, cumulus→pezcumulus)
- Add GitHub actions, issue templates, and configs
- Removed unnecessary workflows (fork-sync, gitspiegel, upstream-tracker, sync-templates, backport)
- Renamed zombienet test files to match new naming convention
2025-12-19 22:51:57 +03:00

589 lines
18 KiB
Bash
Executable File
Raw Blame History

This file contains invisible Unicode characters
This file contains invisible Unicode characters that are indistinguishable to humans but may be processed differently by a computer. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
#!/bin/sh
api_base="https://api.github.com/repos"
# Function to take 2 git tags/commits and get any lines from commit messages
# that contain something that looks like a PR reference: e.g., (#1234)
sanitised_git_logs(){
git --no-pager log --pretty=format:"%s" "$1...$2" |
# Only find messages referencing a PR
grep -E '\(#[0-9]+\)' |
# Strip any asterisks
sed 's/^* //g'
}
# Checks whether a tag on github has been verified
# repo: 'organization/repo'
# tagver: 'v1.2.3'
# Usage: check_tag $repo $tagver
check_tag () {
repo=$1
tagver=$2
if [ -n "$GITHUB_RELEASE_TOKEN" ]; then
echo '[+] Fetching tag using privileged token'
tag_out=$(curl -H "Authorization: token $GITHUB_RELEASE_TOKEN" -s "$api_base/$repo/git/refs/tags/$tagver")
else
echo '[+] Fetching tag using unprivileged token'
tag_out=$(curl -H "Authorization: token $GITHUB_PR_TOKEN" -s "$api_base/$repo/git/refs/tags/$tagver")
fi
tag_sha=$(echo "$tag_out" | jq -r .object.sha)
object_url=$(echo "$tag_out" | jq -r .object.url)
if [ "$tag_sha" = "null" ]; then
return 2
fi
echo "[+] Tag object SHA: $tag_sha"
verified_str=$(curl -H "Authorization: token $GITHUB_RELEASE_TOKEN" -s "$object_url" | jq -r .verification.verified)
if [ "$verified_str" = "true" ]; then
# Verified, everything is good
return 0
else
# Not verified. Bad juju.
return 1
fi
}
# Checks whether a given PR has a given label.
# repo: 'organization/repo'
# pr_id: 12345
# label: B1-silent
# Usage: has_label $repo $pr_id $label
has_label(){
repo="$1"
pr_id="$2"
label="$3"
# These will exist if the function is called in Gitlab.
# If the function's called in Github, we should have GITHUB_ACCESS_TOKEN set
# already.
if [ -n "$GITHUB_RELEASE_TOKEN" ]; then
GITHUB_TOKEN="$GITHUB_RELEASE_TOKEN"
elif [ -n "$GITHUB_PR_TOKEN" ]; then
GITHUB_TOKEN="$GITHUB_PR_TOKEN"
fi
out=$(curl -H "Authorization: token $GITHUB_TOKEN" -s "$api_base/$repo/pulls/$pr_id")
[ -n "$(echo "$out" | tr -d '\r\n' | jq ".labels | .[] | select(.name==\"$label\")")" ]
}
github_label () {
echo
echo "# run github-api job for labeling it ${1}"
curl -sS -X POST \
-F "token=${CI_JOB_TOKEN}" \
-F "ref=master" \
-F "variables[LABEL]=${1}" \
-F "variables[PRNO]=${CI_COMMIT_REF_NAME}" \
-F "variables[PROJECT]=pezkuwichain/pezkuwi" \
"${GITLAB_API}/projects/${GITHUB_API_PROJECT}/trigger/pipeline"
}
# Formats a message into a JSON string for posting to Matrix
# message: 'any plaintext message'
# formatted_message: '<strong>optional message formatted in <em>html</em></strong>'
# Usage: structure_message $content $formatted_content (optional)
structure_message() {
if [ -z "$2" ]; then
body=$(jq -Rs --arg body "$1" '{"msgtype": "m.text", $body}' < /dev/null)
else
body=$(jq -Rs --arg body "$1" --arg formatted_body "$2" '{"msgtype": "m.text", $body, "format": "org.matrix.custom.html", $formatted_body}' < /dev/null)
fi
echo "$body"
}
# Post a message to a matrix room
# body: '{body: "JSON string produced by structure_message"}'
# room_id: !fsfSRjgjBWEWffws:matrix.parity.io
# access_token: see https://matrix.org/docs/guides/client-server-api/
# Usage: send_message $body (json formatted) $room_id $access_token
send_message() {
curl -XPOST -d "$1" "https://m.parity.io/_matrix/client/r0/rooms/$2/send/m.room.message?access_token=$3"
}
# Pretty-printing functions
boldprint () { printf "|\n| \033[1m%s\033[0m\n|\n" "${@}"; }
boldcat () { printf "|\n"; while read -r l; do printf "| \033[1m%s\033[0m\n" "${l}"; done; printf "|\n" ; }
skip_if_companion_pr() {
url="https://api.github.com/repos/pezkuwichain/pezkuwi/pulls/${CI_COMMIT_REF_NAME}"
echo "[+] API URL: $url"
pr_title=$(curl -sSL -H "Authorization: token ${GITHUB_PR_TOKEN}" "$url" | jq -r .title)
echo "[+] PR title: $pr_title"
if echo "$pr_title" | grep -qi '^companion'; then
echo "[!] PR is a companion PR. Build is already done in substrate"
exit 0
else
echo "[+] PR is not a companion PR. Proceeding test"
fi
}
# Fetches the tag name of the latest release from a repository
# repo: 'organisation/repo'
# Usage: latest_release 'pezkuwichain/pezkuwi'
latest_release() {
curl -s "$api_base/$1/releases/latest" | jq -r '.tag_name'
}
# Check for runtime changes between two commits. This is defined as any changes
# to /primitives/src/* and any *production* chains under /runtime
has_runtime_changes() {
from=$1
to=$2
if git diff --name-only "${from}...${to}" \
| grep -q -e '^runtime/pezkuwi' -e '^runtime/kusama' -e '^primitives/src/' -e '^runtime/common'
then
return 0
else
return 1
fi
}
# given a bootnode and the path to a chainspec file, this function will create a new chainspec file
# with only the bootnode specified and test whether that bootnode provides peers
# The optional third argument is the index of the bootnode in the list of bootnodes, this is just used to pick an ephemeral
# port for the node to run on. If you're only testing one, it'll just use the first ephemeral port
# BOOTNODE: /dns/pezkuwi-connect-0.parity.io/tcp/443/wss/p2p/12D3KooWEPmjoRpDSUuiTjvyNDd8fejZ9eNWH5bE965nyBMDrB4o
# CHAINSPEC_FILE: /path/to/pezkuwi.json
check_bootnode(){
BOOTNODE=$1
BASE_CHAINSPEC=$2
RUNTIME=$(basename "$BASE_CHAINSPEC" | cut -d '.' -f 1)
MIN_PEERS=1
# Generate a temporary chainspec file containing only the bootnode we care about
TMP_CHAINSPEC_FILE="$RUNTIME.$(echo "$BOOTNODE" | tr '/' '_').tmp.json"
jq ".bootNodes = [\"$BOOTNODE\"] " < "$CHAINSPEC_FILE" > "$TMP_CHAINSPEC_FILE"
# Grab an unused port by binding to port 0 and then immediately closing the socket
# This is a bit of a hack, but it's the only way to do it in the shell
RPC_PORT=$(python -c "import socket; s=socket.socket(); s.bind(('', 0)); print(s.getsockname()[1]); s.close()")
echo "[+] Checking bootnode $BOOTNODE"
pezkuwi --chain "$TMP_CHAINSPEC_FILE" --no-mdns --rpc-port="$RPC_PORT" --tmp > /dev/null 2>&1 &
# Wait a few seconds for the node to start up
sleep 5
PEZKUWI_PID=$!
MAX_POLLS=10
TIME_BETWEEN_POLLS=3
for _ in $(seq 1 "$MAX_POLLS"); do
# Check the health endpoint of the RPC node
PEERS="$(curl -s -X POST -H "Content-Type: application/json" --data '{"jsonrpc":"2.0","method":"system_health","params":[],"id":1}' http://localhost:"$RPC_PORT" | jq -r '.result.peers')"
# Sometimes due to machine load or other reasons, we don't get a response from the RPC node
# If $PEERS is an empty variable, make it 0 so we can still do the comparison
if [ -z "$PEERS" ]; then
PEERS=0
fi
if [ "$PEERS" -ge $MIN_PEERS ]; then
echo "[+] $PEERS peers found for $BOOTNODE"
echo " Bootnode appears contactable"
kill $PEZKUWI_PID
# Delete the temporary chainspec file now we're done running the node
rm "$TMP_CHAINSPEC_FILE"
return 0
fi
sleep "$TIME_BETWEEN_POLLS"
done
kill $PEZKUWI_PID
# Delete the temporary chainspec file now we're done running the node
rm "$TMP_CHAINSPEC_FILE"
echo "[!] No peers found for $BOOTNODE"
echo " Bootnode appears unreachable"
return 1
}
# Assumes the ENV are set:
# - RELEASE_ID
# - GITHUB_TOKEN
# - REPO in the form pezkuwichain/pezkuwi
fetch_release_artifacts() {
echo "Release ID : $RELEASE_ID"
echo "Repo : $REPO"
echo "Binary : $BINARY"
OUTPUT_DIR=${OUTPUT_DIR:-"./release-artifacts/${BINARY}"}
echo "OUTPUT_DIR : $OUTPUT_DIR"
echo "Fetching release info..."
curl -L -s \
-H "Accept: application/vnd.github+json" \
-H "Authorization: Bearer ${GITHUB_TOKEN}" \
-H "X-GitHub-Api-Version: 2022-11-28" \
https://api.github.com/repos/${REPO}/releases/${RELEASE_ID} > release.json
echo "Extract asset ids..."
ids=($(jq -r '.assets[].id' < release.json ))
echo "Extract asset count..."
count=$(jq '.assets|length' < release.json )
# Fetch artifacts
mkdir -p "$OUTPUT_DIR"
pushd "$OUTPUT_DIR" > /dev/null
echo "Fetching assets..."
iter=1
for id in "${ids[@]}"
do
echo " - $iter/$count: downloading asset id: $id..."
curl -s -OJ -L -H "Accept: application/octet-stream" \
-H "Authorization: Token ${GITHUB_TOKEN}" \
"https://api.github.com/repos/${REPO}/releases/assets/$id"
iter=$((iter + 1))
done
pwd
ls -al --color
popd > /dev/null
}
# Fetch rpm package from S3.
fetch_rpm_package_from_s3() {
BINARY=$1
OUTPUT_DIR=${OUTPUT_DIR:-"./release-artifacts/${BINARY}"}
echo "--- Preparing to fetch RPM package ---"
echo "Git Tag (VERSION): $VERSION"
echo "Code Version (NODE_VERSION): $NODE_VERSION"
URL_BASE=$(get_s3_url_base $BINARY)
# CORRECTED FILENAME: Changed underscore to hyphen to match the uploaded file.
FILENAME="${BINARY}-${NODE_VERSION}-1.x86_64.rpm"
URL="${URL_BASE}/${VERSION}/x86_64-unknown-linux-gnu/${FILENAME}"
echo "Constructed URL: $URL"
echo "------------------------------------"
mkdir -p "$OUTPUT_DIR"
pushd "$OUTPUT_DIR" > /dev/null
echo "Fetching rpm package..."
# This curl command will now succeed because the URL is correct.
curl --fail --progress-bar -LO "$URL"
echo "Download successful."
ls -al
popd > /dev/null
}
# Fetch deb package from S3. Assumes the ENV are set:
# - RELEASE_ID
# - GITHUB_TOKEN
# - REPO in the form pezkuwichain/pezkuwi
fetch_debian_package_from_s3() {
BINARY=$1
echo "Version : $NODE_VERSION"
echo "Repo : $REPO"
echo "Binary : $BINARY"
echo "Tag : $VERSION"
OUTPUT_DIR=${OUTPUT_DIR:-"./release-artifacts/${BINARY}"}
echo "OUTPUT_DIR : $OUTPUT_DIR"
URL_BASE=$(get_s3_url_base $BINARY)
echo "URL_BASE=$URL_BASE"
URL=$URL_BASE/$VERSION/x86_64-unknown-linux-gnu/${BINARY}_${NODE_VERSION}_amd64.deb
mkdir -p "$OUTPUT_DIR"
pushd "$OUTPUT_DIR" > /dev/null
echo "Fetching deb package..."
echo "Fetching %s" "$URL"
curl --progress-bar -LO "$URL" || echo "Missing $URL"
pwd
ls -al --color
popd > /dev/null
}
# Fetch the release artifacts like binary and signatures from S3. Assumes the ENV are set:
# inputs: binary (pezkuwi), target(aarch64-apple-darwin)
fetch_release_artifacts_from_s3() {
BINARY=$1
TARGET=$2
OUTPUT_DIR=${OUTPUT_DIR:-"./release-artifacts/${TARGET}/${BINARY}"}
echo "OUTPUT_DIR : $OUTPUT_DIR"
URL_BASE=$(get_s3_url_base $BINARY)
echo "URL_BASE=$URL_BASE"
URL_BINARY=$URL_BASE/$VERSION/$TARGET/$BINARY
URL_SHA=$URL_BASE/$VERSION/$TARGET/$BINARY.sha256
URL_ASC=$URL_BASE/$VERSION/$TARGET/$BINARY.asc
# Fetch artifacts
mkdir -p "$OUTPUT_DIR"
pushd "$OUTPUT_DIR" > /dev/null
echo "Fetching artifacts..."
for URL in $URL_BINARY $URL_SHA $URL_ASC; do
echo "Fetching %s" "$URL"
curl --progress-bar -LO "$URL" || echo "Missing $URL"
done
pwd
ls -al --color
popd > /dev/null
}
# Pass the name of the binary as input, it will
# return the s3 base url
function get_s3_url_base() {
name=$1
case $name in
pezkuwi | pezkuwi-execute-worker | pezkuwi-prepare-worker )
printf "https://releases.parity.io/pezkuwi"
;;
pezkuwi-teyrchain)
printf "https://releases.parity.io/pezkuwi-teyrchain"
;;
pezkuwi-omni-node)
printf "https://releases.parity.io/pezkuwi-omni-node"
;;
chain-spec-builder)
printf "https://releases.parity.io/chain-spec-builder"
;;
frame-omni-bencher)
printf "https://releases.parity.io/frame-omni-bencher"
;;
*)
printf "UNSUPPORTED BINARY $name"
exit 1
;;
esac
}
# Check the checksum for a given binary
function check_sha256() {
echo "Checking SHA256 for $1"
shasum -qc $1.sha256
}
# Import GPG keys of the release team members
function import_gpg_keys() {
GPG_KEYSERVER=${GPG_KEYSERVER:-"hkps://keyserver.ubuntu.com"}
SEC="9D4B2B6EB8F97156D19669A9FF0812D491B96798"
EGOR="E6FC4D4782EB0FA64A4903CCDB7D3555DD3932D3"
MORGAN="2E92A9D8B15D7891363D1AE8AF9E6C43F7F8C4CF"
PARITY_RELEASES="90BD75EBBB8E95CB3DA6078F94A4029AB4B35DAE"
PARITY_RELEASES_SIGN_COMMITS="D8018FBB3F534D866A45998293C5FB5F6A367B51"
echo "Importing GPG keys from $GPG_KEYSERVER"
for key in $SEC $EGOR $MORGAN $PARITY_RELEASES $PARITY_RELEASES_SIGN_COMMITS; do
(
echo "Importing GPG key $key"
gpg --no-tty --quiet --keyserver $GPG_KEYSERVER --recv-keys $key
echo -e "5\ny\n" | gpg --no-tty --command-fd 0 --expert --edit-key $key trust;
)
done
wait
gpg -k
}
# Check the GPG signature for a given binary
function check_gpg() {
echo "Checking GPG Signature for $1"
gpg --no-tty --verify -q $1.asc $1
}
# GITHUB_REF will typically be like:
# - refs/heads/release-v1.2.3
# - refs/heads/release-pezkuwi-v1.2.3-rc2
# This function extracts the version
function get_version_from_ghref() {
GITHUB_REF=$1
stripped=${GITHUB_REF#refs/heads/release-}
re="v([0-9]+\.[0-9]+\.[0-9]+)"
if [[ $stripped =~ $re ]]; then
echo ${BASH_REMATCH[0]};
return 0
else
return 1
fi
}
# Get latest rc tag based on the release version and product
function get_latest_rc_tag() {
version=$1
product=$2
if [[ "$product" == "pezkuwi" ]]; then
last_rc=$(git tag -l "$version-rc*" | sort -V | tail -n 1)
elif [[ "$product" == "pezkuwi-teyrchain" ]]; then
last_rc=$(git tag -l "pezkuwi-teyrchains-$version-rc*" | sort -V | tail -n 1)
fi
echo "${last_rc}"
}
# Increment rc tag number based on the value of a suffix of the current rc tag
function increment_rc_tag() {
last_rc=$1
suffix=$(echo "$last_rc" | grep -Eo '[0-9]+$')
((suffix++))
echo $suffix
}
function relative_parent() {
echo "$1" | sed -E 's/(.*)\/(.*)\/\.\./\1/g'
}
# Find all the runtimes, it returns the result as JSON object, compatible to be
# used as Github Workflow Matrix. This call is exposed by the `scan` command and can be used as:
# podman run --rm -it -v /.../fellowship-runtimes:/build docker.io/chevdor/srtool:1.70.0-0.11.1 scan
function find_runtimes() {
libs=($(git grep -I -r --cached --max-depth 20 --files-with-matches '[frame_support::runtime]!' -- '*lib.rs'))
re=".*-runtime$"
JSON=$(jq --null-input '{ "include": [] }')
# EXCLUDED_RUNTIMES is a space separated list of runtime names (without the -runtime postfix)
# EXCLUDED_RUNTIMES=${EXCLUDED_RUNTIMES:-"substrate-test"}
IFS=' ' read -r -a exclusions <<< "$EXCLUDED_RUNTIMES"
for lib in "${libs[@]}"; do
crate_dir=$(dirname "$lib")
cargo_toml="$crate_dir/../Cargo.toml"
name=$(toml get -r $cargo_toml 'package.name')
chain=${name//-runtime/}
if [[ "$name" =~ $re ]] && ! [[ ${exclusions[@]} =~ $chain ]]; then
lib_dir=$(dirname "$lib")
runtime_dir=$(relative_parent "$lib_dir/..")
ITEM=$(jq --null-input \
--arg chain "$chain" \
--arg name "$name" \
--arg runtime_dir "$runtime_dir" \
'{ "chain": $chain, "crate": $name, "runtime_dir": $runtime_dir }')
JSON=$(echo $JSON | jq ".include += [$ITEM]")
fi
done
echo $JSON
}
# Filter the version matches the particular pattern and return it.
# input: version (v1.8.0 or v1.8.0-rc1)
# output: none
filter_version_from_input() {
version=$1
regex="^(v)?[0-9]+\.[0-9]+\.[0-9]+(-rc[0-9]+)?$"
if [[ $version =~ $regex ]]; then
echo $version
else
echo "Invalid version: $version"
exit 1
fi
}
# Check if the release_id is valid number
# input: release_id
# output: release_id or exit 1
check_release_id() {
input=$1
release_id=$(echo "$input" | sed 's/[^0-9]//g')
if [[ $release_id =~ ^[0-9]+$ ]]; then
echo "$release_id"
else
echo "Invalid release_id from input: $input"
exit 1
fi
}
# Get latest release tag
#
# input: none
# output: latest_release_tag
get_latest_release_tag() {
TOKEN="Authorization: Bearer $GITHUB_TOKEN"
latest_release_tag=$(curl -s -H "$TOKEN" $api_base/pezkuwichain/pezkuwi-sdk/releases/latest | jq -r '.tag_name')
printf $latest_release_tag
}
function get_pezkuwi_node_version_from_code() {
# list all the files with node version
git grep -e "\(NODE_VERSION[^=]*= \)\".*\"" |
# fetch only the one we need
grep "primitives/src/lib.rs:" |
# Print only the version
awk '{ print $7 }' |
# Remove the quotes
sed 's/"//g' |
# Remove the semicolon
sed 's/;//g'
}
validate_stable_tag() {
tag="$1"
pattern="^(pezkuwi-)?stable[0-9]{4}(-[0-9]+)?(-rc[0-9]+)?$"
if [[ $tag =~ $pattern ]]; then
echo $tag
else
echo "The input '$tag' does not match the pattern."
exit 1
fi
}
# Prepare docker stable tag from the pezkuwi stable tag
#
# input: tag (pezkuwi-stableYYMM(-X) or pezkuwi-stableYYMM(-X)-rcX)
# output: stableYYMM(-X) or stableYYMM(-X)-rcX
prepare_docker_stable_tag() {
tag="$1"
if [[ "$tag" =~ stable[0-9]{4}(-[0-9]+)?(-rc[0-9]+)? ]]; then
echo "${BASH_REMATCH[0]}"
else
echo "Tag is invalid: $tag"
exit 1
fi
}
# Parse names of the branches from the github labels based on the pattern
#
# input: labels (array of lables like ("A3-backport" "RO-silent" "A4-backport-stable2407" "A4-backport-stable2503"))
# output: BRANCHES (array of the branch names)
parse_branch_names_from_backport_labels() {
labels="$1"
BRANCHES=""
for label in $labels; do
if [[ "$label" =~ ^A4-backport-(stable|unstable)[0-9]{4}$ ]]; then
branch_name=$(sed 's/A4-backport-//' <<< "$label")
BRANCHES+=" ${branch_name}"
fi
done
BRANCHES=$(echo "$BRANCHES" | sed 's/^ *//')
echo "$BRANCHES"
}
# Extract the PR number from the PR title
#
# input: PR_TITLE
# output: PR_NUMBER or exit 1 if the PR title does not contain the PR number
extract_pr_number_from_pr_title() {
PR_TITLE=$1
if [[ "$PR_TITLE" =~ \#([0-9]+) ]]; then
PR_NUMBER="${BASH_REMATCH[1]}"
else
echo "⚠️ The PR title does not contain original PR number. PR title should be in form: [stableBranchName] Backport #originalPRNumber"
exit 1
fi
echo $PR_NUMBER
}