Move scripts used in CI to the new location (#5198)

* Move CI scripts and update references

* Update paths in .gitlab-ci.yml

* Removed outdated entries from CODEOWNERS
This commit is contained in:
Sergejs Kostjucenko
2022-04-26 08:39:31 +03:00
committed by GitHub
parent 9a840bb12a
commit 631a5db536
61 changed files with 46 additions and 48 deletions
+82
View File
@@ -0,0 +1,82 @@
#!/usr/bin/env bash
set -e
# Include the common functions library
#shellcheck source=../common/lib.sh
. "$(dirname "${0}")/../common/lib.sh"
HEAD_BIN=./artifacts/polkadot
HEAD_WS=ws://localhost:9944
RELEASE_WS=ws://localhost:9945
runtimes=(
"westend"
"kusama"
"polkadot"
)
# First we fetch the latest released binary
latest_release=$(latest_release 'paritytech/polkadot')
RELEASE_BIN="./polkadot-$latest_release"
echo "[+] Fetching binary for Polkadot version $latest_release"
curl -L "https://github.com/paritytech/polkadot/releases/download/$latest_release/polkadot" > "$RELEASE_BIN" || exit 1
chmod +x "$RELEASE_BIN"
for RUNTIME in "${runtimes[@]}"; do
echo "[+] Checking runtime: ${RUNTIME}"
release_transaction_version=$(
git show "origin/release:runtime/${RUNTIME}/src/lib.rs" | \
grep 'transaction_version'
)
current_transaction_version=$(
grep 'transaction_version' "./runtime/${RUNTIME}/src/lib.rs"
)
echo "[+] Release: ${release_transaction_version}"
echo "[+] Ours: ${current_transaction_version}"
if [ ! "$release_transaction_version" = "$current_transaction_version" ]; then
echo "[+] Transaction version for ${RUNTIME} has been bumped since last release."
exit 0
fi
# Start running the nodes in the background
$HEAD_BIN --chain="$RUNTIME-local" --tmp &
$RELEASE_BIN --chain="$RUNTIME-local" --ws-port 9945 --tmp &
jobs
# Sleep a little to allow the nodes to spin up and start listening
TIMEOUT=5
for i in $(seq $TIMEOUT); do
sleep 1
if [ "$(lsof -nP -iTCP -sTCP:LISTEN | grep -c '994[45]')" == 2 ]; then
echo "[+] Both nodes listening"
break
fi
if [ "$i" == $TIMEOUT ]; then
echo "[!] Both nodes not listening after $i seconds. Exiting"
exit 1
fi
done
sleep 5
changed_extrinsics=$(
polkadot-js-metadata-cmp "$RELEASE_WS" "$HEAD_WS" \
| sed 's/^ \+//g' | grep -e 'idx: [0-9]\+ -> [0-9]\+' || true
)
if [ -n "$changed_extrinsics" ]; then
echo "[!] Extrinsics indexing/ordering has changed in the ${RUNTIME} runtime! If this change is intentional, please bump transaction_version in lib.rs. Changed extrinsics:"
echo "$changed_extrinsics"
exit 1
fi
echo "[+] No change in extrinsics ordering for the ${RUNTIME} runtime"
jobs -p | xargs kill; sleep 5
done
# Sleep a little to let the jobs die properly
sleep 5
+204
View File
@@ -0,0 +1,204 @@
#!/usr/bin/env bash
# Check for any changes in any runtime directories (e.g., ^runtime/polkadot) as
# well as directories common to all runtimes (e.g., ^runtime/common). If there
# are no changes, check if the Substrate git SHA in Cargo.lock has been
# changed. If so, pull the repo and verify if {spec,impl}_versions have been
# altered since the previous Substrate version used.
#
# If there were changes to any runtimes or common dirs, we iterate over each
# runtime (defined in the $runtimes() array), and check if {spec,impl}_version
# have been changed since the last release.
set -e # fail on any error
#Include the common functions library
#shellcheck source=../common/lib.sh
. "$(dirname "${0}")/../common/lib.sh"
SUBSTRATE_REPO="https://github.com/paritytech/substrate"
SUBSTRATE_REPO_CARGO="git\+${SUBSTRATE_REPO}"
SUBSTRATE_VERSIONS_FILE="bin/node/runtime/src/lib.rs"
# figure out the latest release tag
boldprint "make sure we have all tags (including those from the release branch)"
git fetch --depth="${GIT_DEPTH:-100}" origin release
git fetch --depth="${GIT_DEPTH:-100}" origin 'refs/tags/*:refs/tags/*'
LATEST_TAG="$(git tag -l | grep -E '^v[0-9]+\.[0-9]+\.[0-9]+-?[0-9]*$' | sort -V | tail -n 1)"
boldprint "latest release tag ${LATEST_TAG}"
boldprint "latest 10 commits of ${CI_COMMIT_REF_NAME}"
git --no-pager log --graph --oneline --decorate=short -n 10
boldprint "make sure the master branch is available in shallow clones"
git fetch --depth="${GIT_DEPTH:-100}" origin master
runtimes=(
"kusama"
"polkadot"
"westend"
"rococo"
)
common_dirs=(
"common"
)
# Helper function to join elements in an array with a multi-char delimiter
# https://stackoverflow.com/questions/1527049/how-can-i-join-elements-of-an-array-in-bash
function join_by { local d=$1; shift; echo -n "$1"; shift; printf "%s" "${@/#/$d}"; }
boldprint "check if the wasm sources changed since ${LATEST_TAG}"
if ! has_runtime_changes "${LATEST_TAG}" "${CI_COMMIT_SHA}"; then
boldprint "no changes to any runtime source code detected"
# continue checking if Cargo.lock was updated with a new substrate reference
# and if that change includes a {spec|impl}_version update.
SUBSTRATE_REFS_CHANGED="$(
git diff "refs/tags/${LATEST_TAG}...${CI_COMMIT_SHA}" Cargo.lock \
| sed -n -r "s~^[\+\-]source = \"${SUBSTRATE_REPO_CARGO}#([a-f0-9]+)\".*$~\1~p" | sort -u | wc -l
)"
# check Cargo.lock for substrate ref change
case "${SUBSTRATE_REFS_CHANGED}" in
(0)
boldprint "substrate refs not changed in Cargo.lock"
exit 0
;;
(2)
boldprint "substrate refs updated since ${LATEST_TAG}"
;;
(*)
boldprint "check unsupported: more than one commit targeted in repo ${SUBSTRATE_REPO_CARGO}"
exit 1
esac
SUBSTRATE_PREV_REF="$(
git diff "refs/tags/${LATEST_TAG}...${CI_COMMIT_SHA}" Cargo.lock \
| sed -n -r "s~^\-source = \"${SUBSTRATE_REPO_CARGO}#([a-f0-9]+)\".*$~\1~p" | sort -u | head -n 1
)"
SUBSTRATE_NEW_REF="$(
git diff "refs/tags/${LATEST_TAG}...${CI_COMMIT_SHA}" Cargo.lock \
| sed -n -r "s~^\+source = \"${SUBSTRATE_REPO_CARGO}#([a-f0-9]+)\".*$~\1~p" | sort -u | head -n 1
)"
boldcat <<EOT
previous substrate commit id ${SUBSTRATE_PREV_REF}
new substrate commit id ${SUBSTRATE_NEW_REF}
EOT
# okay so now need to fetch the substrate repository and check whether spec_version or impl_version has changed there
SUBSTRATE_CLONE_DIR="$(mktemp -t -d substrate-XXXXXX)"
trap 'rm -rf "${SUBSTRATE_CLONE_DIR}"' INT QUIT TERM ABRT EXIT
git clone --depth="${GIT_DEPTH:-100}" --no-tags \
"${SUBSTRATE_REPO}" "${SUBSTRATE_CLONE_DIR}"
# check if there are changes to the spec|impl versions
git -C "${SUBSTRATE_CLONE_DIR}" diff \
"${SUBSTRATE_PREV_REF}..${SUBSTRATE_NEW_REF}" "${SUBSTRATE_VERSIONS_FILE}" \
| grep -E '^[\+\-][[:space:]]+(spec|impl)_version: +([0-9]+),$' || exit 0
boldcat <<EOT
spec_version or or impl_version have changed in substrate after updating Cargo.lock
please make sure versions are bumped in polkadot accordingly
EOT
fi
failed_runtime_checks=()
# Iterate over each runtime defined at the start of the script
for RUNTIME in "${runtimes[@]}"
do
# Check if there were changes to this specific runtime or common directories.
# If not, we can skip to the next runtime
regex="^runtime/$(join_by '|^runtime/' "$RUNTIME" "${common_dirs[@]}")"
if ! git diff --name-only "refs/tags/${LATEST_TAG}...${CI_COMMIT_SHA}" \
| grep -E -q -e "$regex"; then
continue
fi
# check for spec_version updates: if the spec versions changed, then there is
# consensus-critical logic that has changed. the runtime wasm blobs must be
# rebuilt.
add_spec_version="$(
git diff "refs/tags/${LATEST_TAG}...${CI_COMMIT_SHA}" "runtime/${RUNTIME}/src/lib.rs" \
| sed -n -r "s/^\+[[:space:]]+spec_version: +([0-9]+),$/\1/p"
)"
sub_spec_version="$(
git diff "refs/tags/${LATEST_TAG}...${CI_COMMIT_SHA}" "runtime/${RUNTIME}/src/lib.rs" \
| sed -n -r "s/^\-[[:space:]]+spec_version: +([0-9]+),$/\1/p"
)"
# see if the version and the binary blob changed
if [ "${add_spec_version}" != "${sub_spec_version}" ]
then
boldcat <<EOT
## RUNTIME: ${RUNTIME} ##
changes to the ${RUNTIME} runtime sources and changes in the spec version.
spec_version: ${sub_spec_version} -> ${add_spec_version}
EOT
continue
else
# check for impl_version updates: if only the impl versions changed, we assume
# there is no consensus-critical logic that has changed.
add_impl_version="$(
git diff refs/tags/"${LATEST_TAG}...${CI_COMMIT_SHA}" "runtime/${RUNTIME}/src/lib.rs" \
| sed -n -r 's/^\+[[:space:]]+impl_version: +([0-9]+),$/\1/p'
)"
sub_impl_version="$(
git diff refs/tags/"${LATEST_TAG}...${CI_COMMIT_SHA}" "runtime/${RUNTIME}/src/lib.rs" \
| sed -n -r 's/^\-[[:space:]]+impl_version: +([0-9]+),$/\1/p'
)"
# see if the impl version changed
if [ "${add_impl_version}" != "${sub_impl_version}" ]
then
boldcat <<EOT
## RUNTIME: ${RUNTIME} ##
changes to the ${RUNTIME} runtime sources and changes in the impl version.
impl_version: ${sub_impl_version} -> ${add_impl_version}
EOT
continue
fi
failed_runtime_checks+=("$RUNTIME")
fi
done
if [ ${#failed_runtime_checks} -gt 0 ]; then
boldcat <<EOT
wasm source files changed or the spec version in the substrate reference in
the Cargo.lock but not the spec/impl version. If changes made do not alter
logic, just bump 'impl_version'. If they do change logic, bump
'spec_version'.
source file directories:
- runtime
version files: ${failed_runtime_checks[@]}
EOT
exit 1
fi
exit 0
+330
View File
@@ -0,0 +1,330 @@
150
2D
A&V
accessor/MS
AccountId
activations
acyclic
adversary/SM
allocator/SM
annualised
anonymize/D
Apache-2.0/M
API
APIs
arg/MS
assignee/SM
async
asynchrony
autogenerated
backable
backend/MS
benchmark/DSMG
BFT/M
bitfield/MS
bitwise
blake2/MS
blockchain/MS
borked
broadcast/UDSMG
BTC/S
canonicalization
canonicalize/D
CentOS
CLI/MS
codebase/SM
codec/SM
commit/D
comparator
computable
conclude/UD
config/MS
could've
crowdfund
crowdloan/MSG
crypto/MS
CSM
Cucumber/MS
customizable/B
DDoS
Debian/M
decodable/MS
decrement
deduplicated
deinitializing
dequeue/SD
dequeuing
deregister
deserialize/G
DHT
disincentivize/D
dispatchable/SM
DLEQ
DM
DMP/SM
DMQ
DoS
DOT
DOTs
ECDSA
ed25519
encodable
enqueue/D
enqueue/DMSG
entrypoint/MS
enum
ERC-20
ETH/S
ethereum/MS
externality/MS
extrinsic
extrinsics
fedora/M
finalize/B
FRAME/MS
FSMs
functor
fungibility
gameable
getter/MS
GiB/S
GKE
GNUNet
GPL/M
GPLv3/M
Grafana/MS
Gurke/MS
gurke/MS
Handler/MS
HMP/SM
HRMP
HSM
https
iff
implementer/MS
includable
include/BG
increment/DSMG
inherent
inherents
initialize/CRG
initializer
instantiate/B
instantiation/SM
intrinsic
intrinsics
invariant/MS
invariants
inverter/MS
invertible
io
IP/S
isn
isolatable
isolate/BG
iterable
jaeger/MS
js
judgement/S
keccak256/M
keypair/MS
keystore/MS
Kovan
KSM/S
Kubernetes/MS
kusama/S
KYC/M
lib
libp2p
lifecycle/MS
lookahead/MS
lookup/MS
LRU
mainnet/MS
malus/MS
MB/M
Mbit
merkle/MS
Merklized
metadata/M
middleware/MS
Millau
misbehavior/SM
misbehaviors
misvalidate/D
MIT/M
MMR
modularity
mpsc
MPSC
MQC/SM
msg
multisig/S
multivalidator/SM
mutex
natively
NFA
NFT/SM
no_std
nonces
NPoS
NTB
offboard/DMSG
onboard/DMSG
oneshot/MS
onwards
OOM/S
OPENISH
others'
ourself
overseer/MS
ownerless
p2p
parablock/MS
parachain/MS
ParaId
parameterization
parameterize/D
parathread/MS
participations
passthrough
PDK
peerset/MS
permission/D
pessimization
phragmen
picosecond/SM
PoA/MS
polkadot/MS
Polkadot/MS
PoS/MS
PoV/MS
PoW/MS
PR
precheck
preconfigured
preimage/MS
preopen
prepend/G
prevalidation
preverify/G
programmatically
prometheus/MS
provisioner/MS
proxy/DMSG
proxy/G
proxying
PRs
PVF/S
README/MS
redhat/M
register/CD
relayer
repo/MS
reservable
responder/SM
retriability
reverify
roundtrip/MS
routable
rpc
RPC/MS
runtime/MS
rustc/MS
SAFT
scalability
scalable
Schnorr
schnorrkel
SDF
sending/S
sharding
shareable
Simnet/MS
spawn/SR
spawner
sr25519
SS58
SSL
startup/MS
stateful
str
struct/MS
subcommand/SM
substream
subsystem/MS
subsystems'
supermajority
SURI
sybil
systemwide
taskmanager/MS
TCP
teleport/D
teleport/RG
teleportation/SM
teleporter/SM
teleporters
template/GSM
testnet/MS
tera/M
teleports
timeframe
timestamp/MS
topologies
tradeoff
transitionary
trie/MS
trustless/Y
TTL
tuple/SM
typesystem
ubuntu/M
UDP
UI
unassign
unconcluded
unfinalize/B
unfinalized
union/MSG
unordered
unreceived
unreserve
unreserving
unroutable
unservable/B
untrusted
untyped
unvested
URI
utilize
v0
v1
v2
validator/SM
ve
vec
verifier
verify/R
versa
version/DMSG
versioned
VMP/SM
VPS
VRF/SM
w3f/MS
wakeup
wakeups
warming/S
wasm/M
wasmtime
Westend/M
wildcard/MS
WND/S
Wococo
WS
XCM/S
XCMP/M
yeet
yml
zsh
@@ -0,0 +1,27 @@
[hunspell]
lang = "en_US"
search_dirs = ["."]
extra_dictionaries = ["lingua.dic"]
skip_os_lookups = true
use_builtin = true
[hunspell.quirks]
# He tagged it as 'TheGreatestOfAllTimes'
transform_regex = [
# `Type`'s
"^'([^\\s])'$",
# 5x
# 10.7%
"^[0-9_]+(?:\\.[0-9]*)?(x|%)$",
# Transforms'
"^(.*)'$",
# backslashes
"^\\+$",
"^[0-9]*+k|MB|Mb|ms|Mbit|nd|th|rd$",
# single char `=` `>` `%` ..
"^=|>|<|%$",
# 22_100
"^(?:[0-9]+_)+[0-9]+$"
]
allow_concatenation = true
allow_dashes = true
+15
View File
@@ -0,0 +1,15 @@
#!/usr/bin/env bash
#shellcheck source=../common/lib.sh
source "$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )/../common/lib.sh"
# build runtime
WASM_BUILD_NO_COLOR=1 cargo build --verbose --release -p kusama-runtime -p polkadot-runtime -p westend-runtime
# make checksum
sha256sum target/release/wbuild/*-runtime/target/wasm32-unknown-unknown/release/*.wasm > checksum.sha256
# clean up - FIXME: can we reuse some of the artifacts?
cargo clean
# build again
WASM_BUILD_NO_COLOR=1 cargo build --verbose --release -p kusama-runtime -p polkadot-runtime -p westend-runtime
# confirm checksum
sha256sum -c checksum.sha256