fix(ci): catch server_timeout_running/no_updates_running, fix test cold-build timeout

Three root causes from pipeline 2480399247 (b756499b):

1. kurditv runner failures: `server_timeout_running` and `no_updates_running`
   are internal GitLab failure codes NOT caught by stuck_or_timeout_failure or
   runner_system_failure. Added unknown_failure to all retry conditions so
   retries trigger and land on a healthy runner.

2. Test partition job_execution_timeout (150m): tests run with --release so
   C deps (jemalloc/rocksdb) take hours to compile on cold VPS cache. CARGO_TARGET_DIR
   was per-CI_JOB_NAME so each partition compiled independently. Changed to shared
   test-release dir — C deps compiled once per runner per cache key, all partitions
   reuse. Timeout bumped from 150m to 300m to survive a full cold build.

3. wasm-check: no explicit timeout → hit 60m GitLab default. Added timeout: 2h
   and retry conditions.

Also: test-doc/build-rustdoc/build-implementers-guide got retry conditions;
build-implementers-guide got explicit 15m timeout (mdbook is fast).

Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
Kurdistan Tech MinistryandClaude Sonnet 4.6 committed 2026-04-26 23:04:11 +03:00
1 parent b756499b66
commit 0354c96a84
1 file changed
+86 -16
+86 -16
View File
@@ -64,7 +64,10 @@ fmt:
stage: quick-checks
retry:
max: 1
when: stuck_or_timeout_failure
when:
- stuck_or_timeout_failure
- runner_system_failure
- unknown_failure
script:
- cargo fmt --all -- --check
@@ -74,7 +77,10 @@ check-toml-format:
CARGO_HOME: /root/.cargo
retry:
max: 1
when: stuck_or_timeout_failure
when:
- stuck_or_timeout_failure
- runner_system_failure
- unknown_failure
script:
- taplo format --check --config .config/taplo.toml
- echo "Run 'taplo format --config .config/taplo.toml' to fix TOML issues"
@@ -83,7 +89,10 @@ check-dependency-rules:
stage: quick-checks
retry:
max: 1
when: stuck_or_timeout_failure
when:
- stuck_or_timeout_failure
- runner_system_failure
- unknown_failure
script:
- cd bizinikiwi/ && ../.gitlab/ensure-deps.sh
@@ -91,7 +100,10 @@ check-fail-ci:
stage: quick-checks
retry:
max: 1
when: stuck_or_timeout_failure
when:
- stuck_or_timeout_failure
- runner_system_failure
- unknown_failure
script:
- |
if rg --line-number --hidden --type rust --glob '!{.git,target}' "FAIL-CI" .; then
@@ -106,7 +118,10 @@ check-workspace-integrity:
CARGO_HOME: /root/.cargo
retry:
max: 1
when: stuck_or_timeout_failure
when:
- stuck_or_timeout_failure
- runner_system_failure
- unknown_failure
script:
- pip3 install toml "cargo-workspace>=1.2.6" --quiet
- |
@@ -118,7 +133,10 @@ check-zepter:
stage: quick-checks
retry:
max: 1
when: stuck_or_timeout_failure
when:
- stuck_or_timeout_failure
- runner_system_failure
- unknown_failure
script:
- cargo metadata --format-version=1 --locked > /dev/null
- cargo install zepter@1.82.1 --force
@@ -130,7 +148,10 @@ test-rust-features:
stage: quick-checks
retry:
max: 1
when: stuck_or_timeout_failure
when:
- stuck_or_timeout_failure
- runner_system_failure
- unknown_failure
script:
- bash .gitlab/rust-features.sh .
@@ -138,7 +159,10 @@ check-umbrella:
stage: quick-checks
retry:
max: 1
when: stuck_or_timeout_failure
when:
- stuck_or_timeout_failure
- runner_system_failure
- unknown_failure
script:
- pip3 install "cargo-workspace>=1.2.4" toml --quiet
- cargo install zepter@1.82.1 --force --quiet
@@ -167,7 +191,10 @@ check-licenses-headers:
stage: quick-checks
retry:
max: 1
when: stuck_or_timeout_failure
when:
- stuck_or_timeout_failure
- runner_system_failure
- unknown_failure
script:
- |
FAIL=0
@@ -229,7 +256,10 @@ check-core-crypto-features:
needs: [fmt]
retry:
max: 1
when: stuck_or_timeout_failure
when:
- stuck_or_timeout_failure
- runner_system_failure
- unknown_failure
script:
- rustup target add wasm32v1-none
- |
@@ -265,7 +295,10 @@ cargo-deny-licenses:
before_script: []
retry:
max: 1
when: stuck_or_timeout_failure
when:
- stuck_or_timeout_failure
- runner_system_failure
- unknown_failure
script:
- cargo install cargo-deny --locked -q
- cargo deny check licenses --config deny.toml
@@ -280,7 +313,10 @@ cargo-deny-sources:
before_script: []
retry:
max: 1
when: stuck_or_timeout_failure
when:
- stuck_or_timeout_failure
- runner_system_failure
- unknown_failure
script:
- cargo install cargo-deny --locked -q
- cargo deny check sources --config deny.toml
@@ -295,7 +331,10 @@ cargo-audit:
before_script: []
retry:
max: 1
when: stuck_or_timeout_failure
when:
- stuck_or_timeout_failure
- runner_system_failure
- unknown_failure
script:
- cargo install cargo-audit --locked -q
- cargo audit --json 2>/dev/null > /tmp/audit.json || true
@@ -331,15 +370,22 @@ cargo-audit:
- rustc --version && cargo --version
- rustup target add wasm32v1-none
- export CACHE_KEY="$(cat Cargo.lock rust-toolchain.toml | md5sum | cut -c1-12)"
- export CARGO_TARGET_DIR="/cache/cargo-target/${CACHE_KEY}/${CI_JOB_NAME}"
# All test-partition and test-no-try-runtime jobs share one target dir so
# C deps (jemalloc, rocksdb) are compiled once per runner per cache key.
# cargo fingerprinting handles different feature sets within the same dir.
- export CARGO_TARGET_DIR="/cache/cargo-target/${CACHE_KEY}/test-release"
- mkdir -p "$CARGO_TARGET_DIR"
- find /cache/cargo-target -mindepth 1 -maxdepth 1 -type d ! -name "${CACHE_KEY}" -exec rm -rf {} + 2>/dev/null || true
- echo "CARGO_TARGET_DIR=$CARGO_TARGET_DIR (disk $(df -h /cache/cargo-target | tail -1 | awk '{print $3 "/" $2 " used, " $4 " free"}'))"
cache: []
timeout: 150m
# 300m: cold release build on VPS (jemalloc+rocksdb C compilation) can take 3h+.
# Warm cache (second run on same runner) completes in ~30m.
timeout: 300m
retry:
max: 1
when: runner_system_failure
when:
- runner_system_failure
- unknown_failure
rules:
- if: $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH
- if: $CI_PIPELINE_SOURCE == "schedule"
@@ -432,6 +478,13 @@ wasm-check:
- find /cache/cargo-target -mindepth 1 -maxdepth 1 -type d ! -name "${CACHE_KEY}" -exec rm -rf {} + 2>/dev/null || true
- echo "CARGO_TARGET_DIR=$CARGO_TARGET_DIR"
cache: []
timeout: 2h
retry:
max: 1
when:
- stuck_or_timeout_failure
- runner_system_failure
- unknown_failure
script:
- cargo check -p pezkuwichain-runtime --locked --target wasm32v1-none
rules:
@@ -480,6 +533,11 @@ test-doc:
- echo "CARGO_TARGET_DIR=$CARGO_TARGET_DIR"
cache: []
timeout: 5h
retry:
max: 1
when:
- runner_system_failure
- unknown_failure
script:
- git config --global --add safe.directory '*'
- |
@@ -503,6 +561,11 @@ build-rustdoc:
--extend-css ./docs/sdk/assets/theme.css
--html-after-content ./docs/sdk/assets/after-content.html
timeout: 4h
retry:
max: 1
when:
- runner_system_failure
- unknown_failure
script:
- git config --global --add safe.directory '*'
- |
@@ -524,6 +587,13 @@ build-implementers-guide:
stage: docs
needs: []
before_script: []
timeout: 15m
retry:
max: 1
when:
- stuck_or_timeout_failure
- runner_system_failure
- unknown_failure
script:
- mdbook build ./pezkuwi/roadmap/implementers-guide
artifacts: