From 0354c96a8444f8ee3ac8bb4222c4017c232203d5 Mon Sep 17 00:00:00 2001 From: Kurdistan Tech Ministry Date: Sun, 26 Apr 2026 23:04:11 +0300 Subject: [PATCH] fix(ci): catch server_timeout_running/no_updates_running, fix test cold-build timeout MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Three root causes from pipeline 2480399247 (b756499b): 1. kurditv runner failures: `server_timeout_running` and `no_updates_running` are internal GitLab failure codes NOT caught by stuck_or_timeout_failure or runner_system_failure. Added unknown_failure to all retry conditions so retries trigger and land on a healthy runner. 2. Test partition job_execution_timeout (150m): tests run with --release so C deps (jemalloc/rocksdb) take hours to compile on cold VPS cache. CARGO_TARGET_DIR was per-CI_JOB_NAME so each partition compiled independently. Changed to shared test-release dir — C deps compiled once per runner per cache key, all partitions reuse. Timeout bumped from 150m to 300m to survive a full cold build. 3. wasm-check: no explicit timeout → hit 60m GitLab default. Added timeout: 2h and retry conditions. Also: test-doc/build-rustdoc/build-implementers-guide got retry conditions; build-implementers-guide got explicit 15m timeout (mdbook is fast). Co-Authored-By: Claude Sonnet 4.6 --- .gitlab-ci.yml | 102 +++++++++++++++++++++++++++++++++++++++++-------- 1 file changed, 86 insertions(+), 16 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 6b97b49e..8a731789 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -64,7 +64,10 @@ fmt: stage: quick-checks retry: max: 1 - when: stuck_or_timeout_failure + when: + - stuck_or_timeout_failure + - runner_system_failure + - unknown_failure script: - cargo fmt --all -- --check @@ -74,7 +77,10 @@ check-toml-format: CARGO_HOME: /root/.cargo retry: max: 1 - when: stuck_or_timeout_failure + when: + - stuck_or_timeout_failure + - runner_system_failure + - unknown_failure script: - taplo format --check --config .config/taplo.toml - echo "Run 'taplo format --config .config/taplo.toml' to fix TOML issues" @@ -83,7 +89,10 @@ check-dependency-rules: stage: quick-checks retry: max: 1 - when: stuck_or_timeout_failure + when: + - stuck_or_timeout_failure + - runner_system_failure + - unknown_failure script: - cd bizinikiwi/ && ../.gitlab/ensure-deps.sh @@ -91,7 +100,10 @@ check-fail-ci: stage: quick-checks retry: max: 1 - when: stuck_or_timeout_failure + when: + - stuck_or_timeout_failure + - runner_system_failure + - unknown_failure script: - | if rg --line-number --hidden --type rust --glob '!{.git,target}' "FAIL-CI" .; then @@ -106,7 +118,10 @@ check-workspace-integrity: CARGO_HOME: /root/.cargo retry: max: 1 - when: stuck_or_timeout_failure + when: + - stuck_or_timeout_failure + - runner_system_failure + - unknown_failure script: - pip3 install toml "cargo-workspace>=1.2.6" --quiet - | @@ -118,7 +133,10 @@ check-zepter: stage: quick-checks retry: max: 1 - when: stuck_or_timeout_failure + when: + - stuck_or_timeout_failure + - runner_system_failure + - unknown_failure script: - cargo metadata --format-version=1 --locked > /dev/null - cargo install zepter@1.82.1 --force @@ -130,7 +148,10 @@ test-rust-features: stage: quick-checks retry: max: 1 - when: stuck_or_timeout_failure + when: + - stuck_or_timeout_failure + - runner_system_failure + - unknown_failure script: - bash .gitlab/rust-features.sh . @@ -138,7 +159,10 @@ check-umbrella: stage: quick-checks retry: max: 1 - when: stuck_or_timeout_failure + when: + - stuck_or_timeout_failure + - runner_system_failure + - unknown_failure script: - pip3 install "cargo-workspace>=1.2.4" toml --quiet - cargo install zepter@1.82.1 --force --quiet @@ -167,7 +191,10 @@ check-licenses-headers: stage: quick-checks retry: max: 1 - when: stuck_or_timeout_failure + when: + - stuck_or_timeout_failure + - runner_system_failure + - unknown_failure script: - | FAIL=0 @@ -229,7 +256,10 @@ check-core-crypto-features: needs: [fmt] retry: max: 1 - when: stuck_or_timeout_failure + when: + - stuck_or_timeout_failure + - runner_system_failure + - unknown_failure script: - rustup target add wasm32v1-none - | @@ -265,7 +295,10 @@ cargo-deny-licenses: before_script: [] retry: max: 1 - when: stuck_or_timeout_failure + when: + - stuck_or_timeout_failure + - runner_system_failure + - unknown_failure script: - cargo install cargo-deny --locked -q - cargo deny check licenses --config deny.toml @@ -280,7 +313,10 @@ cargo-deny-sources: before_script: [] retry: max: 1 - when: stuck_or_timeout_failure + when: + - stuck_or_timeout_failure + - runner_system_failure + - unknown_failure script: - cargo install cargo-deny --locked -q - cargo deny check sources --config deny.toml @@ -295,7 +331,10 @@ cargo-audit: before_script: [] retry: max: 1 - when: stuck_or_timeout_failure + when: + - stuck_or_timeout_failure + - runner_system_failure + - unknown_failure script: - cargo install cargo-audit --locked -q - cargo audit --json 2>/dev/null > /tmp/audit.json || true @@ -331,15 +370,22 @@ cargo-audit: - rustc --version && cargo --version - rustup target add wasm32v1-none - export CACHE_KEY="$(cat Cargo.lock rust-toolchain.toml | md5sum | cut -c1-12)" - - export CARGO_TARGET_DIR="/cache/cargo-target/${CACHE_KEY}/${CI_JOB_NAME}" + # All test-partition and test-no-try-runtime jobs share one target dir so + # C deps (jemalloc, rocksdb) are compiled once per runner per cache key. + # cargo fingerprinting handles different feature sets within the same dir. + - export CARGO_TARGET_DIR="/cache/cargo-target/${CACHE_KEY}/test-release" - mkdir -p "$CARGO_TARGET_DIR" - find /cache/cargo-target -mindepth 1 -maxdepth 1 -type d ! -name "${CACHE_KEY}" -exec rm -rf {} + 2>/dev/null || true - echo "CARGO_TARGET_DIR=$CARGO_TARGET_DIR (disk $(df -h /cache/cargo-target | tail -1 | awk '{print $3 "/" $2 " used, " $4 " free"}'))" cache: [] - timeout: 150m + # 300m: cold release build on VPS (jemalloc+rocksdb C compilation) can take 3h+. + # Warm cache (second run on same runner) completes in ~30m. + timeout: 300m retry: max: 1 - when: runner_system_failure + when: + - runner_system_failure + - unknown_failure rules: - if: $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH - if: $CI_PIPELINE_SOURCE == "schedule" @@ -432,6 +478,13 @@ wasm-check: - find /cache/cargo-target -mindepth 1 -maxdepth 1 -type d ! -name "${CACHE_KEY}" -exec rm -rf {} + 2>/dev/null || true - echo "CARGO_TARGET_DIR=$CARGO_TARGET_DIR" cache: [] + timeout: 2h + retry: + max: 1 + when: + - stuck_or_timeout_failure + - runner_system_failure + - unknown_failure script: - cargo check -p pezkuwichain-runtime --locked --target wasm32v1-none rules: @@ -480,6 +533,11 @@ test-doc: - echo "CARGO_TARGET_DIR=$CARGO_TARGET_DIR" cache: [] timeout: 5h + retry: + max: 1 + when: + - runner_system_failure + - unknown_failure script: - git config --global --add safe.directory '*' - | @@ -503,6 +561,11 @@ build-rustdoc: --extend-css ./docs/sdk/assets/theme.css --html-after-content ./docs/sdk/assets/after-content.html timeout: 4h + retry: + max: 1 + when: + - runner_system_failure + - unknown_failure script: - git config --global --add safe.directory '*' - | @@ -524,6 +587,13 @@ build-implementers-guide: stage: docs needs: [] before_script: [] + timeout: 15m + retry: + max: 1 + when: + - stuck_or_timeout_failure + - runner_system_failure + - unknown_failure script: - mdbook build ./pezkuwi/roadmap/implementers-guide artifacts: