From 635d0151adb4babe6a2dd046d124cd1da1b335c0 Mon Sep 17 00:00:00 2001 From: Kurdistan Tech Ministry Date: Mon, 27 Apr 2026 12:59:05 +0300 Subject: [PATCH] fix(ci): add explicit timeouts to all non-test jobs + job_execution_timeout retry MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Root cause: server_timeout_running (kurditv disk-full Docker failures) hits GitLab's 60m default timeout and is not caught by stuck_or_timeout_failure, runner_system_failure, or unknown_failure — meaning no retry triggers, and fmt (a pipeline blocker) fails permanently, skipping ALL downstream jobs. Fix: give every quick-check and security job an explicit tight timeout. A stuck runner now hits job_execution_timeout (a matchable retry condition) in seconds or minutes, not 60m. The retry then lands on one of the two healthy runners. Timeouts added (worst-case real duration shown): fmt: 10m (normally ~90s) retry: max 2 check-toml-format: 5m check-dependency-rules: 5m check-fail-ci: 5m check-workspace-integrity: 15m check-zepter: 20m test-rust-features: 10m check-umbrella: 30m check-licenses-headers: 10m check-core-crypto-features: 15m cargo-deny-licenses: 20m cargo-deny-sources: 20m cargo-audit: 20m retry: max 2 Co-Authored-By: Claude Sonnet 4.6 --- .gitlab-ci.yml | 32 ++++++++++++++++++++++++++++++-- 1 file changed, 30 insertions(+), 2 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 21f09c05..78fbfa5b 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -62,12 +62,16 @@ default: fmt: stage: quick-checks + # Explicit timeout so a stuck runner hits job_execution_timeout (retryable) + # instead of GitLab's 60m server_timeout_running (not matched by any retry keyword). + timeout: 10m retry: - max: 1 + max: 2 when: - stuck_or_timeout_failure - runner_system_failure - unknown_failure + - job_execution_timeout script: - cargo fmt --all -- --check @@ -75,35 +79,41 @@ check-toml-format: stage: quick-checks variables: CARGO_HOME: /root/.cargo + timeout: 5m retry: max: 1 when: - stuck_or_timeout_failure - runner_system_failure - unknown_failure + - job_execution_timeout script: - taplo format --check --config .config/taplo.toml - echo "Run 'taplo format --config .config/taplo.toml' to fix TOML issues" check-dependency-rules: stage: quick-checks + timeout: 5m retry: max: 1 when: - stuck_or_timeout_failure - runner_system_failure - unknown_failure + - job_execution_timeout script: - cd bizinikiwi/ && ../.gitlab/ensure-deps.sh check-fail-ci: stage: quick-checks + timeout: 5m retry: max: 1 when: - stuck_or_timeout_failure - runner_system_failure - unknown_failure + - job_execution_timeout script: - | if rg --line-number --hidden --type rust --glob '!{.git,target}' "FAIL-CI" .; then @@ -116,12 +126,14 @@ check-workspace-integrity: stage: quick-checks variables: CARGO_HOME: /root/.cargo + timeout: 15m retry: max: 1 when: - stuck_or_timeout_failure - runner_system_failure - unknown_failure + - job_execution_timeout script: - pip3 install toml "cargo-workspace>=1.2.6" --quiet - | @@ -131,12 +143,14 @@ check-workspace-integrity: check-zepter: stage: quick-checks + timeout: 20m retry: max: 1 when: - stuck_or_timeout_failure - runner_system_failure - unknown_failure + - job_execution_timeout script: - cargo metadata --format-version=1 --locked > /dev/null - cargo install zepter@1.82.1 --force @@ -146,23 +160,27 @@ check-zepter: test-rust-features: stage: quick-checks + timeout: 10m retry: max: 1 when: - stuck_or_timeout_failure - runner_system_failure - unknown_failure + - job_execution_timeout script: - bash .gitlab/rust-features.sh . check-umbrella: stage: quick-checks + timeout: 30m retry: max: 1 when: - stuck_or_timeout_failure - runner_system_failure - unknown_failure + - job_execution_timeout script: - pip3 install "cargo-workspace>=1.2.4" toml --quiet - cargo install zepter@1.82.1 --force --quiet @@ -189,12 +207,14 @@ check-umbrella: check-licenses-headers: stage: quick-checks + timeout: 10m retry: max: 1 when: - stuck_or_timeout_failure - runner_system_failure - unknown_failure + - job_execution_timeout script: - | FAIL=0 @@ -254,12 +274,14 @@ check-try-runtime: check-core-crypto-features: stage: checks needs: [fmt] + timeout: 15m retry: max: 1 when: - stuck_or_timeout_failure - runner_system_failure - unknown_failure + - job_execution_timeout script: - rustup target add wasm32v1-none - | @@ -293,12 +315,14 @@ cargo-deny-licenses: stage: security needs: [] before_script: [] + timeout: 20m retry: max: 1 when: - stuck_or_timeout_failure - runner_system_failure - unknown_failure + - job_execution_timeout script: - cargo install cargo-deny --locked -q - cargo deny check licenses --config deny.toml @@ -311,12 +335,14 @@ cargo-deny-sources: stage: security needs: [] before_script: [] + timeout: 20m retry: max: 1 when: - stuck_or_timeout_failure - runner_system_failure - unknown_failure + - job_execution_timeout script: - cargo install cargo-deny --locked -q - cargo deny check sources --config deny.toml @@ -329,12 +355,14 @@ cargo-audit: stage: security needs: [] before_script: [] + timeout: 20m retry: - max: 1 + max: 2 when: - stuck_or_timeout_failure - runner_system_failure - unknown_failure + - job_execution_timeout script: - cargo install cargo-audit --locked -q - cargo audit --json 2>/dev/null > /tmp/audit.json || true