fix(ci): add explicit timeouts to all non-test jobs + job_execution_timeout retry

Root cause: server_timeout_running (kurditv disk-full Docker failures) hits
GitLab's 60m default timeout and is not caught by stuck_or_timeout_failure,
runner_system_failure, or unknown_failure — meaning no retry triggers, and fmt
(a pipeline blocker) fails permanently, skipping ALL downstream jobs.

Fix: give every quick-check and security job an explicit tight timeout. A stuck
runner now hits job_execution_timeout (a matchable retry condition) in seconds
or minutes, not 60m. The retry then lands on one of the two healthy runners.

Timeouts added (worst-case real duration shown):
  fmt: 10m (normally ~90s)                     retry: max 2
  check-toml-format: 5m
  check-dependency-rules: 5m
  check-fail-ci: 5m
  check-workspace-integrity: 15m
  check-zepter: 20m
  test-rust-features: 10m
  check-umbrella: 30m
  check-licenses-headers: 10m
  check-core-crypto-features: 15m
  cargo-deny-licenses: 20m
  cargo-deny-sources: 20m
  cargo-audit: 20m                              retry: max 2

Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
Kurdistan Tech MinistryandClaude Sonnet 4.6 committed 2026-04-27 12:59:05 +03:00
1 parent 1486a3c2c8
commit 635d0151ad
1 file changed
+30 -2
+30 -2
View File
@@ -62,12 +62,16 @@ default:
fmt:
stage: quick-checks
# Explicit timeout so a stuck runner hits job_execution_timeout (retryable)
# instead of GitLab's 60m server_timeout_running (not matched by any retry keyword).
timeout: 10m
retry:
max: 1
max: 2
when:
- stuck_or_timeout_failure
- runner_system_failure
- unknown_failure
- job_execution_timeout
script:
- cargo fmt --all -- --check
@@ -75,35 +79,41 @@ check-toml-format:
stage: quick-checks
variables:
CARGO_HOME: /root/.cargo
timeout: 5m
retry:
max: 1
when:
- stuck_or_timeout_failure
- runner_system_failure
- unknown_failure
- job_execution_timeout
script:
- taplo format --check --config .config/taplo.toml
- echo "Run 'taplo format --config .config/taplo.toml' to fix TOML issues"
check-dependency-rules:
stage: quick-checks
timeout: 5m
retry:
max: 1
when:
- stuck_or_timeout_failure
- runner_system_failure
- unknown_failure
- job_execution_timeout
script:
- cd bizinikiwi/ && ../.gitlab/ensure-deps.sh
check-fail-ci:
stage: quick-checks
timeout: 5m
retry:
max: 1
when:
- stuck_or_timeout_failure
- runner_system_failure
- unknown_failure
- job_execution_timeout
script:
- |
if rg --line-number --hidden --type rust --glob '!{.git,target}' "FAIL-CI" .; then
@@ -116,12 +126,14 @@ check-workspace-integrity:
stage: quick-checks
variables:
CARGO_HOME: /root/.cargo
timeout: 15m
retry:
max: 1
when:
- stuck_or_timeout_failure
- runner_system_failure
- unknown_failure
- job_execution_timeout
script:
- pip3 install toml "cargo-workspace>=1.2.6" --quiet
- |
@@ -131,12 +143,14 @@ check-workspace-integrity:
check-zepter:
stage: quick-checks
timeout: 20m
retry:
max: 1
when:
- stuck_or_timeout_failure
- runner_system_failure
- unknown_failure
- job_execution_timeout
script:
- cargo metadata --format-version=1 --locked > /dev/null
- cargo install zepter@1.82.1 --force
@@ -146,23 +160,27 @@ check-zepter:
test-rust-features:
stage: quick-checks
timeout: 10m
retry:
max: 1
when:
- stuck_or_timeout_failure
- runner_system_failure
- unknown_failure
- job_execution_timeout
script:
- bash .gitlab/rust-features.sh .
check-umbrella:
stage: quick-checks
timeout: 30m
retry:
max: 1
when:
- stuck_or_timeout_failure
- runner_system_failure
- unknown_failure
- job_execution_timeout
script:
- pip3 install "cargo-workspace>=1.2.4" toml --quiet
- cargo install zepter@1.82.1 --force --quiet
@@ -189,12 +207,14 @@ check-umbrella:
check-licenses-headers:
stage: quick-checks
timeout: 10m
retry:
max: 1
when:
- stuck_or_timeout_failure
- runner_system_failure
- unknown_failure
- job_execution_timeout
script:
- |
FAIL=0
@@ -254,12 +274,14 @@ check-try-runtime:
check-core-crypto-features:
stage: checks
needs: [fmt]
timeout: 15m
retry:
max: 1
when:
- stuck_or_timeout_failure
- runner_system_failure
- unknown_failure
- job_execution_timeout
script:
- rustup target add wasm32v1-none
- |
@@ -293,12 +315,14 @@ cargo-deny-licenses:
stage: security
needs: []
before_script: []
timeout: 20m
retry:
max: 1
when:
- stuck_or_timeout_failure
- runner_system_failure
- unknown_failure
- job_execution_timeout
script:
- cargo install cargo-deny --locked -q
- cargo deny check licenses --config deny.toml
@@ -311,12 +335,14 @@ cargo-deny-sources:
stage: security
needs: []
before_script: []
timeout: 20m
retry:
max: 1
when:
- stuck_or_timeout_failure
- runner_system_failure
- unknown_failure
- job_execution_timeout
script:
- cargo install cargo-deny --locked -q
- cargo deny check sources --config deny.toml
@@ -329,12 +355,14 @@ cargo-audit:
stage: security
needs: []
before_script: []
timeout: 20m
retry:
max: 1
max: 2
when:
- stuck_or_timeout_failure
- runner_system_failure
- unknown_failure
- job_execution_timeout
script:
- cargo install cargo-audit --locked -q
- cargo audit --json 2>/dev/null > /tmp/audit.json || true