From 6b368d97353a52b2f40fd9c814696161f7af24bd Mon Sep 17 00:00:00 2001 From: Satoshi Qazi Muhammed Date: Thu, 30 Jul 2026 16:23:38 -0700 Subject: [PATCH 1/3] ci: widen the deploy approval window from 30 minutes to 6 hours MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit #25 was merged, the approval request went out at 22:14, nobody saw it in time, and at 22:44 the gate cancelled the deploy. Nothing shipped and the only trace was a failed job — the repair migration sat merged but unapplied. Thirty minutes assumes the approver is at their phone when the request lands. A meeting or a night's sleep loses the deploy, and the failure mode is silent unless someone goes looking at Actions. Six hours, with the deadline now stated in the Telegram message itself so the approver knows what they are working against. Job timeout-minutes is set to 370 explicitly: without it the job relies on GitHub's 6h default, which would kill the job at almost exactly the moment the script is still waiting. Worth being honest about the tradeoff: this job holds a pwap-runner slot for the entire wait, so six hours of waiting is six hours that runner cannot do anything else. The right fix is a GitHub Environment with required reviewers — that holds no runner while it waits and allows up to 30 days. This loop is a hand-rolled stand-in for that mechanism; widening it buys time but does not remove the cost. --- .github/workflows/quality-gate.yml | 19 +++++++++++++++---- 1 file changed, 15 insertions(+), 4 deletions(-) diff --git a/.github/workflows/quality-gate.yml b/.github/workflows/quality-gate.yml index 573f8538..d8596290 100644 --- a/.github/workflows/quality-gate.yml +++ b/.github/workflows/quality-gate.yml @@ -307,6 +307,9 @@ jobs: name: Telegram deploy approval runs-on: pwap-runner needs: [web, security-audit] + # Must exceed the in-script wait below, otherwise the job is killed first and + # the approval window is silently shorter than advertised. + timeout-minutes: 370 if: github.ref == 'refs/heads/main' && (github.event_name == 'push' || github.event_name == 'workflow_dispatch') steps: @@ -331,7 +334,7 @@ jobs: -d "{ \"chat_id\": \"${CEO_CHAT_ID}\", \"parse_mode\": \"Markdown\", - \"text\": \"🚀 *pwap/web Deploy Approval*\\n\\n\`${SHORT}\` — ${ACTOR}\\n\\n_${SAFE_MSG}_\\n\\nTargets: app.pezkuwichain.io + pex.mom\", + \"text\": \"🚀 *pwap/web Deploy Approval*\\n\\n\`${SHORT}\` — ${ACTOR}\\n\\n_${SAFE_MSG}_\\n\\nTargets: app.pezkuwichain.io + pex.mom\\n\\n_Approve within 6 hours or the deploy is cancelled._\", \"reply_markup\": { \"inline_keyboard\": [[ {\"text\": \"✅ Approve\", \"callback_data\": \"deploy_approve:${SHORT}\"}, @@ -340,8 +343,16 @@ jobs: } }" - echo "Waiting for Telegram approval (max 30 min)..." - TIMEOUT=1800 + # 30 minutes meant an approval request sent at a bad moment expired + # unseen and the deploy cancelled itself with nothing shipped. + # + # NOTE: this job holds a pwap-runner slot for the whole wait. Six hours + # of waiting is six hours that runner cannot do anything else. The + # proper fix is a GitHub Environment with required reviewers, which + # holds no runner at all while it waits; this loop is a hand-rolled + # stand-in for that. + echo "Waiting for Telegram approval (max 6 hours)..." + TIMEOUT=21600 ELAPSED=0 while [ $ELAPSED -lt $TIMEOUT ]; do if [ -f "$GATE_DIR/$SHORT" ]; then @@ -358,7 +369,7 @@ jobs: sleep 10 ELAPSED=$((ELAPSED + 10)) done - echo "No approval received within 30 minutes — deploy cancelled." + echo "No approval received within 6 hours — deploy cancelled." exit 1 # ======================================== From e4abb38757a613d8c530fd447923fad4dc3e376a Mon Sep 17 00:00:00 2001 From: Satoshi Qazi Muhammed Date: Thu, 30 Jul 2026 18:36:12 -0700 Subject: [PATCH 2/3] ci: gate deploys on a GitHub Environment instead of a hand-rolled wait loop MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Approval was implemented as a job that polled /tmp/pexsec-gates for 30 minutes while holding a pwap-runner slot. On 2026-07-30 the request went out at 22:14, was not seen in time, and at 22:44 the gate cancelled the deploy — #25 stayed merged but never reached the database, and the only trace was a failed job. Widening that timeout was the wrong fix, so this replaces the mechanism. Deploys now gate on the `production` environment, which has SatoshiQaziMuhammed as a required reviewer and is restricted to protected branches. GitHub holds the run until it is approved: - no runner is occupied while waiting (the loop burned one for the full window) - the window is 30 days, not minutes, so a missed notification costs nothing - the approval is recorded — who approved which SHA, in deployment history - approval state lives outside the workflow, so a runner restart cannot lose it telegram-gate becomes notify-deploy-pending: it sends one message and exits in seconds, and cannot block anything. The message now links to the run rather than carrying Approve/Cancel buttons, since approval happens in GitHub. Only the four jobs that touch production carry the environment. Image builds and the version bump stay ungated and run in parallel with the wait; nothing they produce is user-visible until a deploy job runs. --- .github/workflows/quality-gate.yml | 93 +++++++++++------------------- 1 file changed, 35 insertions(+), 58 deletions(-) diff --git a/.github/workflows/quality-gate.yml b/.github/workflows/quality-gate.yml index d8596290..55b6afcd 100644 --- a/.github/workflows/quality-gate.yml +++ b/.github/workflows/quality-gate.yml @@ -153,7 +153,7 @@ jobs: build-image: name: Build & Push Image runs-on: pwap-runner - needs: [web, telegram-gate] + needs: [web, notify-deploy-pending] if: | github.ref == 'refs/heads/main' && (github.event_name == 'push' || @@ -235,7 +235,7 @@ jobs: build-image-backend: name: Build & Push Backend Image runs-on: pwap-runner - needs: [backend, telegram-gate] + needs: [backend, notify-deploy-pending] if: | (github.ref == 'refs/heads/main' || startsWith(github.ref, 'refs/tags/')) && (github.event_name == 'push' || @@ -298,34 +298,35 @@ jobs: echo "✅ Backend image signed (transparency log: rekor.sigstore.dev)" # ======================================== - # TELEGRAM CEO APPROVAL GATE - # Runs on self-hosted pwap-runner (DEV VPS) where pexsec-bot.service - # writes the gate file to /tmp/pexsec-gates/ when CEO clicks - # Approve/Cancel in Telegram. 30-minute timeout = deploy cancelled. + # DEPLOY APPROVAL + # The approval is a GitHub Environment protection rule on `production`, not + # something implemented here. No runner is held while it waits, the window is + # 30 days instead of minutes, and who approved which SHA is recorded in the + # deployment history. + # + # This job only notifies; it sends one message and exits in seconds and cannot + # block anything. It replaces a loop that polled /tmp/pexsec-gates for 30 + # minutes while occupying a pwap-runner slot — on 2026-07-30 that window + # expired unseen and cancelled a deploy with nothing shipped. # ======================================== - telegram-gate: - name: Telegram deploy approval + notify-deploy-pending: + name: Notify approver runs-on: pwap-runner needs: [web, security-audit] - # Must exceed the in-script wait below, otherwise the job is killed first and - # the approval window is silently shorter than advertised. - timeout-minutes: 370 if: github.ref == 'refs/heads/main' && (github.event_name == 'push' || github.event_name == 'workflow_dispatch') + timeout-minutes: 5 steps: - - name: Send approval request and wait + - name: Send Telegram notification env: BOT_TOKEN: ${{ secrets.PEXSEC_BOT_TOKEN }} CEO_CHAT_ID: ${{ secrets.TELEGRAM_CEO_CHAT_ID }} SHA: ${{ github.sha }} ACTOR: ${{ github.actor }} MESSAGE: ${{ github.event.head_commit.message }} + RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} run: | SHORT="${SHA:0:7}" - GATE_DIR="/tmp/pexsec-gates" - mkdir -p "$GATE_DIR" 2>/dev/null || true - rm -f "$GATE_DIR/$SHORT" 2>/dev/null || true - # Strip Markdown special chars to prevent Telegram parse errors SAFE_MSG=$(echo "${MESSAGE}" | head -1 | tr -d '_*`[]()#|{}!' | cut -c1-120) @@ -334,43 +335,15 @@ jobs: -d "{ \"chat_id\": \"${CEO_CHAT_ID}\", \"parse_mode\": \"Markdown\", - \"text\": \"🚀 *pwap/web Deploy Approval*\\n\\n\`${SHORT}\` — ${ACTOR}\\n\\n_${SAFE_MSG}_\\n\\nTargets: app.pezkuwichain.io + pex.mom\\n\\n_Approve within 6 hours or the deploy is cancelled._\", + \"text\": \"🚀 *pwap/web Deploy Approval*\\n\\n\`${SHORT}\` — ${ACTOR}\\n\\n_${SAFE_MSG}_\\n\\nTargets: app.pezkuwichain.io + pex.mom\\n\\nApprove in GitHub — the deploy waits until you do.\", \"reply_markup\": { \"inline_keyboard\": [[ - {\"text\": \"✅ Approve\", \"callback_data\": \"deploy_approve:${SHORT}\"}, - {\"text\": \"❌ Cancel\", \"callback_data\": \"deploy_cancel:${SHORT}\"} + {\"text\": \"🔎 Review \& Approve\", \"url\": \"${RUN_URL}\"} ]] } - }" + }" > /dev/null - # 30 minutes meant an approval request sent at a bad moment expired - # unseen and the deploy cancelled itself with nothing shipped. - # - # NOTE: this job holds a pwap-runner slot for the whole wait. Six hours - # of waiting is six hours that runner cannot do anything else. The - # proper fix is a GitHub Environment with required reviewers, which - # holds no runner at all while it waits; this loop is a hand-rolled - # stand-in for that. - echo "Waiting for Telegram approval (max 6 hours)..." - TIMEOUT=21600 - ELAPSED=0 - while [ $ELAPSED -lt $TIMEOUT ]; do - if [ -f "$GATE_DIR/$SHORT" ]; then - DECISION=$(cat "$GATE_DIR/$SHORT") - rm -f "$GATE_DIR/$SHORT" 2>/dev/null || true - if [ "$DECISION" = "approved" ]; then - echo "Deploy approved." - exit 0 - else - echo "Deploy cancelled." - exit 1 - fi - fi - sleep 10 - ELAPSED=$((ELAPSED + 10)) - done - echo "No approval received within 6 hours — deploy cancelled." - exit 1 + echo "Approver notified; deployment waits on the 'production' environment." # ======================================== # VERSION BUMP (RUNS BEFORE BOTH DEPLOYS) @@ -378,7 +351,7 @@ jobs: bump-version: name: Bump Version runs-on: pwap-runner - needs: [web, security-audit, telegram-gate, build-image] + needs: [web, security-audit, notify-deploy-pending, build-image] # Skip on rollback (workflow_dispatch with rollback_to set) if: | github.ref == 'refs/heads/main' && @@ -424,10 +397,11 @@ jobs: deploy-app: name: Deploy app.pezkuwichain.io runs-on: pwap-runner - needs: [telegram-gate, bump-version, build-image] + environment: production + needs: [notify-deploy-pending, bump-version, build-image] if: | always() && - needs.telegram-gate.result == 'success' && + needs.notify-deploy-pending.result == 'success' && ((github.event_name == 'push' && needs.build-image.result == 'success' && needs.bump-version.result == 'success') || (github.event_name == 'workflow_dispatch' && github.event.inputs.rollback_to != '')) permissions: @@ -593,10 +567,11 @@ jobs: deploy-pex: name: Deploy pex.mom runs-on: pwap-runner - needs: [telegram-gate, bump-version, build-image] + environment: production + needs: [notify-deploy-pending, bump-version, build-image] if: | always() && - needs.telegram-gate.result == 'success' && + needs.notify-deploy-pending.result == 'success' && ((github.event_name == 'push' && needs.build-image.result == 'success' && needs.bump-version.result == 'success') || (github.event_name == 'workflow_dispatch' && github.event.inputs.rollback_to != '')) permissions: @@ -755,7 +730,7 @@ jobs: # Runs a container from the SHA-tagged GHCR image on the backend VPS. # Same discipline as the web deploy: cosign verify → deploy → health check → # auto-rollback to previous SHA → Telegram notify. Gated behind the SAME - # telegram-gate approval as the web deploy. Only main/tags, never fork PRs. + # same environment approval as the web deploy. Only main/tags, never fork PRs. # # STATEFUL DB: the sqlite file lives in the named docker volume # `pwap-indexer-db` (mounted at /data). Deploy replaces the container/image @@ -788,12 +763,13 @@ jobs: deploy-backend: name: Deploy Backend (indexer) runs-on: pwap-runner - needs: [telegram-gate, build-image-backend, backend-cfg] + environment: production + needs: [notify-deploy-pending, build-image-backend, backend-cfg] if: | always() && needs.backend-cfg.outputs.configured == 'true' && (github.ref == 'refs/heads/main' || startsWith(github.ref, 'refs/tags/')) && - needs.telegram-gate.result == 'success' && + needs.notify-deploy-pending.result == 'success' && ((github.event_name == 'push' && needs.build-image-backend.result == 'success') || (github.event_name == 'workflow_dispatch' && github.event.inputs.rollback_to != '')) permissions: @@ -952,10 +928,11 @@ jobs: deploy-supabase: name: Deploy Supabase (functions + migrations) runs-on: pwap-runner - needs: [telegram-gate] + environment: production + needs: [notify-deploy-pending] if: | always() && - needs.telegram-gate.result == 'success' && + needs.notify-deploy-pending.result == 'success' && github.event_name == 'push' && github.ref == 'refs/heads/main' permissions: contents: read From 7482421f3d16b8ee0010c98324f85fd3795666fc Mon Sep 17 00:00:00 2001 From: Satoshi Qazi Muhammed Date: Fri, 31 Jul 2026 02:07:21 -0700 Subject: [PATCH 3/3] ops: capture branch protection and the deploy environment as code MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Both were configured by hand through the GitHub API today. Nothing in the repo reflected them, so they were invisible here, unversioned, and removable without a trace — which undercuts their purpose, since they exist so that nothing reaches main or production unreviewed. apply-repo-settings.sh is now the source of truth for both, and idempotent: each call PUTs the full desired state rather than stacking changes. --check reports drift without touching anything, which is what makes this useful later — a setting silently relaxed is otherwise only discovered when it matters. Documented alongside it: why CI Gate ✅ is the only required check (it is an aggregate that already inspects the others, and listing each job would mean a rename quietly drops a requirement), and why enforce_admins stays false (an incident needs an escape hatch — rare, visible, recorded). This is the same class of problem as the migrations fixed earlier today: state that only exists because someone once clicked something, with no record of what or why. --- ops/README.md | 75 +++++++++++++++++++++ ops/apply-repo-settings.sh | 135 +++++++++++++++++++++++++++++++++++++ 2 files changed, 210 insertions(+) create mode 100644 ops/README.md create mode 100755 ops/apply-repo-settings.sh diff --git a/ops/README.md b/ops/README.md new file mode 100644 index 00000000..cc583ac2 --- /dev/null +++ b/ops/README.md @@ -0,0 +1,75 @@ +# ops + +Operational configuration that lives outside the application code. + +## `apply-repo-settings.sh` + +The repository's GitHub protections, as code. + +Branch protection and the `production` environment are configured through the +GitHub API. Nothing in the repository reflects them, so by default they are +invisible here, unversioned, and gone without a trace if someone removes them — +which undercuts the point, since both exist precisely so that nothing reaches +`main` or production unreviewed. + +```bash +./ops/apply-repo-settings.sh # apply the desired state +./ops/apply-repo-settings.sh --check # report drift, change nothing +``` + +Idempotent: each call PUTs the full desired state, so repeated runs converge. +Requires `gh` authenticated with admin rights on the repo. + +### What it enforces + +**Branch protection on `main`** + +| setting | value | why | +|---|---|---| +| required check | `CI Gate ✅` | aggregate job — fails if web, backend or security-audit did not succeed | +| strict | true | branch must be up to date with `main` before merging | +| approvals | 1 | no unreviewed code on `main` | +| dismiss stale reviews | true | an approval does not carry over to new commits | +| force pushes | blocked | history cannot be rewritten | +| deletions | blocked | `main` cannot be deleted | +| conversation resolution | required | review comments cannot be merged past | + +Only `CI Gate ✅` is listed as a required check, deliberately. It runs with +`if: always()` and inspects the results of the jobs it needs, so it already +covers them. Listing each job here as well would mean editing this file whenever +one is renamed — and a rename would then quietly drop a requirement instead of +failing loudly. + +`enforce_admins` stays `false` on purpose: an admin needs a way through during a +real incident. It is an escape hatch, not the normal path — rare, visible, and +recorded in the branch's protection history. + +**`production` environment** + +Deployment approval is an environment protection rule, not something the workflow +implements. GitHub holds the run until a reviewer approves it: + +- no runner is occupied while waiting +- the window is 30 days, so a missed notification costs nothing +- who approved which SHA is recorded in the deployment history +- approval state survives a runner restart + +The deploy jobs in `quality-gate.yml` opt in with `environment: production`. +`notify-deploy-pending` only sends a Telegram message pointing at the run; it +cannot block anything. + +This replaced a job that polled `/tmp/pexsec-gates` for 30 minutes while holding +a runner slot. On 2026-07-30 that window expired unseen and cancelled a deploy — +a merged migration never reached the database and the only trace was a failed +job. + +### Reviewer + +`REVIEWER_ID` is a numeric user id rather than a login, because the API takes ids +and a login rename would silently break the rule instead of erroring. + +Override any of it via environment variables: + +```bash +REPO=owner/repo BRANCH=main REVIEWER_ID=123456 ./ops/apply-repo-settings.sh +``` diff --git a/ops/apply-repo-settings.sh b/ops/apply-repo-settings.sh new file mode 100755 index 00000000..85bf3a5a --- /dev/null +++ b/ops/apply-repo-settings.sh @@ -0,0 +1,135 @@ +#!/usr/bin/env bash +# +# apply-repo-settings.sh — the repo's GitHub protections, as code. +# +# Branch protection and the production environment are configured through +# GitHub's API, not through anything in this repository. That means they are +# invisible here, unversioned, and gone without trace if someone removes them. +# The whole point of both is that nobody can push to main or deploy unreviewed, +# so leaving them as undocumented clicks in a settings page undercuts them. +# +# This script is the source of truth. It is idempotent — every call is a PUT of +# the full desired state, so running it repeatedly converges rather than stacks. +# +# ./ops/apply-repo-settings.sh # apply +# ./ops/apply-repo-settings.sh --check # report drift, change nothing +# +# Requires gh with admin rights on the repo. +set -euo pipefail + +REPO="${REPO:-pezkuwichain/pwap}" +BRANCH="${BRANCH:-main}" +ENVIRONMENT="${ENVIRONMENT:-production}" + +# The reviewer who approves deployments. Numeric id rather than login, because +# the API takes ids and a login rename would silently break the rule. +REVIEWER_ID="${REVIEWER_ID:-224622464}" # SatoshiQaziMuhammed + +CHECK_ONLY=0 +[[ "${1:-}" == "--check" ]] && CHECK_ONLY=1 + +info() { printf ' %s\n' "$*"; } +ok() { printf ' \033[32m✔\033[0m %s\n' "$*"; } +bad() { printf ' \033[31m✗\033[0m %s\n' "$*"; } + +# ── Branch protection ──────────────────────────────────────────────────────── +# CI Gate ✅ is the only required check on purpose: it is an aggregate job that +# runs with if:always() and fails if web, backend or security-audit did not +# succeed. Listing the individual jobs here as well would mean this file has to +# be edited every time one is renamed, and a rename would silently drop a +# requirement rather than fail loudly. +read -r -d '' PROTECTION <<'JSON' || true +{ + "required_status_checks": { "strict": true, "contexts": ["CI Gate ✅"] }, + "enforce_admins": false, + "required_pull_request_reviews": { + "dismiss_stale_reviews": true, + "require_code_owner_reviews": false, + "required_approving_review_count": 1, + "require_last_push_approval": false + }, + "restrictions": null, + "allow_force_pushes": false, + "allow_deletions": false, + "required_conversation_resolution": true +} +JSON + +# enforce_admins stays false deliberately: an admin needs a way out during a real +# incident. It is an escape hatch, not the normal path — using it should be rare +# and visible, and it is recorded in the branch's protection history. + +# ── Deployment approval ────────────────────────────────────────────────────── +# Approval is an environment protection rule, not something the workflow +# implements. GitHub holds the run without occupying a runner, waits up to 30 +# days, and records who approved which SHA. The workflow's deploy jobs opt in +# with `environment: production`. +read -r -d '' ENV_CONFIG </dev/null || echo '{}')" + if [[ "$cur" == "{}" ]]; then + bad "branch is NOT protected" + else + python3 - "$cur" <<'PY' +import json, sys +d = json.loads(sys.argv[1]) +checks = (d.get('required_status_checks') or {}) +rev = (d.get('required_pull_request_reviews') or {}) +want = { + 'required checks': (checks.get('contexts'), ['CI Gate ✅']), + 'strict': (checks.get('strict'), True), + 'approvals': (rev.get('required_approving_review_count'), 1), + 'dismiss stale': (rev.get('dismiss_stale_reviews'), True), + 'force pushes blocked': (not (d.get('allow_force_pushes') or {}).get('enabled'), True), + 'deletions blocked': (not (d.get('allow_deletions') or {}).get('enabled'), True), + 'conversation resolution': ((d.get('required_conversation_resolution') or {}).get('enabled'), True), +} +for label, (got, exp) in want.items(): + mark = '\033[32m✔\033[0m' if got == exp else '\033[31m✗\033[0m' + extra = '' if got == exp else f' (expected {exp}, got {got})' + print(f' {mark} {label}{extra}') +PY + fi + + echo "── environment ($ENVIRONMENT)" + env_cur="$(gh api "repos/$REPO/environments/$ENVIRONMENT" 2>/dev/null || echo '{}')" + python3 - "$env_cur" <<'PY' +import json, sys +d = json.loads(sys.argv[1]) +if not d: + print(' \033[31m✗\033[0m environment does not exist') + raise SystemExit +rules = d.get('protection_rules') or [] +reviewers = [] +for r in rules: + if r.get('type') == 'required_reviewers': + reviewers = [x['reviewer'].get('login') for x in r.get('reviewers', [])] +mark = '\033[32m✔\033[0m' if reviewers else '\033[31m✗\033[0m' +print(f' {mark} required reviewers: {reviewers or "NONE — deploys are not gated"}') +pol = d.get('deployment_branch_policy') or {} +mark = '\033[32m✔\033[0m' if pol.get('protected_branches') else '\033[31m✗\033[0m' +print(f' {mark} protected branches only: {pol.get("protected_branches")}') +PY + exit 0 +fi + +echo "── applying branch protection ($BRANCH)" +gh api -X PUT "repos/$REPO/branches/$BRANCH/protection" --input - <<<"$PROTECTION" >/dev/null +ok "protected: 1 approval, CI Gate required, no force push, no deletion" + +echo "── applying environment ($ENVIRONMENT)" +gh api -X PUT "repos/$REPO/environments/$ENVIRONMENT" --input - <<<"$ENV_CONFIG" >/dev/null +ok "deployments require review, protected branches only" + +echo "✔ done — verify with: $0 --check"