name: Quality Gate & Deploy on: push: branches: [ main, develop ] pull_request: branches: [ main, develop ] workflow_dispatch: inputs: rollback_to: description: 'Rollback to git SHA (skips build, redeploys old image). Empty = normal deploy.' required: false default: '' concurrency: group: ${{ github.workflow }}-${{ github.ref }} cancel-in-progress: true permissions: contents: write # version bump commit packages: write # GHCR push env: VITE_SUPABASE_URL: ${{ secrets.VITE_SUPABASE_URL }} VITE_SUPABASE_ANON_KEY: ${{ secrets.VITE_SUPABASE_ANON_KEY }} REGISTRY: ghcr.io IMAGE_NAME: pezkuwichain/pwap-web BACKEND_IMAGE_NAME: pezkuwichain/pwap-indexer # Deploy targets. Each deploy key is pinned on its host to one fixed script # in ops/host (authorized_keys restrict,command=), so these are the only # places CI can write, and only through those scripts. APP_HOST: 89.117.59.63 INDEXER_HOST: 89.117.59.63 PEX_HOST: 217.77.6.126 SUPABASE_HOST: 217.77.6.126 # Host keys, read from the hosts themselves (ops/known_hosts); a deploy to a # host that presents any other key fails. DEPLOY_KNOWN_HOSTS: | 89.117.59.63 ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIGBR32BTb3hC4stTUvoCN/ZKZcAretzHttdkMOodgS4W 217.77.6.126 ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIMxxRmMEYyng+B9O3jcUCMnUa31SyWQ+l7aj18/soWtJ jobs: # ======================================== # WEB APP - LINT, TEST & BUILD # ======================================== web: name: Web App runs-on: ubuntu-latest steps: - name: Checkout code uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0 - name: Setup Node.js uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4.4.0 with: node-version: '24' - name: Cache npm dependencies uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830 # v4.3.0 with: path: web/node_modules key: ${{ runner.os }}-web-${{ hashFiles('web/package-lock.json') }} restore-keys: | ${{ runner.os }}-web- - name: Install dependencies working-directory: ./web run: npm ci - name: Run Linter working-directory: ./web run: npm run lint # Typecheck gate, held to a ratchet: the type errors that predate the gate # are recorded in ops/typecheck-baseline.txt, and this fails when any file # gains one, or when errors were fixed without lowering the baseline. - name: Typecheck (ratchet) run: python3 ops/typecheck-ratchet.py - name: Run Tests working-directory: ./web run: npm run test - name: Build Project working-directory: ./web run: npm run build env: VITE_NETWORK: MAINNET VITE_WS_ENDPOINT: wss://rpc.pezkuwichain.io VITE_WS_ENDPOINT_FALLBACK_1: wss://mainnet.pezkuwichain.io VITE_ASSET_HUB_ENDPOINT: wss://asset-hub-rpc.pezkuwichain.io VITE_PEOPLE_CHAIN_ENDPOINT: wss://people-rpc.pezkuwichain.io VITE_WALLETCONNECT_PROJECT_ID: 8292a793b7640e8364c378e331e76d04 VITE_TURNSTILE_SITE_KEY: 0x4AAAAAAEDkUkbsUgw2D9zN - name: Upload build artifact uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 with: name: web-dist path: web/dist/ # ======================================== # BACKEND - INDEXER SERVICE (lint/smoke/audit) # No build step exists (plain ESM node service); we lockfile-enforce install, # syntax-check the entry point, and audit. # ======================================== backend: name: Backend Indexer runs-on: ubuntu-latest steps: - name: Checkout code uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0 - name: Setup Node.js uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4.4.0 with: node-version: '24' - name: Cache npm dependencies uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830 # v4.3.0 with: path: backend/node_modules key: ${{ runner.os }}-backend-${{ hashFiles('backend/package-lock.json') }} restore-keys: | ${{ runner.os }}-backend- - name: Install dependencies working-directory: ./backend run: npm ci - name: Syntax check (entry point) working-directory: ./backend run: node --check src/index.js # Real, offline test suite (node:test). Uses in-memory sqlite and a # mocked chain/Supabase — NO live node, NO network. The integration-tests/ # *.live.test.js suites are intentionally EXCLUDED (they require a live # chain + Supabase and are not run in CI); the `test` script scopes to # test/*.test.js only. - name: Run tests (offline) working-directory: ./backend run: npm test - name: npm audit (high + critical, production deps) working-directory: ./backend run: npm audit --audit-level=high --omit=dev # ======================================== # SIGN - MULTISIG SIGNING PORTAL (typecheck + build) # sign/ compiles ../shared/lib with tsc before vite builds it, so this is also # the check that shared/ still typechecks against the chain's augmented API. # ======================================== sign: name: Signing Portal runs-on: ubuntu-latest steps: - name: Checkout code uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0 - name: Setup Node.js uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4.4.0 with: node-version: '24' - name: Install dependencies working-directory: ./sign run: npm ci - name: Typecheck and build working-directory: ./sign run: npm run build - name: npm audit (high + critical, production deps) working-directory: ./sign run: npm audit --audit-level=high --omit=dev # ======================================== # BUILD & PUSH DOCKER IMAGE TO GHCR # Immutable artifact for audit + rollback (vs ephemeral GHA artifact). # Tagged with git SHA so any commit can be redeployed by SHA. # ======================================== build-image: name: Build & Push Image runs-on: ubuntu-latest needs: [web, notify-deploy-pending] if: | github.ref == 'refs/heads/main' && (github.event_name == 'push' || (github.event_name == 'workflow_dispatch' && github.event.inputs.rollback_to == '')) permissions: contents: read packages: write id-token: write # cosign keyless signing via Sigstore OIDC outputs: image_sha: ${{ steps.meta.outputs.image_sha }} image_digest: ${{ steps.build.outputs.digest }} steps: - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0 - uses: docker/setup-buildx-action@8d2750c68a42422c14e847fe6c8ac0403b4cbd6f # v3.12.0 - name: Install cosign uses: sigstore/cosign-installer@7e8b541eb2e61bf99390e1afd4be13a184e9ebc5 # v3.10.1 with: cosign-release: 'v2.4.1' - name: Log in to GHCR uses: docker/login-action@c94ce9fb468520275223c153574b00df6fe4bcc9 # v3.7.0 with: registry: ${{ env.REGISTRY }} username: ${{ github.actor }} password: ${{ secrets.GITHUB_TOKEN }} - name: Extract image metadata id: meta run: | SHORT_SHA="${GITHUB_SHA:0:7}" echo "short_sha=$SHORT_SHA" >> $GITHUB_OUTPUT echo "image_sha=$SHORT_SHA" >> $GITHUB_OUTPUT echo "image=${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}" >> $GITHUB_OUTPUT - name: Build and push id: build uses: docker/build-push-action@10e90e3645eae34f1e60eeb005ba3a3d33f178e8 # v6.19.2 with: context: ./ file: ./web/Dockerfile push: true tags: | ${{ steps.meta.outputs.image }}:${{ steps.meta.outputs.short_sha }} ${{ steps.meta.outputs.image }}:latest build-args: | VITE_NETWORK=MAINNET VITE_WS_ENDPOINT=wss://rpc.pezkuwichain.io VITE_WS_ENDPOINT_FALLBACK_1=wss://mainnet.pezkuwichain.io VITE_ASSET_HUB_ENDPOINT=wss://asset-hub-rpc.pezkuwichain.io VITE_PEOPLE_CHAIN_ENDPOINT=wss://people-rpc.pezkuwichain.io VITE_WALLETCONNECT_PROJECT_ID=8292a793b7640e8364c378e331e76d04 VITE_TURNSTILE_SITE_KEY=0x4AAAAAAEDkUkbsUgw2D9zN VITE_SUPABASE_URL=${{ secrets.VITE_SUPABASE_URL }} VITE_SUPABASE_ANON_KEY=${{ secrets.VITE_SUPABASE_ANON_KEY }} cache-from: type=registry,ref=${{ steps.meta.outputs.image }}:cache cache-to: type=registry,ref=${{ steps.meta.outputs.image }}:cache,mode=max provenance: false - name: Sign image with cosign (keyless, Sigstore Fulcio) env: COSIGN_EXPERIMENTAL: '1' run: | IMAGE_DIGEST="${{ steps.meta.outputs.image }}@${{ steps.build.outputs.digest }}" # cosign needs its own registry auth — docker/login-action only writes # ~/.docker/config.json which cosign on self-hosted runner can't read echo "${{ secrets.GITHUB_TOKEN }}" | cosign login ghcr.io -u "${{ github.actor }}" --password-stdin echo "Signing $IMAGE_DIGEST" cosign sign --yes "$IMAGE_DIGEST" echo "✅ Image signed (transparency log: rekor.sigstore.dev)" # ======================================== # BUILD & PUSH BACKEND (INDEXER) IMAGE TO GHCR # Runnable Node service image (vs the web static-dist image). SHA-tagged + # cosign-signed for the same audit/rollback discipline as the web image. # Gated to main/tags, never fork PRs (needs packages:write + the telegram gate). # ======================================== build-image-backend: name: Build & Push Backend Image runs-on: ubuntu-latest needs: [backend, notify-deploy-pending] if: | (github.ref == 'refs/heads/main' || startsWith(github.ref, 'refs/tags/')) && (github.event_name == 'push' || (github.event_name == 'workflow_dispatch' && github.event.inputs.rollback_to == '')) permissions: contents: read packages: write id-token: write # cosign keyless signing via Sigstore OIDC outputs: image_sha: ${{ steps.meta.outputs.image_sha }} image_digest: ${{ steps.build.outputs.digest }} steps: - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0 - uses: docker/setup-buildx-action@8d2750c68a42422c14e847fe6c8ac0403b4cbd6f # v3.12.0 - name: Install cosign uses: sigstore/cosign-installer@7e8b541eb2e61bf99390e1afd4be13a184e9ebc5 # v3.10.1 with: cosign-release: 'v2.4.1' - name: Log in to GHCR uses: docker/login-action@c94ce9fb468520275223c153574b00df6fe4bcc9 # v3.7.0 with: registry: ${{ env.REGISTRY }} username: ${{ github.actor }} password: ${{ secrets.GITHUB_TOKEN }} - name: Extract image metadata id: meta run: | SHORT_SHA="${GITHUB_SHA:0:7}" echo "short_sha=$SHORT_SHA" >> $GITHUB_OUTPUT echo "image_sha=$SHORT_SHA" >> $GITHUB_OUTPUT echo "image=${{ env.REGISTRY }}/${{ env.BACKEND_IMAGE_NAME }}" >> $GITHUB_OUTPUT - name: Build and push id: build uses: docker/build-push-action@10e90e3645eae34f1e60eeb005ba3a3d33f178e8 # v6.19.2 with: context: ./backend file: ./backend/Dockerfile push: true tags: | ${{ steps.meta.outputs.image }}:${{ steps.meta.outputs.short_sha }} ${{ steps.meta.outputs.image }}:latest cache-from: type=registry,ref=${{ steps.meta.outputs.image }}:cache cache-to: type=registry,ref=${{ steps.meta.outputs.image }}:cache,mode=max provenance: false - name: Sign image with cosign (keyless, Sigstore Fulcio) env: COSIGN_EXPERIMENTAL: '1' run: | IMAGE_DIGEST="${{ steps.meta.outputs.image }}@${{ steps.build.outputs.digest }}" echo "${{ secrets.GITHUB_TOKEN }}" | cosign login ghcr.io -u "${{ github.actor }}" --password-stdin echo "Signing $IMAGE_DIGEST" cosign sign --yes "$IMAGE_DIGEST" echo "✅ Backend image signed (transparency log: rekor.sigstore.dev)" # ======================================== # DEPLOY APPROVAL # The approval is a GitHub Environment protection rule on `production`, not # something implemented here. No runner is held while it waits, the window is # 30 days instead of minutes, and who approved which SHA is recorded in the # deployment history. # # This job only notifies; it sends one message and exits in seconds and cannot # block anything. It replaces a loop that polled /tmp/pexsec-gates for 30 # minutes while occupying a pwap-runner slot — on 2026-07-30 that window # expired unseen and cancelled a deploy with nothing shipped. # ======================================== notify-deploy-pending: name: Notify approver runs-on: ubuntu-latest needs: [web, security-audit] if: github.ref == 'refs/heads/main' && (github.event_name == 'push' || github.event_name == 'workflow_dispatch') timeout-minutes: 5 steps: - name: Send Telegram notification env: BOT_TOKEN: ${{ secrets.PEXSEC_BOT_TOKEN }} CEO_CHAT_ID: ${{ secrets.TELEGRAM_CEO_CHAT_ID }} SHA: ${{ github.sha }} ACTOR: ${{ github.actor }} MESSAGE: ${{ github.event.head_commit.message }} RUN_ID: ${{ github.run_id }} RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} run: | SHORT="${SHA:0:7}" # Strip Markdown special chars to prevent Telegram parse errors SAFE_MSG=$(echo "${MESSAGE}" | head -1 | tr -d '_*`[]()#|{}!' | cut -c1-120) # The buttons answer the environment's pending deployment through # @pexsecBot, so approving is one tap from a phone. The environment rule # is still what holds the deploy — the bot only presses the button on the # reviewer's behalf, and the GitHub link stays as the fallback for when # the bot or its host is down. # Capture the reply. A notification step that discards it cannot fail: # a stale token, a wrong chat id or a rejected message all end with # the step printing success and the approver never hearing about the # deploy. That happened on 2026-08-01 — the run sat waiting and the # log said "Approver notified". RESP=$(curl -s -X POST "https://api.telegram.org/bot${BOT_TOKEN}/sendMessage" \ -H "Content-Type: application/json" \ -d "{ \"chat_id\": \"${CEO_CHAT_ID}\", \"parse_mode\": \"Markdown\", \"text\": \"🚀 *pwap/web Deploy Approval*\\n\\n\`${SHORT}\` — ${ACTOR}\\n\\n_${SAFE_MSG}_\\n\\nTargets: app.pezkuwichain.io + pex.mom\\n\\nOnaylayana kadar deploy bekler.\", \"reply_markup\": { \"inline_keyboard\": [ [ {\"text\": \"✅ Onayla\", \"callback_data\": \"pwap_approve:${RUN_ID}\"}, {\"text\": \"🛑 Reddet\", \"callback_data\": \"pwap_reject:${RUN_ID}\"} ], [ {\"text\": \"🔎 GitHub'da incele\", \"url\": \"${RUN_URL}\"} ] ] } }") if ! echo "$RESP" | grep -q '"ok":true'; then echo "::error::Telegram rejected the approval prompt: ${RESP:0:300}" exit 1 fi echo "Approver notified; deployment waits on the 'production' environment." # ======================================== # APPROVAL GATE — ONE JOB, ONE APPROVAL # ======================================== # Every deploy hangs off this job. It exists so that a run needs exactly one # approval, not several. # # It used to be that all four deploy jobs carried `environment: production` # themselves. GitHub opens an approval request for whichever jobs are eligible # at that moment, and these four are not eligible at the same moment — # deploy-supabase only waits on the notification, while the rest wait on image # builds that take minutes. So the run asked twice, several minutes apart. On # 2026-07-31 the first approval shipped Supabase and the run then sat waiting # again, unnoticed: the schema had moved and the app serving it had not. # # Deployment protection lives here now and nowhere else, so approving once # releases the whole set. The deploy jobs are held by their dependency on this # job instead of by their own environment — workflow-guard enforces that edge # so a new deploy job cannot quietly skip the gate. # # Waiting for approval costs no runner: GitHub holds the job before dispatch. approve-deploy: name: Await deploy approval runs-on: ubuntu-latest environment: production needs: [notify-deploy-pending] timeout-minutes: 5 steps: - name: Record approval run: | echo "Deploy of ${GITHUB_SHA:0:7} approved. Releasing deploy jobs." # ======================================== # VERSION BUMP (RUNS BEFORE BOTH DEPLOYS) # ======================================== bump-version: name: Bump Version runs-on: ubuntu-latest needs: [web, security-audit, notify-deploy-pending, build-image] # Skip on rollback (workflow_dispatch with rollback_to set) if: | github.ref == 'refs/heads/main' && (github.event_name == 'push' || (github.event_name == 'workflow_dispatch' && github.event.inputs.rollback_to == '')) outputs: new_version: ${{ steps.bump.outputs.version }} steps: - name: Checkout code uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0 with: fetch-depth: 0 token: ${{ secrets.GITHUB_TOKEN }} - name: Setup Node.js uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4.4.0 with: node-version: '24' - name: Configure Git run: | git config user.name "github-actions[bot]" git config user.email "github-actions[bot]@users.noreply.github.com" - name: Bump version id: bump working-directory: ./web run: | npm version patch --no-git-tag-version VERSION=$(node -p "require('./package.json').version") echo "version=$VERSION" >> $GITHUB_OUTPUT cd .. git add web/package.json git commit -m "chore(web): bump version to $VERSION [skip ci]" || echo "No version change" git push || echo "Nothing to push" # ======================================== # DEPLOY TO app.pezkuwichain.io (DEV VPS) # Pulls SHA-tagged image from GHCR, extracts /dist, scp to VPS. # Health check + auto-rollback to .deploy-tag-prev on failure. # ======================================== deploy-app: name: Deploy app.pezkuwichain.io runs-on: ubuntu-latest needs: [approve-deploy, notify-deploy-pending, bump-version, build-image] if: | always() && needs.approve-deploy.result == 'success' && needs.notify-deploy-pending.result == 'success' && ((github.event_name == 'push' && needs.build-image.result == 'success' && needs.bump-version.result == 'success') || (github.event_name == 'workflow_dispatch' && github.event.inputs.rollback_to != '')) permissions: contents: read packages: read env: DOMAIN: app.pezkuwichain.io TARGET_PATH: /var/www/subdomains/app steps: - name: Determine image SHA id: sha run: | if [ -n "${{ github.event.inputs.rollback_to }}" ]; then echo "sha=${{ github.event.inputs.rollback_to }}" >> $GITHUB_OUTPUT echo "Rolling back to: ${{ github.event.inputs.rollback_to }}" else echo "sha=${{ needs.build-image.outputs.image_sha }}" >> $GITHUB_OUTPUT fi - name: Capture currently-live SHA (for auto-rollback) id: prev run: | # /.deploy-sha is written into every deploy; read what's live now PREV=$(curl -sf --max-time 5 "https://${{ env.DOMAIN }}/.deploy-sha" | head -c 40 | tr -dc 'a-f0-9' || echo "") echo "Previous live SHA: ${PREV:-unknown}" echo "prev=$PREV" >> $GITHUB_OUTPUT - name: Log in to GHCR uses: docker/login-action@c94ce9fb468520275223c153574b00df6fe4bcc9 # v3.7.0 with: registry: ${{ env.REGISTRY }} username: ${{ github.actor }} password: ${{ secrets.GITHUB_TOKEN }} - name: Install cosign (for verify) uses: sigstore/cosign-installer@7e8b541eb2e61bf99390e1afd4be13a184e9ebc5 # v3.10.1 with: cosign-release: 'v2.4.1' - name: Verify image signature (cosign keyless) env: COSIGN_EXPERIMENTAL: '1' run: | IMAGE="${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:${{ steps.sha.outputs.sha }}" echo "${{ secrets.GITHUB_TOKEN }}" | cosign login ghcr.io -u "${{ github.actor }}" --password-stdin echo "Verifying signature for $IMAGE" cosign verify "$IMAGE" \ --certificate-identity-regexp "^https://github.com/pezkuwichain/pwap/.github/workflows/quality-gate.yml@" \ --certificate-oidc-issuer "https://token.actions.githubusercontent.com" \ > /dev/null echo "✅ Signature valid — image was built by trusted CI" - name: Extract /dist from image run: | IMAGE="${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:${{ steps.sha.outputs.sha }}" docker pull "$IMAGE" CID=$(docker create "$IMAGE") mkdir -p dist docker cp "$CID:/dist/." dist/ docker rm "$CID" >/dev/null # Stamp this build's SHA into dist so future deploys can read PREV echo "${{ steps.sha.outputs.sha }}" > dist/.deploy-sha ls -la dist/ | head -10 - name: Deploy to app.pezkuwichain.io (NEW-7) env: DEPLOY_KEY: ${{ secrets.PWAP_APP_DEPLOY_KEY }} run: | umask 077 printf '%s\n' "$DEPLOY_KEY" > "$RUNNER_TEMP/deploy_key" printf '%s\n' "$DEPLOY_KNOWN_HOSTS" > "$RUNNER_TEMP/known_hosts" # The key is pinned to pwap-web-receive on the host (ops/host): it # takes the site tar on stdin and can do nothing else. tar -C dist -cf - . | ssh -i "$RUNNER_TEMP/deploy_key" -o IdentitiesOnly=yes -o BatchMode=yes -o StrictHostKeyChecking=yes -o UserKnownHostsFile="$RUNNER_TEMP/known_hosts" app-deploy@"$APP_HOST" deploy - name: Health check (60s window) id: healthcheck run: | for i in 1 2 3 4 5 6; do if curl -sf --max-time 10 "https://${{ env.DOMAIN }}/" >/dev/null; then echo "✅ ${{ env.DOMAIN }} healthy" exit 0 fi echo "Attempt $i/6 failed, retrying in 10s..." sleep 10 done echo "❌ Health check failed for ${{ env.DOMAIN }}" exit 1 # ── Automatic rollback: pull PREV SHA image, redeploy, recheck ── - name: Auto-rollback to previous SHA id: rollback if: failure() && steps.healthcheck.conclusion == 'failure' && steps.prev.outputs.prev != '' run: | PREV="${{ steps.prev.outputs.prev }}" echo "🔄 Rolling back ${{ env.DOMAIN }} to $PREV" IMAGE="${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:$PREV" docker pull "$IMAGE" CID=$(docker create "$IMAGE") rm -rf dist && mkdir dist docker cp "$CID:/dist/." dist/ docker rm "$CID" >/dev/null echo "$PREV" > dist/.deploy-sha echo "rollback_sha=$PREV" >> $GITHUB_OUTPUT - name: SCP rollback artifact if: steps.rollback.outcome == 'success' env: DEPLOY_KEY: ${{ secrets.PWAP_APP_DEPLOY_KEY }} run: | umask 077 printf '%s\n' "$DEPLOY_KEY" > "$RUNNER_TEMP/deploy_key" printf '%s\n' "$DEPLOY_KNOWN_HOSTS" > "$RUNNER_TEMP/known_hosts" # The key is pinned to pwap-web-receive on the host (ops/host): it # takes the site tar on stdin and can do nothing else. tar -C dist -cf - . | ssh -i "$RUNNER_TEMP/deploy_key" -o IdentitiesOnly=yes -o BatchMode=yes -o StrictHostKeyChecking=yes -o UserKnownHostsFile="$RUNNER_TEMP/known_hosts" app-deploy@"$APP_HOST" deploy - name: Re-health-check after rollback if: steps.rollback.outcome == 'success' id: healthcheck_rb run: | for i in 1 2 3 4 5 6; do if curl -sf --max-time 10 "https://${{ env.DOMAIN }}/" >/dev/null; then echo "✅ Rolled back successfully — ${{ env.DOMAIN }} healthy on ${{ steps.rollback.outputs.rollback_sha }}" exit 0 fi sleep 10 done echo "❌ Rollback also failed!" exit 1 - name: Post-deploy notification if: success() run: | echo "✅ Deployed image ${{ steps.sha.outputs.sha }} to ${{ env.DOMAIN }}" - name: Notify failure (Telegram) if: failure() env: BOT_TOKEN: ${{ secrets.PEXSEC_BOT_TOKEN }} CEO_CHAT_ID: ${{ secrets.TELEGRAM_CEO_CHAT_ID }} NEW_SHA: ${{ steps.sha.outputs.sha }} PREV_SHA: ${{ steps.prev.outputs.prev }} ROLLBACK_OUTCOME: ${{ steps.rollback.outcome }} RECHECK_OUTCOME: ${{ steps.healthcheck_rb.outcome }} run: | if [ "$RECHECK_OUTCOME" = "success" ]; then MSG="⚠️ pwap/web ${{ env.DOMAIN }}: deploy ($NEW_SHA) failed health check, AUTO-ROLLED-BACK to $PREV_SHA. Site healthy." elif [ "$ROLLBACK_OUTCOME" = "success" ]; then MSG="🚨 pwap/web ${{ env.DOMAIN }}: deploy ($NEW_SHA) failed AND rollback to $PREV_SHA also failed. Manual intervention needed." elif [ -z "$PREV_SHA" ]; then MSG="❌ pwap/web ${{ env.DOMAIN }}: deploy ($NEW_SHA) failed. No previous SHA available (first deploy?). Manual rollback: gh workflow run quality-gate.yml -f rollback_to=" else MSG="❌ pwap/web ${{ env.DOMAIN }}: deploy ($NEW_SHA) failed. Auto-rollback was not attempted. Manual: gh workflow run quality-gate.yml -f rollback_to=$PREV_SHA" fi curl -s -X POST "https://api.telegram.org/bot${BOT_TOKEN}/sendMessage" \ -d "chat_id=${CEO_CHAT_ID}" --data-urlencode "text=$MSG" # ======================================== # DEPLOY TO pex.mom (VPS3 — geo-redundant mirror) # ======================================== deploy-pex: name: Deploy pex.mom runs-on: ubuntu-latest needs: [approve-deploy, notify-deploy-pending, bump-version, build-image] if: | always() && needs.approve-deploy.result == 'success' && needs.notify-deploy-pending.result == 'success' && ((github.event_name == 'push' && needs.build-image.result == 'success' && needs.bump-version.result == 'success') || (github.event_name == 'workflow_dispatch' && github.event.inputs.rollback_to != '')) permissions: contents: read packages: read env: DOMAIN: pex.mom TARGET_PATH: /var/www/pex.mom steps: - name: Determine image SHA id: sha run: | if [ -n "${{ github.event.inputs.rollback_to }}" ]; then echo "sha=${{ github.event.inputs.rollback_to }}" >> $GITHUB_OUTPUT else echo "sha=${{ needs.build-image.outputs.image_sha }}" >> $GITHUB_OUTPUT fi - name: Capture currently-live SHA (for auto-rollback) id: prev run: | PREV=$(curl -sf --max-time 5 "https://${{ env.DOMAIN }}/.deploy-sha" | head -c 40 | tr -dc 'a-f0-9' || echo "") echo "Previous live SHA: ${PREV:-unknown}" echo "prev=$PREV" >> $GITHUB_OUTPUT - name: Log in to GHCR uses: docker/login-action@c94ce9fb468520275223c153574b00df6fe4bcc9 # v3.7.0 with: registry: ${{ env.REGISTRY }} username: ${{ github.actor }} password: ${{ secrets.GITHUB_TOKEN }} - name: Install cosign (for verify) uses: sigstore/cosign-installer@7e8b541eb2e61bf99390e1afd4be13a184e9ebc5 # v3.10.1 with: cosign-release: 'v2.4.1' - name: Verify image signature (cosign keyless) env: COSIGN_EXPERIMENTAL: '1' run: | IMAGE="${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:${{ steps.sha.outputs.sha }}" echo "Verifying signature for $IMAGE" cosign verify "$IMAGE" \ --certificate-identity-regexp "^https://github.com/pezkuwichain/pwap/.github/workflows/quality-gate.yml@" \ --certificate-oidc-issuer "https://token.actions.githubusercontent.com" \ > /dev/null echo "✅ Signature valid — image was built by trusted CI" - name: Extract /dist from image run: | IMAGE="${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:${{ steps.sha.outputs.sha }}" docker pull "$IMAGE" CID=$(docker create "$IMAGE") mkdir -p dist docker cp "$CID:/dist/." dist/ docker rm "$CID" >/dev/null echo "${{ steps.sha.outputs.sha }}" > dist/.deploy-sha - name: Deploy to VPS3 env: DEPLOY_KEY: ${{ secrets.PWAP_PEX_DEPLOY_KEY }} run: | umask 077 printf '%s\n' "$DEPLOY_KEY" > "$RUNNER_TEMP/deploy_key" printf '%s\n' "$DEPLOY_KNOWN_HOSTS" > "$RUNNER_TEMP/known_hosts" # The key is pinned to pwap-web-receive on the host (ops/host): it # takes the site tar on stdin and can do nothing else. tar -C dist -cf - . | ssh -i "$RUNNER_TEMP/deploy_key" -o IdentitiesOnly=yes -o BatchMode=yes -o StrictHostKeyChecking=yes -o UserKnownHostsFile="$RUNNER_TEMP/known_hosts" pex-deploy@"$PEX_HOST" deploy - name: Health check (60s window) id: healthcheck run: | for i in 1 2 3 4 5 6; do if curl -sf --max-time 10 "https://${{ env.DOMAIN }}/" >/dev/null; then echo "✅ ${{ env.DOMAIN }} healthy" exit 0 fi echo "Attempt $i/6 failed, retrying in 10s..." sleep 10 done echo "❌ Health check failed for ${{ env.DOMAIN }}" exit 1 - name: Auto-rollback to previous SHA id: rollback if: failure() && steps.healthcheck.conclusion == 'failure' && steps.prev.outputs.prev != '' run: | PREV="${{ steps.prev.outputs.prev }}" echo "🔄 Rolling back ${{ env.DOMAIN }} to $PREV" IMAGE="${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:$PREV" docker pull "$IMAGE" CID=$(docker create "$IMAGE") rm -rf dist && mkdir dist docker cp "$CID:/dist/." dist/ docker rm "$CID" >/dev/null echo "$PREV" > dist/.deploy-sha echo "rollback_sha=$PREV" >> $GITHUB_OUTPUT - name: SCP rollback artifact if: steps.rollback.outcome == 'success' env: DEPLOY_KEY: ${{ secrets.PWAP_PEX_DEPLOY_KEY }} run: | umask 077 printf '%s\n' "$DEPLOY_KEY" > "$RUNNER_TEMP/deploy_key" printf '%s\n' "$DEPLOY_KNOWN_HOSTS" > "$RUNNER_TEMP/known_hosts" # The key is pinned to pwap-web-receive on the host (ops/host): it # takes the site tar on stdin and can do nothing else. tar -C dist -cf - . | ssh -i "$RUNNER_TEMP/deploy_key" -o IdentitiesOnly=yes -o BatchMode=yes -o StrictHostKeyChecking=yes -o UserKnownHostsFile="$RUNNER_TEMP/known_hosts" pex-deploy@"$PEX_HOST" deploy - name: Re-health-check after rollback if: steps.rollback.outcome == 'success' id: healthcheck_rb run: | for i in 1 2 3 4 5 6; do if curl -sf --max-time 10 "https://${{ env.DOMAIN }}/" >/dev/null; then echo "✅ Rolled back successfully — ${{ env.DOMAIN }} healthy on ${{ steps.rollback.outputs.rollback_sha }}" exit 0 fi sleep 10 done echo "❌ Rollback also failed!" exit 1 - name: Post-deploy notification if: success() run: | echo "✅ Deployed image ${{ steps.sha.outputs.sha }} to ${{ env.DOMAIN }}" - name: Notify failure (Telegram) if: failure() env: BOT_TOKEN: ${{ secrets.PEXSEC_BOT_TOKEN }} CEO_CHAT_ID: ${{ secrets.TELEGRAM_CEO_CHAT_ID }} NEW_SHA: ${{ steps.sha.outputs.sha }} PREV_SHA: ${{ steps.prev.outputs.prev }} ROLLBACK_OUTCOME: ${{ steps.rollback.outcome }} RECHECK_OUTCOME: ${{ steps.healthcheck_rb.outcome }} run: | if [ "$RECHECK_OUTCOME" = "success" ]; then MSG="⚠️ pwap/web ${{ env.DOMAIN }}: deploy ($NEW_SHA) failed health check, AUTO-ROLLED-BACK to $PREV_SHA. Site healthy." elif [ "$ROLLBACK_OUTCOME" = "success" ]; then MSG="🚨 pwap/web ${{ env.DOMAIN }}: deploy ($NEW_SHA) failed AND rollback to $PREV_SHA also failed. Manual intervention needed." elif [ -z "$PREV_SHA" ]; then MSG="❌ pwap/web ${{ env.DOMAIN }}: deploy ($NEW_SHA) failed. No previous SHA available (first deploy?). Manual rollback: gh workflow run quality-gate.yml -f rollback_to=" else MSG="❌ pwap/web ${{ env.DOMAIN }}: deploy ($NEW_SHA) failed. Auto-rollback was not attempted. Manual: gh workflow run quality-gate.yml -f rollback_to=$PREV_SHA" fi curl -s -X POST "https://api.telegram.org/bot${BOT_TOKEN}/sendMessage" \ -d "chat_id=${CEO_CHAT_ID}" --data-urlencode "text=$MSG" # ======================================== # DEPLOY BACKEND (INDEXER) TO ITS HOST # Runs a container from the SHA-tagged GHCR image on the backend VPS. # Same discipline as the web deploy: cosign verify → deploy → health check → # auto-rollback to previous SHA → Telegram notify. Gated behind the SAME # same environment approval as the web deploy. Only main/tags, never fork PRs. # # STATEFUL DB: the sqlite file lives in the named docker volume # `pwap-indexer-db` (mounted at /data). Deploy replaces the container/image # but NEVER the volume — indexer state survives every deploy and rollback. # # Host param via secret BACKEND_VPS_HOST (no hardcoded IP). If the backend # shares the web DEV VPS, set BACKEND_VPS_HOST = VPS_HOST (see backend/DEPLOY.md). # ======================================== # Config gate: the backend indexer only deploys once its host is provisioned. # Until BACKEND_VPS_HOST is set, deploy-backend skips cleanly (neutral, no false # failure/alert) instead of erroring on "missing server host" every push. backend-cfg: name: Backend deploy config check runs-on: ubuntu-latest if: (github.ref == 'refs/heads/main' || startsWith(github.ref, 'refs/tags/')) && (github.event_name == 'push' || github.event_name == 'workflow_dispatch') outputs: configured: ${{ steps.c.outputs.configured }} steps: - id: c env: BH: ${{ secrets.PWAP_INDEXER_DEPLOY_KEY }} run: | if [ -n "$BH" ]; then echo "configured=true" >> "$GITHUB_OUTPUT" else echo "::notice::PWAP_INDEXER_DEPLOY_KEY unset — backend indexer deploy skipped (provision the host to enable)" echo "configured=false" >> "$GITHUB_OUTPUT" fi deploy-backend: name: Deploy Backend (indexer) runs-on: ubuntu-latest needs: [approve-deploy, notify-deploy-pending, build-image-backend, backend-cfg] if: | always() && needs.approve-deploy.result == 'success' && needs.backend-cfg.outputs.configured == 'true' && (github.ref == 'refs/heads/main' || startsWith(github.ref, 'refs/tags/')) && needs.notify-deploy-pending.result == 'success' && ((github.event_name == 'push' && needs.build-image-backend.result == 'success') || (github.event_name == 'workflow_dispatch' && github.event.inputs.rollback_to != '')) permissions: contents: read packages: read steps: - name: Determine image SHA id: sha run: | if [ -n "${{ github.event.inputs.rollback_to }}" ]; then echo "sha=${{ github.event.inputs.rollback_to }}" >> $GITHUB_OUTPUT echo "Rolling back backend to: ${{ github.event.inputs.rollback_to }}" else echo "sha=${{ needs.build-image-backend.outputs.image_sha }}" >> $GITHUB_OUTPUT fi - name: Install cosign (for verify) uses: sigstore/cosign-installer@7e8b541eb2e61bf99390e1afd4be13a184e9ebc5 # v3.10.1 with: cosign-release: 'v2.4.1' - name: Log in to GHCR (runner-side, for verify) uses: docker/login-action@c94ce9fb468520275223c153574b00df6fe4bcc9 # v3.7.0 with: registry: ${{ env.REGISTRY }} username: ${{ github.actor }} password: ${{ secrets.GITHUB_TOKEN }} - name: Verify image signature (cosign keyless) env: COSIGN_EXPERIMENTAL: '1' run: | IMAGE="${{ env.REGISTRY }}/${{ env.BACKEND_IMAGE_NAME }}:${{ steps.sha.outputs.sha }}" echo "${{ secrets.GITHUB_TOKEN }}" | cosign login ghcr.io -u "${{ github.actor }}" --password-stdin echo "Verifying signature for $IMAGE" cosign verify "$IMAGE" \ --certificate-identity-regexp "^https://github.com/pezkuwichain/pwap/.github/workflows/quality-gate.yml@" \ --certificate-oidc-issuer "https://token.actions.githubusercontent.com" \ > /dev/null echo "✅ Signature valid — backend image was built by trusted CI" - name: Deploy container to backend host (health-checked, auto-rollback) id: deploy env: DEPLOY_KEY: ${{ secrets.PWAP_INDEXER_DEPLOY_KEY }} NEW_SHA: ${{ steps.sha.outputs.sha }} run: | umask 077 printf '%s\n' "$DEPLOY_KEY" > "$RUNNER_TEMP/deploy_key" printf '%s\n' "$DEPLOY_KNOWN_HOSTS" > "$RUNNER_TEMP/known_hosts" # The key is pinned to pwap-indexer-ssh on the host (ops/host): the # image SHA is its only input. The host pulls the public image, runs # it with its own settings (/etc/pwap-indexer.env), health-checks it # and rolls back to the previous SHA if it does not come up. ssh -i "$RUNNER_TEMP/deploy_key" -o IdentitiesOnly=yes -o BatchMode=yes -o StrictHostKeyChecking=yes -o UserKnownHostsFile="$RUNNER_TEMP/known_hosts" indexer-deploy@"$INDEXER_HOST" "$NEW_SHA" - name: Post-deploy notification if: success() run: echo "✅ Deployed backend image ${{ steps.sha.outputs.sha }} to backend host" - name: Notify failure (Telegram) if: failure() env: BOT_TOKEN: ${{ secrets.PEXSEC_BOT_TOKEN }} CEO_CHAT_ID: ${{ secrets.TELEGRAM_CEO_CHAT_ID }} NEW_SHA: ${{ steps.sha.outputs.sha }} run: | MSG="❌ pwap/backend indexer: deploy ($NEW_SHA) failed. If a previous SHA existed the host auto-rolled-back to it (DB volume untouched). Manual: gh workflow run quality-gate.yml -f rollback_to=" curl -s -X POST "https://api.telegram.org/bot${BOT_TOKEN}/sendMessage" \ -d "chat_id=${CEO_CHAT_ID}" --data-urlencode "text=$MSG" # ======================================== # DEPLOY - SUPABASE (edge functions + migrations) # Same Telegram-gated cutover as the frontend/backend: syncs the edge functions # into the self-hosted edge-runtime volume (hot-reloaded, no restart) and applies # any un-recorded migrations transactionally against the fund-custody Postgres. # Runs in parallel with deploy-app so functions, migrations and the new frontend # go live in the same post-approval window (minimal fail-closed gap). # ======================================== deploy-supabase: name: Deploy Supabase (functions + migrations) runs-on: ubuntu-latest needs: [approve-deploy, notify-deploy-pending] if: | always() && needs.approve-deploy.result == 'success' && needs.notify-deploy-pending.result == 'success' && github.event_name == 'push' && github.ref == 'refs/heads/main' permissions: contents: read steps: - name: Checkout code uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0 - name: Pack supabase tree run: | # __tests__ are Deno tests — kept out of the runtime bundle entirely. tar czf supabase-deploy.tgz -C web/supabase \ --exclude='__tests__' functions migrations deploy - name: Sync edge functions + apply migrations (health-checked) id: apply env: DEPLOY_KEY: ${{ secrets.PWAP_SUPABASE_DEPLOY_KEY }} run: | umask 077 printf '%s\n' "$DEPLOY_KEY" > "$RUNNER_TEMP/deploy_key" printf '%s\n' "$DEPLOY_KNOWN_HOSTS" > "$RUNNER_TEMP/known_hosts" # The key is pinned to pwap-supabase-ssh on the host (ops/host): the # bundle on stdin is its only input. pwap-supabase-deploy syncs the # functions, applies the migrations and verifies the guards. ssh -i "$RUNNER_TEMP/deploy_key" -o IdentitiesOnly=yes -o BatchMode=yes -o StrictHostKeyChecking=yes -o UserKnownHostsFile="$RUNNER_TEMP/known_hosts" supabase-deploy@"$SUPABASE_HOST" deploy < supabase-deploy.tgz - name: Telegram notify if: always() env: BOT_TOKEN: ${{ secrets.PEXSEC_BOT_TOKEN }} CEO_CHAT_ID: ${{ secrets.TELEGRAM_CEO_CHAT_ID }} OUTCOME: ${{ steps.apply.outcome }} run: | if [ "$OUTCOME" = "success" ]; then MSG="✅ pwap Supabase: edge functions synced + migrations applied & verified (${{ github.sha }})." else MSG="🚨 pwap Supabase deploy FAILED (${{ github.sha }}) — functions/migrations may be partially applied. Check the run." fi curl -s -X POST "https://api.telegram.org/bot${BOT_TOKEN}/sendMessage" \ -d chat_id="${CEO_CHAT_ID}" -d text="$MSG" >/dev/null || true # ======================================== # SECURITY CHECKS (BLOCKING) # npm audit (high + critical) + TruffleHog secret scan # ======================================== security-audit: name: Security Audit runs-on: ubuntu-latest needs: [web] steps: - name: Checkout code uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0 with: fetch-depth: ${{ github.event_name == 'pull_request' && 0 || 1 }} - name: Setup Node.js uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4.4.0 with: node-version: '24' - name: Web — npm audit (high + critical, production deps only) working-directory: ./web run: | npm ci # Audit only production dependencies. Build tooling (vite, esbuild, # vite-plugin-node-polyfills → elliptic, etc.) ships to no user, and # advisories on those dev deps kept blocking production deploys. npm audit --audit-level=high --omit=dev - name: TruffleHog — PR diff (verified secrets only) if: github.event_name == 'pull_request' uses: trufflesecurity/trufflehog@e76dff8a7e57d6df0c60ad8cbf5d5cba505a8fc9 # v3.99.2 with: base: ${{ github.event.pull_request.base.sha }} head: ${{ github.event.pull_request.head.sha }} extra_args: --only-verified - name: TruffleHog — full repo scan (verified secrets only) if: github.event_name != 'pull_request' uses: trufflesecurity/trufflehog@e76dff8a7e57d6df0c60ad8cbf5d5cba505a8fc9 # v3.99.2 with: path: ./ extra_args: --only-verified # ======================================== # CI GATE — explicit merge-block # All required checks must succeed (or be skipped, e.g. for rollback path). # Branch protection on main should require this job's success. # ======================================== # Applies every migration to an empty Postgres, so a migration that cannot run # from scratch fails here rather than on the production host. # # This was not possible before 2026-08-01: the old set could not be replayed at # all. Five migrations failed on an empty database, partly because the legacy # 0NN filenames sort before the 14-digit timestamps they depend on # ("013" < "20241117054600"), and the schema they produced did not match # production anyway. The baseline replaced them with a dump of the live schema, # which is what makes this check meaningful. migration-test: name: Migration test runs-on: ubuntu-latest services: postgres: image: postgres:17 env: POSTGRES_PASSWORD: postgres POSTGRES_DB: migration_test ports: - 5432:5432 options: >- --health-cmd pg_isready --health-interval 10s --health-timeout 5s --health-retries 5 steps: - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0 - name: Install psql run: sudo apt-get update -qq && sudo apt-get install -y -qq postgresql-client # Stand-ins for auth.uid(), auth.users, storage.objects and the Supabase # roles. Without them the run would fail on the environment rather than on # anything a migration got wrong. - name: Apply Supabase stubs env: PGPASSWORD: postgres run: | psql -h localhost -U postgres -d migration_test -v ON_ERROR_STOP=1 -q \ -f web/supabase/deploy/test-bootstrap.sql echo "stubs applied" - name: Apply migrations in order env: PGPASSWORD: postgres run: | set -euo pipefail failed=0 for f in $(ls web/supabase/migrations/*.sql | sort); do base="$(basename "$f")" case "$base" in COMBINED*) echo " skip $base"; continue;; esac # --single-transaction so a failure leaves nothing half-applied, # matching how apply-migrations.sh runs them in production if psql -h localhost -U postgres -d migration_test \ -v ON_ERROR_STOP=1 --single-transaction -q -f "$f" 2>/tmp/err.log; then echo " ✔ $base" else failed=$((failed+1)) echo " ✗ $base" grep -E "ERROR" /tmp/err.log | head -3 | sed 's/^/ /' fi done if [ $failed -gt 0 ]; then echo "::error::$failed migration(s) cannot be applied to an empty database" exit 1 fi echo "all migrations applied from scratch" # A migration set that applies cleanly but produces nothing is still broken. - name: Verify the schema was actually built env: PGPASSWORD: postgres run: | set -euo pipefail read -r tables functions <<<"$(psql -h localhost -U postgres -d migration_test -tAq -c \ "SELECT (SELECT count(*) FROM pg_tables WHERE schemaname='public'), (SELECT count(*) FROM pg_proc p JOIN pg_namespace n ON n.oid=p.pronamespace WHERE n.nspname='public')" | tr '|' ' ')" echo "built: $tables tables, $functions functions" if [ "$tables" -lt 50 ] || [ "$functions" -lt 20 ]; then echo "::error::schema looks incomplete — expected the full public schema" exit 1 fi # Deploy approval is enforced by a dependency edge (see approve-deploy), and an # edge can be dropped by a later edit without anyone noticing until a deploy # goes out unapproved. This job is what keeps that from being possible: it # fails the PR, not production. workflow-guard: name: Deploy gate wiring runs-on: ubuntu-latest steps: - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0 - name: Every deploy job must sit behind the approval gate run: python3 ops/check-deploy-gate.py # The scripts the deploy keys are pinned to (ops/host). They are the # whole of what CI can do on the hosts, so they are checked here. - name: Host deploy scripts run: | shellcheck -S warning ops/host/pwap-* ops/host/test-web-receive.sh bash ops/host/test-web-receive.sh ci-gate: name: CI Gate ✅ runs-on: ubuntu-latest needs: [web, backend, sign, security-audit, migration-test, workflow-guard] if: always() steps: - name: Verify all required jobs succeeded or were intentionally skipped run: | results='${{ toJSON(needs) }}' echo "$results" | python3 -c " import json, sys needs = json.load(sys.stdin) failed = [name for name, job in needs.items() if job['result'] not in ('success', 'skipped')] if failed: print('❌ Required jobs failed: ' + ', '.join(failed)) sys.exit(1) print('✅ All required CI jobs passed or skipped') "