ci: deploy over a pinned, single-purpose key instead of root

deploy-vps reached subquery1 as root, through appleboy/ssh-action referenced
by tag, with a key whose comment (mamostehp@MamostePC) made it look like a
person's. It now uses plain ssh with SUBQUERY_DEPLOY_KEY, which the host pins
(authorized_keys restrict,command=) to ops/host/subquery-deploy-ssh: the
commit SHA is its only input. subquery-deploy, run through sudo, does what
the workflow's script did, with three changes:

- It deploys exactly the SHA CI was triggered for, and only a commit on main.
- The host's local edits to tracked files (the indexer manifests) are put
  back whether the deploy succeeds or fails. Before, a failed build exited
  before `stash pop` and left them stashed.
- Only tracked files are stashed. Stashing untracked ones too (.yarn/, which
  yarn install recreates) made every pop fail half-way; that is how 13 stash
  entries piled up on the host.

test-deploy.sh covers this against a scratch clone (red with the restore
trap removed); CI runs it with shellcheck. subquery1's host key is pinned.
The indexer endpoints themselves are unchanged. Actions pinned to SHAs; the
secret scanner's own AGPL licence is allowed in dependency review.

.github/workflows/auto-pr.yml

@@ -12,7 +12,7 @@ jobs:
       pull-requests: write
     steps:
       - name: Checkout
-        uses: actions/checkout@v4
+        uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0
 
       - name: Create or update PR
         env:

.github/workflows/ci.yml

@@ -12,10 +12,10 @@ jobs:
 
     steps:
       - name: Checkout
-        uses: actions/checkout@v4
+        uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0
 
       - name: Setup Node.js
-        uses: actions/setup-node@v4
+        uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4.4.0
         with:
           node-version: '18'
 
@@ -52,7 +52,7 @@ jobs:
 
     steps:
       - name: Checkout
-        uses: actions/checkout@v4
+        uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0
 
       - name: Build the image
         run: docker build -t bot ${{ matrix.bot }}
@@ -78,3 +78,14 @@ jobs:
           echo "$out"
           [ "$code" -ne 0 ] && [ "$code" -ne 124 ] || { echo "expected the bot to exit with an error, got $code"; exit 1; }
           grep -qF "$EXPECT" <<<"$out" || { echo "expected: $EXPECT"; exit 1; }
+
+  # The scripts the deploy key is pinned to are the whole of what CI can do on
+  # the host, so they are checked with every change.
+  host-scripts:
+    name: Host deploy scripts
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0
+      - run: |
+          shellcheck -S warning ops/host/subquery-deploy ops/host/subquery-deploy-ssh ops/host/test-deploy.sh
+          bash ops/host/test-deploy.sh

.github/workflows/code-quality.yml

@@ -13,10 +13,10 @@ jobs:
 
     steps:
       - name: Checkout
-        uses: actions/checkout@v4
+        uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0
 
       - name: Setup Node.js
-        uses: actions/setup-node@v4
+        uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4.4.0
         with:
           node-version: '18'
 
@@ -36,10 +36,10 @@ jobs:
 
     steps:
       - name: Checkout
-        uses: actions/checkout@v4
+        uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0
 
       - name: Setup Node.js
-        uses: actions/setup-node@v4
+        uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4.4.0
         with:
           node-version: '18'
 
@@ -58,10 +58,10 @@ jobs:
 
     steps:
       - name: Checkout
-        uses: actions/checkout@v4
+        uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0
 
       - name: Setup Node.js
-        uses: actions/setup-node@v4
+        uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4.4.0
         with:
           node-version: '18'
 
@@ -83,10 +83,10 @@ jobs:
 
     steps:
       - name: Checkout
-        uses: actions/checkout@v4
+        uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0
 
       - name: Setup Node.js
-        uses: actions/setup-node@v4
+        uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4.4.0
         with:
           node-version: '18'
 
@@ -107,10 +107,10 @@ jobs:
 
     steps:
       - name: Checkout
-        uses: actions/checkout@v4
+        uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0
 
       - name: Setup Node.js
-        uses: actions/setup-node@v4
+        uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4.4.0
         with:
           node-version: '18'
 

.github/workflows/deploy-vps.yml

@@ -2,24 +2,31 @@ name: Deploy to VPS
 on:
   push:
     branches: [main]
+
+concurrency:
+  group: deploy-vps
+  cancel-in-progress: false
+
+permissions:
+  contents: read
+
 jobs:
   deploy:
     runs-on: ubuntu-latest
     steps:
+      # The deploy key is pinned on the host to ops/host/subquery-deploy-ssh
+      # (authorized_keys restrict,command=): the commit SHA is its only input.
+      # subquery-deploy, run through sudo, brings /opt/subquery to exactly that
+      # commit of main, keeps the host's local edits, and rebuilds the stack.
       - name: Deploy via SSH
-        uses: appleboy/ssh-action@v1
+        env:
-        with:
+          DEPLOY_KEY: ${{ secrets.SUBQUERY_DEPLOY_KEY }}
-          host: ${{ secrets.VPS_HOST }}
+          # subquery1's host key, read from the host itself; any other key fails.
-          username: root
+          KNOWN_HOST: "158.220.93.26 ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAINtnaMWZ0RZDD6yt+6WdtU/rpEz4pyHmo7otkbmgTpkA"
-          key: ${{ secrets.VPS_SSH_KEY }}
+        run: |
-          script: |
+          umask 077
-            set -e
+          printf '%s\n' "$DEPLOY_KEY" > "$RUNNER_TEMP/deploy_key"
-            cd /opt/subquery
+          printf '%s\n' "$KNOWN_HOST" > "$RUNNER_TEMP/known_hosts"
-            git stash --include-untracked || true
+          ssh -i "$RUNNER_TEMP/deploy_key" -o IdentitiesOnly=yes -o BatchMode=yes \
-            git pull origin main
+            -o StrictHostKeyChecking=yes -o UserKnownHostsFile="$RUNNER_TEMP/known_hosts" \
-            yarn install --frozen-lockfile
+            subquery-deploy@158.220.93.26 "$GITHUB_SHA"
-            ./node_modules/.bin/subql codegen pezkuwi-governance.yaml
-            ./node_modules/.bin/subql build pezkuwi.yaml
-            docker compose -f docker-compose.prod.yml build
-            docker compose -f docker-compose.prod.yml up -d
-            git stash pop || true

.github/workflows/deploy.yml

@@ -25,10 +25,10 @@ jobs:
 
     steps:
       - name: Checkout
-        uses: actions/checkout@v4
+        uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0
 
       - name: Setup Node.js
-        uses: actions/setup-node@v4
+        uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4.4.0
         with:
           node-version: '18'
 
@@ -53,10 +53,10 @@ jobs:
 
     steps:
       - name: Checkout
-        uses: actions/checkout@v4
+        uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0
 
       - name: Setup Node.js
-        uses: actions/setup-node@v4
+        uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4.4.0
         with:
           node-version: '18'
 

.github/workflows/security.yml

@@ -18,10 +18,10 @@ jobs:
 
     steps:
       - name: Checkout
-        uses: actions/checkout@v4
+        uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0
 
       - name: Setup Node.js
-        uses: actions/setup-node@v4
+        uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4.4.0
         with:
           node-version: '18'
 
@@ -49,20 +49,23 @@ jobs:
 
     steps:
       - name: Checkout
-        uses: actions/checkout@v4
+        uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0
 
       - name: Dependency Review
-        uses: actions/dependency-review-action@v4
+        uses: actions/dependency-review-action@2031cfc080254a8a887f58cffee85186f0e49e48 # v4.9.0
         with:
           fail-on-severity: high
           deny-licenses: GPL-3.0, AGPL-3.0
           # The light client is an optional dependency of @pezkuwi/rpc-provider,
           # so it is listed in the bots' lockfiles, but their images are built
           # with --omit=optional and never contain it (checked in ci.yml, bots).
+          # TruffleHog (AGPL-3.0) is the secret scanner this workflow runs; it
+          # executes in CI and is never shipped, served or distributed.
           allow-dependencies-licenses: >-
             pkg:npm/@bizinikiwi/connect,
             pkg:npm/@bizinikiwi/connect-extension-protocol,
-            pkg:npm/@bizinikiwi/connect-known-chains
+            pkg:npm/@bizinikiwi/connect-known-chains,
+            pkg:githubactions/trufflesecurity/trufflehog
 
   secret-scan:
     name: Secret Scan
@@ -70,12 +73,12 @@ jobs:
 
     steps:
       - name: Checkout
-        uses: actions/checkout@v4
+        uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0
         with:
           fetch-depth: 0
 
       - name: TruffleHog Secret Scan
-        uses: trufflesecurity/trufflehog@main
+        uses: trufflesecurity/trufflehog@e76dff8a7e57d6df0c60ad8cbf5d5cba505a8fc9 # v3.99.2
         with:
           path: ./
           base: ${{ github.event.repository.default_branch }}

ops/host/subquery-deploy

@@ -0,0 +1,44 @@
+#!/bin/bash
+# Brings /opt/subquery to a commit of main and rebuilds and restarts the
+# compose stack (indexers, noter and payout bots). Installed root-owned at
+# /usr/local/sbin/subquery-deploy; reached only through subquery-deploy-ssh.
+#
+# The host keeps local edits to tracked files (the indexer manifests point at
+# a host-specific endpoint). They are stashed for the pull and always put
+# back, whether the deploy succeeds or not. Only tracked files are stashed:
+# stashing untracked ones too (.yarn/, which yarn install recreates) made
+# every pop fail half-way and left a stash entry behind on each deploy.
+set -euo pipefail
+
+SHA="${1:-}"
+if ! [[ "$SHA" =~ ^[0-9a-f]{40}$ ]]; then
+  echo "subquery-deploy: expected the 40-character commit SHA to deploy" >&2
+  exit 2
+fi
+
+cd /opt/subquery
+
+stashed=
+if ! git diff --quiet; then
+  git stash push -q -m "subquery-deploy $SHA"
+  stashed=1
+fi
+restore () {
+  if [ -n "$stashed" ]; then
+    git stash pop -q || echo "subquery-deploy: could not restore the host's local edits; they are in 'git stash list'" >&2
+  fi
+}
+trap restore EXIT
+
+git fetch -q origin main
+# Deploy exactly what CI tested, and only a commit that is on main.
+git merge-base --is-ancestor "$SHA" origin/main || { echo "subquery-deploy: $SHA is not on main" >&2; exit 1; }
+git merge -q --ff-only origin/main
+[ "$(git rev-parse HEAD)" = "$SHA" ] || { echo "subquery-deploy: main is at $(git rev-parse --short HEAD), not ${SHA:0:7}; a newer deploy will follow" >&2; exit 1; }
+
+yarn install --frozen-lockfile
+./node_modules/.bin/subql codegen pezkuwi-governance.yaml
+./node_modules/.bin/subql build pezkuwi.yaml
+docker compose -f docker-compose.prod.yml build
+docker compose -f docker-compose.prod.yml up -d
+echo "subquery-deploy: deployed ${SHA:0:7}"

ops/host/subquery-deploy-ssh

@@ -0,0 +1,18 @@
+#!/bin/bash
+# Forced command for the subquery deploy key, installed root-owned at
+# /usr/local/sbin/subquery-deploy-ssh:
+#
+#   restrict,command="/usr/local/sbin/subquery-deploy-ssh" ssh-ed25519 ...
+#
+# CI runs `ssh subquery-deploy@<host> <commit sha>`. The SHA is the only
+# input; subquery-deploy, which sudo lets this user run and nothing else,
+# does the work.
+set -euo pipefail
+
+SHA="${SSH_ORIGINAL_COMMAND:-}"
+if ! [[ "$SHA" =~ ^[0-9a-f]{40}$ ]]; then
+  echo "subquery-deploy-ssh: expected the 40-character commit SHA to deploy" >&2
+  exit 2
+fi
+
+exec sudo -n /usr/local/sbin/subquery-deploy "$SHA"

ops/host/test-deploy.sh

@@ -0,0 +1,66 @@
+#!/bin/bash
+# The rc variables are read inside the check expressions, which check() runs
+# with eval; shellcheck cannot see into those strings.
+# shellcheck disable=SC2034
+# Exercises subquery-deploy's git handling against a scratch clone: the host's
+# local edits come back whether the deploy succeeds or fails, no stash entry
+# is left behind, untracked files (.yarn/) do not get in the way, and only a
+# commit on main is deployed. yarn, subql and docker are replaced with a
+# command that succeeds, or with one that fails.
+set -uo pipefail
+
+HERE=$(cd "$(dirname "$0")" && pwd)
+T=$(mktemp -d)
+trap 'rm -rf "$T"' EXIT
+git() { command git -c user.email=t@t -c user.name=t "$@"; }
+
+git init -q --bare -b main "$T/origin.git"
+git clone -q "$T/origin.git" "$T/work" 2>/dev/null
+( cd "$T/work" && echo 'endpoint: repo' > pezkuwi.yaml && git add . && git commit -qm one && git push -q origin main )
+git clone -q "$T/origin.git" "$T/host" 2>/dev/null
+( cd "$T/host" && echo 'endpoint: host-local' > pezkuwi.yaml && mkdir .yarn && echo cache > .yarn/x )
+
+make_script () {  # make_script <build command>
+  sed -e "s#^cd /opt/subquery\$#cd $T/host#" \
+      -e "s#^yarn install --frozen-lockfile\$#$1#" \
+      -e 's#^\./node_modules/\.bin/subql .*$#true#' \
+      -e 's#^docker compose .*$#true#' \
+      "$HERE/subquery-deploy" > "$T/deploy"
+  chmod +x "$T/deploy"
+}
+push_commit () { ( cd "$T/work" && echo "$1" > other.txt && git add . && git commit -qm "$1" && git push -q origin main && git rev-parse HEAD ); }
+
+pass=0; fail=0
+check () { if eval "$2"; then echo "ok    $1"; pass=$((pass+1)); else echo "FAIL  $1"; fail=$((fail+1)); fi; }
+host_edit () { grep -q host-local "$T/host/pezkuwi.yaml"; }
+stashes () { (cd "$T/host" && git stash list | wc -l); }
+
+make_script true
+S2=$(push_commit two)
+"$T/deploy" "$S2" >/dev/null 2>&1; rc=$?
+check "a deploy of main's head succeeds"                '[ $rc = 0 ] && [ "$(cd "$T/host" && git rev-parse HEAD)" = "$S2" ]'
+check "the host's local edit is back"                   'host_edit'
+check "no stash entry is left"                          '[ "$(stashes)" = 0 ]'
+check "untracked files are untouched"                   '[ -f "$T/host/.yarn/x" ]'
+
+make_script false
+S3=$(push_commit three)
+"$T/deploy" "$S3" >/dev/null 2>&1; rc=$?
+check "a failing build fails the deploy"                '[ $rc != 0 ]'
+check "the local edit is back after a failure"          'host_edit && [ "$(stashes)" = 0 ]'
+
+make_script true
+S4=$(push_commit four); S5=$(push_commit five)
+"$T/deploy" "$S4" >/dev/null 2>&1; rc=$?
+check "a SHA behind main's head is not deployed"        '[ $rc != 0 ] && host_edit && [ "$(stashes)" = 0 ]'
+
+( cd "$T/work" && git checkout -qb side && echo x > side.txt && git add . && git commit -qm side && git push -q origin side )
+SIDE=$(cd "$T/work" && git rev-parse HEAD)
+"$T/deploy" "$SIDE" >/dev/null 2>&1; rc=$?
+check "a commit that is not on main is refused"          '[ $rc != 0 ] && [ "$(cd "$T/host" && git rev-parse HEAD)" != "$SIDE" ] && host_edit'
+
+"$T/deploy" 'abc; id' >/dev/null 2>&1; rc=$?
+check "anything but a full SHA is refused"               '[ $rc = 2 ]'
+
+echo "RESULT: $pass passed, $fail failed"
+[ "$fail" = 0 ]