fix(noter): use the public endpoints and fail loudly when scans stop (#1)

The bot pointed at the block-producing nodes' RPC ports. Those bind to
localhost, so once they stopped being exposed to the internet the bot lost
every chain at once and has submitted nothing since. `staking_score` gates
the entire trust score, so a noter that cannot reach the chains does not
degrade the score — it zeroes it for every tracked account.

Point it at the public endpoints instead, which is what an external client
should use and what the code already defaulted to. They are TLS-terminated
rather than plaintext ws:// across the internet, and this drops a hardcoded
node address from the repository.

Reaching the chains again is not enough on its own: the outage lasted three
weeks because reconnecting forever looks identical to working. The bot now
writes a heartbeat when a scan *completes*, a container healthcheck reads it,
and a watchdog exits once it goes stale so the restart policy turns a silent
stall into a visibly crash-looping container.

docker-compose.prod.yml

@@ -154,10 +154,25 @@ services:
       - noter_mnemonic
     environment:
       TZ: UTC
-      RELAY_RPC: ws://217.77.6.126:9944
-      ASSET_HUB_RPC: ws://217.77.6.126:40944
-      PEOPLE_RPC: ws://217.77.6.126:41944
+      # The block-producing nodes bind their RPC to localhost by design — reaching
+      # them from here needed the ports open to the internet, and closing them (as
+      # they should be) silently cut this bot off from every chain. These are the
+      # public endpoints, which is what an external client is supposed to use, and
+      # they are TLS-terminated rather than plaintext ws:// across the internet.
+      RELAY_RPC: wss://rpc.pezkuwichain.io
+      ASSET_HUB_RPC: wss://asset-hub-rpc.pezkuwichain.io
+      PEOPLE_RPC: wss://people-rpc.pezkuwichain.io
       SCAN_INTERVAL_MS: "300000"
+    # The bot writes a heartbeat when a scan completes, not when it merely stays
+    # up: an endless reconnect loop kept the container "Up" for three weeks while
+    # no staking data reached the People chain.
+    healthcheck:
+      test: ["CMD", "node", "-e", "const fs=require('fs');const i=parseInt(process.env.SCAN_INTERVAL_MS||'300000',10);let t=0;try{t=Number(fs.readFileSync(process.env.HEARTBEAT_FILE||'/tmp/noter-heartbeat','utf8'))}catch{};process.exit(Number.isFinite(t)&&Date.now()-t<i*3?0:1)"]
+      interval: 60s
+      timeout: 10s
+      retries: 3
+      # The first scan has to walk every tracked account before it can report.
+      start_period: 10m
 
   payout-bot:
     container_name: payout-pezkuwi

noter/bot.js

@@ -33,10 +33,42 @@ const PEOPLE_RPC   = process.env.PEOPLE_RPC   || 'wss://people-rpc.pezkuwichain.
 const SCAN_INTERVAL = parseInt(process.env.SCAN_INTERVAL_MS || '300000', 10); // 5 min
 const UNITS = BigInt('1000000000000'); // 10^12
 
+// Liveness. A scan that throws is logged and retried, which is right — a chain can
+// be briefly unreachable. What is not right is doing that forever in silence: this
+// bot once spent three weeks reconnecting to endpoints that had been closed, while
+// every tracked account's staking score sat at zero. `staking_score` gates the whole
+// trust score, so a mute noter is not a degraded service, it is a wrong answer for
+// every citizen.
+//
+// The heartbeat records the last scan that actually completed. The container
+// healthcheck reads it, and the watchdog below exits once it goes stale, so the
+// restart policy turns a silent stall into a visibly crash-looping container.
+const HEARTBEAT_FILE = process.env.HEARTBEAT_FILE || '/tmp/noter-heartbeat';
+const STALE_AFTER = SCAN_INTERVAL * 3;
+
 // ========================================
 // LOGGING
 // ========================================
 
+function touchHeartbeat() {
+  try {
+    require('fs').writeFileSync(HEARTBEAT_FILE, String(Date.now()));
+  } catch (err) {
+    log('WARN', 'Could not write heartbeat', { error: err.message });
+  }
+}
+
+/// Milliseconds since the last completed scan, or `Infinity` if none has completed
+/// yet — an unwritable or missing heartbeat counts as stale rather than healthy.
+function heartbeatAge() {
+  try {
+    const t = Number(require('fs').readFileSync(HEARTBEAT_FILE, 'utf8'));
+    return Number.isFinite(t) ? Date.now() - t : Infinity;
+  } catch {
+    return Infinity;
+  }
+}
+
 function log(level, msg, data) {
   const ts = new Date().toISOString();
   const entry = data ? `${ts} [${level}] ${msg} ${JSON.stringify(data)}` : `${ts} [${level}] ${msg}`;
@@ -563,6 +595,7 @@ async function main() {
   // already been promoted to CachedStakingDetails this cycle.
   await finalizeMaturedPending(peopleApi, noterKeypair);
   await fullScan(relayApi, assetHubApi, peopleApi, noterKeypair);
+  touchHeartbeat();
 
   // Start event listener for real-time processing
   await startEventListener(relayApi, assetHubApi, peopleApi, noterKeypair);
@@ -572,10 +605,25 @@ async function main() {
   setInterval(() => {
     finalizeMaturedPending(peopleApi, noterKeypair)
       .then(() => fullScan(relayApi, assetHubApi, peopleApi, noterKeypair))
+      .then(() => touchHeartbeat())
       .catch(err => {
         log('ERROR', 'Periodic scan failed', { error: err.message });
       });
   }, SCAN_INTERVAL);
+
+  // Exiting is the point. Reconnecting forever looks like the bot is coping; a
+  // container that keeps dying does not, and the restart policy makes that
+  // visible in `docker ps` without anyone having to read the logs.
+  setInterval(() => {
+    const age = heartbeatAge();
+    if (age > STALE_AFTER) {
+      log('ERROR', 'No scan completed within the staleness window — exiting so the restart policy takes over', {
+        stale_for_ms: Number.isFinite(age) ? age : null,
+        stale_after_ms: STALE_AFTER,
+      });
+      process.exit(1);
+    }
+  }, Math.min(SCAN_INTERVAL, 60_000)).unref?.();
 }
 
 main().catch(err => {