usdt-bridge: never drop a withdrawal record on a transient RPC failure

process_withdrawal_block's reserve/circuit-breaker check used to `continue`
past a Polkadot connect failure without ever calling db.add_withdrawal() -
a transient RPC error (confirmed live: a public-endpoint rate limit) meant
an already-finalized Pezkuwi withdrawal was silently untracked, with no
pending_multisig_approval entry for the signatories to ever see. It now
always records the withdrawal; an unreachable reserve check is treated as
"unverified" (never auto-paid, flagged distinctly from a confirmed
shortfall) rather than as a reason to drop the record.

Also add bounded exponential backoff (2s..32s, 6 attempts) to
connect_to_chain instead of failing on the first error - every caller
either bubbles the failure through a listener's outer reconnect loop or
out of main() to a systemd restart, so a single blip was tearing down and
fully re-initializing both listeners instead of just waiting it out.

Finally, support POLKADOT_RPC_URL/PEZKUWI_RPC_URL env var overrides for
the configured RPC endpoints, so a dedicated/API-keyed provider URL (e.g.
Dwellir, adopted after the public Polkadot Asset Hub endpoint started
rate-limiting us) can live in a tightly-permissioned EnvironmentFile
instead of the git-tracked bridge_config.json - same pattern already used
for PEZBRIDGE_TELEGRAM_TOKEN and AUTOMATION_KEY_MNEMONIC.

tools/usdt-bridge/src/main.rs

@@ -529,12 +529,38 @@ fn init_db(db_path: &PathBuf) -> Result<()> {
 // Chain Operations
 // ============================================================================
 
+/// Retries with exponential backoff (2s, 4s, 8s, 16s, 32s; capped, 6 attempts total) rather than
+/// failing on the first transient error. This matters because every caller here either bubbles a
+/// failure straight up through a `?` to the listener's outer reconnect loop (see listen_deposits/
+/// listen_withdrawals) or all the way out of main() to a systemd restart - a single blip (a rate
+/// limit, a DNS hiccup) would otherwise tear down and fully re-initialize both listeners instead
+/// of just waiting the blip out. Observed live 2026-07-16: the public Polkadot Asset Hub RPC
+/// briefly 429'd, which without this retry crashed the whole process and then kept re-crashing
+/// every RestartSec, hammering the same rate-limited endpoint harder instead of backing off.
 async fn connect_to_chain(url: &str) -> Result<OnlineClient<SubstrateConfig>> {
-    info!("Connecting to {}...", url);
+    const MAX_ATTEMPTS: u32 = 6;
-    let client = OnlineClient::<SubstrateConfig>::from_url(url).await
+    let mut delay = Duration::from_secs(2);
-        .context(format!("Failed to connect to {}", url))?;
+
-    info!("Connected successfully");
+    for attempt in 1..=MAX_ATTEMPTS {
-    Ok(client)
+        info!("Connecting to {}... (attempt {attempt}/{MAX_ATTEMPTS})", url);
+
+        match OnlineClient::<SubstrateConfig>::from_url(url).await {
+            Ok(client) => {
+                info!("Connected successfully");
+                return Ok(client);
+            }
+            Err(e) if attempt < MAX_ATTEMPTS => {
+                warn!("Connect attempt {attempt}/{MAX_ATTEMPTS} to {url} failed: {e} - retrying in {}s", delay.as_secs());
+                tokio::time::sleep(delay).await;
+                delay = (delay * 2).min(Duration::from_secs(60));
+            }
+            Err(e) => {
+                return Err(e).context(format!("Failed to connect to {url} after {MAX_ATTEMPTS} attempts"));
+            }
+        }
+    }
+
+    unreachable!("loop always returns via Ok or the final Err arm")
 }
 
 /// Get asset balance using dynamic API
@@ -1433,21 +1459,36 @@ async fn process_withdrawal_block(
                             // wallet actually holds. If this ever trips, the peg is already
                             // under-collateralized (e.g. wUSDT minted without a matching real
                             // deposit) - that must halt and page someone, not degrade silently.
-                            let polkadot_client = match connect_to_chain(&config.polkadot_rpc).await {
+                            //
-                                Ok(c) => c,
+                            // Note this reserve check is best-effort only: the withdrawal itself
-                                Err(e) => {
+                            // is already finalized on Pezkuwi by the time we get here, so a
-                                    error!("   ❌ Failed to connect to Polkadot: {}", e);
+                            // transient RPC hiccup while trying to verify the Polkadot-side
-                                    continue;
+                            // reserve must NEVER `continue` past recording it - that would drop
-                                }
+                            // a real, already-committed withdrawal on the floor with no
+                            // pending_multisig_approval entry anywhere (exactly the silent-loss
+                            // failure mode fixed for the subscription-stall bug). The signatories
+                            // verify the real reserve themselves in pwap-web before approving, so
+                            // an unverified reserve here just means "never auto-pay, always
+                            // human-review" - it does not block recording the request.
+                            let polkadot_client = connect_to_chain(&config.polkadot_rpc).await.ok();
+                            let reserve_known = polkadot_client.is_some();
+
+                            let reserve = match &polkadot_client {
+                                Some(client) => get_asset_balance(
+                                    client,
+                                    config.polkadot_usdt_asset_id,
+                                    polkadot_bridge_addr,
+                                ).await.unwrap_or(0),
+                                None => 0,
                             };
 
-                            let reserve = get_asset_balance(
+                            if !reserve_known {
-                                &polkadot_client,
+                                error!("   ❌ Failed to connect to Polkadot to verify reserve - recording withdrawal as pending_multisig_approval anyway (reserve NOT confirmed, signatories must check manually)");
-                                config.polkadot_usdt_asset_id,
+                                notify_telegram(config, &format!(
-                                polkadot_bridge_addr,
+                                    "⚠️ Could not verify Polkadot reserve for a {} USDT withdrawal from {} (RPC connect failed). Recorded as pending_multisig_approval - reserve NOT confirmed, signatories must check manually before approving.",
-                            ).await.unwrap_or(0);
+                                    amount as f64 / 1_000_000.0, from_str
-
+                                )).await;
-                            if reserve < net_amount {
+                            } else if reserve < net_amount {
                                 let shortfall_msg = format!(
                                     "🚨 RESERVE SHORTFALL: custody holds {} USDT, need {} USDT to honor a pending {} USDT withdrawal from {}. \
                                      This must be reviewed by the signatories before any approval - do NOT approve as-is.",
@@ -1462,9 +1503,9 @@ async fn process_withdrawal_block(
                             match convert_ss58_prefix(&from_str, 0) {
                                 Ok(polkadot_recipient) => {
                                     let now = Utc::now();
-                                    // Never attempt auto-pay against an already-known reserve shortfall -
+                                    // Never attempt auto-pay against an unverified or known-short reserve -
-                                    // that's exactly the case that must always go to human review.
+                                    // both cases must always go to human review.
-                                    let auto_pay_attempt = if reserve < net_amount {
+                                    let auto_pay_attempt = if !reserve_known || reserve < net_amount {
                                         None
                                     } else {
                                         match (from_bytes_opt, &automation.polkadot_signing_client, &automation.polkadot_keypair) {
@@ -1560,8 +1601,12 @@ async fn process_withdrawal_block(
                                                 "      3 of 5 signatories must approve: burn {} wUSDT, then Assets.transfer({}, {}, {}) on Polkadot",
                                                 amount as f64 / 1_000_000.0, config.polkadot_usdt_asset_id, polkadot_recipient, net_amount
                                             );
-                                            info!("      Reserve check at detection time: {} USDT available. Use pwap-web's multisig UI.",
+                                            if reserve_known {
-                                                reserve as f64 / 1_000_000.0);
+                                                info!("      Reserve check at detection time: {} USDT available. Use pwap-web's multisig UI.",
+                                                    reserve as f64 / 1_000_000.0);
+                                            } else {
+                                                info!("      Reserve could NOT be verified (Polkadot RPC unreachable at detection time) - signatories must check manually via pwap-web.");
+                                            }
 
                                             db.add_withdrawal(WithdrawalRecord {
                                                 id: db.next_withdrawal_id(),
@@ -1573,7 +1618,9 @@ async fn process_withdrawal_block(
                                                 fee,
                                                 net_amount,
                                                 polkadot_tx_hash: None,
-                                                status: if reserve < net_amount {
+                                                status: if !reserve_known {
+                                                    "pending_multisig_approval_reserve_unverified".to_string()
+                                                } else if reserve < net_amount {
                                                     "pending_multisig_approval_reserve_shortfall".to_string()
                                                 } else {
                                                     "pending_multisig_approval".to_string()
@@ -1822,7 +1869,7 @@ async fn main() -> Result<()> {
     let cli = Cli::parse();
 
     // Load or create config
-    let config: BridgeConfig = if cli.config.exists() {
+    let mut config: BridgeConfig = if cli.config.exists() {
         let content = std::fs::read_to_string(&cli.config)?;
         serde_json::from_str(&content)?
     } else {
@@ -1833,6 +1880,17 @@ async fn main() -> Result<()> {
         default_config
     };
 
+    // Dedicated/API-keyed RPC endpoints (e.g. a Dwellir URL) carry a credential in the URL itself,
+    // so - same reasoning as PEZBRIDGE_TELEGRAM_TOKEN/AUTOMATION_KEY_MNEMONIC below - they belong
+    // in an env var backed by a tightly-permissioned EnvironmentFile, not in the git-tracked JSON
+    // config. Unset means "use whatever bridge_config.json says" (the public endpoint by default).
+    if let Ok(url) = std::env::var("POLKADOT_RPC_URL") {
+        config.polkadot_rpc = url;
+    }
+    if let Ok(url) = std::env::var("PEZKUWI_RPC_URL") {
+        config.pezkuwi_rpc = url;
+    }
+
     match cli.command {
         Commands::GenerateWallet { output } => {
             generate_wallet(&output)?;