From 177116279364fbd23e172173e03f85eb2b0cbd6d Mon Sep 17 00:00:00 2001 From: Satoshi Qazi Muhammed Date: Thu, 16 Jul 2026 08:51:23 -0700 Subject: [PATCH] usdt-bridge: never drop a withdrawal record on a transient RPC failure process_withdrawal_block's reserve/circuit-breaker check used to `continue` past a Polkadot connect failure without ever calling db.add_withdrawal() - a transient RPC error (confirmed live: a public-endpoint rate limit) meant an already-finalized Pezkuwi withdrawal was silently untracked, with no pending_multisig_approval entry for the signatories to ever see. It now always records the withdrawal; an unreachable reserve check is treated as "unverified" (never auto-paid, flagged distinctly from a confirmed shortfall) rather than as a reason to drop the record. Also add bounded exponential backoff (2s..32s, 6 attempts) to connect_to_chain instead of failing on the first error - every caller either bubbles the failure through a listener's outer reconnect loop or out of main() to a systemd restart, so a single blip was tearing down and fully re-initializing both listeners instead of just waiting it out. Finally, support POLKADOT_RPC_URL/PEZKUWI_RPC_URL env var overrides for the configured RPC endpoints, so a dedicated/API-keyed provider URL (e.g. Dwellir, adopted after the public Polkadot Asset Hub endpoint started rate-limiting us) can live in a tightly-permissioned EnvironmentFile instead of the git-tracked bridge_config.json - same pattern already used for PEZBRIDGE_TELEGRAM_TOKEN and AUTOMATION_KEY_MNEMONIC. --- tools/usdt-bridge/src/main.rs | 108 ++++++++++++++++++++++++++-------- 1 file changed, 83 insertions(+), 25 deletions(-) diff --git a/tools/usdt-bridge/src/main.rs b/tools/usdt-bridge/src/main.rs index 0b30ff7f..23a6a693 100644 --- a/tools/usdt-bridge/src/main.rs +++ b/tools/usdt-bridge/src/main.rs @@ -529,12 +529,38 @@ fn init_db(db_path: &PathBuf) -> Result<()> { // Chain Operations // ============================================================================ +/// Retries with exponential backoff (2s, 4s, 8s, 16s, 32s; capped, 6 attempts total) rather than +/// failing on the first transient error. This matters because every caller here either bubbles a +/// failure straight up through a `?` to the listener's outer reconnect loop (see listen_deposits/ +/// listen_withdrawals) or all the way out of main() to a systemd restart - a single blip (a rate +/// limit, a DNS hiccup) would otherwise tear down and fully re-initialize both listeners instead +/// of just waiting the blip out. Observed live 2026-07-16: the public Polkadot Asset Hub RPC +/// briefly 429'd, which without this retry crashed the whole process and then kept re-crashing +/// every RestartSec, hammering the same rate-limited endpoint harder instead of backing off. async fn connect_to_chain(url: &str) -> Result> { - info!("Connecting to {}...", url); - let client = OnlineClient::::from_url(url).await - .context(format!("Failed to connect to {}", url))?; - info!("Connected successfully"); - Ok(client) + const MAX_ATTEMPTS: u32 = 6; + let mut delay = Duration::from_secs(2); + + for attempt in 1..=MAX_ATTEMPTS { + info!("Connecting to {}... (attempt {attempt}/{MAX_ATTEMPTS})", url); + + match OnlineClient::::from_url(url).await { + Ok(client) => { + info!("Connected successfully"); + return Ok(client); + } + Err(e) if attempt < MAX_ATTEMPTS => { + warn!("Connect attempt {attempt}/{MAX_ATTEMPTS} to {url} failed: {e} - retrying in {}s", delay.as_secs()); + tokio::time::sleep(delay).await; + delay = (delay * 2).min(Duration::from_secs(60)); + } + Err(e) => { + return Err(e).context(format!("Failed to connect to {url} after {MAX_ATTEMPTS} attempts")); + } + } + } + + unreachable!("loop always returns via Ok or the final Err arm") } /// Get asset balance using dynamic API @@ -1433,21 +1459,36 @@ async fn process_withdrawal_block( // wallet actually holds. If this ever trips, the peg is already // under-collateralized (e.g. wUSDT minted without a matching real // deposit) - that must halt and page someone, not degrade silently. - let polkadot_client = match connect_to_chain(&config.polkadot_rpc).await { - Ok(c) => c, - Err(e) => { - error!(" ❌ Failed to connect to Polkadot: {}", e); - continue; - } + // + // Note this reserve check is best-effort only: the withdrawal itself + // is already finalized on Pezkuwi by the time we get here, so a + // transient RPC hiccup while trying to verify the Polkadot-side + // reserve must NEVER `continue` past recording it - that would drop + // a real, already-committed withdrawal on the floor with no + // pending_multisig_approval entry anywhere (exactly the silent-loss + // failure mode fixed for the subscription-stall bug). The signatories + // verify the real reserve themselves in pwap-web before approving, so + // an unverified reserve here just means "never auto-pay, always + // human-review" - it does not block recording the request. + let polkadot_client = connect_to_chain(&config.polkadot_rpc).await.ok(); + let reserve_known = polkadot_client.is_some(); + + let reserve = match &polkadot_client { + Some(client) => get_asset_balance( + client, + config.polkadot_usdt_asset_id, + polkadot_bridge_addr, + ).await.unwrap_or(0), + None => 0, }; - let reserve = get_asset_balance( - &polkadot_client, - config.polkadot_usdt_asset_id, - polkadot_bridge_addr, - ).await.unwrap_or(0); - - if reserve < net_amount { + if !reserve_known { + error!(" ❌ Failed to connect to Polkadot to verify reserve - recording withdrawal as pending_multisig_approval anyway (reserve NOT confirmed, signatories must check manually)"); + notify_telegram(config, &format!( + "⚠️ Could not verify Polkadot reserve for a {} USDT withdrawal from {} (RPC connect failed). Recorded as pending_multisig_approval - reserve NOT confirmed, signatories must check manually before approving.", + amount as f64 / 1_000_000.0, from_str + )).await; + } else if reserve < net_amount { let shortfall_msg = format!( "🚨 RESERVE SHORTFALL: custody holds {} USDT, need {} USDT to honor a pending {} USDT withdrawal from {}. \ This must be reviewed by the signatories before any approval - do NOT approve as-is.", @@ -1462,9 +1503,9 @@ async fn process_withdrawal_block( match convert_ss58_prefix(&from_str, 0) { Ok(polkadot_recipient) => { let now = Utc::now(); - // Never attempt auto-pay against an already-known reserve shortfall - - // that's exactly the case that must always go to human review. - let auto_pay_attempt = if reserve < net_amount { + // Never attempt auto-pay against an unverified or known-short reserve - + // both cases must always go to human review. + let auto_pay_attempt = if !reserve_known || reserve < net_amount { None } else { match (from_bytes_opt, &automation.polkadot_signing_client, &automation.polkadot_keypair) { @@ -1560,8 +1601,12 @@ async fn process_withdrawal_block( " 3 of 5 signatories must approve: burn {} wUSDT, then Assets.transfer({}, {}, {}) on Polkadot", amount as f64 / 1_000_000.0, config.polkadot_usdt_asset_id, polkadot_recipient, net_amount ); - info!(" Reserve check at detection time: {} USDT available. Use pwap-web's multisig UI.", - reserve as f64 / 1_000_000.0); + if reserve_known { + info!(" Reserve check at detection time: {} USDT available. Use pwap-web's multisig UI.", + reserve as f64 / 1_000_000.0); + } else { + info!(" Reserve could NOT be verified (Polkadot RPC unreachable at detection time) - signatories must check manually via pwap-web."); + } db.add_withdrawal(WithdrawalRecord { id: db.next_withdrawal_id(), @@ -1573,7 +1618,9 @@ async fn process_withdrawal_block( fee, net_amount, polkadot_tx_hash: None, - status: if reserve < net_amount { + status: if !reserve_known { + "pending_multisig_approval_reserve_unverified".to_string() + } else if reserve < net_amount { "pending_multisig_approval_reserve_shortfall".to_string() } else { "pending_multisig_approval".to_string() @@ -1822,7 +1869,7 @@ async fn main() -> Result<()> { let cli = Cli::parse(); // Load or create config - let config: BridgeConfig = if cli.config.exists() { + let mut config: BridgeConfig = if cli.config.exists() { let content = std::fs::read_to_string(&cli.config)?; serde_json::from_str(&content)? } else { @@ -1833,6 +1880,17 @@ async fn main() -> Result<()> { default_config }; + // Dedicated/API-keyed RPC endpoints (e.g. a Dwellir URL) carry a credential in the URL itself, + // so - same reasoning as PEZBRIDGE_TELEGRAM_TOKEN/AUTOMATION_KEY_MNEMONIC below - they belong + // in an env var backed by a tightly-permissioned EnvironmentFile, not in the git-tracked JSON + // config. Unset means "use whatever bridge_config.json says" (the public endpoint by default). + if let Ok(url) = std::env::var("POLKADOT_RPC_URL") { + config.polkadot_rpc = url; + } + if let Ok(url) = std::env::var("PEZKUWI_RPC_URL") { + config.pezkuwi_rpc = url; + } + match cli.command { Commands::GenerateWallet { output } => { generate_wallet(&output)?;