From 82657941503e2abf395448e560878ccee39b07be Mon Sep 17 00:00:00 2001 From: Satoshi Qazi Muhammed Date: Wed, 15 Jul 2026 21:44:48 -0700 Subject: [PATCH] Retry auto-pay once with a fresh connection before falling back to manual The automation key's signing clients (pezkuwi_signing_client/ polkadot_signing_client) are created once at startup and reused for every auto-pay call. Same failure class as the listener's live subscription (silent WS death, no error until the next real use, see LIVE_SUBSCRIPTION_STALL_TIMEOUT's comment): a real withdrawal auto-pay hit "Connection reset by peer... restart required" twice in a row today, sending both to manual 3-of-5 review even though the automation key's Polkadot approval had just been granted and should have covered them. Both execute_wusdt_auto_payout/execute_usdt_auto_payout call sites now retry once against a genuinely fresh client on failure before giving up - cheap for an occasional per-event call (unlike the listener's persistent subscription, no reason to avoid reconnecting every time). Falls back to manual review exactly as before if the retry also fails - never a new failure mode, just a narrower one. --- tools/usdt-bridge/src/main.rs | 37 +++++++++++++++++++++++++++++++++-- 1 file changed, 35 insertions(+), 2 deletions(-) diff --git a/tools/usdt-bridge/src/main.rs b/tools/usdt-bridge/src/main.rs index 83f1e705..0b30ff7f 100644 --- a/tools/usdt-bridge/src/main.rs +++ b/tools/usdt-bridge/src/main.rs @@ -1085,7 +1085,25 @@ async fn process_deposit_block( .map(Into::<[u8; 32]>::into) .map_err(|e| format!("bad custody_multisig_pezkuwi config: {e:?}")); match owner_bytes { - Ok(owner_bytes) => Some(execute_wusdt_auto_payout(client, keypair, config.pezkuwi_wusdt_asset_id, owner_bytes, dest_bytes, net_amount).await), + Ok(owner_bytes) => { + let first = execute_wusdt_auto_payout(client, keypair, config.pezkuwi_wusdt_asset_id, owner_bytes, dest_bytes, net_amount).await; + // A cached signing client's WS connection can die silently between + // uses (same failure class as the listener's live subscription - + // see LIVE_SUBSCRIPTION_STALL_TIMEOUT's doc) with no error until the + // next real use - retry once against a genuinely fresh connection + // before falling back to manual review, rather than requiring a + // full process restart to recover auto-pay. + let result = if first.is_err() { + warn!(" ⚠️ Auto-pay attempt failed ({:?}) - retrying once with a fresh connection", first.as_ref().err()); + match PezkuwiSigningClient::::from_insecure_url(&config.pezkuwi_rpc).await { + Ok(fresh_client) => execute_wusdt_auto_payout(&fresh_client, keypair, config.pezkuwi_wusdt_asset_id, owner_bytes, dest_bytes, net_amount).await, + Err(_) => first, + } + } else { + first + }; + Some(result) + } Err(reason) => { warn!(" ⚠️ Auto-pay skipped: {reason}"); None } } } @@ -1457,7 +1475,22 @@ async fn process_withdrawal_block( .map(Into::<[u8; 32]>::into) .map_err(|e| format!("bad custody_multisig_polkadot config: {e:?}")); match owner_bytes { - Ok(owner_bytes) => Some(execute_usdt_auto_payout(client, keypair, config.polkadot_usdt_asset_id, owner_bytes, dest_bytes, net_amount).await), + Ok(owner_bytes) => { + let first = execute_usdt_auto_payout(client, keypair, config.polkadot_usdt_asset_id, owner_bytes, dest_bytes, net_amount).await; + // Same reconnect-and-retry as the wUSDT leg - see its comment. This + // exact failure ("Connection reset by peer... restart required") + // was observed live twice on this leg before this fix. + let result = if first.is_err() { + warn!(" ⚠️ Auto-pay attempt failed ({:?}) - retrying once with a fresh connection", first.as_ref().err()); + match OnlineClient::::from_url(&config.polkadot_rpc).await { + Ok(fresh_client) => execute_usdt_auto_payout(&fresh_client, keypair, config.polkadot_usdt_asset_id, owner_bytes, dest_bytes, net_amount).await, + Err(_) => first, + } + } else { + first + }; + Some(result) + } Err(reason) => { warn!(" ⚠️ Auto-pay skipped: {reason}"); None } } }