Retry auto-pay once with a fresh connection before falling back to manual

The automation key's signing clients (pezkuwi_signing_client/
polkadot_signing_client) are created once at startup and reused for every
auto-pay call. Same failure class as the listener's live subscription
(silent WS death, no error until the next real use, see
LIVE_SUBSCRIPTION_STALL_TIMEOUT's comment): a real withdrawal auto-pay hit
"Connection reset by peer... restart required" twice in a row today,
sending both to manual 3-of-5 review even though the automation key's
Polkadot approval had just been granted and should have covered them.

Both execute_wusdt_auto_payout/execute_usdt_auto_payout call sites now
retry once against a genuinely fresh client on failure before giving up -
cheap for an occasional per-event call (unlike the listener's persistent
subscription, no reason to avoid reconnecting every time). Falls back to
manual review exactly as before if the retry also fails - never a new
failure mode, just a narrower one.
This commit is contained in:
Satoshi Qazi Muhammed committed 2026-07-15 21:44:48 -07:00
1 parent f02f831edf
commit 8265794150
1 file changed
+35 -2
+35 -2
View File
@@ -1085,7 +1085,25 @@ async fn process_deposit_block(
.map(Into::<[u8; 32]>::into) .map(Into::<[u8; 32]>::into)
.map_err(|e| format!("bad custody_multisig_pezkuwi config: {e:?}")); .map_err(|e| format!("bad custody_multisig_pezkuwi config: {e:?}"));
match owner_bytes { match owner_bytes {
Ok(owner_bytes) => Some(execute_wusdt_auto_payout(client, keypair, config.pezkuwi_wusdt_asset_id, owner_bytes, dest_bytes, net_amount).await), Ok(owner_bytes) => {
let first = execute_wusdt_auto_payout(client, keypair, config.pezkuwi_wusdt_asset_id, owner_bytes, dest_bytes, net_amount).await;
// A cached signing client's WS connection can die silently between
// uses (same failure class as the listener's live subscription -
// see LIVE_SUBSCRIPTION_STALL_TIMEOUT's doc) with no error until the
// next real use - retry once against a genuinely fresh connection
// before falling back to manual review, rather than requiring a
// full process restart to recover auto-pay.
let result = if first.is_err() {
warn!(" ⚠️ Auto-pay attempt failed ({:?}) - retrying once with a fresh connection", first.as_ref().err());
match PezkuwiSigningClient::<PezkuwiConfig>::from_insecure_url(&config.pezkuwi_rpc).await {
Ok(fresh_client) => execute_wusdt_auto_payout(&fresh_client, keypair, config.pezkuwi_wusdt_asset_id, owner_bytes, dest_bytes, net_amount).await,
Err(_) => first,
}
} else {
first
};
Some(result)
}
Err(reason) => { warn!(" ⚠️ Auto-pay skipped: {reason}"); None } Err(reason) => { warn!(" ⚠️ Auto-pay skipped: {reason}"); None }
} }
} }
@@ -1457,7 +1475,22 @@ async fn process_withdrawal_block(
.map(Into::<[u8; 32]>::into) .map(Into::<[u8; 32]>::into)
.map_err(|e| format!("bad custody_multisig_polkadot config: {e:?}")); .map_err(|e| format!("bad custody_multisig_polkadot config: {e:?}"));
match owner_bytes { match owner_bytes {
Ok(owner_bytes) => Some(execute_usdt_auto_payout(client, keypair, config.polkadot_usdt_asset_id, owner_bytes, dest_bytes, net_amount).await), Ok(owner_bytes) => {
let first = execute_usdt_auto_payout(client, keypair, config.polkadot_usdt_asset_id, owner_bytes, dest_bytes, net_amount).await;
// Same reconnect-and-retry as the wUSDT leg - see its comment. This
// exact failure ("Connection reset by peer... restart required")
// was observed live twice on this leg before this fix.
let result = if first.is_err() {
warn!(" ⚠️ Auto-pay attempt failed ({:?}) - retrying once with a fresh connection", first.as_ref().err());
match OnlineClient::<SubstrateConfig>::from_url(&config.polkadot_rpc).await {
Ok(fresh_client) => execute_usdt_auto_payout(&fresh_client, keypair, config.polkadot_usdt_asset_id, owner_bytes, dest_bytes, net_amount).await,
Err(_) => first,
}
} else {
first
};
Some(result)
}
Err(reason) => { warn!(" ⚠️ Auto-pay skipped: {reason}"); None } Err(reason) => { warn!(" ⚠️ Auto-pay skipped: {reason}"); None }
} }
} }