From 82543bdfab5b4ff89ac94a531b49278636254bc2 Mon Sep 17 00:00:00 2001 From: Satoshi Qazi Muhammed Date: Thu, 16 Jul 2026 12:39:13 -0700 Subject: [PATCH] usdt-bridge: record over-cap swaps, throttle backfill, gate routine Telegram noise Same silent-loss pattern as the previous connect-failure fix, found while auditing further: amount > max_single_tx (both deposit and withdrawal) warned and Telegram-alerted but never called db.add_deposit/add_withdrawal - "not auto-recorded" was literally in the log line. Now always recorded (pending_multisig_approval_amount_exceeds_cap), same as every other manual-review path. Backfill throttle: catching up many blocks after real downtime fires a tight burst of RPC calls that can trip a metered/shared endpoint's rate limit hard enough to poison the whole connection - observed live. A 150ms pause between backfilled blocks avoids it. Telegram notification threshold: at real user volume, a message for every small successful auto-pay would bury the ones that need a human (failures, over-cap, reserve problems, anything falling to manual 3-of-5) under noise. Added notify_threshold (default 500 USDT) gating ONLY the routine "auto-paid successfully" message on both legs - everything else keeps notifying unconditionally regardless of amount. Also found and fixed a real asymmetry: the withdrawal leg's "awaiting multisig approval" case never notified Telegram at all (the deposit leg always did) - a withdrawal needing manual review must always page someone. --- tools/usdt-bridge/Cargo.lock | 2 +- tools/usdt-bridge/src/main.rs | 99 ++++++++++++++++++++++++++++------- 2 files changed, 82 insertions(+), 19 deletions(-) diff --git a/tools/usdt-bridge/Cargo.lock b/tools/usdt-bridge/Cargo.lock index b7578a04..f465b363 100644 --- a/tools/usdt-bridge/Cargo.lock +++ b/tools/usdt-bridge/Cargo.lock @@ -3278,7 +3278,7 @@ dependencies = [ "once_cell", "socket2", "tracing", - "windows-sys 0.52.0", + "windows-sys 0.61.2", ] [[package]] diff --git a/tools/usdt-bridge/src/main.rs b/tools/usdt-bridge/src/main.rs index 23a6a693..635e9e71 100644 --- a/tools/usdt-bridge/src/main.rs +++ b/tools/usdt-bridge/src/main.rs @@ -135,12 +135,23 @@ pub struct BridgeConfig { /// Same, for the ops group chat. #[serde(default)] pub ops_chat_id: i64, + /// Below this, a SUCCESSFUL auto-pay stays silent on Telegram - at real user volume, a + /// message per small routine swap would bury the notifications that actually need a human's + /// attention (failures, amounts over max_single_tx, reserve problems, anything that fell to + /// manual 3-of-5 review) under noise. Those all notify unconditionally regardless of amount - + /// only the routine "everything worked, nothing to look at" case is gated by this. + #[serde(default = "default_notify_threshold")] + pub notify_threshold: u128, } fn default_max_single_tx() -> u128 { 50_000_000_000 // 50,000 USDT (6 decimals) - small/mid-size swaps clear without waiting on 3-of-5 } +fn default_notify_threshold() -> u128 { + 500_000_000 // 500 USDT (6 decimals) +} + fn default_multisig_polkadot() -> String { "15sF76THfpefUaKomHZSpssayRbsp6Yt6ESgMrLjzJCmpe66".to_string() } @@ -181,6 +192,7 @@ impl Default for BridgeConfig { auto_pay_daily_cap: default_auto_pay_daily_cap(), circuit_breaker_window_minutes: default_circuit_breaker_window_minutes(), circuit_breaker_fraction: default_circuit_breaker_fraction(), + notify_threshold: default_notify_threshold(), } } } @@ -1085,14 +1097,30 @@ async fn process_deposit_block( warn!(" ⚠️ Already processed ({}), skipping - refusing to mint twice", tx_hash); } else if amount > config.max_single_tx { warn!( - " ⚠️ Amount {} exceeds max_single_tx cap {} - leaving for manual multi-party review, NOT auto-processing", + " ⚠️ Amount {} exceeds max_single_tx cap {} - recording as pending_multisig_approval_amount_exceeds_cap, needs manual multi-party review", amount as f64 / 1_000_000.0, config.max_single_tx as f64 / 1_000_000.0 ); notify_telegram(config, &format!( - "📥 Deposit {} USDT from {} exceeds max_single_tx ({} USDT) - needs manual multi-party review, not auto-recorded.", + "📥 Deposit {} USDT from {} exceeds max_single_tx ({} USDT) - recorded, needs manual multi-party review.", amount as f64 / 1_000_000.0, from_str, config.max_single_tx as f64 / 1_000_000.0 )).await; + let fee = calculate_fee(amount, config.fee_basis_points); + let net_amount = amount - fee; + db.add_deposit(DepositRecord { + id: db.next_deposit_id(), + polkadot_tx_hash: tx_hash.clone(), + polkadot_block: block_number as u64, + sender_address: from_str.clone(), + amount, + fee, + net_amount, + pezkuwi_tx_hash: None, + status: "pending_multisig_approval_amount_exceeds_cap".to_string(), + created_at: Utc::now().to_rfc3339(), + processed_at: None, + }); + db.save(&config.db_path)?; } else if amount >= config.min_deposit { let fee = calculate_fee(amount, config.fee_basis_points); let net_amount = amount - fee; @@ -1158,7 +1186,7 @@ async fn process_deposit_block( db.save(&config.db_path)?; if let Some(reason) = breaker_tripped { notify_telegram(config, &format!("🚨 CIRCUIT BREAKER TRIPPED (deposit/wUSDT leg): {reason}. All further deposits now queue for manual 3-of-5 review until `reset-circuit-breaker` is run after human review.")).await; - } else { + } else if amount >= config.notify_threshold { notify_telegram(config, &format!( "✅ Auto-paid deposit: {} USDT from {} -> {} wUSDT (net of fee) to {}. tx {}", amount as f64 / 1_000_000.0, from_str, net_amount as f64 / 1_000_000.0, pezkuwi_recipient, pezkuwi_tx_hash @@ -1303,6 +1331,13 @@ async fn listen_deposits(config: &BridgeConfig, automation: &AutomationContext) info!("Backfilling Polkadot blocks {} to {head_number} before going live (resuming from checkpoint)", last_processed + 1); for n in (last_processed + 1)..=head_number { backfill_deposit_block(config, automation, &client, &legacy_rpc, n).await?; + // A public/shared RPC endpoint (or a metered provider's free tier) can rate-limit + // a tight burst of per-block calls hard enough to poison the whole connection + // ("background task closed... restart required"), killing this entire listener + // over what would otherwise be a routine catch-up after any real downtime - + // observed live 2026-07-16. A small per-block pause costs little against a large + // gap and avoids that failure mode entirely. + tokio::time::sleep(Duration::from_millis(150)).await; } head_number } else { @@ -1358,6 +1393,7 @@ async fn listen_deposits(config: &BridgeConfig, automation: &AutomationContext) )).await; for n in (last_seen + 1)..block_number { backfill_deposit_block(config, automation, &client, &legacy_rpc, n).await?; + tokio::time::sleep(Duration::from_millis(150)).await; } } @@ -1434,19 +1470,6 @@ async fn process_withdrawal_block( continue; } - if amount > config.max_single_tx { - warn!( - " ⚠️ Amount {} exceeds max_single_tx cap {} - leaving for manual multi-party review, NOT auto-processing", - amount as f64 / 1_000_000.0, - config.max_single_tx as f64 / 1_000_000.0 - ); - notify_telegram(config, &format!( - "📤 Withdrawal {} USDT from {} exceeds max_single_tx ({} USDT) - needs manual multi-party review, not auto-recorded.", - amount as f64 / 1_000_000.0, from_str, config.max_single_tx as f64 / 1_000_000.0 - )).await; - continue; - } - if amount < config.min_withdraw { warn!(" ⚠️ Amount below minimum, skipping"); continue; @@ -1455,6 +1478,35 @@ async fn process_withdrawal_block( let fee = calculate_fee(amount, config.fee_basis_points); let net_amount = amount - fee; + if amount > config.max_single_tx { + warn!( + " ⚠️ Amount {} exceeds max_single_tx cap {} - recording as pending_multisig_approval_amount_exceeds_cap, needs manual multi-party review", + amount as f64 / 1_000_000.0, + config.max_single_tx as f64 / 1_000_000.0 + ); + notify_telegram(config, &format!( + "📤 Withdrawal {} USDT from {} exceeds max_single_tx ({} USDT) - recorded, needs manual multi-party review.", + amount as f64 / 1_000_000.0, from_str, config.max_single_tx as f64 / 1_000_000.0 + )).await; + let destination_address = convert_ss58_prefix(&from_str, 0).unwrap_or_default(); + db.add_withdrawal(WithdrawalRecord { + id: db.next_withdrawal_id(), + pezkuwi_tx_hash: tx_hash.clone(), + pezkuwi_block: block_number as u64, + sender_address: from_str.clone(), + destination_address, + amount, + fee, + net_amount, + polkadot_tx_hash: None, + status: "pending_multisig_approval_amount_exceeds_cap".to_string(), + created_at: Utc::now().to_rfc3339(), + processed_at: None, + }); + db.save(&config.db_path)?; + continue; + } + // Circuit breaker: never release more real USDT than the custody // wallet actually holds. If this ever trips, the peg is already // under-collateralized (e.g. wUSDT minted without a matching real @@ -1568,7 +1620,7 @@ async fn process_withdrawal_block( db.save(&config.db_path)?; if let Some(reason) = breaker_tripped { notify_telegram(config, &format!("🚨 CIRCUIT BREAKER TRIPPED (withdrawal/USDT leg): {reason}. All further withdrawals now queue for manual 3-of-5 review until `reset-circuit-breaker` is run after human review.")).await; - } else { + } else if amount >= config.notify_threshold { notify_telegram(config, &format!( "✅ Auto-paid withdrawal: {} wUSDT from {} -> {} USDT (net of fee) to {}. tx {}. wUSDT burn still pending - batch via pwap-web when convenient.", amount as f64 / 1_000_000.0, from_str, net_amount as f64 / 1_000_000.0, polkadot_recipient, polkadot_tx_hash @@ -1613,7 +1665,7 @@ async fn process_withdrawal_block( pezkuwi_tx_hash: tx_hash.clone(), pezkuwi_block: block_number as u64, sender_address: from_str.clone(), - destination_address: polkadot_recipient, + destination_address: polkadot_recipient.clone(), amount, fee, net_amount, @@ -1629,6 +1681,17 @@ async fn process_withdrawal_block( processed_at: None, }); db.save(&config.db_path)?; + + // Unlike the two branches above (reserve_unverified/reserve_shortfall already + // notify further up), the plain "automation not configured/eligible" case had no + // notification at all until now - a withdrawal needing manual 3-of-5 review must + // always page someone, the same as the deposit leg already does. + if reserve_known && reserve >= net_amount { + notify_telegram(config, &format!( + "📤 New withdrawal pending approval: {} USDT from {} -> release {} USDT (net of fee) to {}. Sign at pwap-web's /multisig/pending.", + amount as f64 / 1_000_000.0, from_str, net_amount as f64 / 1_000_000.0, polkadot_recipient + )).await; + } } } }